NLTK11《Python自然语言处理》code10 分析语句的含义

分析语句的含义

# -*- coding: utf-8 -*-
# win10 python3.5.3/python3.6.1 nltk3.2.4
# 《Python自然语言处理》 10 分析语句的含义
# pnlp10.py
# 更官方的参考 http://www.nltk.org/book/ch10.html

# 需要特别关注prover9的安装
# pip3 install prove
# http://www.cs.unm.edu/~mccune/prover9/gui/Prover9-Mace4-v05-setup.exe
"""
 # C:\Python36\Lib\site-packages\nltk\inference\prover9.py
 def binary_locations(self):
 return ['/usr/local/bin/prover9',
 '/usr/local/bin/prover9/bin',
 '/usr/local/bin',
 '/usr/bin',
 '/usr/local/prover9',
 '/usr/local/share/prover9',
 'C:/Program Files (x86)/Prover9-Mace4/bin-win32'] # 添加了这个路径，特注
"""

import nltk
# 10.1 自然语言理解
# 查询数据库
# nltk.data.show_cfg('grammars/book_grammars/sql0.fcfg')
from nltk import load_parser
cp = load_parser('grammars/book_grammars/sql0.fcfg')
query = 'What cities are located in China'
for tree in cp.parse(query.split()):
    print(tree)
"""
(S[SEM=(SELECT, City FROM city_table, WHERE, , , Country="china")]
 (NP[SEM=(SELECT, City FROM city_table)]
 (Det[SEM='SELECT'] What)
 (N[SEM='City FROM city_table'] cities))
 (VP[SEM=(, , Country="china")]
 (IV[SEM=''] are)
 (AP[SEM=(, Country="china")]
 (A[SEM=''] located)
 (PP[SEM=(, Country="china")]
 (P[SEM=''] in)
 (NP[SEM='Country="china"'] China)))))
"""
#trees = cp.parse(query.split())
trees = next(cp.parse(query.split()))
answer = trees[0].label()
q = answer['SEM']
print(q) # (SELECT, City FROM city_table)

from nltk.sem import chat80
q = "select city from city_table where country='china'"
rows = chat80.sql_query('corpora/city_database/city.db', q)
for r in rows: print(r[0], end=" ")
"""canton chungking dairen harbin kowloon mukden peking shanghai sian tientsin """

# 自然语言、语义和逻辑

# 10.2 命题逻辑
nltk.boolean_ops()
"""
conjunction &
disjunction |
implication ->
equivalence <->
"""

lp = nltk.sem.Expression.fromstring('-(P & Q)')
print(lp) # -(P & Q)

# 这个实验没有找到Prover9，没安装prover9
prover = nltk.Prover9()
NotFnS = nltk.sem.Expression.fromstring('-FnS')
SnF = nltk.sem.Expression.fromstring('SnF')
R = nltk.sem.Expression.fromstring('SnF -> -FnS')
res = prover.prove(NotFnS, [SnF, R])
print(res) # True

val = nltk.Valuation([('P', True), ('Q', True), ('R', False)])
print(val['P']) # True
dom = set([])
g = nltk.Assignment(dom)
m = nltk.Model(dom, val)
print(m.evaluate('(P & Q)', g)) # True
print(m.evaluate('-(P & Q)', g)) # False
print(m.evaluate('(P & R)', g)) # False
print(m.evaluate('(P | R)', g)) # True

# 10.3 一阶逻辑
#tlp = nltk.LogicParser()
#parsed = tlp.parse('walk(angus)')
parsed = nltk.sem.Expression.fromstring('walk(angus)', type_check=True)
print(parsed.argument) # angus
print(parsed.argument.type) # e
print(parsed.function) # walk

sig = {'walk': '<e, t>'}
parsed = nltk.sem.Expression.fromstring('walk(angus)', type_check=True, signature=sig)
print(parsed.function.type) # <e,t>

lp = nltk.sem.Expression.fromstring('dog(cyril)')
lp.free()
print(lp) # dog(cyril)

lp = nltk.sem.Expression.fromstring('dog(cyril)')
lp = lp.free()
print(lp) # set()

lp = nltk.sem.Expression.fromstring('dog(cyril)').free()
print(lp) # set()

lp = nltk.sem.Expression.fromstring('dog(x)').free()
print(lp) # {Variable('x')}

lp = nltk.sem.Expression.fromstring('own(angus, cyril)').free()
print(lp) # set()

lp = nltk.sem.Expression.fromstring('exists x.dog(x)').free()
print(lp) # set()

lp = nltk.sem.Expression.fromstring('((some x. walk(x)) -> sing(x))').free()
print(lp) # {Variable('x')}

lp = nltk.sem.Expression.fromstring('exists x.own(y, x)').free()
print(lp) # {Variable('y')}

# 一阶定理证明
NotFnS = nltk.sem.Expression.fromstring('-north_of(f, s)')
SnF = nltk.sem.Expression.fromstring('north_of(s, f)')
R = nltk.sem.Expression.fromstring('all x. all y. (north_of(x, y) -> -north_of(y, x))')
prover = nltk.Prover9()
prover.prove(NotFnS, [SnF, R])
FnS = nltk.sem.Expression.fromstring('north_of(f, s)')
res = prover.prove(FnS, [SnF, R])
print(res) # False

# 一阶逻辑语言总结
# 真值模型
dom = set(['b', 'o', 'c'])
v = """
bertie => b
olive => o
cyril => c
boy => {b}
girl => {o}
dog => {c}
walk => {o, c}
see => {(b, o), (c, b), (o, c)}
"""
val = nltk.sem.Valuation.fromstring(v)
print(val)
"""
{'bertie': 'b',
 'boy': {('b',)},
 'cyril': 'c',
 'dog': {('c',)},
 'girl': {('o',)},
 'olive': 'o',
 'see': {('o', 'c'), ('b', 'o'), ('c', 'b')},
 'walk': {('c',), ('o',)}}
 """

print(('o', 'c') in val['see']) # True
print(('b',) in val['boy']) # True

# 独立变量和赋值
g = nltk.Assignment(dom, [('x', 'o'), ('y', 'c')])
print(g) # g[c/y][o/x]
m = nltk.Model(dom, val)
res = m.evaluate('see(olive, y)', g)
print(res) # True
print(g['y']) # c
g.purge()
print(g) # g
print(m.evaluate('see(olive, y)', g)) # Undefined

res = m.evaluate('see(bertie, olive) & boy(bertie) & -walk(bertie)', g)
print(res) # True

# 量化
res = m.evaluate('exists x.(girl(x) & walk(x))', g)
print(res) # True

res = m.evaluate('girl(x) & walk(x)', g.add('x', 'o'))
print(res) # True

fmla1 = nltk.sem.Expression.fromstring('girl(x) | boy(x)')
res = m.satisfiers(fmla1, 'x', g)
print(res) # {'o', 'b'}
fmla2 = nltk.sem.Expression.fromstring('girl(x) -> walk(x)')
res = m.satisfiers(fmla2, 'x', g)
print(res) # {'o', 'b', 'c'}
fmla3 = nltk.sem.Expression.fromstring('walk(x) -> girl(x)')
res = m.satisfiers(fmla3, 'x', g)
print(res) # {'o', 'b'}

res = m.evaluate('all x.(girl(x) -> walk(x))', g)
print(res) # True

# 量词范围歧义
v2 = """
bruce => b
cyril => c
elspeth => e
julia => j
matthew => m
person => {b, e, j, m}
admire => {(j, b), (b, b), (m, e), (e, m), (c, a)}
"""
val2 = nltk.sem.Valuation.fromstring(v2)
dom2 = val2.domain
m2 = nltk.Model(dom2, val2)
g2 = nltk.Assignment(dom2)
fmla4 = nltk.sem.Expression.fromstring('(person(x) -> exists y. (person(y) & admire(x, y)))')
res = m2.satisfiers(fmla4, 'x', g2)
print(res) # {'a', 'm', 'j', 'e', 'b', 'c'}
fmla5 = nltk.sem.Expression.fromstring('(person(y) & all x.(person(x) -> admire(x, y)))')
res = m2.satisfiers(fmla5, 'y', g2)
print(res) # set()
fmla6 = nltk.sem.Expression.fromstring('(person(y) & all x.((x=bruce | x= julia) -> admire(x, y)))')
res = m2.satisfiers(fmla6, 'y', g2)
print(res) # {'b'}

# 模型的建立
a3 = nltk.sem.Expression.fromstring('exists x.(man(x) & walks(x))')
c1 = nltk.sem.Expression.fromstring('mortal(socrates)')
c2 = nltk.sem.Expression.fromstring('-mortal(socrates)')
mb = nltk.Mace(5)
print(mb.build_model(None, [a3, c1])) # True
print(mb.build_model(None, [a3, c2])) # True
print(mb.build_model(None, [c1, c2])) # False

a4 = nltk.sem.Expression.fromstring('exists y. (woman(y) & all x. (man(x) -> love(x, y)))')
a5 = nltk.sem.Expression.fromstring('man(adam)')
a6 = nltk.sem.Expression.fromstring('woman(eve)')
g = nltk.sem.Expression.fromstring('love(adam, eve)')
mc = nltk.MaceCommand(g, assumptions=[a4, a5, a6])
res = mc.build_model()
print(res) # True

print(mc.valuation)
"""
{'C1': 'b',
 'adam': 'a',
 'eve': 'a',
 'love': {('a', 'b')},
 'man': {('a',)},
 'woman': {('b',), ('a',)}}
"""

a7 = nltk.sem.Expression.fromstring('all x. (max(x) -> woman(x))')
g = nltk.sem.Expression.fromstring('love(adam, eve)')
mc = nltk.MaceCommand(g, assumptions=[a4, a5, a6, a7])
res = mc.build_model()
print(res) # True
print(mc.valuation)
"""
{'C1': 'b',
 'adam': 'a',
 'eve': 'a',
 'love': {('a', 'b')},
 'man': {('a',)},
 'max': set(),
 'woman': {('a',), ('b',)}}
"""

# 10.4 英语语句的语义
lp = nltk.sem.Expression.fromstring(r'\x.(walk(x) & chew_gum(x))')
print(lp) # \x.(walk(x) & chew_gum(x))
print(lp.free()) # set()
lp = nltk.sem.Expression.fromstring(r'\x.(walk(x) & chew_gum(y))')
print(lp) # \x.(walk(x) & chew_gum(y))

lp = nltk.sem.Expression.fromstring(r'\x.(walk(x) & chew_gum(x))(gerald)')
print(lp) # \x.(walk(x) & chew_gum(x))(gerald)
print(lp.simplify()) # (walk(gerald) & chew_gum(gerald))

lp = nltk.sem.Expression.fromstring(r'\x.\y.(dog(x) & own(y, x))(cyril)').simplify()
print(lp) # \y.(dog(cyril) & own(y,cyril))
lp = nltk.sem.Expression.fromstring(r'\x y.(dog(x) & own(y, x))(cyril, angus)').simplify()
print(lp) # (dog(cyril) & own(angus,cyril))

e1 = nltk.sem.Expression.fromstring('exists x.P(x)')
print(e1)
e2 = e1.alpha_convert(nltk.Variable('z'))
print(e2) # exists z.P(z)
print(e1 == e2) # True

e3 = nltk.sem.Expression.fromstring('\P.exists x.P(x)(\y.see(y, x))')
print(e3) # \P.exists x.P(x,\y.see(y,x))
print(e3.simplify()) # \P.exists x.P(x,\y.see(y,x)) # 注意，和原书说明不同

# 量化的NP
# 及物动词
lp = nltk.sem.Expression.fromstring(r'\X x.X(\y.chase(x,y))')
np = nltk.sem.Expression.fromstring(r'(\P.exists x.(dog(x) & P(x)))')
vp = nltk.ApplicationExpression(lp, np)
print(vp) # (\X x.X(\y.chase(x,y)))(\P.exists x.(dog(x) & P(x)))
print(vp.simplify()) # \x.exists z1.(dog(z1) & chase(x,z1))

from nltk import load_parser
parser = load_parser('grammars/book_grammars/simple-sem.fcfg', trace=0)
sentence = 'Angus gives a bone to every dog'
tokens = sentence.split()
trees = parser.parse(tokens)
for tree in trees:
    print(tree.label()['SEM'])
"""all z3.(dog(z3) -> exists z2.(bone(z2) & give(angus,z2,z3)))"""

v = """
bertie => b
olive => o
cyril => c
boy => {b}
girl => {o}
dog => {c}
walk => {o, c}
see => {(b, o), (c, b), (o, c)}
"""
val = val2 = nltk.sem.Valuation.fromstring(v)
g = nltk.Assignment(val.domain)
m = nltk.Model(val.domain, val)
sent = 'Cyril sees every boy'
grammar_file = 'grammars/book_grammars/simple-sem.fcfg'
#results = nltk.batch_evaluate([sent], grammar_file, m, g)[0]
results = nltk.evaluate_sents([sent], grammar_file, m, g)[0]
for (syntree, semrel, value) in results:
    print(semrel)
    print(value)
"""
all z4.(boy(z4) -> see(cyril,z4))
True
"""

# 重述量词歧义
from nltk.sem import cooper_storage as cs
sentence = 'every girl chases a dog'
trees = cs.parse_with_bindops(sentence, grammar='grammars/book_grammars/storage.fcfg')
semrep = trees[0].label()['SEM']
cs_semrep = cs.CooperStore(semrep)
print(cs_semrep.core) # chase(z2,z3)
for bo in cs_semrep.store:
    print(bo)
"""
bo(\P.all x.(girl(x) -> P(x)),z2)
bo(\P.exists x.(dog(x) & P(x)),z3)
"""

res = cs_semrep.s_retrieve(trace=True)
print(res)
"""
Permutation 1
 (\P.all x.(girl(x) -> P(x)))(\z2.chase(z2,z3))
 (\P.exists x.(dog(x) & P(x)))(\z3.all x.(girl(x) -> chase(x,z3)))
Permutation 2
 (\P.exists x.(dog(x) & P(x)))(\z3.chase(z2,z3))
 (\P.all x.(girl(x) -> P(x)))(\z2.exists x.(dog(x) & chase(z2,x)))
None
"""
for reading in cs_semrep.readings:
    print(reading)
"""
exists x.(dog(x) & all z7.(girl(z7) -> chase(z7,x)))
all x.(girl(x) -> exists z8.(dog(z8) & chase(x,z8)))
"""

# 10.5 段落语义层
# 段落表示理论
dp = nltk.sem.DrtExpression.fromstring('([x, y], [angus(x), dog(y), own(x, y)])')
print(dp) # ([x,y],[angus(x), dog(y), own(x,y)])
dp.draw()

print(dp.fol()) # exists x y.(angus(x) & dog(y) & own(x,y))

"""
# 此处有错误
drs2 = nltk.sem.Expression.fromstring('([x], [walk(x)]) + ([y], [run(y)])') # 有错误
print(drs2)
print(drs2.simplify())

drs3 = nltk.sem.Expression.fromstring('([], [(([x], [dog(x)]) -> ([y], [ankle(y), bite(x, y)]))])')
print(drs3.fol())

drs4 = nltk.sem.Expression.fromstring('([x, y], [angus(x), dog(y), own(x, y)])')
drs5 = nltk.sem.Expression.fromstring('([u, z], [PRO(u), irene(z), bite(u, z)])')
drs6 = drs4 + drs5
print(drs6.simplify())
"""

from nltk import load_parser
parser = load_parser('grammars/book_grammars/drt.fcfg', logic_parser=nltk.sem.drt.DrtParser())
trees = parser.parse('Angus owns a dog'.split())
#for tree in trees: print(tree)
tree = next(trees)
print(tree[0].label()['SEM']) # \P.(([x],[Angus(x)]) + P(x))
print(tree[0].label()['SEM'].simplify()) # \P.(([x],[Angus(x)]) + P(x))

# 段落处理
dt = nltk.DiscourseTester(['A student dances', 'Every student is a person'])
dt.readings()
"""
s0 readings:

s0-r0: exists z1.(student(z1) & dance(z1))

s1 readings:

s1-r0: all z1.(student(z1) -> person(z1))
"""

dt.add_sentence('No person dances', consistchk=True)
"""
Inconsistent discourse: d0 ['s0-r0', 's1-r0', 's2-r0']:
 s0-r0: exists z1.(student(z1) & dance(z1))
 s1-r0: all z1.(student(z1) -> person(z1))
 s2-r0: -exists z1.(person(z1) & dance(z1))
"""

dt.retract_sentence('No person dances', verbose=True)
"""
Current sentences are 
s0: A student dances
s1: Every student is a person
"""

dt.add_sentence('A person dances', 'informchk=True')
"""
Sentence 'A person dances' under reading 'exists x.(person(x) & dance(x))':
Not informative relative to thread 'd0'
"""

"""
# 此处有错误未处理
tagger = nltk.tag.RegexpTagger(
[('^(chases|runs)$', 'VB'),
('^(a)$', 'ex_guant'),
('^(every)$', 'univ_quant'),
('^(dog|boy)$', 'NN'),
('^(He)$', 'PRP')])
# 使用nltk.MaltParser需要下载maltparser的jar包，并正确指定路径parser_dirname
# http://www.maltparser.org/download.html
# http://maltparser.org/dist/maltparser-1.9.1.tar.gz
# http://maltparser.org/dist/maltparser-1.9.1.zip
rc = nltk.DrtGlueReadingCommand(depparser=nltk.MaltParser(
 parser_dirname="D:/nltk_data/maltparser-1.9.1", tagger=tagger))
dt = nltk.DiscourseTester(['Every dog chases a boy', 'He runs'], rc)
#dt.readings() # 出错
dt.readings(show_thread_readings=True, filter=True) # 出错
"""