语言数据管理
import nltk
phonetic = nltk.corpus.timit.phones('dr1-fvmh0/sa1')
print(phonetic)
res = nltk.corpus.timit.word_times('dr1-fvmh0/sa1')
print(res)
timitdict = nltk.corpus.timit.transcription_dict()
res = timitdict['greasy'] + timitdict['wash'] + timitdict['water']
print(res)
print(phonetic[17:30])
res = nltk.corpus.timit.spkrinfo('dr1-fvmh0')
print(res)
s1 = "00000010000000001000000"
s2 = "00000001000000010000000"
s3 = "00001000000000000001000"
res = nltk.windowdiff(s1, s1, 3)
print(res)
res = nltk.windowdiff(s1, s2, 3)
print(res)
res = nltk.windowdiff(s2, s3, 3)
print(res)
"""dict.htm.html
<p class=MsoNormal>sleep
<span style='mso-spacerun:yes'></span>
[<span class=S;ellE>sli:p</span>]
<span style='mso-spancerun:yes'></span>
<b><span style='font-size:11.0pt'>v.i</span></b>
<span style='mso-spacerun:yes'></span>
<i>a condition of body and mind ...<o:p></o:p></i>
</p>
"""
import re
legal_pos = set(['n', 'v.t.', 'v.i.', 'adj', 'det'])
pattern = re.compile(r"'font-size:11.0pt'>([a-z.]+)<")
document = open('dict.htm.html').read()
used_pos = set(re.findall(pattern, document))
illegal_pos = used_pos.difference(legal_pos)
print(list(illegal_pos))
import bs4, lxml
def lexical_data(html_file):
SEP = '_INTRY'
html = open(html_file).read()
html = re.sub(r'<p', SEP + '<p', html)
text = bs4.BeautifulSoup(html, "lxml").get_text()
text = ' '.join(text.split())
for entry in text.split(SEP):
if entry.count(' ') > 2:
yield entry.split(' ', 3)
import csv
writer = csv.writer(open("dict1.csv", "w"))
writer.writerows(lexical_data("dict.htm.html"))
"""
# dict.csv
"sleep", "sli:p", "v.i", "a condition of body and mind ..."
"walk", "wo:k", "v.intr", "progress by lifting and setting down each foot ..."
"wake", "weik", "intrans", "cease to sleep"
"""
import csv
lexicon = csv.reader(open("dict.csv"))
pairs = [(lexeme, defn) for (lexeme, _, _, defn) in lexicon]
lexemes, defns = zip(*pairs)
defn_words = set(w for defn in defns for w in defn.split())
res = sorted(defn_words.difference(lexemes))
print(res)
"""
['"a', '"cease', '"progress', '..."', 'and', 'body', 'by', 'condition', 'down',
'each', 'foot', 'lifting', 'mind', 'of', 'setting', 'sleep"', 'to']
"""
idx = nltk.Index((defn_word, lexeme)
for (lexeme, defn) in pairs
for defn_word in nltk.word_tokenize(defn)
if len(defn_word) > 3)
idx_file = open("dict.idx", "w")
for word in sorted(idx):
idx_words = ', '.join(idx[word])
idx_line = "%s: %s\n" % (word, idx_words)
idx_file.write(idx_line)
idx_file.close()
"""dict.idx
body: sleep
cease: wake
condition: sleep
down: walk
each: walk
foot: walk
lifting: walk
mind: sleep
progress: walk
setting: walk
sleep: wake
"""
mappings = [('ph', 'f'), ('ght', 't'), ('^kn', 'n'), ('qu', 'kw'),
('[aeiou]+', 'a'), (r'(.)\1', r'\1')]
def signature(word):
for patt, repl in mappings:
word = re.sub(patt, repl, word)
pieces = re.findall('[aeiou]+', word)
return ''.join(char for piece in pieces for char in sorted(piece))[:8]
print(signature('illefent'))
print(signature('ebsekwieous'))
print(signature('nuculerr'))
signatures = nltk.Index((signature(w), w) for w in nltk.corpus.words.words())
res = signatures[signature('nuculerr')]
print(res)
from nltk.metrics import edit_distance
def rank(word, wordlist):
ranked = sorted((edit_distance(word, w), w) for w in wordlist)
return [word for (_, word) in ranked]
def fuzzy_spell(word):
sig = signature(word)
if sig in signatures:
return rank(word, signatures[sig])
else:
return []
print(fuzzy_spell('illefent'))
print(fuzzy_spell('ebsekwieous'))
print(fuzzy_spell('nucular'))
