19-python 自己建立词库并实现文章汉语词频统计

首先在网上下载一个汉语词典的txt文件，汉语词典

1.用正则去掉词语的解释，即提取出所有汉语词语；

import re

def getHanYuCi(st):

   p = re.compile(r'【.*?】') # 挑选出: [汉字]

   rt = p.findall(st)

   p = re.compile(r'[\u4E00-\u9FA5]+') # 去掉【】:只保留汉字;  *:前一个字符0次或无限次; +:表示1次或无限制

   rt = p.findall(str(rt))

   #print(str[0:1000])

   return rt

def test_1():

    path = r'C:\Users\sss\Desktop\hanyucidian.txt'

    with open(path, 'rb') as f:

        st = f.read().decode('gb18030')

    rt = getHanYuCi(st)

    dict = {}

    for x in rt:

        dict[x] = 0

    #print('+++++++++++++')

    #print(len(rt))

    #print(rt) #  由于rt特别大，直接print不会读出任何东西,但只读某一段时可以读出来

    #print(rt[1:10])

    path = r'C:\Users\sss\Desktop\hanyu_ci.txt'

    with open(path, 'w') as f:

        f.write(str(dict))

test_1()