Python大数据：jieba 中文分词，词频统计

# -*- coding: UTF-8 -*-

import sys

import numpy as np

import pandas as pd

import jieba

import jieba.analyse

import codecs

#设置pd的显示长度

pd.set_option('max_colwidth',500)

#载入数据

rows=pd.read_csv('datas1.csv', header=0,encoding='utf-8',dtype=str)

#载入停用词

jieba.analyse.set_stop_words('stoped.txt')

# 保存全局分词，用于词频统计

segments = []

# 保存每行分词，用于关联分析

results = []

index = 0

for row in rows.index:

    content = rows[index:index+1]['content'].to_string()

    # 分词操作

    # words = jieba.cut(content)

    # TF-IDF关键词抽取，关键词数量设置为topK，不能过滤标点和数字

    #words = jieba.analyse.extract_tags(content, topK=20)

    #TextRank 关键词抽取，只获取固定词性

    words = jieba.analyse.textrank(content, topK=20,withWeight=False,allowPOS=('ns', 'n', 'vn', 'v'))

    splitedStr = ''

    for word in words:

        # 记录全局分词

        segments.append({'word':word, 'count':1})

        splitedStr += word + ' '

    # 记录行结果

    results.append({'text':content, 'words': splitedStr})

    index = index + 1

# 将结果数组转为df序列

dfSg = pd.DataFrame(segments)

# 词频统计

dfWord = dfSg.groupby('word')['count'].sum()

#导出csv

dfWord.to_csv('keywords.csv',encoding='utf-8')

dfRet = pd.DataFrame(results)

dfRet.to_csv('result.csv',encoding='utf-8')
秒客网

Python大数据：jieba 中文分词，词频统计

相关文章
