liblinear是libsvm的线性核的改进版本，专门适用于百万数据量的分类。正好适用于我这次数据挖掘的实验。
liblinear用法和libsvm很相似，我是用的是.exe文件，利用python的subprocess向控制台发送命令即可完成本次试验。
其中核心两句即
train train.txt
predict test.txt train.txt.model output.txt
由于是线性核，没有设置参数c、g
对于50W篇文章模型训练仅需340秒，50W篇文章的预测仅需6秒
from subprocess import *
import time time = time.time start_time = time()
print("训练")
cmd = "train train.txt"
Popen(cmd, shell = True, stdout = PIPE).communicate()
print("训练结束",str(time() - start_time)) start_time = time()
print("预测")
cmd = "predict test.txt train.txt.model output.txt"
Popen(cmd, shell = True).communicate()
print("预测结束",str(time() - start_time)) #进行统计
#读测试集真实label
start_time = time()
print("统计")
test_filename = "test.txt"
f = open(test_filename,"r",encoding = "utf-8")
real_class = []
for line in f:
real_class.append(line[0]) #总样本
total_sample = len(real_class) #读预测结果label
predict_filename = "output.txt"
f_predict = open(predict_filename,"r",encoding = "utf-8")
s = f_predict.read()
predict_class = s.split() #对预测正确的文章进行计数
T = 0
for real, predict in zip(real_class,predict_class):
if int(real) == int(predict):
T += 1
accuracy = T / total_sample * 100
print("正确率 为", str(accuracy) + "%") # class_label = ["0","1","2","3","4","5","6","7","8","9"]
num_to_cate = {0:"it",1:"体育",2:"军事",3:"金融",4:"健康",5:"汽车",6:"房产",7:"文化",8:"教育",9:"娱乐"} class_label = ["it","体育","军事","金融","健康","汽车","房产","文化","教育","娱乐"] predict_precision = dict.fromkeys(class_label,1.0)
predict_true = dict.fromkeys(class_label,1.0) predict_recall = dict.fromkeys(class_label,1.0)
predict_F = dict.fromkeys(class_label,0.0)
# print(str(predict_precision))
# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))
mat = dict.fromkeys(class_label,{})
for k,v in mat.items():
mat[k] = dict.fromkeys(class_label,0) # print(str(mat)) for real, predict in zip(real_class,predict_class):
real = int(real)
predict = int(predict)
# print(num_to_cate[real])
# print(num_to_cate[predict])
mat[num_to_cate[real]][num_to_cate[predict]] += 1
predict_precision[num_to_cate[predict]] += 1
predict_recall[num_to_cate[real]] += 1 if int(real) == int(predict):
predict_true[num_to_cate[predict]] += 1 # print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true)) #输出混淆矩阵
for k, v in mat.items():
print(k + ":" + str(v)) #计算精确率和召回率
for x in range(len(class_label)):
# x = str(x)
predict_precision[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_precision[num_to_cate[x]]
predict_recall[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_recall[num_to_cate[x]] # print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true)) #计算F测度
for x in range(len(class_label)):
# x = str(x)
predict_F[num_to_cate[x]] = 2 * predict_recall[num_to_cate[x]] * predict_precision[num_to_cate[x]] / (predict_precision[num_to_cate[x]] + predict_recall[num_to_cate[x]]) print("统计结束",str(time() - start_time))
print("精确率为",str(predict_precision))
print("召回率为",str(predict_recall))
print("F测度为",str(predict_F)) print("保存结果")
final_result_filename = "./finalresult.txt"
f = open(final_result_filename,"w",encoding = "utf-8")
for k, v in mat.items():
f.write(k + ":" + str(v) + "\n") f.write("\n")
f.write("正确率为" + str(accuracy) + "%" + "\n\n")
f.write("精确率为" + str(predict_precision) + "\n\n")
f.write("召回率为" + str(predict_recall) + "\n\n")
f.write("F测度为" + str(predict_F) + "\n\n")
print("保存结果结束") # cate_to_num = {"it":0,"体育":1,"军事":2,"华人":3,"国内":4,"国际":5,"房产":6,"文娱":7,"社会":8,"财经":9}
# num_to_cate = {0:"it",1:"体育",2:"军事",3:"华人",4:"国内",5:"国际",6:"房产",7:"文娱",8:"社会",9:"财经"}