新闻文本分类

	可以明显地看出：

Sub Test() 
    Dim Sht As Worksheet 
    For Each Sht In Sheets 
        Sht.Copy 
        ActiveWorkbook.SaveAs ThisWorkbook.Path & "\" & Sht.Name & ".xlsx" 
        ActiveWorkbook.Close 
    Next 
End Sub

import pandas as pd
train_df = pd . read_csv ( '/rootData/train_copy.csv' )
train_df.head()

# 显示各新闻的长度分布 这里只是粗略显示 句号分割
train_df['text_len'] = train_df['content'].apply(lambda x: len(str(x).split('。')))
train_df['text_len'].describe()

# 统计文本长度 生成直方图
import matplotlib.pyplot as plt
_ = plt.hist(train_df['text_len'], bins=240)
plt.xlabel('Text char count')
plt.title("Histogram of char count")

with open("/data/hit_stopwords.txt", 'r', encoding='utf-8') as f:
    remove_words = [w.strip('\n') for w in f.readlines()]

for word in seg_list_exact: # 循环读出每个分词
    if word not in remove_words: # 如果不在去除词库中
        object_list.append(word) # 分词追加到列表

import csv

header = ['label', 'text']
with open('/rootData/train_set.csv', 'w', encoding="utf-8", newline='') as f: # 解决空行的问题
    writer = csv.writer(f)
    writer.writerow(header)
    writer.writerows(Endlist)
f.close()

# Count Vectors + RidgeClassifier
# 词袋 + 特征值提取
import pandas as pd

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import RidgeClassifier
from sklearn.metrics import f1_score

train_df = pd.read_csv('/data/train_set.csv',encoding='ANSI')

vectorizer = CountVectorizer(max_features=3000)
train_test = vectorizer.fit_transform(train_df['text'].values.astype('U'))

clf = RidgeClassifier()
clf.fit(train_test[:10000], train_df['label'].values[:10000])

val_pred = clf.predict(train_test[10000:])

print(f1_score(train_df['label'].values[10000:], val_pred, average='macro'))
print(type(val_pred))

# 0.10343472451465849
# 0.10357460616527975 
# 一定要打乱训练的顺序
# 0.8491012463815872
# 0.8207100395680607
# 0.9201683620360309