<jupyter_start><jupyter_text>文件加载与预览<jupyter_code>import sys
import json
import math
import numbers
import numpy as np
import pandas as pd

%load_ext autoreload
%autoreload 2

from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
# set up display area to show dataframe in jupyter qtconsole

import matplotlib
%matplotlib inline

import matplotlib.pyplot as plt
# 使用notepad++ 修改文件格式和编码：utf-8
path_train = r'train.csv'
path_test = r'test.csv'

##文件不规则，行尾有分隔符，则可以设定index_col=False 来是的pandas不适用第一列作为行索引。
train_pdf = pd.read_csv(path_train,encoding='utf-8',engine='python',dtype='object',index_col=False)
test_pdf = pd.read_csv(path_test,encoding='utf-8',engine='python',dtype='object')

pd.set_option('max_columns',1000) 
pd.set_option('max_row',300) 
pd.set_option('display.float_format', lambda x: '%.5f' % x)
train_pdf.head(10)
train_pdf.describe()
test_pdf.head(10)
test_pdf.describe()<jupyter_output><empty_output><jupyter_text>文章分词处理 pipeline<jupyter_code>import re
import os
import numpy as np
import pandas as pd
import jieba
import jieba.analyse  

#打开stopwords文件  r"""D:\code\python\csdn_nlp\NLP_DEMO\Resources\stopwords\all_stopwords.txt"""
def open_stop(file_stop):
    stopwords = [line.strip() for line in open(file_stop, 'r', encoding='utf-8').readlines()]
    return stopwords

stopwords = open_stop(r"""D:\code\python\csdn_nlp\NLP_DEMO\Resources\stopwords\cn_stopwords.txt""")


# 针对train文件每行（每篇文章）处理的 pipeline，返回汉语分词列表
def extract_words_one_line(str_line):
    
    chinesewords_sentence = []

    #只保留中文字符
    def remove(text):
        remove_chars = r'[^\u4e00-\u9fa5]'
        return re.sub(remove_chars, '', text)

    #利用jieba进行分词
    def seg_sentence(sentence):
        sentence_seged = jieba.cut(sentence.strip())
        outstr = ''
        for word in sentence_seged:
            if word not in stopwords:
                if word != '\t':
                    chinesewords_sentence.append(word)

        return chinesewords_sentence


    inputs = str_line
    #获取每一句中的分词
    words_in_sentence = []
   
    line_delete = remove(inputs)
    line_seg = seg_sentence(line_delete)
    

    return chinesewords_sentence
    
# 调用样例
extract_words_one_line("今天天气不错，挺风和日丽的.我们下午没有课,sometime")<jupyter_output>Building prefix dict from the default dictionary ...
Dumping model to file cache C:\Users\ADMINI~1\AppData\Local\Temp\jieba.cache
Loading model cost 0.801 seconds.
Prefix dict has been built successfully.<jupyter_text>并行分词处理并行分词处理有两种方案：1. 参考 https://zhuanlan.zhihu.com/p/132669978 并行化apply 加速，使用第三方包：pandarallel ，挺好用但是 windows 不支持2. 采用第二种方法 手写并行处理后拼接dataframefrom joblib import Parallel,delayed参考：https://blog.csdn.net/Jerr__y/article/details/71425298?utm_source=blogxgwz1<jupyter_code>import time
from joblib import Parallel,delayed
def tmp_func(df1):
    df1['分词'] = df1['文章'].apply(extract_words_one_line)
    return df1

def apply_parallel(df_grouped,func):
    results = Parallel(n_jobs=8)(delayed(func)(group) for name,group in df_grouped)
    return pd.concat(results)



time0 = time.time()
#  训练集
df_grouped = train_pdf.groupby(train_pdf['分类'])
train_df_all = apply_parallel(df_grouped, tmp_func)

#测试集
df_grouped = test_pdf.groupby(test_pdf['分类'])
test_df_all = apply_parallel(df_grouped, tmp_func)

print ('time costed {0:.2f}'.format(time.time() - time0))

#train_pdf['分词'] = train_pdf['文章'].apply(extract_words_one_line)
train_df_all.head(10)
test_df_all.head(10)
test = test_df_all.groupby(test_df_all['分类'])
test.describe()
# 重新定义标签结果
#定义标签替换字典

label2category = {
0:'财经',
1:'房地产',
2:'健康',
3:'教育',
4:'科技',
5:'旅游',
6:'女人',
7:'汽车',
8:'体育',
9:'文化',
10:'新闻',
11:'娱乐'
}
category2label = dict(zip(label2category.values(), label2category.keys()))
print(category2label)

train_df_all['label'] = train_df_all['分类'].replace(category2label)
test_df_all['label'] = test_df_all['分类'].replace(category2label)

train_df_all = train_df_all.reset_index(drop=True)
test_df_all = test_df_all.reset_index(drop=True)
#保存中间结果以防丢失

train_df_all.to_csv('train2.csv', encoding="utf-8", index=False)
test_df_all.to_csv('test2.csv', encoding="utf-8", index=False)
# Word2Vec模型<jupyter_output><empty_output><jupyter_text>使用gensim 会出现各种问题： 以下原则：（1）先用pip uninstall +包名命令依次将numpy，scipy，gensim卸载。 （2）再按numpy，scipy，gensim顺序依次安装，能够解决这一问题。 pip install numpy -i https://pypi.douban.com/simple pip install scipy -i https://pypi.douban.com/simple pip install gensim -i https://pypi.douban.com/simple<jupyter_code>import gensim
from gensim.models import Word2Vec
#导入数据

time0 = time.time()

sent_words = train_df_all['分词'].values.tolist()

# 参数说明 https://blog.csdn.net/u011748542/article/details/85880852
# 训练维数 128 维
my_model = gensim.models.Word2Vec(sent_words, sg=0, size=128, window=3,iter=5,min_count=3, negative=3, sample=0.001, hs=1)
#模型保存
my_model.wv.save_word2vec_format('./word2vec_model.txt', binary=False)
print ('time costed {0:.2f}'.format(time.time() - time0))
import gensim
import numpy as np

#导入数据
time0 = time.time()
# 首先初始化一个word2vec 模型：
w2v_model = gensim.models.Word2Vec(size=300, window = 3,sg=1, min_count=3)
w2v_model.build_vocab(sent_words)
# 再加载第三方预训练模型：
third_model = gensim.models.KeyedVectors.load_word2vec_format(r'./sgns.sogounews.bigram-char/sgns.sogounews.bigram-char', binary=False)
# 通过 intersect_word2vec_format()方法merge词向量：
w2v_model.build_vocab([list(third_model.vocab.keys())], update=True)
w2v_model.intersect_word2vec_format(r'./sgns.sogounews.bigram-char/sgns.sogounews.bigram-char', binary=False, lockf=1.0)
w2v_model.train(sent_words, total_examples=w2v_model.corpus_count, epochs=5)

print("Model training finished.")

w2v_model.wv.save_word2vec_format('./word2vec_ensemble.txt', binary=False)
print("Model saved.")
print ('time costed {0:.2f}'.format(time.time() - time0))<jupyter_output>Model training finished.
Model saved.
time costed 659.90<jupyter_text>svm 进行训练将每篇文章的词向量融合成一个向量，使用计算平均词向量来表示文档的方法。将文章内包含的所有词的向量相加再取平均，这样，我们就可以得到一个和词向量维度相同的指示文章信息的向量。<jupyter_code>import numpy as np
#num_features表示的文本单词维度大小

w2v_model = gensim.models.KeyedVectors.load_word2vec_format('./word2vec_ensemble.txt',binary=False)

def average_word_vectors(words,model,vocabulary,num_features):
    feature_vector=np.zeros((num_features,),dtype='float64')
    nwords=0
    for word in words:
        if word in vocabulary:
            nwords=nwords+1
            feature_vector=np.add(feature_vector,model[word])
    if nwords:
        feature_vector=np.divide(feature_vector,nwords)
    return feature_vector

def averaged_word_vectorizer(corpus,model,num_features):
    #get the all vocabulary
    vocabulary=set(model.wv.index2word)
    features=[average_word_vectors(tokenized_sentence,model,vocabulary,num_features) for tokenized_sentence in corpus]
    return np.array(features)

def get_word_vectors(data):
    words_art=[]
    for i in range(len(data)):
        words_art.append((data.loc[i]))
    return averaged_word_vectorizer(words_art,model=w2v_model,num_features=300)
train_df_all['分词'].loc[0]
print(type(train_df_all['分词'].loc[0]))
print( range(len(train_df_all['分词'])))
X_train = get_word_vectors(train_df_all['分词'])
y_train = train_df_all.label

X_test = get_word_vectors(test_df_all['分词'])
y_test = test_df_all.label
from sklearn import svm
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import f1_score

time0 = time.time()
clf = svm.SVC()
grid_values = {'gamma': [0.001, 0.01, 0.05, 0.1, 1, 10],
               'C':[0.01, 0.1, 1, 10, 100]}


grid_clf = GridSearchCV(clf, param_grid = grid_values,scoring = 'f1_macro')
grid_clf.fit(X_train, y_train)
y_grid_pred = grid_clf.predict(X_test)

print('Test set F1: ', f1_score(y_test,y_grid_pred,average='macro'))
print('Grid best parameter (max. f1): ', grid_clf.best_params_)
print('Grid best score (accuracy): ', grid_clf.best_score_)
print ('time costed {0:.2f}'.format(time.time() - time0))
import joblib

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

time0 = time.time()

clf = svm.SVC(C=10, gamma= 1,probability=True).fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))
joblib.dump(clf, "download_w2v_svm.m")
print ('time costed {0:.2f}'.format(time.time() - time0))
import joblib

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

time0 = time.time()

clf = svm.SVC(C=100, gamma= 1,probability=True).fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))
joblib.dump(clf, "download_w2v_svm10.m")
print ('time costed {0:.2f}'.format(time.time() - time0))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13309435707678074, 'fraction_numerical': 0.02682701202590194, 'mean_word_length': 6.037428803905614, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '551343', 'license': 'MIT', 'n_tokens_mistral': 3726, 'n_tokens_neox': 3707, 'n_words': 542, 'path': 'src/Texts_Classification/文本分类-SVM.ipynb', 'repo_name': 'wynshiter/NLP_DEMO', 'size': '8648'}