<jupyter_start><jupyter_text>读数据<jupyter_code>df=pd.read_csv('./data/biquge_2500.csv',encoding='UTF-8-sig')
df.head()<jupyter_output><empty_output><jupyter_text>获得id和标题2id的字典<jupyter_code>df['id']=np.array([df['link'][i][-9:-5] for i in range(len(df['link']))])
#df.drop(columns=['link'],inplace=True)
df.head()
t2id={}
id2t={}
for i in range(len(df)):
    t2id[df.loc[i]['title']]=df.loc[i]['id']
    id2t[df.loc[i]['id']]=df.loc[i]['title']
t2id['重生之极品赘婿']
df.set_index('id',inplace=True)
df.head()<jupyter_output><empty_output><jupyter_text>描述信息分词，去停用词<jupyter_code>def get_stopwords_list():
    stopwords = [line.strip() for line in open('./stopwords.txt',encoding='UTF-8-sig').readlines()]
    return stopwords
stopwords_list=get_stopwords_list()
stopwords_list[:5]
def remove_digits(input_str):
    punc = u'0123456789.'
    output_str = re.sub(r'[{}]+'.format(punc), '', input_str)
    return output_str
def move_stopwords(sentence_list, stopwords_list):
    out_list = []#results
    for word in sentence_list:
        if word not in stopwords_list:
            if not remove_digits(word):
                continue
            if word != '\t':
                out_list.append(word)
    return ' '.join(out_list)
#df['split_title']=df['title'].map(lambda x:move_stopwords(list(jieba.cut(x)),stopwords_list))
df['discription']=df['discription'].astype('str')
df.head()
df['discription']=df['discription'].map(lambda x:move_stopwords(list(jieba.cut(x.replace(u'\xa0', u''))),stopwords_list))
df.head()
df['discription'][0]<jupyter_output><empty_output><jupyter_text>向量化&计算余弦相似度<jupyter_code>count = CountVectorizer()
count_matrix = count.fit_transform(df['discription'])
cosine_sim = cosine_similarity(count_matrix, count_matrix)
cosine_sim
indices = pd.Series(df.index)
indices[:5]<jupyter_output><empty_output><jupyter_text>选择10部值得推荐的小说<jupyter_code>def recommendations(title, cosine_sim = cosine_sim):    
    recommended_movies = []
    # gettin the index of the movie that matches the title
    idx = indices[indices == title].index[0]
    # creating a Series with the similarity scores in descending order
    score_series = pd.Series(cosine_sim[idx]).sort_values(ascending = False)
    # getting the indexes of the 10 most similar movies
    top_10_indexes = list(score_series.iloc[1:10].index)
    # populating the list with the titles of the best 10 matching movies
    for i in top_10_indexes:
        recommended_movies.append(list(df.index)[i])
    return recommended_movies
rec_list=recommendations('7260')
print('《'+id2t['7260']+"》"+'的相关推荐小说是：')
print()
for v in rec_list:
    print(id2t[v])
df.head()
df['rec']=df.index.map(lambda x:' '.join([id2t[y] for y in recommendations(x)]))
df.head()
df.drop(columns=['discription'],inplace=True)<jupyter_output><empty_output><jupyter_text>保存数据<jupyter_code>df.to_csv("./results/biquge2500_results.csv",index=False, sep=',',encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16780354706684858, 'fraction_numerical': 0.019781718963165076, 'mean_word_length': 6.2063882063882065, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '13805', 'license': 'MIT', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1150, 'n_words': 161, 'path': '.ipynb_checkpoints/RecSys-checkpoint.ipynb', 'repo_name': 'sherpahu/novel_recommendation_system', 'size': '2932'}