<jupyter_start><jupyter_text>tf-idf<jupyter_code>from functools import partial

f = partial(text_precess, cut_method=ltp.segmentor.segment)

content['q_processed'] = content.question.apply(f)
content.head()
content.to_csv('qa_processed', encoding='utf-8', index=False)
# content_1 = pd.read_csv('qa_processed', encoding='utf-8')
# content_1.head()
# 注意，自定义了analyzer，忽略了stopwords
def split_func(s):
    return s.split()

vectorized = TfidfVectorizer(max_features=12000, tokenizer=split_func, stop_words=stopwords)
X = vectorized.fit_transform(content['q_processed'])

print(len(vectorized.vocabulary_))
print(X.shape)
word_2_id = vectorized.vocabulary_

save_pickle('data/tfidf_word2vec.bin', word_2_id)
vectorizer_path = 'model/tfidf.model'

# pickle不能保存lambda
fileObject = open(vectorizer_path, 'wb')
pickle.dump(vectorized, fileObject)
fileObject.close()
from copy import deepcopy

word_2_id = deepcopy(vectorized.vocabulary_)
id_2_word = {d: w for w, d in word_2_id.items()}
len(word_2_id)
# tfidf的词表忽略了一些词，这些词在进行word vec之类的模型时，可以加入使用。Elmo等
for q in content.q_processed:
    for w in q.split():
        if w not in word_2_id:
            # print("adding", w)
            word_2_id[w] = len(word_2_id)
            
print(len(word_2_id))
# 单独保存全量词表
save_pickle('data/full_word2id.bin', word_2_id)
# save_pickle('data/full_id2word.bin', id_2_word)
from operator import and_
from functools import reduce

# 倒排索引召回
inverse_idx = X.transpose()

and_pos_set = {'n', 'v', 'm', 'nh', 'ni', 'nl', 'ns', 'nt', 'ws'}


def boolen_search(doc_word):
    inverse_data = doc_word.transpose()
    inverse_idx = []
    for arr in inverse_data:
        inverse_idx.append(set(np.nonzero(arr.toarray()[0])[0]))
    return inverse_idx
    

inverse_table = boolen_search(X)    
from dataclasses import dataclass, field
from typing import Any

@dataclass(order=True)
class PrioritizedItem:
    priority: int
    item: Any=field(compare=False)
k_largest_heap = KLargest(10, [])
def search_tfidf(query):
    """"""
    words = text_precess(query, ltp.segmentor.segment)
    cws_list = words.split()
    postags = ltp.postagger.postag(cws_list)


    to_and = []
    for i, postag in enumerate(postags):
        if postag in and_pos_set:
            to_and.append(cws_list[i])
    print(to_and)
    try:
        candidates_ids = [word_2_id[w] for w in to_and]
    except KeyError:
        pass
    
    # 布尔搜索：找到同时包含关键词的文档
    documents_ids = [
         inverse_table[_id] for _id in candidates_ids
    ]
    merged_documents = reduce(and_, documents_ids)
    
    query_vec = vectorized.transform([words]).toarray()
    # 计算相似性排序
    k_largest_heap = KLargest(10, [])
    for i in merged_documents:
        a = PrioritizedItem(-distance(query_vec, X[i].toarray()), i)
        k_largest_heap.add(a)
        
    sorted_docuemtns_id = [i.item for i in sorted(k_largest_heap.pool, reverse=True)]
    
#     sorted_docuemtns_id = sorted(merged_documents, key=lambda i: distance(query_vec, X[i].toarray()))
#     a = [distance(query_vec, X[i].toarray()) for i in sorted_docuemtns_id]
    
    return sorted_docuemtns_id
search_tfidf('办理龙卡')
search_tfidf('办理龙卡')
save_pickle('data/doc_tfidf_vec.bin', X)
save_pickle('data/inverse_idx_table.bin', inverse_table)<jupyter_output><empty_output><jupyter_text>BM25<jupyter_code>from collections import Counter
from math import log

def get_tf(sent_i, sent_j):
    """计算bm25的term frequence. sent来自预处理的sent_of_words列表。"""
    freq = {}
    sent_i_counts = Counter(sent_i)
    # 计算i句中的词，在j句中的tf
    for w in sent_j:
        # if not self.filte_words(w_item):
        #     continue
        if w in sent_i_counts:
            freq[w] = sent_i_counts[w]
        else:
            freq[w] = 0
    total = len(sent_i)
    return {word: count / total for word, count in freq.items()}


def get_idf(content):
    """计算inverse document frequence. 这里计算句子的相似度，所以计算inverse sentence frequence"""
    total_sent = len(content) + 1 # 假设有一个句子包含所有词
    avg_len = 0
    doc_freq = {}
    for sent in content:
        sent = sent.split()
        avg_len += len(sent)
        words = list({w for w in sent})
        for word in words:
            # 假设有一个句子包含所有词
            count = doc_freq.setdefault(word, 1)
            doc_freq[word] = count + 1
    avg_len /= total_sent
    # sklearn中的实现方式
    idf = {word: log(total_sent / df) + 1 for word, df in doc_freq.items()}
    return idf, avg_len


def sent_corelation_func(sent_i, sent_j, idf, avg_len, k1=1.5, b=0.75):
    """计算bm25。

    sent_i ： 与query对比的句子，在文档中进行遍历，D
    sent_j : query的句子, Q
    """
    tf = get_tf(sent_i, sent_j)

    K = k1 * (1 - b + b * len(sent_i) / avg_len)
    bm25 = 0
    for j_word in sent_j:
        bm25 += idf[j_word] * tf[j_word] * (k1 + 1) / (tf[j_word] + K)
    return bm25
idf, avg_len = get_idf(content.q_processed)
save_pickle('data/idf_and_avglen.bin', (idf, avg_len))
sent_corelation_func(content.q_processed[21231].lower().split(), content.q_processed[21230].lower().split(), idf, avg_len)
for i, j in content[['qid', 'question']].iterrows():
    print(j)
    break<jupyter_output>qid             1
question    补发网银盾
Name: 0, dtype: object<jupyter_text>词向量模型<jupyter_code>from gensim.models.fasttext import load_facebook_model


fb_model  = load_facebook_model('cc.zh.300.bin')

new_sent = list(map(lambda s: s.lower().split(), content.q_processed))
len(new_sent)

fb_model.build_vocab(new_sent, update=True)
fb_model.train(sentences=new_sent, total_examples=len(new_sent), epochs=5)
selected = []
for w in word_2_id.keys():
    vec = fb_model.wv[w]
    selected.append((w, vec))
with open('selected_words_fasttext_300.txt', 'w', encoding='utf-8') as f:
    first_line = str(len(selected)) + '\t' + str(300)
    f.write(first_line)
    for w, v in selected:
        line =  '\n' + str(w) + '\t' + array_to_string(v)
        f.write(line)
# 上次处理，保存格式不对，所以再处理。。
with open('selected_words_fasttext_300.txt', 'r', encoding='utf-8') as f:
    with open('wordvec_fasttext_300.txt', 'w', encoding='utf-8') as ff:
        file = f.readlines()
        ff.write(file[0].replace('\t', ' '))
        for line in file[1: ]:
            ff.write(line.replace('\t', ' '))
from gensim.models import KeyedVectors

word_vectors = KeyedVectors.load_word2vec_format('wordvec_fasttext_300.txt')<jupyter_output><empty_output><jupyter_text>索引 当然可以使用kmeans之类的聚类方法，但是此处利用annoy（基于树的向量聚类）进行稠密向量的索引和聚类<jupyter_code>from annoy import AnnoyIndex
dim = 300

index = AnnoyIndex(dim, 'angular')
sent_avg_vec(content.q_processed[2], word_vectors).shape
from operator import or_


def search_sentvec(query):
    """"""
    words = text_precess(query, ltp.segmentor.segment)
    cws_list = words.split()
    postags = ltp.postagger.postag(cws_list)

    to_and = []
    for i, postag in enumerate(postags):
        if postag in and_pos_set:
            to_and.append(cws_list[i])
    print(to_and)
    
    try:
        candidates_ids = [word_2_id[w] for w in to_and]
    except KeyError:
        pass

    documents_ids = [
         set(np.nonzero(inverse_idx[_id].toarray()[0])[0]) for _id in candidates_ids
    ]

    merged_documents = reduce(or_, documents_ids)
    sorted_docuemtns_id = sorted(merged_documents, 
                                 key=lambda i: distance(sent_avg_vec(cws_list, word_vectors), 
                                                                          sent_avg_vec(content.q_processed[i].split(), word_vectors)))

    return sorted_docuemtns_id
search_sentvec('办理龙卡')
search_sentvec('办理龙卡')
search_sentvec('办理信用卡')
def show_content(idx):
    print(content.iloc[idx])
show_content(344)
show_content(20460)
vec_list = get_wordvec_dict(content.q_processed, word_vectors)
len(vec_list)
def build_annoy(n_trees=10):
    for i, vec in enumerate(vec_list):
        index.add_item(i, vec)
    index.build(n_trees)
build_annoy()
# test
q = sent_avg_vec('办理 龙卡', word_vectors)

index.get_nns_by_vector(q, 20, search_k=-1, include_distances=True)<jupyter_output><empty_output><jupyter_text>可以看到 annoy 返回结果是和暴力搜索的结果基本一致的。 另一种采用kmeans预处理，划分搜索范围的方法，依赖于算法划分的准确度，一旦在某一个子集中搜索，不会在另一个子集中搜索。因为预处理时，是将kmeans预处理结果作为标签，保存下来的，所以固化了搜索范围。<jupyter_code>index.save('data/sentvec.ann')
def search_sentvec(query, top_k=20, include_distances=False):
    words = text_precess(query, ltp.segmentor.segment)
    
    q_vec = sent_avg_vec(words, word_vectors)
    
    sorted_docuemtns_id = index.get_nns_by_vector(q_vec, top_k, search_k=-1, include_distances=include_distances)
    
    return sorted_docuemtns_id


# 直接用库
import heapq
class KthLargest_2:
    def __init__(self, k: int, nums):
        self.pool = heapq.nlargest(k, nums)
        heapq.heapify(self.pool)
        self.k = k

    def add(self, val: int) -> int:
        if len(self.pool) < self.k:
            heapq.heappush(self.pool,val)
        else:
            heapq.heappushpop(self.pool,val)
        return self.pool[0]


# Your KthLargest object will be instantiated and called as such:
if __name__ == "__main__":
    k = 2
    nums = []
    obj = KthLargest_2(k, nums)
    for i in range(10):
        a = PrioritizedItem(i, 'a')
        param_1 = obj.add(a)
        print(param_1)
        print(obj.pool)<jupyter_output>PrioritizedItem(priority=0, item='a')
[PrioritizedItem(priority=0, item='a')]
PrioritizedItem(priority=0, item='a')
[PrioritizedItem(priority=0, item='a'), PrioritizedItem(priority=1, item='a')]
PrioritizedItem(priority=1, item='a')
[PrioritizedItem(priority=1, item='a'), PrioritizedItem(priority=2, item='a')]
PrioritizedItem(priority=2, item='a')
[PrioritizedItem(priority=2, item='a'), PrioritizedItem(priority=3, item='a')]
PrioritizedItem(priority=3, item='a')
[PrioritizedItem(priority=3, item='a'), PrioritizedItem(priority=4, item='a')]
PrioritizedItem(priority=4, item='a')
[PrioritizedItem(priority=4, item='a'), PrioritizedItem(priority=5, item='a')]
PrioritizedItem(priority=5, item='a')
[PrioritizedItem(priority=5, item='a'), PrioritizedItem(priority=6, item='a')]
PrioritizedItem(priority=6, item='a')
[PrioritizedItem(priority=6, item='a'), PrioritizedItem(priority=7, item='a')]
PrioritizedItem(priority=7, item='a')
[PrioritizedItem(priority=7, item='a'), PrioritizedItem(priority=[...]<jupyter_text>Bert<jupyter_code>from bert_serving.client import BertClient
bc = BertClient()
bc.encode(['你好', '你好啊', '早上好'])
list(zip(*[(1, '1'), (2, "2")]))<jupyter_output><empty_output><jupyter_text>Eval<jupyter_code>import pandas as pd
data = pd.read_csv('./data/eval_data.csv')
def check_manul(data):
    score = 0
    for row in data.iterrows():
        print(row)
        v = int(input('对或错：'))
        score += v
    return score / 100
check_manul(data)<jupyter_output>(0, Unnamed: 0             0
top0          我记忆力不好拉~呵呵
top1              别老爱撒娇啊
top2                你识字吗
top3                学口语。
top4               你有妈妈吗
top5             梳头为啥能健脑
top6          赞美是一种说话的艺术
top7               你妈妈是谁
top8              说话要有礼貌
top9                  健谈
input         我记忆力不好拉~呵呵
model           my_model
Name: 0, dtype: object)
对或错：1
(1, Unnamed: 0               1
top0          下雪的原理是什么你知道吗
top1          下雨的原理是什么你知道吗
top2            你知道为什么会下雪吗
top3            你知道为什么会下雨吗
top4               你是啥都知道吗
top5           今天的天气如何你知道吗
top6           你知道今天是什么日子吗
top7               你知道支付宝吗
top8                  我知道啊
top9            今天有什么事情要做的
input         下雪的原理是什么你知道吗
model             my_model
Name: 1, dtype: object)
对或错：1
(2, Unnamed: 0                              2
top0                           龙支付可以在哪里使用
top1                            如何使用龙支付付款
top2                        如何使用龙支付“收款”功能
top3                       如何使用龙支付“向朋友付款”
top4                       如何在善融商城使用龙支付购物
[...]
{'fraction_non_alphanumeric': 0.12451662799690642, 'fraction_numerical': 0.017186560109993986, 'mean_word_length': 3.383427495291902, 'pattern_counts': {'":': 1, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '634129', 'license': 'MIT', 'n_tokens_mistral': 4842, 'n_tokens_neox': 4861, 'n_words': 796, 'path': 'jupyter_notebooks/train.ipynb', 'repo_name': 'RacleRay/Bank_FAQ_ChatBot', 'size': '11637'}