<jupyter_start><jupyter_text>def efitf(X):    count = CountVectorizer(stop_words='english')    X_train_count = count.fit_transform(X)    tfidf = TfidfTransformer(use_idf=True,smooth_idf=True,sublinear_tf=True)    X_train_tfidf = tfidf.fit_transform(X_train_count)    tf_feature_names = count.get_feature_names()    X_train_tfidf = [list(i) for i in list(X_train_tfidf.toarray())]    EFITF = defaultdict(dict)    for Type,values in enumerate(X_train_tfidf):        for index,value in enumerate(values):            if value > 0.0:                EFITF[Type].update({tf_feature_names[index]:value})     return EFITF X = [] X = df_events['des_clean'].tolist() EFITF = efitf(X)<jupyter_code>def class_similarity(class_text,span):
    return nlp(class_text).similarity(nlp(span))
def get_query(doc,class_text,doc_index,doc_date):
    doc_date = datetime.strptime(doc_date,'%Y-%m-%d')
    date_0 = doc_date.strftime('%Y-%m-%d')
    date_0_ = (doc_date+timedelta(days=-3)).strftime('%Y-%m-%d')
    date_1 = (doc_date+timedelta(days=1)).strftime('%Y-%m-%d')
    date_1_ = date_0
    doc = nlp(doc)
    kws = []
    for i in doc.ents:
        kws.append(i.text)
    triggers = []
    for token in doc:
        if not token.is_stop and token.tag_.startswith('V'):
            if token.text.lower() not in list_stopWords:
                triggers.append((token.text,token.tag_,str(class_similarity(class_text,token.text))))
    triggers = sorted(triggers,key=lambda x:x[2],reverse=True)[:3]
    for i in triggers:
        kws.append(i[0])
    noun_chunks = []
    for i in doc.noun_chunks:
        noun_chunks.append((i.text,str(class_similarity(class_text,i.text))))
    try:
        kws.append(sorted(noun_chunks,key=lambda x:x[1],reverse=True)[0][0].split(' ')[-1])
    except:
        pass
    kws = [w for w in kws if not w in list_stopWords]
    kws = list(set(kws))
    query = [i for i in itertools.combinations(kws,2)]
    query = ['"'+i[0]+'"'+' '+'"'+i[1]+'"'+' '+'since:'+date_0_+' '+'until:'+date_0 for i in query]+['"'+i[0]+'"'+' '+'"'+i[1]+'"'+' '+'since:'+date_1_+' '+'until:'+date_1 for i in query]
    print(query)
    return query
queries = []
for event in df_events.iterrows():
    doc_index = event[0]
    doc_date = event[1]['date']
    doc_class = event[1]['class']
    doc_title = event[1]['title']
    doc = event[1]['des_clean']
    class_text = doc_class.replace('and','')
    query = get_query(doc,class_text,doc_index,doc_date)
    queries.append(query)
query = db.current_event.find({},{'_id':1})
ids = []
for i in query:
    ids.append(i['_id'])
requests = [UpdateOne({'_id': _id}, {'$set': {'queries':queries[index]}}) for index,_id in tqdm(enumerate(ids))]
try:
    result = db.current_event.bulk_write(requests)
    pprint(result.bulk_api_result)
except BulkWriteError as bwe:
    pprint(bwe.details)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17356241234221598, 'fraction_numerical': 0.010869565217391304, 'mean_word_length': 4.149819494584838, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '233201', 'license': 'MIT', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1097, 'n_words': 182, 'path': 'experiment/ea/ce_query.ipynb', 'repo_name': 'iecasszyjy/tweet_search-master', 'size': '2852'}