<jupyter_start><jupyter_text>データの読み込み<jupyter_code>all_title_abstract = pd.read_feather(os.path.join(INPUT_DIR, 'all_title_abstract_df.feather'))
all_title_abstract.head()
file = os.path.join(INPUT_DIR, 'w2v_model_abstract.dill')
abstract_w2vmodel = dill.load(open(file,'rb'))
# ベクトル化
tfidf_vectorizer = TfidfVectorizer(min_df=20)
tfidf_vectorizer.fit(all_title_abstract['clean_abstract'].values)
abstract_vectors = abstract_w2vmodel.wv.vectors
gmm = GaussianMixture(n_components=clusters_num, 
                      covariance_type='tied', max_iter=50, random_state=SEED)
gmm.fit(abstract_vectors)
idf_dic = dict(zip(tfidf_vectorizer.get_feature_names(), tfidf_vectorizer._tfidf.idf_))
assign_dic = dict(zip(abstract_w2vmodel.wv.index2word, gmm.predict(abstract_vectors)))
soft_assign_dic = dict(zip(abstract_w2vmodel.wv.index2word, gmm.predict_proba(abstract_vectors)))
 
word_topic_vecs = {}
for word in assign_dic:
    word_topic_vecs[word] = np.zeros(features_num*clusters_num, dtype=np.float32)
    for i in range(0, clusters_num):
        try:
            word_topic_vecs[word][i*features_num:(i+1)*features_num] \
            = abstract_w2vmodel.wv[word]*soft_assign_dic[word][i]*idf_dic[word]
        except:
            continue
def analyzer(text):
    
    stop_words = ['i', 'a', 'an', 'the', 'to', 'and', 'or', 'if', 'is', 'are', 'am', 'it', 'this', 'that', 'of', 'from', 'in', 'on']
    text = text.lower() # 小文字化
    text = text.replace('\n', '') # 改行削除
    text = text.replace('\t', '') # タブ削除
    text = re.sub(re.compile(r'[!-\/:-@[-`{-~]'), ' ', text) # 記号をスペースに置き換え
    text = text.split(' ') # スペースで区切る
    
    words = []
    for word in text:
        if (re.compile(r'^.*[0-9]+.*$').fullmatch(word) is not None): # 数字が含まれるものは除外
            continue
        if word in stop_words: # ストップワードに含まれるものは除外
            continue
        if len(word) < 2: #  1文字、0文字（空文字）は除外
            continue
        words.append(word)
        
    return words
train_title_abstract = all_title_abstract.iloc[:851524, :]
test_title_abstract = all_title_abstract.iloc[851524:, :]
second_all_title_abstract \
= pd.concat([train_title_abstract[train_title_abstract['cites'].notnull()],
             test_title_abstract], axis=0).reset_index(drop=True)
print(train_title_abstract.shape, test_title_abstract.shape, second_all_title_abstract.shape)
del train_title_abstract, test_title_abstract, all_title_abstract
gc.collect()
scdvs = np.zeros((len(second_all_title_abstract.clean_abstract), clusters_num*features_num), 
                 dtype=np.float32)
 
a_min = 0
a_max = 0
 
for i, text in tqdm(enumerate(second_all_title_abstract.clean_abstract), 
                    total=len(second_all_title_abstract.clean_abstract)):
    tmp = np.zeros(clusters_num*features_num, dtype=np.float32)
    words = analyzer(text)
    for word in words:
        if word in word_topic_vecs:
            tmp += word_topic_vecs[word]
    norm = np.sqrt(np.sum(tmp**2))
    if norm > 0:
        tmp /= norm
    a_min += min(tmp)
    a_max += max(tmp)
    scdvs[i] = tmp

p = 0.04
a_min = a_min*1.0 / len(second_all_title_abstract.clean_abstract)
a_max = a_max*1.0 / len(second_all_title_abstract.clean_abstract)
thres = (abs(a_min)+abs(a_max)) / 2
thres *= p
 
scdvs[abs(scdvs) < thres] = 0
scdvs.shape # (2800, 12000)
pca = PCA(n_components=20, random_state=SEED)
pca_abstract_df = pd.DataFrame(pca.fit_transform(scdvs))
pca_abstract_df.columns = ['pca_scdv_abstract_' + str(col) for col in pca_abstract_df.columns]
pca_abstract_df.to_feather(os.path.join(INPUT_DIR, 'scdv_abstract.feather'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13905079100749376, 'fraction_numerical': 0.017207882320288647, 'mean_word_length': 4.269005847953216, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '331363', 'license': 'MIT', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1421, 'n_words': 202, 'path': 'feature_engineering/make-abstract-scdv-features.ipynb', 'repo_name': 'Pirototo/ProbSpace-citation_prediction-1st-solution', 'size': '3603'}