<jupyter_start><jupyter_text>Самые частые слова<jupyter_code>from nltk import FreqDist
n_types = []
n_tokens = []
tokens = []
fd = FreqDist()
for index, row in df.iterrows():
    tokens = row['message'].split()
    fd.update(tokens)
    n_types.append(len(fd))
    n_tokens.append(sum(fd.values()))
for i in fd.most_common(10):
    print(i)
from nltk.corpus import stopwords
# print(stopwords.words('russian'))
with open("/Users/romakindmitriy/PycharmProjects/TelegramParser/docs/stopwords/fullstopwords.txt", 'r') as f:
    stopw = f.readlines()

# print(stopw)
v_stopwords = list(set([x[:-1] for x in stopw]))
print(len(v_stopwords))
mystopwords = stopwords.words('russian') + v_stopwords
mystopwords = list(set(mystopwords))
def remove_stopwords(text, mystopwords = mystopwords):
    try:
        return " ".join([token for token in text.split() if not token in mystopwords])
    except:
        return ""
 
from pymystem3 import Mystem

m = Mystem()
def lemmatize(text, mystem=m):
    try:
        return "".join(m.lemmatize(text)).strip()  
    except:
        return " "
mystoplemmas = ['который','прошлый','сей', 'свой', 'наш', 'мочь', 'лс', 'год']

def remove_stoplemmas(text, mystoplemmas = mystoplemmas):
    try:
        return " ".join([token for token in text.split() if not token in mystoplemmas])
    except:
        return ""
df.message = df.message.apply(remove_stopwords) 
df.message = df.message.apply(lemmatize)
df.message = df.message.apply(remove_stoplemmas) 
lemmata = []
for index, row in df.iterrows():
    lemmata += row['message'].split()
fd = FreqDist(lemmata)
for i in fd.most_common(100):
    print(i)
tokens_by_topic = []
for topic in df['class'].unique():
    tokens = []
    sample = df[df['class']==topic]
    for i in range(len(sample)):
        tokens += sample.message.iloc[i].split()
    tokens_by_topic.append(tokens)
df['class'].unique()<jupyter_output><empty_output><jupyter_text>Выберем событие, из текстов про которое будем извлекать ключевые слова:<jupyter_code>event_id = 0
df['class'].unique()[event_id]
# %%time 
import nltk
from nltk.collocations import *
N_best = 100 # число извлекаемых биграм

bigram_measures = nltk.collocations.BigramAssocMeasures() # класс для мер ассоциации биграм
finder = BigramCollocationFinder.from_words(tokens_by_topic[event_id]) # класс для хранения и извлечения биграм
finder.apply_freq_filter(3) # избавимся от биграм, которые встречаются реже трех раз
raw_freq_ranking = [' '.join(i) for i in finder.nbest(bigram_measures.raw_freq, N_best)] # выбираем топ-10 биграм по частоте 
tscore_ranking = [' '.join(i) for i in finder.nbest(bigram_measures.student_t, N_best)] # выбираем топ-100 биграм по каждой мере 
pmi_ranking =  [' '.join(i) for i in finder.nbest(bigram_measures.pmi, N_best)]
llr_ranking = [' '. join(i) for i in finder.nbest(bigram_measures.likelihood_ratio, N_best)]
chi2_ranking =  [' '.join(i) for i in finder.nbest(bigram_measures.chi_sq, N_best)]<jupyter_output><empty_output><jupyter_text>Результаты:<jupyter_code>rankings = pd.DataFrame({ 'chi2': chi2_ranking, 'llr':llr_ranking, 't-score' : tscore_ranking, 'pmi': pmi_ranking, 'raw_freq':raw_freq_ranking})
rankings = rankings[['raw_freq', 'pmi', 't-score', 'chi2', 'llr']]
rankings.head(10)<jupyter_output><empty_output><jupyter_text>Похожи ли списки биграм?<jupyter_code>from scipy.stats import spearmanr
import seaborn as sns
%matplotlib inline
corr = spearmanr(rankings).correlation
sns.heatmap(corr, annot=True, xticklabels = list(rankings), yticklabels = list(rankings))
# %%time
from gensim.summarization import keywords
text = ' '.join(tokens_by_topic[event_id])
kw = keywords(text)<jupyter_output><empty_output><jupyter_text>Результаты:<jupyter_code>rankings = pd.DataFrame({'Text Rank': kw.split('\n')})
rankings.head(10)<jupyter_output><empty_output><jupyter_text>RAKEДля RAKE нужны сырые тексты со стоп-словами:<jupyter_code>dd = pd.read_csv('./dataset_classes.csv')
dd.head()
filter_1 = d['class'] == 1.
filter_2 = d['class'] == 2.
filter_3 = d['class'] == 3.
filter_4 = d['class'] == 4.
filter_5 = d['class'] == 5.
filter_6 = d['class'] == 5

raw_df = dd.loc[filter_1 | filter_2 | filter_3 | filter_4 | filter_5 | filter_6]
raw_df.message = raw_df.message.str.lower()
raw_df.message = raw_df.message.apply(words_only)
raw_df.message = raw_df.message.apply(lemmatize)
raw_df.head()
text = ' '.join(raw_df[raw_df['class'] == raw_df['class'].unique()[event_id]].message.tolist())
len(text)<jupyter_output><empty_output><jupyter_text>Результаты RAKE:<jupyter_code>import RAKE

# /Users/romakindmitriy/PycharmProjects/TelegramParser/docs/stopwords/fullstopwords.txt
# Rake = RAKE.Rake('./stopwords.txt')
Rake = RAKE.Rake('/Users/romakindmitriy/PycharmProjects/TelegramParser/docs/stopwords/fullstopwords.txt')
kp = [i[0] for i in Rake.run(text)]
rankings = pd.DataFrame({'RAKE': kp})
rankings.head(10)<jupyter_output><empty_output><jupyter_text>Вычисление сходства и Gensim Представление данных в Gensim словарем и корпусом:<jupyter_code>from gensim.corpora import *
df.sort_values(['class'], inplace = True)
texts = [df.message.iloc[i].split() for i in range(len(df))]
dictionary = Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]<jupyter_output>/Users/romakindmitriy/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>Вычисление сходства по косинусной мере на векторах $tf-idf$:<jupyter_code># %%time
from gensim.models import  *
tfidf = TfidfModel(corpus)
print(tfidf[corpus])
corpus_tfidf = tfidf[corpus]
from gensim import similarities

index = similarities.MatrixSimilarity(tfidf[corpus])
sims = index[corpus_tfidf]
from pylab import pcolor, show, colorbar, xticks, yticks
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

plt.figure(figsize = (10,10))


sns.heatmap(data=sims, cmap = 'Spectral').set(xticklabels=[],yticklabels=[])

plt.title("Матрица близости")
plt.show()<jupyter_output><empty_output><jupyter_text>TSNE<jupyter_code>from sklearn.manifold import TSNE
from sklearn.decomposition import TruncatedSVD
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Normalizer
from sklearn.feature_extraction.text import *

vectors = TfidfVectorizer().fit_transform(df.message)
X_reduced = TruncatedSVD(n_components=5, random_state=0).fit_transform(vectors)
X_embedded = TSNE(n_components=2, perplexity=5, verbose=0).fit_transform(X_reduced)
vis_df = pd.DataFrame({'X': X_embedded[:, 0], 'Y': X_embedded[:, 1], 'topic' : df['class']})
sns.FacetGrid(vis_df, hue="topic", size=10).map(plt.scatter, "X", "Y").add_legend()<jupyter_output>/Users/romakindmitriy/anaconda3/lib/python3.7/site-packages/seaborn/axisgrid.py:230: UserWarning: The `size` paramter has been renamed to `height`; please update your code.
  warnings.warn(msg, UserWarning)
{'fraction_non_alphanumeric': 0.14053591790193842, 'fraction_numerical': 0.009977194982896237, 'mean_word_length': 6.160204081632653, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '655217', 'license': 'MIT', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2724, 'n_words': 506, 'path': 'src/dataset_maker/Analyse.ipynb', 'repo_name': 'Dmitriy1594/TelegramParser', 'size': '7016'}