<jupyter_start><jupyter_text>Bayes ngram approach for genre classification for ukrpravda corpus Mixed POS-punctuation ngrams as a baseline¶<jupyter_code>from articlehelper import ArticleHandler, RePostprocessor, map_paragraphs, map_category_ukrpravda
import numpy as np
import pandas as pd
from clfhelpers import *
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, mutual_info_score, adjusted_mutual_info_score, accuracy_score
from analyzers import PosAnalyzer, PosFreqWordsAnalyzer, Morphology, PosLexAnalyzer
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.metrics import mutual_info_score
from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import mutual_info_classif

MASK = "data\\news"
zip_fname = MASK + '.zip'
train_fname = MASK + '_train.txt'
test_fname = MASK + '_test.txt'
train = ArticleHandler.read_zip_list(zip_fname, train_fname) 
print(len(train))
test = ArticleHandler.read_zip_list(zip_fname, test_fname)
print(len(test))
df_train = pd.DataFrame.from_dict(map_paragraphs(train, map_category_ukrpravda))
df_train.groupby(['category']).count()

df_test = pd.DataFrame.from_dict(map_paragraphs(test, map_category_ukrpravda))
for doc in test:
    doc['category'] = map_category_ukrpravda(doc)
   
df_test.groupby(['category']).count()
X_train = df_train['text'].values
X_test = df_test['text'].values
y_train = df_train['category'].values
y_test = df_test['category'].values
print(len(X_train), len(X_test), len(y_train), len(y_test))
class ClfFactoryPOS:
    def get_analyzer(self):
        return PosAnalyzer.analyze
    
    def make_classifier(self):
        return MultinomialNB(alpha=0.05)
    
    def make_vectorizer(self):
        return TfidfVectorizer(tokenizer=self.get_analyzer(), ngram_range=(2, 4), min_df=10)

factory = ClfFactoryPOS()
s = X_train[0] +'\n'+ X_train[1]
print(s)
print(factory.get_analyzer()(s))   
tfidf = factory.make_vectorizer()
clf = factory.make_classifier()

vtrain = tfidf.fit_transform(X_train)
vtest = tfidf.transform(X_test)
print(vtrain.shape, vtest.shape)
clf.fit(vtrain, y_train)
y_predicted = clf.predict(vtest)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
show_all_top_features(tfidf, clf, n=40)
percentile = 60
sel_perc = SelectPercentile(mutual_info_classif, percentile)
vtrain_perc = sel_perc.fit_transform(vtrain, y_train)
vtest_perc = sel_perc.transform(vtest)
clf.fit(vtrain_perc, y_train)
y_predicted = clf.predict(vtest_perc)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
plot_clf_classes_distances(clf)
show_all_top_features(tfidf, clf, n=40)
for doc in test:
    doc['category'] = map_category_ukrpravda(doc)
df_test = pd.DataFrame.from_dict(test)
df_test.groupby(['category']).count()
X_test = df_test['text'].values
y_test = df_test['category']
vtest = tfidf.transform(X_test)
vtest_perc = sel_perc.transform(vtest)
y_predicted = clf.predict(vtest_perc.toarray())
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
# from sklearn.externals import joblib
# joblib.dump(clf, 'clf_news_MNB_POS.pkl') 
# joblib.dump(tfidf, 'tfidf_news_MNB_POS.pkl')
# joblib.dump(sel_perc, 'featsel_news_MNB_POS.pkl')<jupyter_output><empty_output><jupyter_text>Mixed POS-punctuation-frequent words ngrams<jupyter_code>class ClfFactoryMix:
    def __init__(self, analyzer=PosFreqWordsAnalyzer(Morphology().getAnalyzer())):
        self.analyzer = analyzer
    
    def get_analyzer(self):
        return self.analyzer.analyze
    
    def make_classifier(self):
        return MultinomialNB(alpha=0.05)
    
    def make_vectorizer(self):
        return TfidfVectorizer(tokenizer=self.get_analyzer(), ngram_range=(2, 4), min_df=10)

freq_words = pd.read_csv('data\\freq_words.txt', sep=' ', header=None, names=['word', 'freq'])['word'].values
print(freq_words)
factory = ClfFactoryMix(PosFreqWordsAnalyzer(Morphology().getAnalyzer(), list(freq_words)))
s = X_train[0] +'\n'+ X_train[1]
print(s)
print(factory.get_analyzer()(s))  
tfidf = factory.make_vectorizer()
clf = factory.make_classifier()

vtrain = tfidf.fit_transform(X_train)
vtest = tfidf.transform(X_test)
clf.fit(vtrain, y_train)
y_predicted = clf.predict(vtest)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
print(clf.coef_.shape)
print(accuracy_score(y_test, y_predicted))
percentile = 70
sel_perc = SelectPercentile(mutual_info_classif, percentile)
vtrain_perc = sel_perc.fit_transform(vtrain, y_train)
vtest_perc = sel_perc.transform(vtest)
clf.fit(vtrain_perc, y_train)
y_predicted = clf.predict(vtest_perc)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
plot_clf_classes_distances(clf)
df_test_doc = pd.DataFrame.from_dict(test)
X_test = df_test_doc['text'].values
y_test = df_test_doc['category']
vtest = tfidf.transform(X_test)
vtest_perc = sel_perc.transform(vtest)
y_predicted = clf.predict(vtest_perc.toarray())
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
print(clf.coef_.shape)
show_all_top_features(tfidf, clf, n=100)
plot_clf_classes_distances(clf, metric='l1', normalization='sum')<jupyter_output><empty_output><jupyter_text>Use lemmatization - for better results<jupyter_code>freq_words = pd.read_csv('data\\freq_words.txt', sep=' ', header=None, names=['word', 'freq'])['word'].values
print(freq_words)
factory = ClfFactoryMix(PosFreqWordsAnalyzer(Morphology().getAnalyzer(), list(freq_words), lemmatize_freq = True))
s = X_train[0] +'\n'+ X_train[1]
print(s)
print(factory.get_analyzer()(s))  
tfidf = factory.make_vectorizer()
clf = factory.make_classifier()

vtrain = tfidf.fit_transform(X_train)
vtest = tfidf.transform(X_test)
clf.fit(vtrain, y_train)
y_predicted = clf.predict(vtest)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
percentile = 70
sel_perc = SelectPercentile(mutual_info_classif, percentile)
vtrain_perc = sel_perc.fit_transform(vtrain, y_train)
vtest_perc = sel_perc.transform(vtest)
clf.fit(vtrain_perc, y_train)
y_predicted = clf.predict(vtest_perc)
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
plot_clf_classes_distances(clf)
df_test_doc = pd.DataFrame.from_dict(test)
X_test = df_test_doc['text'].values
y_test = df_test_doc['category']
vtest = tfidf.transform(X_test)
vtest_perc = sel_perc.transform(vtest)
y_predicted = clf.predict(vtest_perc.toarray())
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
print(clf.coef_.shape)<jupyter_output>(4, 29476)<jupyter_text>Using stylistic bigrams<jupyter_code>class ClfFactoryPosLex:
    def __init__(self, analyzer = None, pwf_analyzer=PosFreqWordsAnalyzer(Morphology().getAnalyzer(), lemmatize_freq = True), 
                 lexngrams = None, ngramsize = 2):
        if analyzer is None:
            self.analyzer = PosLexAnalyzer(pwf_analyzer, lexngrams, ngramsize)
        else:
            self.analyzer = analyzer
    
    def get_analyzer(self):
        return self.analyzer.analyze
    
    def make_classifier(self):
        return MultinomialNB(alpha=0.05)
    
    def make_vectorizer(self):
        return TfidfVectorizer(tokenizer=self.get_analyzer(), ngram_range=(2, 4), min_df=10)

freq_words = pd.read_csv('data\\freq_words.txt', sep=' ', header=None, names=['word', 'freq'])['word'].values
lexngrams = np.loadtxt('data\\news_bigrams.txt', dtype=object, encoding='utf-8')
print('Lexical ngrams count:', len(lexngrams))
factory = ClfFactoryPosLex(None, PosFreqWordsAnalyzer(Morphology().getAnalyzer(), list(freq_words), lemmatize_freq=True), lexngrams)
s = X_train[0] +'\n'+ X_train[1]
print(s)
print(factory.get_analyzer()(s))  
tfidf = factory.make_vectorizer()
clf = factory.make_classifier()

vtrain = tfidf.fit_transform(X_train)
vtest = tfidf.transform(X_test)
clf.fit(vtrain, y_train)
y_predicted = clf.predict(vtest)
print(classification_report(y_test, y_predicted, clf.classes_))
print(adjusted_mutual_info_score(y_test, y_predicted))
print(accuracy_score(y_test, y_predicted))
print(clf.coef_.shape)
percentile = 80
sel_perc = SelectPercentile(mutual_info_classif, percentile)
vtrain_perc = sel_perc.fit_transform(vtrain, y_train)
vtest_perc = sel_perc.transform(vtest)
clf.fit(vtrain_perc, y_train)
y_predicted = clf.predict(vtest_perc)
print(classification_report(y_test, y_predicted, clf.classes_))
print(adjusted_mutual_info_score(y_test, y_predicted))
print(accuracy_score(y_test, y_predicted))
plot_clf_classes_distances(clf, normalization='dim')
print(clf.coef_.shape)<jupyter_output>(4, 36060)<jupyter_text>**Top 40 features**<jupyter_code>show_all_top_features(tfidf, clf, n=40)
from sklearn.externals import joblib
# joblib.dump(clf, 'saves\\clf_news_MNB_poslex.pkl') 
# joblib.dump(tfidf, 'saves\\tfidf_news_MNB_poslex.pkl')
# joblib.dump(sel_perc, 'saves\\featsel_news_MNB_poslex.pkl')
df_test_doc = pd.DataFrame.from_dict(test)
X_test = df_test_doc['text'].values
y_test = df_test_doc['category']
vtest = tfidf.transform(X_test)
vtest_perc = sel_perc.transform(vtest)
y_predicted = clf.predict(vtest_perc.toarray())
print(classification_report(y_test, y_predicted, clf.classes_))
print('Adjusted mutual info score: ', adjusted_mutual_info_score(y_test, y_predicted))
print('Accuracy score: ', accuracy_score(y_test, y_predicted))
print(clf.coef_.shape)<jupyter_output>(4, 36060)<jupyter_text>**Conclusion:** best averaged accuracy score for paragraphs from ukrpravda is 61.07%. After cutting 20% less important features with mutual_info_classif, the accuracy score is 60.62%. For the whole documents accuracy score is 86.67%  Classification of stylistic bigrams with genre classifier<jupyter_code>columns = ['bigram', 'predicted']
columns.extend(clf.classes_)
df_ngrams = pd.DataFrame(columns=columns)
for x in lexngrams:
    row = {}
    s = ' '.join(x)
    row['bigram'] = s
    v = tfidf.transform([s])
    v = sel_perc.transform(v)
    row['predicted'] = clf.predict(v)[0]
    for i, p in enumerate(clf.predict_proba(v)[0]):
        row[clf.classes_[i]] = p
    df_ngrams = df_ngrams.append(row, ignore_index=True)
df_ngrams
counts = df_ngrams.groupby('predicted').count()['bigram']
counts.plot(kind='bar')
pd.set_option("display.max_rows",500)
df_ngrams[df_ngrams['predicted'] == 'news'].sort_values('news', ascending = False)
df_ngrams[df_ngrams['predicted'] == 'blog'].sort_values('blog', ascending = False)
df_ngrams[df_ngrams['predicted'] == 'columns'].sort_values('columns', ascending = False)
df_ngrams[df_ngrams['predicted'] == 'articles'].sort_values('articles', ascending = False)[0:15]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12557154953429298, 'fraction_numerical': 0.007705334462320068, 'mean_word_length': 7.596069868995633, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '377994', 'license': 'MIT', 'n_tokens_mistral': 4526, 'n_tokens_neox': 4350, 'n_words': 707, 'path': 'bayes_news.ipynb', 'repo_name': 'juliamakogon/news_stylistics', 'size': '11810'}