<jupyter_start><jupyter_text>code from fast.ai https://github.com/fastai/fastai/blob/master/courses/dl2/imdb.ipynblanguage models were trained using code and instructions from here https://github.com/fastai/fastai/tree/master/courses/dl2/imdb_scripts**Uses older fastai library version - https://forums.fast.ai/t/fastai-v0-7-install-issues-thread/24652 installation**<jupyter_code>import _pickle as pickle, pandas as pd, numpy as np
from tqdm import tqdm
from pathlib import Path<jupyter_output><empty_output><jupyter_text>Load dataRead data and split it into test and train datasets.<jupyter_code>uk_annotation = pd.read_csv('../uk_annotation.csv')
ru_annotation = pd.read_csv('../ru_annotation.csv')

htmls = pd.read_json('../htmls_sample.jl.bz2', lines=True, chunksize=10000)
uk_ids = []; ru_ids = []
for df in tqdm(htmls):
    uk_ids.append(
        df.loc[df.html_id.isin(uk_annotation.html_id)].copy(
         ).reindex(['html_id', 'word_ids'], axis=1)
    )
    ru_ids.append(
        df.loc[df.html_id.isin(ru_annotation.html_id)].copy(
         ).reindex(['html_id', 'word_ids'], axis=1)
    )
    
uk_ids = pd.concat(uk_ids, ignore_index=True
                  ).merge(uk_annotation, on='html_id', how='left'
                  ).sample(frac=1)
ru_ids = pd.concat(ru_ids, ignore_index=True
                  ).merge(ru_annotation, on='html_id', how='left'
                  ).sample(frac=1)

del uk_annotation, ru_annotation
ru_train_test_mask = np.random.choice([True, False], len(ru_ids), p=[0.2, 0.8])
uk_train_test_mask = np.random.choice([True, False], len(uk_ids), p=[0.2, 0.8])

# ids - lists of token ids in text
ru_trn_ids = ru_ids.word_ids.values[~ru_train_test_mask]
ru_val_ids = ru_ids.word_ids.values[ru_train_test_mask]
uk_trn_ids = uk_ids.word_ids.values[~uk_train_test_mask]
uk_val_ids = uk_ids.word_ids.values[uk_train_test_mask]

#labels
ru_trn_lab = ru_ids[['is_emo', 'is_arg', 'is_other']].values[~ru_train_test_mask]
ru_val_lab = ru_ids[['is_emo', 'is_arg', 'is_other']].values[ru_train_test_mask]
uk_trn_lab = uk_ids[['is_emo', 'is_arg', 'is_other']].values[~uk_train_test_mask]
uk_val_lab = uk_ids[['is_emo', 'is_arg', 'is_other']].values[uk_train_test_mask]
with open('itos_ru.pkl', 'rb') as f:
    itos_ru = pickle.load(f)
with open('itos_uk.pkl', 'rb') as f:
    itos_uk = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Relevant - irrelevant classifierClassify relevant news (Ukrainian politics, economy, important social issues) from irrelevant (sport, weather, celebrities)<jupyter_code>from fastai.text import *
softmax = lambda x : np.exp(x)/np.sum(np.exp(x)) # softmax activation
sigm = lambda x: 1 / (1 + np.exp(-x))
sigmv = np.vectorize(sigm)<jupyter_output><empty_output><jupyter_text>ru<jupyter_code>CLAS_PATH = Path('ru/')
# reads and saves models from this forlder / models
bptt,em_sz,nh,nl = 70,400,1150,3 # bptt - numper of tokens that model prosesses at once; em - embedding size
# nh - n of cells in LSTM layers in pre-trained language model
# nl - n of LSTM layers in language model
vs = len(itos_ru) # len of token dictionary
bs = 8 # batch size - reduce in case of memory error
c = 2 # n of categories<jupyter_output><empty_output><jupyter_text>Data in fastai format. Sortish sampler groups texts of same length in batch, so there will be less padding<jupyter_code>trn_ds = TextDataset(ru_trn_ids, ru_trn_lab[:, -1])
val_ds = TextDataset(ru_val_ids, ru_val_lab[:, -1])
trn_samp = SortishSampler(ru_trn_ids, key=lambda x: len(ru_trn_ids[x]), bs=bs//2)
val_samp = SortSampler(ru_val_ids, key=lambda x: len(ru_val_ids[x]))
trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_samp)
val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_samp)
md = ModelData(CLAS_PATH, trn_dl, val_dl)
dps = np.array([0.4,0.5,0.05,0.3,0.4])*0.6 # dropouts for each level, increased comparing to default

# model definition
m = get_rnn_classifier(bptt, 15*70, c, vs, emb_sz=em_sz,
                       n_hid=nh, n_layers=nl,
                       pad_token=1, #token for padding
                       #final layer. max and average pooling of LSTM states, and the last state
                       layers=[em_sz*3, 50, c], 
                       # dropouts
                       drops=[dps[4], 0.2],
                       dropouti=dps[0],
                       wdrop=dps[1],
                       dropoute=dps[2],
                       dropouth=dps[3])

opt_fn = partial(optim.Adam, betas=(0.7, 0.99)) # Adam optimizer

learn = RNN_Learner(md, TextModel(to_gpu(m)), opt_fn=opt_fn)
learn.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)
learn.clip=25. # maximum value of weights, left as default
learn.metrics = [accuracy]
# learning rates
lr=3e-3
lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])

wd = 1e-7 # weight decay
learn.load_encoder('fwd_ru_finetuned_lm_enc') # load pretrained language model encoder
learn.freeze_to(-1) # freeze all layers (trained), except for the last feed-dorward one (randomly initialized)
# finds the best learning rate
learn.lr_find(lrs/1000)
learn.sched.plot()
# training. 'use_wd_sched' tweaks weight regularisation, as described here
# https://forums.fast.ai/t/new-adamw-optimizer-now-available/8518
# clr decreases learning rate during training
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_other_0')
learn.load('ru_test_other_0')
learn.freeze_to(-2) # more unfreezing

learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8,3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_other_1')
learn.load('ru_test_other_1')
learn.unfreeze()

learn.fit(lrs, 1, wds=wd, cycle_len=3, use_clr=(32,10), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_other_2')<jupyter_output><empty_output><jupyter_text>predict<jupyter_code>val_no_sample_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1)
preds = predict(learn.model, dl_pred)
result = np.apply_along_axis(softmax, 1, preds)[:, 1]
thr = 0.68 # threshold after whisch to consider article irrelevant
print(sklearn.metrics.classification_report(y_pred=result > thr,
                                            y_true=ru_val_lab[:, -1],
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result > thr,
                                      y_true=ru_val_lab[:, -1],
                                      labels=[1,0])

cm / cm.sum(1)[:, None]<jupyter_output><empty_output><jupyter_text>uk<jupyter_code>CLAS_PATH = Path('uk/')
# reads and saves models from this forlder / models
bptt,em_sz,nh,nl = 70,400,1150,3 # bptt - numper of tokens that model prosesses at once; em - embedding size
# nh - n of cells in LSTM layers in pre-trained language model
# nl - n of LSTM layers in language model
vs = len(itos_uk) # len of token dictionary
bs = 32 # batch size - reduce in case of memory error
c = 2 # n of categories
trn_ds = TextDataset(uktrn_ids, uktrn_lab[:, -1])
val_ds = TextDataset(ukval_ids, ukval_lab[:, -1])
trn_samp = SortishSampler(uktrn_ids, key=lambda x: len(uktrn_ids[x]), bs=bs//2)
val_samp = SortSampler(ukval_ids, key=lambda x: len(ukval_ids[x]))
trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_samp)
val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_samp)
md = ModelData(CLAS_PATH, trn_dl, val_dl)
dps = np.array([0.4,0.5,0.05,0.3,0.4])*0.6 # dropouts for each level, increased comparing to default

# model definition
m = get_rnn_classifier(bptt, 25*70, c, vs, emb_sz=em_sz,
                       n_hid=nh, n_layers=nl,
                       pad_token=1, #token for padding
                       #final layer. max and average pooling of LSTM states, and the last state
                       layers=[em_sz*3, 50, c], 
                       # dropouts
                       drops=[dps[4], 0.2],
                       dropouti=dps[0],
                       wdrop=dps[1],
                       dropoute=dps[2],
                       dropouth=dps[3])

opt_fn = partial(optim.Adam, betas=(0.7, 0.99)) # Adam optimizer

learn = RNN_Learner(md, TextModel(to_gpu(m)), opt_fn=opt_fn)
learn.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)
learn.clip=25. # maximum value of weights, left as default
learn.metrics = [accuracy]
# learning rates
lr=3e-3
lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])

wd = 1e-7 # weight decay
learn.load_encoder('fwd_uk_finetuned_lm_enc')
learn.freeze_to(-1) # freeze all layers, except for the last feed-dorward one
# finds the best learning rate
learn.lr_find(lrs/1000)
learn.sched.plot()
# training. 'use_wd_sched' tweaks weight regularisation, as described here
# https://forums.fast.ai/t/new-adamw-optimizer-now-available/8518
# clr decreases learning rate during training
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_other_0')
learn.load('uk_test_other_0')
learn.freeze_to(-2) # more unfreezing

learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8,3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_other_1')
learn.load('uk_test_other_1')
learn.unfreeze()

learn.fit(lrs, 1, wds=wd, cycle_len=3, use_clr=(32,10), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_other_2')<jupyter_output><empty_output><jupyter_text>check<jupyter_code>val_no_sample_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1)
preds = predict(learn.model, dl_pred)
result = np.apply_along_axis(softmax, 1, preds)[:, 1]
thr = 0.55 # threshold after whisch to consider article irrelevant
print(sklearn.metrics.classification_report(y_pred=result > thr,
                                            y_true=uk_val_lab[:, -1],
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result > thr,
                                      y_true=uk_val_lab[:, -1],
                                      labels=[1,0])

cm / cm.sum(1)[:, None]<jupyter_output><empty_output><jupyter_text>Manipulations classifier<jupyter_code># multilabel output layer
# from https://forums.fast.ai/t/multilabel-classification-with-ulmfit/19809/3
class MultiLabelClassifier(nn.Module):
    def __init__(self, y_range=None):
        super().__init__()
        self.y_range = y_range
    
    def forward(self, input):
        x, raw_outputs, outputs = input
        x = F.sigmoid(x)
        # no y_range here
        if (self.y_range):
            x = x * (self.y_range[1] - self.y_range[0])
            x = x + self.y_range[0]
        return x, raw_outputs, outputs
bptt,em_sz,nh,nl = 70,400,1150,3
opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
bs = 8<jupyter_output><empty_output><jupyter_text>uk<jupyter_code>CLAS_PATH = Path('uk/')
vs = len(itos_uk)
# filter out irrelevant articles
rel_uk_trn_ids0 = uk_trn_ids[uk_trn_lab[:, -1] == 0]
rel_uk_trn_lab0 = uk_trn_lab[uk_trn_lab[:, -1] == 0, :-1]
rel_uk_val_ids = uk_val_ids[uk_val_lab[:, -1] == 0]
rel_uk_val_lab = uk_val_lab[uk_val_lab[:, -1] == 0, :-1]

# Multiply manipulative texts to balance training set
rel_uk_trn_ids = np.hstack([rel_uk_trn_ids0,
                      rel_uk_trn_ids0[rel_uk_trn_lab0.any(1)],
                      rel_uk_trn_ids0[rel_uk_trn_lab0.any(1)],])

rel_uk_trn_lab = np.vstack([rel_uk_trn_lab0,
                        rel_uk_trn_lab0[rel_uk_trn_lab0.any(1)],
                        rel_uk_trn_lab0[rel_uk_trn_lab0.any(1)],])

print(Counter(rel_uk_trn_lab.any(1)))
# mix multiplied examples
perm = np.random.permutation(len(rel_uk_trn_ids))
rel_uk_trn_ids = rel_uk_trn_ids[perm]
rel_uk_trn_lab = rel_uk_trn_lab[perm]
c = 2
trn_ds = TextDataset(rel_uk_trn_ids, rel_uk_trn_lab.astype(np.float64))
val_ds = TextDataset(rel_uk_val_ids, rel_uk_val_lab.astype(np.float64))
trn_samp = SortishSampler(rel_uk_trn_ids, key=lambda x: len(rel_uk_trn_ids[x]), bs=bs//2)
val_samp = SortSampler(rel_uk_val_ids, key=lambda x: len(rel_uk_val_ids[x]))
trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_samp)
val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_samp)
md = ModelData(CLAS_PATH, trn_dl, val_dl)
dps = np.array([0.4,0.5,0.05,0.3,0.4])*0.65

m = get_rnn_classifier(bptt, 15*70, c, vs, emb_sz=em_sz,
                       n_hid=nh, n_layers=nl,
                       pad_token=1,
                       layers=[em_sz*3, 50, c],
                       drops=[dps[4], 0.2],
                       dropouti=dps[0],
                       wdrop=dps[1],
                       dropoute=dps[2],
                       dropouth=dps[3])

opt_fn = partial(optim.Adam, betas=(0.7, 0.99))

learn = RNN_Learner(md, TextModel(to_gpu(m)), opt_fn=opt_fn)
learn.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)
learn.clip=25.
    
learn.model.add_module('2', MultiLabelClassifier())
# different loss function - for multilabel with sigmoid activation
learn.crit = F.binary_cross_entropy
learn.metrics = [accuracy_thresh(0.5)]
lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])

wd = 1e-7
# wd = 0
learn.load_encoder('fwd_uk_finetuned_lm_enc')
learn.freeze_to(-1)
learn.lr_find(1/1000)
learn.sched.plot()
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_mult_0')
learn.load('uk_test_mult_0')
learn.freeze_to(-2)
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_mult_1')
learn.lr_find(lr / 1000)
learn.sched.plot()
# change learning rate after checking lr search.
# Ideal value is somewhat before the moment of steep increase in loss
learn.load('uk_test_mult_1')
learn.unfreeze()
learn.fit(lrs/3, 1, wds=wd, cycle_len=1, use_clr=(32, 10), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('uk_test_mult_2')<jupyter_output><empty_output><jupyter_text>check on test settest set is labeled by 2 annotators, the quality is much better, than of general trainig annotation<jupyter_code>m_pred = get_rnn_classifer(bptt, 25*70, 2, vs,
                           emb_sz=em_sz, n_hid=nh, n_layers=nl, pad_token=1,
                           drops=[0,0],
                           layers=[em_sz*3, 50, c])

load_model(m_pred, 'uk/models/uk_emo_arg_cls.h5')

m_pred.cuda()
m_pred.eval()
df_uk_test = pd.read_json('uk_test_set.jl', lines=True)
df_uk_test = df_uk_test.loc[(df_uk_test.other_check == 0)
                            & df_uk_test.is_emo.notnull()
                           ].copy()
ds = TextDataset(df_uk_test.word_ids.values, df_uk_test[['emo_check', 'arg_check']].values.astype(int))
dl = DataLoader(ds, batch_size=8, transpose=True, num_workers=3, pad_idx=1)
pred = predict(m_pred, dl)
result = sigmv(pred)
thr = 0.42 # threshold after which to consider article emotional
print(sklearn.metrics.classification_report(y_pred=result[:, 0] > thr,
                                            y_true=df_uk_test.emo_check,
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result[:, 0] > thr,
                                      y_true=df_uk_test.emo_check,
                                      labels=[1,0])

print('emotionality', '\n', cm / cm.sum(1)[:, None], '\n', cm)

thr = 0.45 # threshold after which to consider article false argumentation
print(sklearn.metrics.classification_report(y_pred=result[:, 1] > thr,
                                            y_true=df_uk_test.arg_check,
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result[:, 1] > thr,
                                      y_true=df_uk_test.arg_check,
                                      labels=[1,0])

print('argumentation', '\n', cm / cm.sum(1)[:, None], '\n', cm)<jupyter_output>precision    recall  f1-score   support

          1       0.52      0.39      0.45        28
          0       0.92      0.95      0.94       216

avg / total       0.88      0.89      0.88       244

emotionality 
 [[0.39286 0.60714]
 [0.0463  0.9537 ]] 
 [[ 11  17]
 [ 10 206]]
             precision    recall  f1-score   support

          1       0.50      0.31      0.38        26
          0       0.92      0.96      0.94       218

avg / total       0.88      0.89      0.88       244

argumentation 
 [[0.30769 0.69231]
 [0.0367  0.9633 ]] 
 [[  8  18]
 [  8 210]]<jupyter_text>ru<jupyter_code>CLAS_PATH = Path('ru/')
vs = len(itos_ru)
# filter out irrelevant articles
rel_ru_trn_ids0 = ru_trn_ids[ru_trn_lab[:, -1] == 0]
rel_ru_trn_lab0 = ru_trn_lab[ru_trn_lab[:, -1] == 0, :-1]
rel_ru_val_ids = ru_val_ids[ru_val_lab[:, -1] == 0]
rel_ru_val_lab = ru_val_lab[ru_val_lab[:, -1] == 0, :-1]

# Multiply manipulative texts to balance training set
rel_ru_trn_ids = np.hstack([rel_ru_trn_ids0,
                            rel_ru_trn_ids0[rel_ru_trn_lab0.any(1)],])

rel_ru_trn_lab = np.vstack([rel_ru_trn_lab0,
                            rel_ru_trn_lab0[rel_ru_trn_lab0.any(1)]
                           ])

print(Counter(rel_ru_trn_lab.any(1)))
# mix multiplied examples
perm = np.random.permutation(len(rel_ru_trn_ids))
rel_ru_trn_ids = rel_ru_trn_ids[perm]
rel_ru_trn_lab = rel_ru_trn_lab[perm]
c = 2
trn_ds = TextDataset(rel_ru_trn_ids, rel_ru_trn_lab)
val_ds = TextDataset(rel_ru_val_ids, rel_ru_val_lab)
trn_samp = SortishSampler(rel_ru_trn_ids, key=lambda x: len(rel_ru_trn_ids[x]), bs=bs//2)
val_samp = SortSampler(rel_ru_val_ids, key=lambda x: len(rel_ru_val_ids[x]))
trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_samp)
val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_samp)
md = ModelData(CLAS_PATH, trn_dl, val_dl)
dps = np.array([0.4,0.5,0.05,0.3,0.4])*0.55

m = get_rnn_classifier(bptt, 25*70, c, vs, emb_sz=em_sz,
                       n_hid=nh, n_layers=nl,
                       pad_token=1,
                       layers=[em_sz*3, 50, c],
                       drops=[dps[4], 0.2],
                       dropouti=dps[0],
                       wdrop=dps[1],
                       dropoute=dps[2],
                       dropouth=dps[3])

opt_fn = partial(optim.Adam, betas=(0.7, 0.99))

learn = RNN_Learner(md, TextModel(to_gpu(m)), opt_fn=opt_fn)
learn.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)
learn.clip=25.
    
learn.model.add_module('2', MultiLabelClassifier())
# different loss function - for multilabel with sigmoid activation
learn.crit = F.binary_cross_entropy
learn.metrics = [accuracy_thresh(0.5)]
??MultiBatchRNN
lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])

wd = 1e-7
# wd = 0
learn.load_encoder('fwd_ru_finetuned_lm_enc')
learn.freeze_to(-1)
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_mult_0')
learn.load('ru_test_mult_0')
learn.freeze_to(-2)
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_mult_1')
learn.load('ru_test_mult_1')
learn.unfreeze()
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8, 3), use_wd_sched=True)
learn.sched.plot_loss()
learn.save('ru_test_mult_2')<jupyter_output><empty_output><jupyter_text>predict<jupyter_code>m_pred = get_rnn_classifer(bptt, 25*70, 2, vs,
                           emb_sz=em_sz, n_hid=nh, n_layers=nl, pad_token=1,
                           drops=[0,0],
                           layers=[em_sz*3, 50, c])

# load_model(m_pred, 'ru/models/ru_test_mult_2')
load_model(m_pred, 'ru/models/ru_arg_emo_cls.h5')

m_pred.cuda()
m_pred.eval()
df_ru_test = pd.read_json('ru_test_set.jl', lines=True)
ds = TextDataset(df_ru_test.word_ids.values, df_ru_test[['emo_check', 'arg_check']].values.astype(int))
dl = DataLoader(ds, batch_size=8, transpose=True, num_workers=3, pad_idx=1)
pred = predict(m_pred, dl)
result = sigmv(pred)
thr = 0.36 # threshold after which to consider article irrelevant
print(sklearn.metrics.classification_report(y_pred=result[:, 0] > thr,
                                            y_true=df_ru_test.emo_check,
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result[:, 0] > thr,
                                      y_true=df_ru_test.emo_check,
                                      labels=[1,0])

print('emotionality', '\n', cm / cm.sum(1)[:, None], '\n', cm)

thr = 0.55 # threshold after which to consider article irrelevant
print(sklearn.metrics.classification_report(y_pred=result[:, 1] > thr,
                                            y_true=df_ru_test.arg_check,
                                            labels=[1,0]))

cm = sklearn.metrics.confusion_matrix(y_pred=result[:, 1] > thr,
                                      y_true=df_ru_test.arg_check,
                                      labels=[1,0])

print('argumentation', '\n', cm / cm.sum(1)[:, None], '\n', cm)<jupyter_output>precision    recall  f1-score   support

          1       0.69      0.61      0.65       111
          0       0.89      0.92      0.91       388

avg / total       0.85      0.85      0.85       499

emotionality 
 [[0.61261 0.38739]
 [0.0799  0.9201 ]] 
 [[ 68  43]
 [ 31 357]]
             precision    recall  f1-score   support

          1       0.77      0.55      0.64       136
          0       0.85      0.94      0.89       363

avg / total       0.83      0.83      0.82       499

argumentation 
 [[0.55147 0.44853]
 [0.06336 0.93664]] 
 [[ 75  61]
 [ 23 340]]
{'fraction_non_alphanumeric': 0.13310993204877594, 'fraction_numerical': 0.04011914735176394, 'mean_word_length': 3.054916021890923, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 63, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '365902', 'license': 'Apache-2.0', 'n_tokens_mistral': 8607, 'n_tokens_neox': 8174, 'n_words': 1490, 'path': 'classifier/training.ipynb', 'repo_name': 'NadiaRom/manipulative_news_methodology', 'size': '21486'}