<jupyter_start><jupyter_text>lab8 - Text classificationThe task concentrates on content-based text the classification.<jupyter_code>from pathlib import Path
from sklearn.model_selection import train_test_split
import numpy as np
import dataclasses as dc
from typing import List
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
import sklearn.metrics as skm
from fasttext import supervised as fasttext
from flair.data import TaggedCorpus, Sentence
from flair.data_fetcher import NLPTaskDataFetcher, NLPTask
from flair.embeddings import WordEmbeddings, FlairEmbeddings, DocumentRNNEmbeddings
from flair.models import TextClassifier
from flair.trainers import ModelTrainer
from pprint import pprint
@dc.dataclass
class Dataset:
    X_train: List
    y_train: List
    X_val: List
    y_val: List    
    X_test: List
    y_test: List
        
    def preprocess(self, fn):
        return dc.replace(
            self,
            X_train = [fn(x) for x in self.X_train],
            X_val = [fn(x) for x in self.X_val],
            X_test = [fn(x) for x in self.X_test],
         )
    
    @property
    def X(self):
        return self.X_train + self.X_val + self.X_test
    
    @property
    def y(self):
        return self.y_train + self.y_val + self.y_test<jupyter_output><empty_output><jupyter_text>Tasks1. Divide the set of bills into two exclusive sets:   1. the set of bills amending other bills (their title starts with `o zmianie ustawy`),   1. the set of bills not amending other bills.<jupyter_code>data_files = Path("../data").glob("*.txt")
bills = [f.open().read().replace("  ", " ").lower() for f in data_files]
def is_changing_bill(text):
    return "o zmianie ustawy" in text.replace("\n", " ")
positives_raw = [b for b in bills if is_changing_bill(b)]
negatives_raw = [b for b in bills if not is_changing_bill(b)]

len(positives_raw), len(negatives_raw)<jupyter_output><empty_output><jupyter_text>2. Change the contents of the bill by removing the date of publication and the title (so the words `o zmianie ustawy`   are removed).<jupyter_code>positives, negatives = [["".join(b.split("art. 1")[1:]) for b in ds] for ds in [positives_raw, negatives_raw]]
X_ = positives + negatives
y_ = ([1] * len(positives)) + ([0] * len(negatives))

X = [x for x in X_ if len(x) > 0]
y = [y for (x, y) in zip(X_, y_) if len(x) > 0 ]<jupyter_output><empty_output><jupyter_text>3. Split the sets of documents into the following groups by randomly selecting the documents:   1. 60% training   1. 20% validation   1. 20% testing<jupyter_code>X_train, X_val_test, y_train, y_val_test = train_test_split(X, y, test_size=0.4)
X_val, X_test, y_val, y_test = train_test_split(X_val_test, y_val_test, test_size=0.5)

original_ds = Dataset(
    X_train,
    y_train,
    X_val,
    y_val,
    X_test,
    y_test
)<jupyter_output><empty_output><jupyter_text>4. Do not change these groups during the following experiments.5. Prepare the following variants of the documents:    a. full text of the document<jupyter_code>def full_text(text):
    return text<jupyter_output><empty_output><jupyter_text>b. randomly selected 10% of the lines of the document<jupyter_code>def tenpercentlines(text):
    lines = [l for l in text.split("\n") if l != ""]
    return "\n".join(np.random.choice(lines, (len(lines) // 10) + 1))<jupyter_output><empty_output><jupyter_text>c. randomly selected 10 lines of the document<jupyter_code>def tenlines(text):
    lines = [l for l in text.split("\n") if l != ""]
    return " ".join(np.random.choice(lines, 10))<jupyter_output><empty_output><jupyter_text>d. randomly selected 1 line of the document<jupyter_code>def oneline(text):
    lines = [l for l in text.split("\n") if l != ""]
    return " ".join(np.random.choice(lines, 1))<jupyter_output><empty_output><jupyter_text>6. Train the following classifiers on the documents:    a. SVM with TF•IDF<jupyter_code>def tfidf_svm(ds: Dataset):
    
    pipeline = Pipeline([
        ('tfidf', TfidfVectorizer().fit(ds.X)),
        ('clf', OneVsRestClassifier(LinearSVC(), n_jobs=1)),
    ])
    parameters = {
        'tfidf__max_df': (0.25, 0.5, 0.75),
        'tfidf__ngram_range': [(1, 1), (1, 2), (1, 3)],
        "clf__estimator__C": [0.01, 0.1, 0.5, 1],
        "clf__estimator__class_weight": ['balanced', None],
    }
    grid_search_tune = GridSearchCV(pipeline, parameters, cv=2, n_jobs=12, verbose=10)
    grid_search_tune.fit(ds.X_train, ds.y_train)
    
    return grid_search_tune.best_estimator_.predict(ds.X_val)<jupyter_output><empty_output><jupyter_text>b. Fasttext<jupyter_code>def fasttext_ds(ds: Dataset):
    out = Path("/tmp") / str(np.random.randn())
    out.mkdir()
    for d in ["train", "val", "test"]:
        X_d, y_d = getattr(ds, f"X_{d}"), getattr(ds, f"y_{d}")
        txt = "\n".join([
          f"__label__{y} {x}"  
            for (x,y) in zip(X_d, y_d)
        ])
        with (out / d).open("w") as f:
            f.write(txt)
    return out
def fasttext_supervised(ds: Dataset):
    ds_path = fasttext_ds(original_ds.preprocess(oneline))
    ft = fasttext(
        str(ds_path / "train"), str(ds_path / "model"),
        epoch=10, 
#         word_ngrams=2, #kills the kernel
    )
    return [int(y[0]) for y in ft.predict(ds.X_val)]<jupyter_output><empty_output><jupyter_text>c. Flair with Polish language model<jupyter_code>def flair_model(ds: Dataset):

    data_folder = fasttext_ds(ds)
    corpus: TaggedCorpus = NLPTaskDataFetcher.load_classification_corpus(
        str(data_folder),
        test_file='test',
        dev_file='val',
        train_file='train'
    )


    # 3. make a list of word embeddings
    word_embeddings = [
        WordEmbeddings('pl'), 
        FlairEmbeddings('polish-forward'),
       FlairEmbeddings('polish-backward')
    ]
        

    # 4. initialize document embedding by passing list of word embeddings
    # Can choose between many RNN types (GRU by default, to change use rnn_type parameter)
    document_embeddings: DocumentRNNmbeddings = DocumentRNNEmbeddings(
        word_embeddings,
        hidden_size=512,
        reproject_words=True,
        reproject_words_dimension=256,
        )

    # 5. create the text classifier
    classifier = TextClassifier(document_embeddings, label_dictionary=corpus.make_label_dictionary(), multi_label=False)

    # 6. initialize the text classifier trainer
    trainer = ModelTrainer(
        classifier, 
        corpus, 
#         optimizer=torch.optim.Adam()
    )

    # 7. start the training
    model_path =  str(data_folder / "model"/ "best-model.pt")
    trainer.train( 
        model_path,
        learning_rate=0.1,
        mini_batch_size=32,
        anneal_factor=0.5,
        patience=5,
        max_epochs=10,
        monitor_train=False
    )    
    return [
        (int(y.labels[0].value) if len(y.labels) > 0 else 0)
        for y in 
        trainer.model.predict([Sentence(x) for x in ds.X_val])
    ]<jupyter_output><empty_output><jupyter_text>7. Report Precision, Recall and F1 for each variant of the experiment (12 variants altogether).<jupyter_code>def metrics(y_gt, y_pred):
    return {
        fn.__name__: fn(y_gt, y_pred)
        for fn in [
            skm.accuracy_score,
            skm.precision_score,
            skm.recall_score,
            skm.f1_score
        ]
    }
results = {
    ds_name: {
        model_fn.__name__: metrics(ds.y_val, model_fn(ds))
        for model_fn in [
#             tfidf_svm,
#             fasttext_supervised,
            flair_model
        ]
    }
    for (ds_name, ds) in {
        fn.__name__: original_ds.preprocess(fn) 
        for fn in
        [
#             oneline,
            tenlines, 
#             tenpercentlines, 
#             full_text, 
        ]
    }.items()
}
pprint(results)
flair_results = {'oneline': {'flair_model': {'accuracy_score': 0.6086956521739131,
                             'f1_score': 0.7527472527472528,
                             'precision_score': 0.6171171171171171,
                             'recall_score': 0.9647887323943662}}}
pprint({'full_text': {'fasttext_supervised': {'accuracy_score': 0.6565217391304348,
                                       'f1_score': 0.7823691460055098,
                                       'precision_score': 0.6425339366515838,
                                       'recall_score': 1.0},
               'tfidf_svm': {'accuracy_score': 0.8826086956521739,
                             'f1_score': 0.9072164948453608,
                             'precision_score': 0.8859060402684564,
                             'recall_score': 0.9295774647887324}},
 'oneline': {'fasttext_supervised': {'accuracy_score': 0.5956521739130435,
                                     'f1_score': 0.7173252279635257,
                                     'precision_score': 0.6310160427807486,
                                     'recall_score': 0.8309859154929577},
             'tfidf_svm': {'accuracy_score': 0.6217391304347826,
                           'f1_score': 0.7642276422764228,
                           'precision_score': 0.6211453744493393,
                           'recall_score': 0.9929577464788732},
             'flair_model': {'accuracy_score': 0.6086956521739131,
                             'f1_score': 0.7527472527472528,
                             'precision_score': 0.6171171171171171,
                             'recall_score': 0.9647887323943662}            
            },
 'tenlines': {'fasttext_supervised': {'accuracy_score': 0.7043478260869566,
                                      'f1_score': 0.8068181818181819,
                                      'precision_score': 0.6761904761904762,
                                      'recall_score': 1.0},
              'tfidf_svm': {'accuracy_score': 0.8130434782608695,
                            'f1_score': 0.8532423208191127,
                            'precision_score': 0.8278145695364238,
                            'recall_score': 0.8802816901408451}},
 'tenpercentlines': {'fasttext_supervised': {'accuracy_score': 0.6217391304347826,
                                             'f1_score': 0.7616438356164384,
                                             'precision_score': 0.6233183856502242,
                                             'recall_score': 0.9788732394366197},
                     'tfidf_svm': {'accuracy_score': 0.7608695652173914,
                                   'f1_score': 0.804270462633452,
                                   'precision_score': 0.8129496402877698,
                                   'recall_score': 0.795774647887324}}})<jupyter_output>{'full_text': {'fasttext_supervised': {'accuracy_score': 0.6565217391304348,
                                       'f1_score': 0.7823691460055098,
                                       'precision_score': 0.6425339366515838,
                                       'recall_score': 1.0},
               'tfidf_svm': {'accuracy_score': 0.8826086956521739,
                             'f1_score': 0.9072164948453608,
                             'precision_score': 0.8859060402684564,
                             'recall_score': 0.9295774647887324}},
 'oneline': {'fasttext_supervised': {'accuracy_score': 0.5956521739130435,
                                     'f1_score': 0.7173252279635257,
                                     'precision_score': 0.6310160427807486,
                                     'recall_score': 0.8309859154929577},
             'flair_model': {'accuracy_score': 0.6086956521739131,
                             'f1_score': 0.7527472527472528,
                            [...]
{'fraction_non_alphanumeric': 0.10765853246643586, 'fraction_numerical': 0.08308705564468462, 'mean_word_length': 2.2781621920841406, 'pattern_counts': {'":': 2, '<': 48, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '530381', 'license': 'MIT', 'n_tokens_mistral': 4532, 'n_tokens_neox': 3867, 'n_words': 807, 'path': 'lab8/lab8.ipynb', 'repo_name': 'mprzewie/nlp_course', 'size': '11843'}