<jupyter_start><jupyter_text>Prediction Preparation of sentences for prediction<jupyter_code>import pandas as pd
from pathlib import Path

PATH_DATA = Path("../data/tagger_evaluation")

gold_standard = pd.read_excel(PATH_DATA.joinpath("annotated_data.xlsx"), header=1)
statements_regulative = gold_standard.loc[(gold_standard['IG syntax (regulative, constitutive)'] == "regulative") & (~gold_standard["Statement"].isna()), "Statement"]
statements_constitutive = gold_standard.loc[(gold_standard['IG syntax (regulative, constitutive)'] == "constitutive") & (~gold_standard["Statement"].isna()), "Statement"]
with open(PATH_DATA.joinpath("statements_regulative.txt"), "w") as f:
    for x in statements_regulative:
        f.write(str(x) + "\n\n")
        
with open(PATH_DATA.joinpath("statements_constitutive.txt"), "w") as f:
    for x in statements_constitutive:
        f.write(str(x) + "\n\n")<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code>#!python ../ig_annotator.py ../data/tagger_evaluation/statements.txt ../data/tagger_evaluation/prediction.tsv en tsv<jupyter_output><empty_output><jupyter_text>Evaluation of IG Tagger<jupyter_code>%load_ext autoreload
%autoreload 2

import pandas as pd

from pathlib import Path
from sklearn import metrics

from utils import load_gs, write_gs, load_evaluation_data


PATH_DATA = Path("../data/tagger_evaluation")

gold_standard = pd.read_excel(PATH_DATA.joinpath("annotated_data.xlsx"), header=1)<jupyter_output><empty_output><jupyter_text>Constitutive<jupyter_code>gold_standard_const = gold_standard.loc[(gold_standard['IG syntax (regulative, constitutive)'] == "constitutive") & (~gold_standard["Statement"].isna()), :].reset_index()
gs = load_gs(gold_standard_const)
write_gs("prediction_constitutive", PATH_DATA, gs)
preds, golds = load_evaluation_data("prediction_constitutive_gs.tsv", PATH_DATA)


from sklearn import metrics
print(metrics.classification_report(golds, preds))<jupyter_output>precision    recall  f1-score   support

                                  0.29      0.24      0.26       566
                (C) Context       0.01      0.02      0.02       205
     (E) Constituted Entity       0.38      0.44      0.41       382
  (F) Constitutive Function       0.46      0.74      0.57        85
                  (M) Modal       0.97      0.82      0.89        40
(P) Constituting Properties       0.57      0.52      0.54      1217

                   accuracy                           0.42      2495
                  macro avg       0.45      0.46      0.45      2495
               weighted avg       0.43      0.42      0.42      2495<jupyter_text>Regulative<jupyter_code>gold_standard_reg = gold_standard.loc[(gold_standard['IG syntax (regulative, constitutive)'] == "regulative") & (~gold_standard["Statement"].isna()), :].reset_index()
gs = load_gs(gold_standard_reg)
write_gs("prediction_regulative", PATH_DATA, gs)
preds, golds = load_evaluation_data("prediction_regulative_gs.tsv", PATH_DATA)


from sklearn import metrics
print(metrics.classification_report(golds, preds))<jupyter_output>precision    recall  f1-score   support

                    0.17      0.08      0.11       264
(A) Attribute       0.83      0.66      0.74       171
   (B) Object       0.69      0.45      0.54       619
  (C) Context       0.44      0.82      0.57       481
  (D) Deontic       0.94      0.92      0.93        36
      (I) Aim       0.93      0.76      0.84       151

     accuracy                           0.55      1722
    macro avg       0.67      0.61      0.62      1722
 weighted avg       0.58      0.55      0.54      1722
{'fraction_non_alphanumeric': 0.108719646799117, 'fraction_numerical': 0.058774834437086095, 'mean_word_length': 2.625, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '82248', 'license': 'MIT', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1179, 'n_words': 241, 'path': 'notebook/tagger_evaluation.ipynb', 'repo_name': 'institutional-grammar-pl/ig-tagger', 'size': '3624'}