<commit_before># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from nalaf.learning.taggers import StubSameSentenceRelationExtractor, StubRelationExtractor
from loctext.learning.train import read_corpus, evaluate_with_argv
from nalaf import print_verbose, print_debug
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
import math
import sys
from nalaf.preprocessing.spliters import NLTKSplitter
from nalaf.preprocessing.tokenizers import NLTK_TOKENIZER


# See conftest.py too
TEST_MIN_CORPUS_PERCENTAGE = 0.4


def test_SS_baseline():
    corpus = read_corpus("LocText")

    # Computation(precision=0.6083150984682714, precision_SE=0.002974704942625582, recall=0.6233183856502242, recall_SE=0.004130201948613626, f_measure=0.6157253599114065, f_measure_SE=0.0030062001054202924)
    EXPECTED_F = 0.6157  # 62
    EXPECTED_F_SE = 0.0030

    annotator_gen_fun = (lambda _: StubSameSentenceRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)

    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("SS Baseline", rel_evaluation)

    return rel_evaluation


def test_count_relations_dists_texts_with_repetitions():
    # Documents 100

    # Texts With Repetitions
    # Counter({'D0': 648, 'D1': 223, 'D2': 174, 'D3': 97, 'D4': 72, 'D5': 52, 'D6': 32, 'D7': 19, 'D8': 15, 'D9': 9, 'D10': 3, 'D11': 1})
    # Counter({'D0': 0.4817843866171004, 'D1': 0.1657992565055762, 'D2': 0.12936802973977696, 'D3': 0.07211895910780669, 'D4': 0.053531598513011154, 'D5': 0.038661710037174724, 'D6': 0.02379182156133829, 'D7': 0.01412639405204461, 'D8': 0.011152416356877323, 'D9': 0.006691449814126394, 'D10': 0.0022304832713754648, 'D11': 0.0007434944237918215})
    # D0 + D1 ~= 0.65

    corpus = read_corpus("LocText")

    sentence_splitter = NLTKSplitter()
    tokenizer = NLTK_TOKENIZER
    sentence_splitter.split(corpus)
    tokenizer.tokenize(corpus)

    print("# Documents", len(corpus))
    (counter_texts_nums, counter_texts_percts) = corpus.compute_stats_relations_distances()
    print(counter_texts_nums)
    print(counter_texts_percts)


def _test_LocText(corpus_percentage, model, EXPECTED_F=None, EXPECTED_F_SE=0.001):
    # Note: EXPECTED_F=None will make the test fail for non-yet verified evaluations
    # Note: the real StdErr's are around ~0.0027-0.0095. Decrease them by default to be more strict with tests

    assert corpus_percentage in [TEST_MIN_CORPUS_PERCENTAGE, 1.0], "corpus_percentage must == {} or 1.0. You gave: {}".format(str(TEST_MIN_CORPUS_PERCENTAGE), str(corpus_percentage))

    corpus = read_corpus("LocText", corpus_percentage)

    rel_evaluation = evaluate_with_argv(['--corpus_percentage', str(corpus_percentage), '--model', model])

    print("LocText " + model, rel_evaluation)
    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1)

    return rel_evaluation


def test_LocText_SS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.6624365482233503, precision_SE=0.0029261497595035445, recall=0.5787139689578714, recall_SE=0.004036629092741261, f_measure=0.6177514792899409, f_measure_SE=0.0027412422752843557)
        EXPECTED_F = 0.6178<commit_msg>fix compute only prots & rels<commit_after># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from nalaf.learning.taggers import StubSameSentenceRelationExtractor, StubRelationExtractor
from loctext.learning.train import read_corpus, evaluate_with_argv
from nalaf import print_verbose, print_debug
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
import math
import sys
from nalaf.preprocessing.spliters import NLTKSplitter
from nalaf.preprocessing.tokenizers import NLTK_TOKENIZER


# See conftest.py too
TEST_MIN_CORPUS_PERCENTAGE = 0.4


def test_SS_baseline():
    corpus = read_corpus("LocText")

    # Computation(precision=0.6083150984682714, precision_SE=0.002974704942625582, recall=0.6233183856502242, recall_SE=0.004130201948613626, f_measure=0.6157253599114065, f_measure_SE=0.0030062001054202924)
    EXPECTED_F = 0.6157  # 62
    EXPECTED_F_SE = 0.0030

    annotator_gen_fun = (lambda _: StubSameSentenceRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)

    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("SS Baseline", rel_evaluation)

    return rel_evaluation


def test_count_relations_dists_texts_with_repetitions():
    # Documents 100

    # Texts With Repetitions
    # Counter({'D0': 351, 'D1': 95, 'D2': 53, 'D3': 23, 'D5': 9, 'D6': 8, 'D4': 7, 'D7': 2, 'D9': 2})
    # Counter({'D0': 0.6381818181818182, 'D1': 0.17272727272727273, 'D2': 0.09636363636363636, 'D3': 0.04181818181818182, 'D5': 0.016363636363636365, 'D6': 0.014545454545454545, 'D4': 0.012727272727272728, 'D9': 0.0036363636363636364, 'D7': 0.0036363636363636364})
    # D0 + D1 ~= 0.81 (just as Shrikant originally indicated)

    corpus = read_corpus("LocText")

    sentence_splitter = NLTKSplitter()
    tokenizer = NLTK_TOKENIZER
    sentence_splitter.split(corpus)
    tokenizer.tokenize(corpus)

    print("# Documents", len(corpus))
    (counter_texts_nums, counter_texts_percts) = corpus.compute_stats_relations_distances(REL_PRO_LOC_ID)
    print(counter_texts_nums)
    print(counter_texts_percts)


def _test_LocText(corpus_percentage, model, EXPECTED_F=None, EXPECTED_F_SE=0.001):
    # Note: EXPECTED_F=None will make the test fail for non-yet verified evaluations
    # Note: the real StdErr's are around ~0.0027-0.0095. Decrease them by default to be more strict with tests

    assert corpus_percentage in [TEST_MIN_CORPUS_PERCENTAGE, 1.0], "corpus_percentage must == {} or 1.0. You gave: {}".format(str(TEST_MIN_CORPUS_PERCENTAGE), str(corpus_percentage))

    corpus = read_corpus("LocText", corpus_percentage)

    rel_evaluation = evaluate_with_argv(['--corpus_percentage', str(corpus_percentage), '--model', model])

    print("LocText " + model, rel_evaluation)
    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1)

    return rel_evaluation


def test_LocText_SS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.6624365482233503, precision_SE=0.0029261497595035445, recall=0.5787139689578714, recall_SE=0.004036629092741261, f_measure=0.6177514792899409, f_measure_SE=0.0027412422752843557)
        EXPECTED_F = 0.6178
{'fraction_non_alphanumeric': 0.1004084859665305, 'fraction_numerical': 0.13387798128870734, 'mean_word_length': 5.764705882352941, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4087793', 'n_tokens_mistral': 3466, 'n_tokens_neox': 2815, 'n_words': 528}