<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from framenetreader import FulltextReader
from conllreader import SyntacticTreeBuilder
import glob
import os
import options


def get_trees(filename):
    # get data from parse trees
    trees = []
    with open("{}/{}.conll".format(options.framenet_parsed, filename)) as conll_file:
        conll_tree = ""
        for l in conll_file.readlines():
            if l != "\n":
                conll_tree += l
            else:
                trees.append(SyntacticTreeBuilder(conll_tree).build_syntactic_tree())
                conll_tree = ""

    return trees

if __name__ == '__main__':
    correct, partial_credit, total = 0, 0, 0

    # get data from FrameNet arguments
    for fulltext_file in glob.glob("{}/*.xml".format(options.fulltext_corpus)):
        reader = FulltextReader(fulltext_file)
        trees = get_trees(".".join(os.path.basename(fulltext_file).split(".")[:-1]))

        # for each argument, see if it is in a parse tree. if yes +1
        for frame in reader.frames:
            frame_text = " ".join([frame.get_word(w) for w in frame.words])
            for t in trees:
                if t.flat() == frame_text:
                    for arg in frame.args:
                        if arg.text != "":
                            total += 1
                            arg_len = len(arg.text.split())
                            match_len = len(t.closest_match(arg.text))
                            partial_credit += 1 - abs(arg_len - match_len) / max(arg_len, match_len)
                            if t.contains(arg.text):
                                correct += 1
                    break
        
    print("{}/{} = {}".format(correct, total, correct/total))
    print("{}/{} = {}".format(partial_credit, total, partial_credit/total))<commit_msg>Print progress<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from framenetreader import FulltextReader
from conllreader import SyntacticTreeBuilder
import glob
import os
import options


def get_trees(filename):
    # get data from parse trees
    trees = []
    with open("{}/{}.conll".format(options.framenet_parsed, filename)) as conll_file:
        conll_tree = ""
        for l in conll_file.readlines():
            if l != "\n":
                conll_tree += l
            else:
                trees.append(SyntacticTreeBuilder(conll_tree).build_syntactic_tree())
                conll_tree = ""

    return trees

if __name__ == '__main__':
    correct, partial_credit, total = 0, 0, 0

    # get data from FrameNet arguments
    for fulltext_file in glob.glob("{}/*.xml".format(options.fulltext_corpus)):
        reader = FulltextReader(fulltext_file)
        trees = get_trees(".".join(os.path.basename(fulltext_file).split(".")[:-1]))

        # for each argument, see if it is in a parse tree. if yes +1
        for frame in reader.frames:
            print('.', end='')
            frame_text = " ".join([frame.get_word(w) for w in frame.words])
            for t in trees:
                if t.flat() == frame_text:
                    for arg in frame.args:
                        if arg.text != "":
                            total += 1
                            arg_len = len(arg.text.split())
                            match_len = len(t.closest_match(arg.text))
                            partial_credit += 1 - abs(arg_len - match_len) / max(arg_len, match_len)
                            if t.contains(arg.text):
                                correct += 1
                    break
        print()
        
    print("{}/{} = {}".format(correct, total, correct/total))
    print("{}/{} = {}".format(partial_credit, total, partial_credit/total))
{'fraction_non_alphanumeric': 0.12144772117962467, 'fraction_numerical': 0.005361930294906166, 'mean_word_length': 1.7678041543026706, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3056595', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1061, 'n_words': 263}