<commit_before>"""Count the numbers of annotated entities and emotional sentences in the
corpus that was manually annotated.

Usage: python annotation_statistics.py <dir containing the folia files with
EmbodiedEmotions annotations>
"""
from lxml import etree
from bs4 import BeautifulSoup
from emotools.bs4_helpers import sentence, note
import argparse
import os
from collections import Counter


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('dir_name', help='the name of the dir containing the '
                        'FoLiA XML files that should be processed.')
    args = parser.parse_args()

    dir_name = args.dir_name

    entity_class = u'EmbodiedEmotions'

    act_tag = '{http://ilk.uvt.nl/folia}div'

    os.chdir(dir_name)

    folia_counter = 0
    num_sent = 0
    num_emotional = 0
    stats = Counter()

    for file_name in os.listdir(dir_name):
        folia_counter += 1
        print '({}) {}'.format(folia_counter, file_name)

        sents = set()
        # load document
        context = etree.iterparse(file_name,
                                  events=('start', 'end'),
                                  tag=act_tag,
                                  huge_tree=True)
        delete = True
        for event, elem in context:
            if event == 'start' and elem.get('class') == 'act':
                delete = False
            if event == 'end' and elem.get('class') == 'act':
                # load act into memory
                act_xml = BeautifulSoup(etree.tostring(elem), 'xml')
                sentences = act_xml.find_all(sentence)
                s = None
                for sent in sentences:
                    if not note(sent.parent):
                        # some t elements appear to be empty (this is not
                        # allowed, but it happens). So, check whether there is
                        # a string to add before adding it.
                        if sent.t:
                            if sent.t.string:
                                s = sent.t.string

                        # calculate stats only for unique sentences in text
                        if s and s not in sents:
                            sents.add(s)
                            num_sent += 1

                            entities = sent.find_all('entity')
                            emotional = False
                            for entity in entities:
                                e = entity.attrs.get('class')
                                if e.startswith(entity_class):
                                    emotional = True
                                    stats[e] += 1
                            if emotional:
                                num_emotional += 1

                delete = True

            # clear memory
            #if delete:
            #    elem.clear()
            #    while elem.getprevious() is not None:
            #        del elem.getparent()[0]
            #        del context

    # print stats
    print '{} sentences in {} files'.format(num_sent, folia_counter)
    print '{} emotional sentences'.format(num_emotional)
    for tag, freq in stats.most_common():
        print '{}\t{}'.format(tag, freq)
<commit_msg>Improve output printed by the annotation_statistics script<commit_after>"""Count the numbers of annotated entities and emotional sentences in the
corpus that was manually annotated.

Usage: python annotation_statistics.py <dir containing the folia files with
EmbodiedEmotions annotations>
"""
from lxml import etree
from bs4 import BeautifulSoup
from emotools.bs4_helpers import sentence, note
import argparse
import os
from collections import Counter


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('dir_name', help='the name of the dir containing the '
                        'FoLiA XML files that should be processed.')
    args = parser.parse_args()

    dir_name = args.dir_name

    entity_class = u'EmbodiedEmotions'

    act_tag = '{http://ilk.uvt.nl/folia}div'

    os.chdir(dir_name)

    folia_counter = 0
    num_sent = 0
    num_emotional = 0
    stats = Counter()

    print 'Files'
    for file_name in os.listdir(dir_name):
        folia_counter += 1
        print '{}'.format(file_name)

        sents = set()
        # load document
        context = etree.iterparse(file_name,
                                  events=('start', 'end'),
                                  tag=act_tag,
                                  huge_tree=True)
        delete = True
        for event, elem in context:
            if event == 'start' and elem.get('class') == 'act':
                delete = False
            if event == 'end' and elem.get('class') == 'act':
                # load act into memory
                act_xml = BeautifulSoup(etree.tostring(elem), 'xml')
                sentences = act_xml.find_all(sentence)
                s = None
                for sent in sentences:
                    if not note(sent.parent):
                        # some t elements appear to be empty (this is not
                        # allowed, but it happens). So, check whether there is
                        # a string to add before adding it.
                        if sent.t:
                            if sent.t.string:
                                s = sent.t.string

                        # calculate stats only for unique sentences in text
                        if s and s not in sents:
                            sents.add(s)
                            num_sent += 1

                            entities = sent.find_all('entity')
                            emotional = False
                            for entity in entities:
                                e = entity.attrs.get('class')
                                if e.startswith(entity_class):
                                    emotional = True
                                    stats[e] += 1
                            if emotional:
                                num_emotional += 1

                delete = True

            # clear memory
            # results in segmentation fault (for some reason)
            #if delete:
            #    elem.clear()
            #    while elem.getprevious() is not None:
            #        del elem.getparent()[0]
            #        del context

    # print stats
    print '\nBasic stats'
    print '{} sentences in {} files'.format(num_sent, folia_counter)
    print '{} emotional sentences'.format(num_emotional)

    print '\nLabel\tFrequency'
    for tag, freq in stats.most_common():
        print '{}\t{}'.format(tag, freq)

{'fraction_non_alphanumeric': 0.07964601769911504, 'fraction_numerical': 0.002999850007499625, 'mean_word_length': 1.3495419309372798, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7186398', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1631, 'n_words': 513}