<commit_before>import re
from collections import namedtuple

Entry = namedtuple('Entry', 'id URI abstract')


def line_to_entry(i, line):
    return Entry(
        i,
        re.match(r'<([^>]+)>', line).group(1),
        re.search(r'"([^"]+)"', line).group(1)
    )


def get_entries(path):
    with open(path, encoding='utf-8') as file:
        i = 0
        list = []
        for line in file:
            i += 1
            if line.startswith('<'):
                list.append(line_to_entry(i, line))
        return list

if __name__ == '__main__':
    entries = get_entries("test.ttl")

    glossary = {}
    re_word = re.compile('\w+', re.UNICODE)
    for entry in entries:
        words = re_word.findall(entry.abstract)
        words = [word.lower() for word in words]
        for word in words:
            if word in glossary:
                glossary[word].append(entry.id)
            else:
                glossary[word] = [entry.id]

    with open('output.txt', 'w', encoding='utf-8') as output:
        for key, value in glossary.items():
            output.write('%s: ' % key)
            output.write(','.join(str(x) for x in value))
            output.write('\n')<commit_msg>eine schleife im parser eingespart :)<commit_after>import re
from collections import namedtuple

Entry = namedtuple('Entry', 'id URI abstract')


def line_to_entry(i, line):
    return Entry(
        i,
        re.match(r'<([^>]+)>', line).group(1),
        re.search(r'"([^"]+)"', line).group(1)
    )


def get_entries(path):
    with open(path, encoding='utf-8') as file:
        i = 0
        list = []
        for line in file:
            i += 1
            if line.startswith('<'):
                list.append(line_to_entry(i, line))
        return list

if __name__ == '__main__':
    entries = get_entries("test.ttl")

    glossary = {}
    re_word = re.compile('\w+', re.UNICODE)
    for entry in entries:
        words = re_word.findall(entry.abstract)
        #words = [word.lower() for word in words]
        for word in words:
            word = word.lower();
            if word in glossary:
                glossary[word].append(entry.id)
            else:
                glossary[word] = [entry.id]

    with open('output.txt', 'w', encoding='utf-8') as output:
        for key, value in glossary.items():
            output.write('%s: ' % key)
            output.write(','.join(str(x) for x in value))
            output.write('\n')
{'fraction_non_alphanumeric': 0.1513911620294599, 'fraction_numerical': 0.004909983633387889, 'mean_word_length': 2.0110837438423643, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '463966', 'n_tokens_mistral': 806, 'n_tokens_neox': 784, 'n_words': 196}