<commit_before>#! /usr/bin/env python3

from sys import stdin

for line in stdin:
    line = line.strip()

    if line == '':
        print('')
    else:
        wf, feats, lemma, label, ann = line.split('\t')

        lemmas = ann
        if ann.find(' ') != -1:
            lemmas = ann[:ann.find(' ')]
            ann = [ann.find(' '):]

        lemma_dict = dict(eval(ann))
        
        if label in lemma_dict:
            lemma = lemma_dict[label]
            lemma = lemma.lower()
            lemma = lemma.replace('#','')

        print('%s\t%s\t%s\t%s\t%s' % (wf, feats, lemma, label, ann))<commit_msg>Choose lemma with fewest parts.<commit_after>#! /usr/bin/env python3

from sys import stdin

def part_count(lemma):
    return lemma.count('#')

def compile_dict(label_lemma_pairs):
    res = {}

    for label, lemma in label_lemma_pairs:
        if label in res:
            old_lemma = res[label]

            if part_count(old_lemma) > part_count(lemma):
                res[label] = lemma
        else:
            res[label] = lemma

    return res

for line in stdin:
    line = line.strip()

    if line == '':
        print('')
    else:
        wf, feats, lemma, label, ann = line.split('\t')

        lemmas = ann
        if ann.find(' ') != -1:
            lemmas = ann[:ann.find(' ')]
            ann = ann[ann.find(' ') + 1:]
        else:
            ann = '_'
        
        lemma_dict = {}
        if lemmas != '_':
            lemma_dict = compile_dict(eval(lemmas))
        
        if label in lemma_dict:
            lemma = lemma_dict[label]
            lemma = lemma.lower()
            lemma = lemma.replace('#','')

        print('%s\t%s\t%s\t%s\t%s' % (wf, feats, lemma, label, ann))
{'fraction_non_alphanumeric': 0.14965197215777262, 'fraction_numerical': 0.002900232018561485, 'mean_word_length': 1.699530516431925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1435342', 'n_tokens_mistral': 608, 'n_tokens_neox': 560, 'n_words': 137}