<commit_before>#!/usr/bin/env python
# Module for reading Longman XML and producing JSON output

import json
import sys

from xml_parser import XMLParser

assert json  # silence pyflakes

u'\xe1rv\xedzt\u0171r\u0151 t\xfck\xf6rf\xfar\xf3g\xe9p\n'

class EkszParser(XMLParser):
    abbreviations = [
        (u"mo.-i", u'magyarorsz\xe1gi'),
        (u"Mo.-on", u'Magyarorsz\xe1gon'),
        (u"Mo.-hoz", u'Magyarorsz\xe1ghoz'),
        (u"Mo.", u'Magyarorsz\xe1g'),
        (u"bp.-i", u"budapesti"),
        (u"vonatk.", u"vonatkoz\xf3"),
        (u"vki", u"valaki"),
        (u"vmi", u"valami"),
        (u"kapcs.", u"kapcsolatos"),
        (u"haszn.", u"haszn\xe1lt"),
        (u"vmely", u"valamely"),
        (u"vmilyen", u"valamilyen"),
        (u".,", u";"),
        (u".;", u";"),
        (u"..", u"."),
        (u".a.", u"a."),  # a single line in the data
        (u"sz.-ban", u"sz\xe1zadban"),
        (u"sz.-beli", u"sz\xe1zadbeli"),
        (u"sz.-i", u"sz\xe1zadi"),
        (u"sz\xf3haszn-\xe1ban", u"sz\xf3haszn\xe1lat\xe1ban"),
        (u" (Na2SO4.10H2O)", u""),
        (u" (CaSO4.2iH2O)", u""),
        (u" MgSO4.7H2O", u""),
        (u"dec. ", u"december "),
    ]

    @staticmethod
    def parse_headword(sense):
        hw = EkszParser.get_section("LEMMA", sense)
        hom_num = int(EkszParser.get_section("HOM", hw))
        hw = EkszParser.remove_sections("HOM", hw)
        return hw, hom_num

    @staticmethod
    def clean_definition(d):<commit_msg>last(?) fixes<commit_after>#!/usr/bin/env python
# Module for reading Longman XML and producing JSON output

import json
import sys

from xml_parser import XMLParser

assert json  # silence pyflakes

u'\xe1rv\xedzt\u0171r\u0151 t\xfck\xf6rf\xfar\xf3g\xe9p\n'

class EkszParser(XMLParser):
    abbreviations = [
        (u"mo.-i", u'magyarorsz\xe1gi'),
        (u"Mo.-on", u'Magyarorsz\xe1gon'),
        (u"Mo.-hoz", u'Magyarorsz\xe1ghoz'),
        (u"Mo.", u'Magyarorsz\xe1g'),
        (u"bp.-i", u"budapesti"),
        (u"vonatk.", u"vonatkoz\xf3"),
        (u"vki", u"valaki"),
        (u"vmi", u"valami"),
        (u"kapcs.", u"kapcsolatos"),
        (u"haszn.", u"haszn\xe1lt"),
        (u"vmely", u"valamely"),
        (u"vmilyen", u"valamilyen"),
        (u".,", u";"),
        (u".;", u";"),
        (u"..", u"."),
        (u".a.", u"a."),  # a single line in the data
        (u"sz.-ban", u"sz\xe1zadban"),
        (u"sz.-beli", u"sz\xe1zadbeli"),
        (u"sz.-i", u"sz\xe1zadi"),
        (u"sz.-nak", u"sz\xe1zadnak"),
        (u"sz\xf3haszn-\xe1ban", u"sz\xf3haszn\xe1lat\xe1ban"),
        (u" (Na2SO4.10H2O)", u""),
        (u" (CaSO4.2H2O)", u""),
        (u" MgSO4.7H2O", u""),
        (u"J\xfan.-", u"J\xfanius"),
        (u"j\xfan.-", u"j\xfanius"),
        (u"szept.-", u"szeptember"),
        (u"dec. ", u"december "),
        (u"h.:", u"hogy:"),
    ]

    @staticmethod
    def parse_headword(sense):
        hw = EkszParser.get_section("LEMMA", sense)
        hom_num = int(EkszParser.get_section("HOM", hw))
        hw = EkszParser.remove_sections("HOM", hw)
        return hw, hom_num

    @staticmethod
    def clean_definition(d):
{'fraction_non_alphanumeric': 0.21403508771929824, 'fraction_numerical': 0.023923444976076555, 'mean_word_length': 2.3793103448275863, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6315858', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1344, 'n_words': 216}