<commit_before>#!/usr/bin/env python3

import argparse

class Block:
    tiers = []

    def __init__(self, tiers = []):
        self.tiers = tiers

    def getTier(self, name):
        for v in self.tiers:
            if v[0] == name:
                return v
        raise "Scheisse"

    def setTier(self, name, value):
        assert value[0] == name
        print(self.tiers)
        for idx, v in enumerate(self.tiers):
            if v[0] == name:
                self.tiers[idx] = value
                return
        raise "Scheisse"

    def doPrint(self):
        for tier in self.tiers:
            print(" ".join(tier))

class FileInstance:
    blocks = []

    def addBlock(self, block):
        self.blocks.append(block)

    def doPrint(self):
        for block in self.blocks:
            block.doPrint()

def parse(f):
    currentBlock = Block()
    ret = FileInstance()

    for line in f:
        line = line.rstrip()
        if len(line) == 0:
            ret.addBlock(currentBlock)
            currentBlock = Block()
        else:
            currentBlock.tiers.append(line.split())

    ret.addBlock(currentBlock)
    return ret

def specialReplace(word, a, b):
    if "question" in word:
        return word
    else:
        return word.replace(a, b)

def createMa(block, function = None):
    mtTier = block.getTier("\\mt")
    maTier = ["\\ma"] + [word for word in mtTier[1:] if word != '/']
    if function:
        maTier = function(maTier)
    block.tiers.append(maTier)
    return block

blockOperations = {
    "QToApostrophe": lambda block: Block([ [specialReplace(word, "q", "'") for word in tier ] for tier in block.tiers]),
    "ApostropheToQ": lambda block: Block([ [specialReplace(word, "'", "q") for word in tier ] for tier in block.tiers]),
    "IvenaToIvaEna": lambda block: createMa(block, lambda tier: (word.replace("ivena", "iva ena") for word in tier)),
    "createMa": createMa
}<commit_msg>Don't use specialReplace when not needed<commit_after>#!/usr/bin/env python3

import argparse

class Block:
    tiers = []

    def __init__(self, tiers = []):
        self.tiers = tiers

    def getTier(self, name):
        for v in self.tiers:
            if v[0] == name:
                return v
        raise "Scheisse"

    def setTier(self, name, value):
        assert value[0] == name
        print(self.tiers)
        for idx, v in enumerate(self.tiers):
            if v[0] == name:
                self.tiers[idx] = value
                return
        raise "Scheisse"

    def doPrint(self):
        for tier in self.tiers:
            print(" ".join(tier))

class FileInstance:
    blocks = []

    def addBlock(self, block):
        self.blocks.append(block)

    def doPrint(self):
        for block in self.blocks:
            block.doPrint()

def parse(f):
    currentBlock = Block()
    ret = FileInstance()

    for line in f:
        line = line.rstrip()
        if len(line) == 0:
            ret.addBlock(currentBlock)
            currentBlock = Block()
        else:
            currentBlock.tiers.append(line.split())

    ret.addBlock(currentBlock)
    return ret

def specialReplace(word, a, b):
    if "question" in word:
        return word
    else:
        return word.replace(a, b)

def createMa(block, function = None):
    mtTier = block.getTier("\\mt")
    maTier = ["\\ma"] + [word for word in mtTier[1:] if word != '/']
    if function:
        maTier = function(maTier)
    block.tiers.append(maTier)
    return block

blockOperations = {
    "QToApostrophe": lambda block: Block([ [specialReplace(word, "q", "'") for word in tier ] for tier in block.tiers]),
    "ApostropheToQ": lambda block: Block([ [word.replace("'", "q") for word in tier ] for tier in block.tiers]),
    "IvenaToIvaEna": lambda block: createMa(block, lambda tier: (word.replace("ivena", "iva ena") for word in tier)),
    "createMa": createMa
}
{'fraction_non_alphanumeric': 0.12817878243000258, 'fraction_numerical': 0.0030824556896994607, 'mean_word_length': 2.2075782537067545, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893252', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1231, 'n_words': 349}