<commit_before>import sys

from machine import Machine
from monoid import Monoid
from control import PosControl as Control
from definition_parser import read
from constructions import read_constructions

def read_order_file(f):
    sentence = []
    for line in f:
        le = line.strip().split("\t")
        if len(le) == 0:
            break
        else:
            sentence.append(tuple(le))
    return sentence

def create_machines(sen):
    machines = []
    for _, pos in sen:
        word, pos = pos.split("/", 1)
        m = Machine(Monoid(word), Control(pos))
        machines.append(m)
    return machines

if __name__ == "__main__":
    order = read_order_file(file(sys.argv[1]))
    definitions = read(file(sys.argv[3]))
    cons = read_constructions(file(sys.argv[2]), definitions)

    machines = create_machines(order)

    while True:
        something = False
        for con in cons:
            l = len(con.rule_right)
            for i in xrange(len(machines) - l + 1):
                result = con.do(machines[i:i+l])
                if result is not None:
                    machines[i:i+l] = result
                    something = True

        if not something:
            break
        else:
            pass
    print " ".join([str(m) for m in machines])
<commit_msg>main operations moved into new run() function<commit_after>import sys

from machine import Machine
from monoid import Monoid
from control import PosControl as Control
from definition_parser import read
from constructions import read_constructions

def read_order_file(f):
    sentence = []
    for line in f:
        le = line.strip().split("\t")
        if len(le) == 0:
            break
        else:
            sentence.append(tuple(le))
    return sentence

def create_machines(sen):
    machines = []
    for _, pos in sen:
        word, pos = pos.split("/", 1)
        m = Machine(Monoid(word), Control(pos))
        machines.append(m)
    return machines

def run(order, definitions, constructions):
    machines = create_machines(order)

    while True:
        something = False
        for con in constructions:
            l = len(con.rule_right)
            for i in xrange(len(machines) - l + 1):
                result = con.do(machines[i:i+l])
                if result is not None:
                    machines[i:i+l] = result
                    something = True

        if not something:
            break
        else:
            pass
    # TODO why [0]?
    return machines[0]

if __name__ == "__main__":
    order = read_order_file(file(sys.argv[1]))
    definitions = read(file(sys.argv[3]))
    cons = read_constructions(file(sys.argv[2]), definitions)
    run(order, definitions, cons)


{'fraction_non_alphanumeric': 0.10022107590272661, 'fraction_numerical': 0.005158437730287398, 'mean_word_length': 2.1242807825086305, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1262367', 'n_tokens_mistral': 801, 'n_tokens_neox': 780, 'n_words': 230}