<commit_before>import collections

OUTPUT=0
INPUT=1
LOOP=2
ENDLOOP=3
MOVE=4
ADD=5
SET=6
MULCOPY=7
SCAN=8
LOADOUT=9
LOADOUTSET=10
IF=11
ENDIF=12

def parse(code):
    code = ''.join(i for i in code if i in '.,[]+-<>')
    tokens = []
    for i in code:
        if i == '+':
            tokens.append((ADD, (0, 1)))
        elif i == '-':
            tokens.append((ADD, (0, -1)))
        elif i == '>':
            tokens.append((MOVE, 1))
        elif i == '<':
            tokens.append((MOVE, -1))
        elif i == '[':
            tokens.append((LOOP, None))
        elif i == ']':
            tokens.append((ENDLOOP, None))
        elif i == ',':
            tokens.append((INPUT, 1))
        if i == '.':
            tokens.append((LOADOUT, (0, 0)))
            tokens.append((OUTPUT, None))
    return tokens

def optimize(tokens):
    # Optimize various things
    newtokens = []
    shift = 0
    # With normal dict, the order sometimes switches
    # in recursion, and the optimized never exits.
    vals = collections.OrderedDict()
    ops = {}
    do_output = False
    for token, value in tokens:
        if token not in (SET, ADD, MOVE, LOADOUT, LOADOUTSET, OUTPUT):
            if do_output:
                newtokens.append((OUTPUT, None))
                do_output = False

            for k, v in vals.items():
                newtokens.append((ops[k], (k, v)))
            vals.clear()
            ops.clear()

        if token in (SET, ADD):
            offset, val = value
            offset += shift
            # ADD/SET then SET does nothing; remove it
            if offset in ops and token == SET:
                ops.pop(offset)
                vals.pop(offset)
            if offset not in ops:
                ops[offset] = token # SET or ADD
            vals[offset] = vals.get(offset, 0) + val
        elif token == MULCOPY:
            src, dest, mul = value
            newtokens.append((MULCOPY, (src+shift, dest+shift, mul)))
        # XXX Deal with shift in if
        elif token == IF:
            offset = shift + value
            newtokens.append((IF, offset))
        elif token == ENDIF:
            newtokens.append((ENDIF, None))
        elif token == MOVE:
            shift += value
        elif token == OUTPUT:
            do_output = True
        elif token == LOADOUT:
            offset, add = value
            offset += shift
            if ops.get(offset) == SET:
                newtokens.append((LOADOUTSET, vals[offset] + add))
            elif ops.get(offset) == ADD:
                newtokens.append((LOADOUT, (offset, vals[offset] + add)))
            else:
                newtokens.append((LOADOUT, (offset, add)))
        elif token == LOADOUTSET:
            newtokens.append((token, value))
        elif token in (LOOP, ENDLOOP, INPUT, SCAN):
            if shift:
                newtokens.append((MOVE, shift))
                shift = 0
            newtokens.append((token, value))
        else:
            raise ValueError('What is this ' + str(token) + ' doing here?')

    # Any remaining add/set/shift is ignored, as it would have no effect
    if do_output:
        newtokens.append((OUTPUT, None))

    newtokens2 = []

    i = 0
    while i < len(newtokens):
        optimized = False

        # Optimize scan loop
        if (i < len(newtokens)-2 and<commit_msg>Move shift code around<commit_after>import collections

OUTPUT=0
INPUT=1
LOOP=2
ENDLOOP=3
MOVE=4
ADD=5
SET=6
MULCOPY=7
SCAN=8
LOADOUT=9
LOADOUTSET=10
IF=11
ENDIF=12

def parse(code):
    code = ''.join(i for i in code if i in '.,[]+-<>')
    tokens = []
    for i in code:
        if i == '+':
            tokens.append((ADD, (0, 1)))
        elif i == '-':
            tokens.append((ADD, (0, -1)))
        elif i == '>':
            tokens.append((MOVE, 1))
        elif i == '<':
            tokens.append((MOVE, -1))
        elif i == '[':
            tokens.append((LOOP, None))
        elif i == ']':
            tokens.append((ENDLOOP, None))
        elif i == ',':
            tokens.append((INPUT, 1))
        if i == '.':
            tokens.append((LOADOUT, (0, 0)))
            tokens.append((OUTPUT, None))
    return tokens

def optimize(tokens):
    # Optimize various things
    newtokens = []
    shift = 0
    # With normal dict, the order sometimes switches
    # in recursion, and the optimized never exits.
    vals = collections.OrderedDict()
    ops = {}
    do_output = False
    for token, value in tokens:
        if token not in (SET, ADD, MOVE, LOADOUT, LOADOUTSET, OUTPUT):
            if do_output:
                newtokens.append((OUTPUT, None))
                do_output = False

            for k, v in vals.items():
                newtokens.append((ops[k], (k, v)))
            vals.clear()
            ops.clear()
        if shift and token in (LOOP, ENDLOOP, INPUT, SCAN):
            newtokens.append((MOVE, shift))
            shift = 0

        if token in (SET, ADD):
            offset, val = value
            offset += shift
            # ADD/SET then SET does nothing; remove it
            if offset in ops and token == SET:
                ops.pop(offset)
                vals.pop(offset)
            if offset not in ops:
                ops[offset] = token # SET or ADD
            vals[offset] = vals.get(offset, 0) + val
        elif token == MULCOPY:
            src, dest, mul = value
            newtokens.append((MULCOPY, (src+shift, dest+shift, mul)))
        # XXX Deal with shift in if
        elif token == IF:
            offset = shift + value
            newtokens.append((IF, offset))
        elif token == MOVE:
            shift += value
        elif token == OUTPUT:
            do_output = True
        elif token == LOADOUT:
            offset, add = value
            offset += shift
            if ops.get(offset) == SET:
                newtokens.append((LOADOUTSET, vals[offset] + add))
            elif ops.get(offset) == ADD:
                newtokens.append((LOADOUT, (offset, vals[offset] + add)))
            else:
                newtokens.append((LOADOUT, (offset, add)))
        elif token in (ENDIF, LOADOUTSET, LOOP, ENDLOOP, INPUT, SCAN):
            newtokens.append((token, value))
        else:
            raise ValueError('What is this ' + str(token) + ' doing here?')

    # Any remaining add/set/shift is ignored, as it would have no effect
    if do_output:
        newtokens.append((OUTPUT, None))

    newtokens2 = []

    i = 0
    while i < len(newtokens):
        optimized = False

        # Optimize scan loop
        if (i < len(newtokens)-2 and
{'fraction_non_alphanumeric': 0.12321482831965967, 'fraction_numerical': 0.009419629292008508, 'mean_word_length': 1.6427137695704537, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587357', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1930, 'n_words': 583}