<commit_before>import os

TESTDIR = 'inriasuite'

def deconstruct_line(line):

    HALT = 'halt,'
    FAIL = 'fail,'
    TRUE = 'true,'
    FLOAT = '0.33'
    ASSIGN = 'X ='
    CUT = '!,'
    line_0_5 = line[0:5]
    H_F_T = (HALT, FAIL, TRUE)
    
    if line_0_5 not in H_F_T and line[0:2] != CUT and line[0:3] != ASSIGN and line[0:4] != FLOAT:
        first_open_par = line.find('(')
        brace_counter = 1
        i = first_open_par
        while brace_counter != 0:
            i += 1
            if line[i] == '(':
                brace_counter += 1
            elif line[i] == ')':
                brace_counter -= 1
        left = line[0:i + 1]
        right = line[i + 2:].strip()
    elif line_0_5 in H_F_T:
        if line_0_5 == FAIL:
            left = 'fail'
        elif line_0_5 == HALT:
            left = 'halt'
        elif line_0_5 == TRUE:
            left = 'true'
        right = line[6:]
    elif line[0:2] == CUT:
        left = '!'
        right = line[3:]
    elif line[0:3] == ASSIGN:
        left = 'X = "fred"'
        right = line[12:]
    elif line[0:4] == FLOAT:
        left = '0.333 =:= 1/3'
        right = line[15:]
    return left, right
    

def get_lines(file_):
    testfile = open(TESTDIR + '/' + file_)
    for test in testfile.readlines():
        if test.endswith('%%SKIP%%\n'):
            continue
        if test.startswith('['):
            last_bracket = test.rfind(']')
            first_bracket = test.find('[') + 1
            relevant = test[first_bracket:last_bracket]
            left, right = deconstruct_line(relevant)
            yield left, right


def deconstruct_list(l):
    pieces = [piece for piece in l.split('], [')]
    pieces[0] = pieces[0][1:]
    return pieces


def get_files():<commit_msg>refactoring in fileutil<commit_after>import os, py

#TESTDIR = py.path.local(__file__).dirpath().join('inriasuite') # use this line instead of the next
TESTDIR = 'inriasuite'

def deconstruct_line(line):

    HALT = 'halt,'
    FAIL = 'fail,'
    TRUE = 'true,'
    FLOAT = '0.33'
    ASSIGN = 'X ='
    CUT = '!,'
    line_0_5 = line[0:5]
    H_F_T = (HALT, FAIL, TRUE)
    
    # use str.startswith instead of [...]
    if line_0_5 in H_F_T:
        if line_0_5 == FAIL:
            left = 'fail'
        elif line_0_5 == HALT:
            left = 'halt'
        elif line_0_5 == TRUE:
            left = 'true'
        right = line[6:]
    elif line[0:2] == CUT:
        left = '!'
        right = line[3:]
    elif line[0:3] == ASSIGN:
        left = 'X = "fred"'
        right = line[12:]
    elif line[0:4] == FLOAT:
        left = '0.333 =:= 1/3'
        right = line[15:]
    else:
        first_open_par = line.find('(')
        brace_counter = 1
        i = first_open_par
        while brace_counter:
            i += 1
            if line[i] == '(':
                brace_counter += 1
            elif line[i] == ')':
                brace_counter -= 1
        left = line[0:i + 1]
        right = line[i + 2:].strip()
    return left, right
    

def get_lines(file_):
    testfile = open(TESTDIR + '/' + file_)
    for test in testfile.readlines():
        if test.endswith('%%SKIP%%\n'):
            continue
        if test.startswith('['):
            last_bracket = test.rfind(']')
            first_bracket = test.find('[') + 1
            relevant = test[first_bracket:last_bracket]
            left, right = deconstruct_line(relevant)
            yield left, right


def deconstruct_list(l):
    pieces = [piece for piece in l.split('], [')]
    pieces[0] = pieces[0][1:]
    return pieces


def get_files():
{'fraction_non_alphanumeric': 0.1463143254520167, 'fraction_numerical': 0.02698191933240612, 'mean_word_length': 1.794094794094794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4219831', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1242, 'n_words': 299}