<commit_before># -*- coding: utf-8 -*-
import shlex


from yamt.core import Task


def old_parser(text):
    """ Very crude parser for a file with syntax somewhat similar to Drake."""
    tasks = []
    code = []
    comments = []
    task_found = False
    for line in text.splitlines():
        if line.startswith("#"):
            if task_found:
                tasks.append(
                    Task(interpreter, inputs, outputs, code, comments))
                comments = []
            if "<-" in line:
                interpreter = "bash"
                inputs = []
                outputs = []
                code = []
                lexer = shlex.shlex(line[1:])
                tokens = [token for token in lexer]
                print tokens
                found = False
                found_interpreter = False
                for token in tokens:
                    if "]" in token:
                        break
                    elif found_interpreter:
                        interpreter = token
                        found_interpreter = False
                    elif "[" in token:
                        found_interpreter = True
                    elif "<" in token or "-" in token:
                        found = True<commit_msg>Removed unneeded print statement in old parser.<commit_after># -*- coding: utf-8 -*-
import shlex


from yamt.core import Task


def old_parser(text):
    """ Very crude parser for a file with syntax somewhat similar to Drake."""
    tasks = []
    code = []
    comments = []
    task_found = False
    for line in text.splitlines():
        if line.startswith("#"):
            if task_found:
                tasks.append(
                    Task(interpreter, inputs, outputs, code, comments))
                comments = []
            if "<-" in line:
                interpreter = "bash"
                inputs = []
                outputs = []
                code = []
                lexer = shlex.shlex(line[1:])
                tokens = [token for token in lexer]
                found = False
                found_interpreter = False
                for token in tokens:
                    if "]" in token:
                        break
                    elif found_interpreter:
                        interpreter = token
                        found_interpreter = False
                    elif "[" in token:
                        found_interpreter = True
                    elif "<" in token or "-" in token:
                        found = True
{'fraction_non_alphanumeric': 0.08501383946223803, 'fraction_numerical': 0.0015816528272044287, 'mean_word_length': 1.097844112769486, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4176212', 'n_tokens_mistral': 625, 'n_tokens_neox': 580, 'n_words': 187}