<commit_before>import json
import sys

import cmdio
from cmdio import Command


def decomment(commands):
    output = []
    for item in commands:
        if item == "":
            output.append("#")
        elif item.startswith("#"):
            output.append("#")  # Magic value, read on line 30
        elif "#" in item:
            output.append(item[:item.find("#")])
        else:
            output.append(item)
    return output

if len(sys.argv) == 2:
    infile = decomment(open(
        sys.argv[1]).read().strip().split("\n"))
else:
    infile = decomment(
        sys.stdin.read().strip().split("\n"))

commands = []


for index, item in enumerate(infile, start=1):
    line = index
    if not item == "#":
        opcode = item.split(" ")[0]
        arguments = item.split(" ")[1:]
        while "" in arguments:
            arguments.remove("")
        intargs = []
        for item in arguments:
            if item.startswith("$"):
                intargs.append(int(item[1:], 16))
            elif item.startswith("%"):
                intargs.append(int(item[1:], 2))
            elif item.startswith("0"):
                try:
                    intargs.append(int(item[1:], 8))
                except ValueError:
                    if item == "0":
                        intargs.append(0)
                    else:
                        raise
                        # The above code was to handle the case where a 0
                        # was being treated like a base specifier.
            else:
                try:
                    intargs.append(int(item))
                except ValueError:
                    intargs.append(item)
        cmd = Command(line, opcode, intargs)
        commands.append(cmd)


sys.stdout.write(cmdio.jsonencode(commands))
<commit_msg>Improve decomment<commit_after>import json
import sys

import cmdio
from cmdio import Command


def decomment(commands):
    output = []
    for item in commands:

        cmtpos = item.find("#")
        if cmtpos == -1:
            output.append(item)
        else:
            output.append(item[:cmtpos])
    return output

if len(sys.argv) == 2:
    infile = decomment(open(
        sys.argv[1]).read().strip().split("\n"))
else:
    infile = decomment(
        sys.stdin.read().strip().split("\n"))

commands = []


for index, item in enumerate(infile, start=1):
    line = index
    if not item == "":
        opcode = item.split(" ")[0]
        arguments = item.split(" ")[1:]
        while "" in arguments:
            arguments.remove("")
        intargs = []
        for item in arguments:
            if item.startswith("$"):
                intargs.append(int(item[1:], 16))
            elif item.startswith("%"):
                intargs.append(int(item[1:], 2))
            elif item.startswith("0"):
                try:
                    intargs.append(int(item[1:], 8))
                except ValueError:
                    if item == "0":
                        intargs.append(0)
                    else:
                        raise
                        # The above code was to handle the case where a 0
                        # was being treated like a base specifier.
            else:
                try:
                    intargs.append(int(item))
                except ValueError:
                    intargs.append(item)
        cmd = Command(line, opcode, intargs)
        commands.append(cmd)


sys.stdout.write(cmdio.jsonencode(commands))

{'fraction_non_alphanumeric': 0.12510760401721666, 'fraction_numerical': 0.010043041606886656, 'mean_word_length': 1.482905982905983, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4238442', 'n_tokens_mistral': 997, 'n_tokens_neox': 951, 'n_words': 241}