<commit_before># -*- coding: utf-8 -*-

import tokenize
import textwrap


__author__ = "<NAME>"


dd = textwrap.dedent


INDENT = "\t"


def is_newline(token, newline_tokens=set((tokenize.NEWLINE, tokenize.NL))):
    return token.type in newline_tokens


def is_indent(token):
    return token.type == tokenize.INDENT


def is_dedent(token):
    return token.type == tokenize.DEDENT


def is_greater(token):
    return token.type == tokenize.OP and token.string == ">"


def is_comma(token):
    return token.type == tokenize.COMMA


def is_name(token):
    return token.type == tokenize.NAME


def is_colon(token):
    return token.string == ":"


def is_comment(token):
    return token.type == tokenize.COMMENT


def is_string(token):
    return token.type == tokenize.STRING


def lineno(token):
    return token.start[0]


class StopAutomaton(Exception):

    def __init__(self, token):
        self.token = token


class TokenAutomaton:

    subautomata = dict()

    def __init__(self, snakefile, base_indent=0, dedent=0, root=True):<commit_msg>Fixed comma detection and made token detection more robust.<commit_after># -*- coding: utf-8 -*-

import tokenize
import textwrap


__author__ = "<NAME>"


dd = textwrap.dedent


INDENT = "\t"


def is_newline(token, newline_tokens=set((tokenize.NEWLINE, tokenize.NL))):
    return token.type in newline_tokens


def is_indent(token):
    return token.type == tokenize.INDENT


def is_dedent(token):
    return token.type == tokenize.DEDENT


def is_op(token):
    return token.type == tokenize.OP


def is_greater(token):
    return is_op(token) and token.string == ">"


def is_comma(token):
    return is_op(token) and token.string == ","


def is_name(token):
    return token.type == tokenize.NAME


def is_colon(token):
    return is_op(token) and token.string == ":"


def is_comment(token):
    return token.type == tokenize.COMMENT


def is_string(token):
    return token.type == tokenize.STRING


def lineno(token):
    return token.start[0]


class StopAutomaton(Exception):

    def __init__(self, token):
        self.token = token


class TokenAutomaton:

    subautomata = dict()

    def __init__(self, snakefile, base_indent=0, dedent=0, root=True):
{'fraction_non_alphanumeric': 0.12167643082469581, 'fraction_numerical': 0.003605227579990987, 'mean_word_length': 3.753747323340471, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3263630', 'n_tokens_mistral': 771, 'n_tokens_neox': 730, 'n_words': 180}