<commit_before>#!/usr/bin/env python
# encoding: utf-8

import sys
import argparse
import yaml

_ignore_symbols = ['\n']


def check(nfae, string, state=None, chain=''):
    if state is None:
        state = nfae['initial']

    nchain = chain + state

    try:
        if None in nfae['transitions'][state]:
            nnchain = '%s [] ' % nchain
            for nstate in nfae['transitions'][state][None]:
                if check(nfae, string, nstate, nnchain):
                    return True
    except KeyError:
        pass

    if string == '':
        if state in nfae['finals']:
            print nchain
            return True
        else:
            return False

    symbol = string[0]
    nstring = string[1:]
    nchain = '%s [%s] ' % (nchain, symbol)

    if symbol in _ignore_symbols:
        return check(nfae, nstring, state, chain)

    elif symbol not in nfae['symbols']:
        return False

    try:
        for nstate in nfae['transitions'][state][symbol]:
            if check(nfae, nstring, nstate, nchain):
                return True
        return False

    except KeyError:
        return False


def main():
    parser = argparse.ArgumentParser(prog='nfae_check', add_help=True)
    parser.add_argument('-f', '--lang-file', required=True, help='file describing the language formatted in YAML', type=file)
    args = parser.parse_args()

    #string = sys.stdin.read()
    string = raw_input()
    for entry in yaml.load_all(args.lang_file):
        nfae = entry['nfae']
        print '%s:' % nfae['name']
        print 'OK!' if check(nfae, string) else 'FAIL!'


if __name__ == '__main__':
    main()
<commit_msg>Better output.<commit_after>#!/usr/bin/env python
# encoding: utf-8

import sys
import argparse
import yaml

_ignore_symbols = ['\n']


def check(nfae, string, state=None, chain=''):
    if state is None:
        state = nfae['initial']

    nchain = chain + state

    try:
        if None in nfae['transitions'][state]:
            nnchain = '%s [] ' % nchain
            for nstate in nfae['transitions'][state][None]:
                if check(nfae, string, nstate, nnchain):
                    return True
    except KeyError:
        pass

    if string == '':
        if state in nfae['finals']:
            print nchain
            return True
        else:
            return False

    symbol = string[0]
    nstring = string[1:]
    nchain = '%s [%s] ' % (nchain, symbol)

    if symbol in _ignore_symbols:
        return check(nfae, nstring, state, chain)

    elif symbol not in nfae['symbols']:
        return False

    try:
        for nstate in nfae['transitions'][state][symbol]:
            if check(nfae, nstring, nstate, nchain):
                return True
        return False

    except KeyError:
        return False


def main():
    parser = argparse.ArgumentParser(prog='nfae_check', add_help=True)
    parser.add_argument('-f', '--lang-file', required=True, help='file describing the language formatted in YAML', type=file)
    args = parser.parse_args()

    #string = sys.stdin.read()
    string = raw_input()
    for entry in yaml.load_all(args.lang_file):
        nfae = entry['nfae']
        print
        print '%s:' % nfae['name']
        print 'OK!' if check(nfae, string) else 'FAIL!'


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.1228493812254754, 'fraction_numerical': 0.0018110473890733474, 'mean_word_length': 2.236328125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5952778', 'n_tokens_mistral': 1064, 'n_tokens_neox': 1020, 'n_words': 294}