<commit_before>import click
import yaml
from tabulate import tabulate
from collections import namedtuple


# Cuba keywords that are excludes from DataContainers
CUBA_DATA_CONTAINER_EXLCUDE = ['Id', 'Position']


@click.group()
def cli():
    """ Auto-generate code from cuba yaml description. """


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def python(input, output):
    """ Create the CUBA Enum for the DataContainer.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'from enum import IntEnum, unique\n',
        '\n',
        '\n',
        '@unique\n',
        'class CUBA(IntEnum):\n',
        '\n']
    template = "    {} = {}\n"
    for keyword in keywords:
        if keyword['name'] in CUBA_DATA_CONTAINER_EXLCUDE:
            continue
        lines.append(template.format(keyword['key'], keyword['number']))
    output.writelines(lines)


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def table(input, output):
    """ Create the CUBA DataContainer Table descriptions.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'import tables\n',
        '\n',
        '\n']

    # create Data table description
    lines.extend([
        'class Data(tables.IsDescription):\n',
        '\n'])
    data_types = {
        'string': 'String',
        'double': 'Float64',
        'integer': 'Int32'}
    position = 0
    for keyword in keywords:
        if keyword['name'] in CUBA_DATA_CONTAINER_EXLCUDE:
            continue
        if keyword['type'] == 'string':
            template = "    {} = tables.{}Col(pos={}, itemsize={})\n"
            shape = keyword['shape'][0]
        else:
            template = "    {} = tables.{}Col(pos={}{})\n"
            shape = keyword['shape']
            if len(shape) == 1:
                if shape[0] == 1:
                    shape = ''
                else:
                    shape = ', shape={}'.format(shape[0])
            else:
                shape = ', shape=({})'.format(
                    ', '.join(map(str, keyword['shape'])))
        lines.append(template.format(
            keyword['key'].lower(),
            data_types[keyword['type']],
            position,
            shape))
        position += 1

    lines.append('\n\n')

    # create Mask column
    mask_size = position  # without index
    lines.extend([
        'class Record(tables.IsDescription):\n',
        '\n',
        '    index = tables.StringCol(itemsize=32, pos=0)\n',
        '    data = Data()\n',
        '    mask = tables.BoolCol(pos=1, shape=({},))\n'.format(mask_size),
        '\n\n'])

    lines.extend([
        'class NoUIDRecord(tables.IsDescription):\n',
        '\n',
        '    data = Data()\n',
        '    mask = tables.BoolCol(pos=1, shape=({},))\n'.format(mask_size)])

    output.writelines(lines)


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def keywords(input, output):
    """ Create a dictionary of CUDS keywords.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'from collections import namedtuple\n',
        '\n',
        'import numpy\n',
        '\n',
        '\n',
        'ATTRIBUTES = [\n'
        '    "name", "description", "domain", "key", "number", "shape", "dtype"]\n'  # noqa
        'Keyword = namedtuple("Keyword", ATTRIBUTES)\n',
        '\n',
        '\n',
        'KEYWORDS = {\n']
    data_types = {
        'string': 'numpy.str',
        'double': 'numpy.float64',
        'integer': 'numpy.int32'}
    template = (
        "    '{key}': Keyword(\n"
        "        name='{name}',\n"
        "        description='{description}',  # noqa\n"
        "        domain={domain},\n"
        "        key='{key}',\n"
        "        number={number},\n"
        "        shape={shape},\n"
        "        dtype={type}),\n")
    for keyword in keywords:
        keyword['type'] = data_types[keyword['type']]
        lines.extend(template.format(**keyword))
    lines.append('}\n')

    output.writelines(lines)


_Column = namedtuple('_Column', ["key", "header"])


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def rst(input, output):
    """ Create an rst document with table describing CUDS keywords.
    """
    keywords = yaml.safe_load(input)

    columns = [_Column("name", "Name"),
               _Column("description", "Description"),
               _Column("domain", "Domain"),
               _Column("key", "Key"),
               _Column("number", "Number"),
               _Column("shape", "Shape"),
               _Column("type", "Type")]

    table_header = [col.header for col in columns]

    table_data = []

    for keyword in keywords:
        # keyword['serialized_type'] = data_types[keyword['type']]
        row = [keyword[col.key] for col in columns]
        table_data.append(row)
    rst = tabulate(table_data, table_header, tablefmt="rst")
    output.write(
        ".. this table was auto-generated by the cuba-generate.py script.")
    output.write(rst)


if __name__ == '__main__':
    cli()
<commit_msg>fix spelling<commit_after>import click
import yaml
from tabulate import tabulate
from collections import namedtuple


# Cuba keywords that are excludes from DataContainers
CUBA_DATA_CONTAINER_EXCLUDE = ['Id', 'Position']


@click.group()
def cli():
    """ Auto-generate code from cuba yaml description. """


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def python(input, output):
    """ Create the CUBA Enum for the DataContainer.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'from enum import IntEnum, unique\n',
        '\n',
        '\n',
        '@unique\n',
        'class CUBA(IntEnum):\n',
        '\n']
    template = "    {} = {}\n"
    for keyword in keywords:
        if keyword['name'] in CUBA_DATA_CONTAINER_EXCLUDE:
            continue
        lines.append(template.format(keyword['key'], keyword['number']))
    output.writelines(lines)


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def table(input, output):
    """ Create the CUBA DataContainer Table descriptions.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'import tables\n',
        '\n',
        '\n']

    # create Data table description
    lines.extend([
        'class Data(tables.IsDescription):\n',
        '\n'])
    data_types = {
        'string': 'String',
        'double': 'Float64',
        'integer': 'Int32'}
    position = 0
    for keyword in keywords:
        if keyword['name'] in CUBA_DATA_CONTAINER_EXCLUDE:
            continue
        if keyword['type'] == 'string':
            template = "    {} = tables.{}Col(pos={}, itemsize={})\n"
            shape = keyword['shape'][0]
        else:
            template = "    {} = tables.{}Col(pos={}{})\n"
            shape = keyword['shape']
            if len(shape) == 1:
                if shape[0] == 1:
                    shape = ''
                else:
                    shape = ', shape={}'.format(shape[0])
            else:
                shape = ', shape=({})'.format(
                    ', '.join(map(str, keyword['shape'])))
        lines.append(template.format(
            keyword['key'].lower(),
            data_types[keyword['type']],
            position,
            shape))
        position += 1

    lines.append('\n\n')

    # create Mask column
    mask_size = position  # without index
    lines.extend([
        'class Record(tables.IsDescription):\n',
        '\n',
        '    index = tables.StringCol(itemsize=32, pos=0)\n',
        '    data = Data()\n',
        '    mask = tables.BoolCol(pos=1, shape=({},))\n'.format(mask_size),
        '\n\n'])

    lines.extend([
        'class NoUIDRecord(tables.IsDescription):\n',
        '\n',
        '    data = Data()\n',
        '    mask = tables.BoolCol(pos=1, shape=({},))\n'.format(mask_size)])

    output.writelines(lines)


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def keywords(input, output):
    """ Create a dictionary of CUDS keywords.
    """
    keywords = yaml.safe_load(input)

    lines = [
        '# code auto-generated by the cuba-generate.py script.\n',
        'from collections import namedtuple\n',
        '\n',
        'import numpy\n',
        '\n',
        '\n',
        'ATTRIBUTES = [\n'
        '    "name", "description", "domain", "key", "number", "shape", "dtype"]\n'  # noqa
        'Keyword = namedtuple("Keyword", ATTRIBUTES)\n',
        '\n',
        '\n',
        'KEYWORDS = {\n']
    data_types = {
        'string': 'numpy.str',
        'double': 'numpy.float64',
        'integer': 'numpy.int32'}
    template = (
        "    '{key}': Keyword(\n"
        "        name='{name}',\n"
        "        description='{description}',  # noqa\n"
        "        domain={domain},\n"
        "        key='{key}',\n"
        "        number={number},\n"
        "        shape={shape},\n"
        "        dtype={type}),\n")
    for keyword in keywords:
        keyword['type'] = data_types[keyword['type']]
        lines.extend(template.format(**keyword))
    lines.append('}\n')

    output.writelines(lines)


_Column = namedtuple('_Column', ["key", "header"])


@cli.command()
@click.argument('input', type=click.File('rb'))
@click.argument('output', type=click.File('wb'))
def rst(input, output):
    """ Create an rst document with table describing CUDS keywords.
    """
    keywords = yaml.safe_load(input)

    columns = [_Column("name", "Name"),
               _Column("description", "Description"),
               _Column("domain", "Domain"),
               _Column("key", "Key"),
               _Column("number", "Number"),
               _Column("shape", "Shape"),
               _Column("type", "Type")]

    table_header = [col.header for col in columns]

    table_data = []

    for keyword in keywords:
        # keyword['serialized_type'] = data_types[keyword['type']]
        row = [keyword[col.key] for col in columns]
        table_data.append(row)
    rst = tabulate(table_data, table_header, tablefmt="rst")
    output.write(
        ".. this table was auto-generated by the cuba-generate.py script.")
    output.write(rst)


if __name__ == '__main__':
    cli()

{'fraction_non_alphanumeric': 0.17109741514120136, 'fraction_numerical': 0.0036795143041118573, 'mean_word_length': 2.188269794721408, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2956117', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3271, 'n_words': 703}