<commit_before>#!/usr/bin/env python

from __future__ import print_function
import copy

HEADER_SYMBOL = "=-*~'"

def process(fh, obj, depth=1):
    obj = copy.deepcopy(obj)
    if '_order' in obj:
        _order = obj['_order']
    else:
        _order = ((None, obj['properties'].keys()),)

    for section, keys in _order:

        if section:
            fh.write('%s\n%s\n\n' % (section, HEADER_SYMBOL[depth]*len(section)))

        for key in keys:
            schema = obj['properties'].pop(key)
            allowed_types = []
            enum = None
            description = None
            has_properties = False
            pattern = None
            minimum = None
            minItems = None
            item_properties = None
            required = True
            nullable = None
            list_property_type = None
            list_property_enum = None
            for sk, sv in schema.iteritems():
                if sk == 'type':
                    allowed_types = sv
                elif sk == 'enum':
                    enum = sv
                elif sk == 'description':
                    description = sv
                elif sk == 'properties':
                    has_properties = True
                elif sk == 'pattern':
                    pattern = sv
                elif sk == 'minimum':
                    minimum = sv
                elif sk == 'minItems':
                    minItems = sv
                elif sk == 'required':
                    required = sv
                elif sk in ('format', 'maximum', 'minimum'):
                    pass
                elif sk in ('items', 'additionalProperties'):
                    if 'properties' in sv:
                        item_properties = sv
                    else:
                        for ik, iv in sv.items():
                            if ik == 'type':
                                list_property_type = iv
                            elif ik == 'enum':
                                list_property_enum = iv
                else:
                    raise ValueError('NEW KEY:', sk)

                if 'null' in allowed_types:
                    nullable = True

                if isinstance(allowed_types, list):
                    allowed_types = ', '.join(allowed_types)

            spaces = '    '*depth

            # Write the property name.
            fh.write('%s**%s** ' % ('    '*(depth-1), key))

            # Write the property type, newline.
            fh.write('(*%s*)\n' % allowed_types)
            if description:
                fh.write(spaces + description)
            else:
                print('no description:', key)

            # If not an array, specify requiredness, nullability.
            if allowed_types == 'array':
                if required:
                    fh.write(' **(required, minItems: %d)**' % (minItems or 0))
            else:
                if required
                    fh.write(' **(required)**')
            fh.write('\n')

            #'\n' + spaces + 'Allowed Types: ' + allowed_types
            if enum is not None:
                fh.write('\n' + spaces + 'Allowed Values:\n')
                for item in enum:
                    fh.write(spaces + '     * ' + item + '\n')
            if minimum is not None:
                fh.write('\n' + spaces + '(minimum value: %s)\n' % minimum)
            if pattern is not None:
                fh.write('\n'+ spaces + '(must match format: ``%s``)\n' % pattern)
            fh.write('\n\n')
            if has_properties:
                process(fh, schema, depth+1)
            if item_properties:
                fh.write(spaces + 'Each element in %s is an object with the following keys: \n\n'
                         % key)
                process(fh, item_properties, depth+1)
            if list_property_type:
                if isinstance(list_property_type, list):
                    list_property_type = '|'.join(list_property_type)
                fh.write(spaces + 'Each element in %s is of type (%s)' % (key, list_property_type))
                if list_property_enum is not None:
                    fh.write('\n' + spaces*2 + 'Allowed Values:\n')
                    for item in list_property_enum:
                        fh.write(spaces + '     * ' + item + '\n')
                fh.write('\n\n')

    if obj['properties']:<commit_msg>missing colon<commit_after>#!/usr/bin/env python

from __future__ import print_function
import copy

HEADER_SYMBOL = "=-*~'"

def process(fh, obj, depth=1):
    obj = copy.deepcopy(obj)
    if '_order' in obj:
        _order = obj['_order']
    else:
        _order = ((None, obj['properties'].keys()),)

    for section, keys in _order:

        if section:
            fh.write('%s\n%s\n\n' % (section, HEADER_SYMBOL[depth]*len(section)))

        for key in keys:
            schema = obj['properties'].pop(key)
            allowed_types = []
            enum = None
            description = None
            has_properties = False
            pattern = None
            minimum = None
            minItems = None
            item_properties = None
            required = True
            nullable = None
            list_property_type = None
            list_property_enum = None
            for sk, sv in schema.iteritems():
                if sk == 'type':
                    allowed_types = sv
                elif sk == 'enum':
                    enum = sv
                elif sk == 'description':
                    description = sv
                elif sk == 'properties':
                    has_properties = True
                elif sk == 'pattern':
                    pattern = sv
                elif sk == 'minimum':
                    minimum = sv
                elif sk == 'minItems':
                    minItems = sv
                elif sk == 'required':
                    required = sv
                elif sk in ('format', 'maximum', 'minimum'):
                    pass
                elif sk in ('items', 'additionalProperties'):
                    if 'properties' in sv:
                        item_properties = sv
                    else:
                        for ik, iv in sv.items():
                            if ik == 'type':
                                list_property_type = iv
                            elif ik == 'enum':
                                list_property_enum = iv
                else:
                    raise ValueError('NEW KEY:', sk)

                if 'null' in allowed_types:
                    nullable = True

                if isinstance(allowed_types, list):
                    allowed_types = ', '.join(allowed_types)

            spaces = '    '*depth

            # Write the property name.
            fh.write('%s**%s** ' % ('    '*(depth-1), key))

            # Write the property type, newline.
            fh.write('(*%s*)\n' % allowed_types)
            if description:
                fh.write(spaces + description)
            else:
                print('no description:', key)

            # If not an array, specify requiredness, nullability.
            if allowed_types == 'array':
                if required:
                    fh.write(' **(required, minItems: %d)**' % (minItems or 0))
            else:
                if required:
                    fh.write(' **(required)**')
            fh.write('\n')

            #'\n' + spaces + 'Allowed Types: ' + allowed_types
            if enum is not None:
                fh.write('\n' + spaces + 'Allowed Values:\n')
                for item in enum:
                    fh.write(spaces + '     * ' + item + '\n')
            if minimum is not None:
                fh.write('\n' + spaces + '(minimum value: %s)\n' % minimum)
            if pattern is not None:
                fh.write('\n'+ spaces + '(must match format: ``%s``)\n' % pattern)
            fh.write('\n\n')
            if has_properties:
                process(fh, schema, depth+1)
            if item_properties:
                fh.write(spaces + 'Each element in %s is an object with the following keys: \n\n'
                         % key)
                process(fh, item_properties, depth+1)
            if list_property_type:
                if isinstance(list_property_type, list):
                    list_property_type = '|'.join(list_property_type)
                fh.write(spaces + 'Each element in %s is of type (%s)' % (key, list_property_type))
                if list_property_enum is not None:
                    fh.write('\n' + spaces*2 + 'Allowed Values:\n')
                    for item in list_property_enum:
                        fh.write(spaces + '     * ' + item + '\n')
                fh.write('\n\n')

    if obj['properties']:
{'fraction_non_alphanumeric': 0.10920917494008901, 'fraction_numerical': 0.0013693940431359123, 'mean_word_length': 1.1833582461385153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1500199', 'n_tokens_mistral': 2329, 'n_tokens_neox': 2150, 'n_words': 628}