<commit_before>import json
from warnings import warn
from .schema_types import SCHEMA_TYPES


class SchemaNode(object):
    """
    Basic schema generator class. SchemaNode objects can be loaded
    up with existing schemas and objects before being serialized.
    """

    def __init__(self):
        self._schema_types = []
        self._unknown_keywords = {}

    def add_schema(self, schema):
        """
        Merges in an existing schema.

        arguments:
        * `schema` (required - `dict` or `SchemaNode`):
          an existing JSON Schema to merge.
        """

        # serialize instances of SchemaNode before parsing
        if isinstance(schema, SchemaNode):
            schema = schema.to_schema()

        # delegate to SchemaType object
        schema_type = self._get_type_for_schema(schema)
        schema_type.add_schema(schema)

        # record any extra keywords
        for keyword, value in schema.items():
            if keyword in schema_type.KEYWORDS:
                continue
            elif keyword not in self._unknown_keywords:
                self._unknown_keywords[keyword] = value
            elif self._unknown_keywords[keyword] != value:
                warn(('Schema incompatible. Keyword {0!r} has '
                      'conflicting values ({1!r} vs. {2!r}). Using '
                      '{1!r}').format(keyword, self._other[keyword], value))

        # return self for easy method chaining
        return self

    def add_object(self, obj):
        """
        Modify the schema to accomodate an object.

        arguments:
        * `obj` (required - `dict`):
          a JSON object to use in generate the schema.
        """

        # delegate to SchemaType object
        schema_type = self._get_type_for_object(obj)
        schema_type.add_object(obj)

        # return self for easy method chaining
        return self

    def to_schema(self):
        """
        Convert the current schema to a `dict`.
        """
        # start with unknown keywords
        schema = dict(self._unknown_keywords)

        types = set()
        type_schemas = []
        for schema_type in self._schema_types:
            type_schema = schema_type.to_schema
            if len(type_schema) == 1:
                types.add(type_schema['type'])
            else:
                type_schemas.append(type_schema)

        if types:
            if len(types) == 1:
                (types,) = types
            type_schemas = [{'type': types}] + type_schemas
        if len(type_schemas) == 1:
            schema.update(type_schemas[0])
        else:
            schema['anyOf'] = type_schemas

        return schema

    def to_json(self, *args, **kwargs):
        """
        Convert the current schema directly to serialized JSON.
        """
        return json.dumps(self.to_dict(), *args, **kwargs)
<commit_msg>get one test to pass<commit_after>import json
from warnings import warn
from .schema_types import SCHEMA_TYPES


class SchemaNode(object):
    """
    Basic schema generator class. SchemaNode objects can be loaded
    up with existing schemas and objects before being serialized.
    """

    def __init__(self):
        self._schema_types = []
        self._unknown_keywords = {}

    def add_schema(self, schema):
        """
        Merges in an existing schema.

        arguments:
        * `schema` (required - `dict` or `SchemaNode`):
          an existing JSON Schema to merge.
        """

        # serialize instances of SchemaNode before parsing
        if isinstance(schema, SchemaNode):
            schema = schema.to_schema()

        if 'type' not in schema:
            warn('Given schema has no "type" key: {0!r}')
            return

        # delegate to SchemaType object
        schema_type = self._get_type_for_schema(schema)
        schema_type.add_schema(schema)

        # record any extra keywords
        for keyword, value in schema.items():
            if keyword in schema_type.KEYWORDS:
                continue
            elif keyword not in self._unknown_keywords:
                self._unknown_keywords[keyword] = value
            elif self._unknown_keywords[keyword] != value:
                warn(('Schema incompatible. Keyword {0!r} has '
                      'conflicting values ({1!r} vs. {2!r}). Using '
                      '{1!r}').format(keyword, self._other[keyword], value))

        # return self for easy method chaining
        return self

    def add_object(self, obj):
        """
        Modify the schema to accomodate an object.

        arguments:
        * `obj` (required - `dict`):
          a JSON object to use in generate the schema.
        """

        # delegate to SchemaType object
        schema_type = self._get_type_for_object(obj)
        schema_type.add_object(obj)

        # return self for easy method chaining
        return self

    def to_schema(self):
        """
        Convert the current schema to a `dict`.
        """
        # start with unknown keywords
        schema = dict(self._unknown_keywords)

        types = set()
        type_schemas = []
        for schema_type in self._schema_types:
            type_schema = schema_type.to_schema
            if len(type_schema) == 1:
                types.add(type_schema['type'])
            else:
                type_schemas.append(type_schema)

        if types:
            if len(types) == 1:
                (types,) = types
            type_schemas = [{'type': types}] + type_schemas
        if len(type_schemas) == 1:
            schema.update(type_schemas[0])
        elif type_schemas:
            schema['anyOf'] = type_schemas

        return schema

    def to_dict(self, recurse=True):
        warn('#to_dict is deprecated in v1.0, and it may soon be removed')
        if recurse is not True:
            warn('the `recurse` option for #to_dict does nothing in v1.0')
        return self.to_schema()

    def to_json(self, *args, **kwargs):
        """
        Convert the current schema directly to serialized JSON.
        """
        return json.dumps(self.to_dict(), *args, **kwargs)

{'fraction_non_alphanumeric': 0.0906111748320498, 'fraction_numerical': 0.0034409306898246766, 'mean_word_length': 1.914995224450812, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515070', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1589, 'n_words': 492}