<commit_before>{ 
    "$schema": "http://json-schema.org/schema#",
    "definitions": {
        "category": {
            "description": "Category, (loosely) modeled after RFC 4287",
            "type": "object",
            "required": ["name"],
            "properties": {
                "label": {"type": "string"},
                "icon": {"$ref": "#/definitions/link"},
                "name": {"type": "string"},
                "scheme": {"type": "string", "format": "uri"}
            }
        },
        "link": {
            "description": "Link as defined by RFC 5988, extended for S3",
            "type": "object",
            "required": ["href"],
            "properties": {
                "bucket": {"type": "string"},
                "key": {"type": "string"},
                "href": {"type": "string", "format": "uri"},
                "hreflang": {"type": "string"},
                "rel": {"type": "string"},
                "rev": {"type": "string"},
                "title": {"type": "string"},
                "type": {"type": "string"}
            }
        },
        "linklist": {
            "description": "List of links is so common, we define separately.",
            "type": "array",
            "items": {"$ref": "#/definitions/link"}
        },
        "person": {
            "type": "object",
            "required": ["name"],
            "properties": {
                "description": {"type": "string", "description": "Bio or about me."},
                "email": {"type": "string", "format": "email"},
                "links": {"$ref": "#/definitions/linklist"},
                "name": {"type": "string", "description": "for display"},
                "role": {"type": "string"},
                "sort": {"type": "string", "description": "for lexical sort"},
                "thumbnail": {"$ref": "#/definitions/link"}
            }
        },
        "organization": {
            "type": "object",
            "required": ["name"],
            "properties": {
                "email": {"type": "string", "format": "email"},
                "links": {"$ref": "#/definitions/linklist"},
                "name": {"type": "string", "description": "for display"},
                "role": {"type": "string"},
                "sort": {"type": "string", "description": "for lexical sort"}
            }
        },
        "rights": {
            "required": ["copyright_notice"],
            "properties": {
                "copyright_notice": {"type": "string"},
                "copyright_holder": {
                    "type": "object",
                    "description": "Person or organization who owns the copyright",
                    "anyOf": [
                        {"$ref": "#/definitions/person"},
                        {"$ref": "#/definitions/organization"}
                    ]
                },
                "copyright_year": {"type": "number"},
                "license": {"$ref": "#/definitions/link"},
                "usage_terms": {"type": "string"}
            }
        },
        "Item": {
            "type": "object",
            "required": [
                "archetype",
                "content_type",
                "guid",
                "item_type",
                "published",
                "rights",
                "title",
                "updated"
            ],
            "properties":{
                "attribution": {
                    "description": "Byline or author/contributor credits",
                    "type": "array",
                    "items": {
                        "anyOf": [
                            {"$ref": "#/definitions/person"},
                            {"$ref": "#/definitions/organization"}
                        ]
                    }
                },
                "category": {
                    "type": "array",
                    "items": {"$ref": "#/definitions/category"}
                },
                "created": {"type": "string", "format": "date-time"},
                "description": {"type": "string"},
                "published": {"type": "string", "format": "date-time"},
                "rights": {"$ref": "#/definitions/rights"},
                "slug": {"type": "string"},
                "thumbnail": {"$ref": "#/definitions/link"},
                "title": {"type": "string"},
                "updated": {"type": "string", "format": "date-time"},

                "archetype": {"$ref": "#/definitions/link"},
                "content_type": {"type": "string"},
                "guid": {"type": "string", "format": "uri"},
                "item_type": {"type": "string"},
                "links": {"$ref": "#/definitions/linklist"}
            }
        }
    },
    "type": "object",
    "required": ["Item"],
    "properties": {
        "Item": {"$ref": "#/definitions/Item"}
    }
}<commit_msg>no underscores<commit_after>{ 
    "$schema": "http://json-schema.org/schema#",
    "definitions": {
        "category": {
            "description": "Category, (loosely) modeled after RFC 4287",
            "type": "object",
            "required": ["name"],
            "properties": {
                "label": {"type": "string"},
                "icon": {"$ref": "#/definitions/link"},
                "name": {"type": "string"},
                "scheme": {"type": "string", "format": "uri"}
            }
        },
        "link": {
            "description": "Link as defined by RFC 5988, extended for S3",
            "type": "object",
            "required": ["href"],
            "properties": {
                "bucket": {"type": "string"},
                "key": {"type": "string"},
                "href": {"type": "string", "format": "uri"},
                "hreflang": {"type": "string"},
                "rel": {"type": "string"},
                "rev": {"type": "string"},
                "title": {"type": "string"},
                "type": {"type": "string"}
            }
        },
        "linklist": {
            "description": "List of links is so common, we define separately.",
            "type": "array",
            "items": {"$ref": "#/definitions/link"}
        },
        "person": {
            "type": "object",
            "required": ["name"],
            "properties": {
                "description": {"type": "string", "description": "Bio or about me."},
                "email": {"type": "string", "format": "email"},
                "links": {"$ref": "#/definitions/linklist"},
                "name": {"type": "string", "description": "for display"},
                "role": {"type": "string"},
                "sort": {"type": "string", "description": "for lexical sort"},
                "thumbnail": {"$ref": "#/definitions/link"}
            }
        },
        "organization": {
            "type": "object",
            "required": ["name"],
            "properties": {
                "email": {"type": "string", "format": "email"},
                "links": {"$ref": "#/definitions/linklist"},
                "name": {"type": "string", "description": "for display"},
                "role": {"type": "string"},
                "sort": {"type": "string", "description": "for lexical sort"}
            }
        },
        "rights": {
            "required": ["copyright_notice"],
            "properties": {
                "copyright_notice": {"type": "string"},
                "copyright_holder": {
                    "type": "object",
                    "description": "Person or organization who owns the copyright",
                    "anyOf": [
                        {"$ref": "#/definitions/person"},
                        {"$ref": "#/definitions/organization"}
                    ]
                },
                "copyright_year": {"type": "number"},
                "license": {"$ref": "#/definitions/link"},
                "usage_terms": {"type": "string"}
            }
        },
        "Item": {
            "type": "object",
            "required": [
                "archetype",
                "contenttype",
                "guid",
                "itemtype",
                "published",
                "rights",
                "title",
                "updated"
            ],
            "properties":{
                "attribution": {
                    "description": "Byline or author/contributor credits",
                    "type": "array",
                    "items": {
                        "anyOf": [
                            {"$ref": "#/definitions/person"},
                            {"$ref": "#/definitions/organization"}
                        ]
                    }
                },
                "category": {
                    "type": "array",
                    "items": {"$ref": "#/definitions/category"}
                },
                "created": {"type": "string", "format": "date-time"},
                "description": {"type": "string"},
                "published": {"type": "string", "format": "date-time"},
                "rights": {"$ref": "#/definitions/rights"},
                "slug": {"type": "string"},
                "thumbnail": {"$ref": "#/definitions/link"},
                "title": {"type": "string"},
                "updated": {"type": "string", "format": "date-time"},

                "archetype": {"$ref": "#/definitions/link"},
                "contenttype": {"type": "string"},
                "guid": {"type": "string", "format": "uri"},
                "itemtype": {"type": "string"},
                "links": {"$ref": "#/definitions/linklist"}
            }
        }
    },
    "type": "object",
    "required": ["Item"],
    "properties": {
        "Item": {"$ref": "#/definitions/Item"}
    }
}
{'fraction_non_alphanumeric': 0.1974673118500978, 'fraction_numerical': 0.001853186451147946, 'mean_word_length': 1.3161659513590844, 'pattern_counts': {'":': 298, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3390042', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2407, 'n_words': 561}