<commit_before> {
    "id": "https://github.com/codeforboston/cornerwise/docs/scraper-schema",
    "$schema": "http://json-schema.org/draft-06/schema#",
    "description": "schema for Cornerwise scrapers",
    "type": "object",
    "properties": {
        "cases": {
            "type": "array",
            "description": "",
            "items": {"$ref": "#/definitions/case" }
        },
        "events": {
            "type": "array",
            "items": { "$ref": "#/definitions/event" }
        }
    },
    "definitions": {
        "case": {
            "required": ["address", "complete", "case_number", "updated_date"],
            "properties": {
                "address": "The primary address",
                "all_addresses": {
                    "type": "array",
                    "items": { "type": "string"}
                },
                "location": {
                    "description": "The coordinates of the ",
                    "type": "object",
                    "properties": {
                        "lat": { "type": "number" },
                        "long": { "type": "number" }
                    },
                    "required": ["lat", "long"]
                },
                "updated_date": {
                    "type": "string",
                    "format": "date-time"
                },
                "region_name": {
                    "type": "string"
                },
                "source": {
                    "type": "string",
                    "format": "uri"
                },
                "status": {
                    "type": { "type": "string" }
                },
                "complete": {
                    "description": "Indicates whether the case is still pending review (false), or if it has been reviewed or withdrawn (true)",
                    "type": "boolean"
                },
                "decision": {
                    "type": "object",
                    "description": "Provides additional details about the decision from the reviewing body",
                    "properties": {
                        "approved": { "type": "boolean" },
                        "votes_for": { "type": "number" },
                        "votes_against": { "type": "number" },
                        "comments": { "type": "array",
                                      "items": { "type": "string" }}
                    },
                    "dependencies": {
                        "votes_for": ["votes_against"]
                    }
                },
                "case_number": {
                    "type": "string",
                    "description": "A unique case number for this case"
                },
                "first_hearing_date": {},
                "documents": {
                    "type": "array",
                    "items": { "$ref": "#/definitions/link"}
                },
                "images": {
                    "type": "array",
                    "items": { "$ref": "#/definitions/link"}
                },
                "attributes": {
                    "type": "array",
                    "items": { "type": "object" }
                }
            }
        },

        "event": {
            "required": [],
            "properties": {
            }
        },

        "link": {
            "required": ["title", "url"],
            "properties": {
                "title": {
                    "type": "string"
                },
                "url": {
                    "type": "string",
                    "format": "uri"
                },
                "content-type": {
                    "description": "MIME type of the document",
                    "type": "string"
                },
                "tags": {
                    "type": "array",
                    "items": { "type": "string"
                             }}
            }
        }
}<commit_msg>Additional work on scraper schema<commit_after>{
    "id": "https://github.com/codeforboston/cornerwise/docs/scraper-schema",
    "$schema": "http://json-schema.org/draft-06/schema#",
    "description": "schema for Cornerwise scrapers",
    "type": "object",
    "properties": {
        "cases": {
            "type": "array",
            "description": "",
            "items": {
                "$ref": "#/definitions/case"
            }
        },
        "events": {
            "type": "array",
            "items": {
                "$ref": "#/definitions/event"
            }
        }
    },
    "definitions": {
        "case": {
            "required": [
                "all_addresses",
                "complete",
                "case_number",
                "updated_date"
            ],
            "properties": {
                "case_number": {
                    "type": "string",
                    "description": "A case number for this case, unique within the scope of this region."
                },
                "first_hearing_date": {
                    "description": "The date and time of the first public hearing",
                    "type": "string",
                    "format": "date-time"
                },
                "all_addresses": {
                    "description": "An array of all the addresses associated with this proposal. The first address is used as the primary address.",
                    "type": "array",
                    "items": {
                        "type": "string"
                    }
                },
                "location": {
                    "description": "The coordinates of the proposal. If omitted, Cornerwise will geocode the primary address.",
                    "type": "object",
                    "properties": {
                        "lat": {
                            "type": "number"
                        },
                        "long": {
                            "type": "number"
                        }
                    },
                    "required": [
                        "lat",
                        "long"
                    ]
                },
                "parcel_id": {
                    "description": "An identifier for the corresponding parcel, unique within the scope of the region.",
                    "type": "string"
                },
                "updated_date": {
                    "description": "The last modification date and time",
                    "type": "string",
                    "format": "date-time"
                },
                "region_name": {
                    "description": "If omitted, Cornerwise will use the region name configured for the importer.",
                    "type": "string"
                },
                "source": {
                    "description": "A site where users can find additional information",
                    "type": "string",
                    "format": "uri"
                },
                "status": {
                    "type": {
                        "type": "string"
                    }
                },
                "complete": {
                    "description": "Indicates whether the case is still pending review (false), or if it has been reviewed or withdrawn (true)",
                    "type": "boolean"
                },
                "decision": {
                    "type": "object",
                    "description": "Provides additional details about the decision from the reviewing body",
                    "properties": {
                        "approved": {
                            "type": "boolean"
                        },
                        "votes_for": {
                            "type": "number"
                        },
                        "votes_against": {
                            "type": "number"
                        },
                        "comments": {
                            "type": "array",
                            "items": {
                                "type": "string"
                            }
                        }
                    },
                    "dependencies": {
                        "votes_for": [
                            "votes_against"
                        ]
                    }
                },
                "events": {
                    "type": "object",
                    "$ref": "#/definitions/event"
                },
                "documents": {
                    "type": "array",
                    "items": {
                        "$ref": "#/definitions/link"
                    }
                },
                "images": {
                    "type": "array",
                    "items": {
                        "$ref": "#/definitions/link"
                    }
                },
                "attributes": {
                    "type": "array",
                    "items": {
                        "type": "object"
                    }
                }
            }
        },
        "event": {
            "required": [
                "title",
                "date"
            ],
            "properties": {
                "title": {
                    "type": "string"
                },
                "date": {
                    "type": "string",
                    "format": "date-time"
                }
            }
        },
        "link": {
            "required": [
                "title",
                "url"
            ],
            "properties": {
                "title": {
                    "type": "string"
                },
                "url": {
                    "type": "string",
                    "format": "uri"
                },
                "content-type": {
                    "description": "MIME type of the document",
                    "type": "string"
                },
                "tags": {
                    "type": "array",
                    "items": {
                        "type": "string"
                    }
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.13075324161140256, 'fraction_numerical': 0.00039592200336533703, 'mean_word_length': 0.7908543069833393, 'pattern_counts': {'":': 209, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3915532', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2177, 'n_words': 500}