<jupyter_start><jupyter_text>The inimitable schema library Part 2By [@stavros](https://twitter.com/stavros)  Structured data is everywhere<jupyter_code>data = {
    "operation": "upload",  # "upload" or "delete"
    "timeout": 3600,  # Optional, how long the sig should be valid for.
    "md5": "deadbeefetc",  # Optional
    "files": {
        "5gbCtxlvljhx5-al": {
            "size": 65536,
            "shred_date": "2015-05-02T00:00:00Z"  # Must be a date from now up to 4 months in the future.
        },
    }
}<jupyter_output><empty_output><jupyter_text>How do we validate it?<jupyter_code>if data.get("operation") not in ["upload", "delete"]:
    raise SomeError("Operation not valid.")

try:
    timeout = int(data.get("timeout"))
except ValueError:
    raise SomeError("Timeout not a number.")

if not 0 < timeout <= 3600:
    raise SomeError("Timeout not up to one hour in the future.")

if data.get("md5") and not isinstance(data["md5"], str):
    raise SomeError("md5 is not a valid MD5 hash.")

if not isinstance(data.get("files"), dict):
    raise SomeError("files must be a dictionary.")

# etc<jupyter_output><empty_output><jupyter_text>Is there a better way?  No.  Just kidding, of course there is. Who asks "is there a better way?" if there's no better way? No one, that's who.  Presenting the `schema` library.<jupyter_code>from schema import Schema, And, Or, Optional, Use, SchemaError

schema = Schema({
    "foo": int,
    Optional("hello"): "hi!",
})

data = {
    "hello": "hi!",
    "foo": 3,
}

schema.validate(data)
schema = Schema({
    "foo": int,
    Optional("hello"): "hi!",
})

data = {
    "foo": 3,
}

schema.validate(data)
schema = Schema({
    "foo": int,
    Optional("hello"): "hi!",
})

data = {
    "hello": "yo",
    "foo": 3,
}

try:
    schema.validate(data)
except SchemaError as e:
    print e<jupyter_output>'hi!' does not match 'yo'<jupyter_text>Tricks<jupyter_code>schema = Schema(And(int, fetch_user_by_id))

data = "/tmp/pythess"

try:
    print schema.validate(data)
except SchemaError as e:
    print(e)
schema = Schema(range(10))

data = [2, 4, 6, 2, 2, 2, 20]

try:
    print schema.validate(data)
except SchemaError as e:
    print(e)
schema = Schema({
    "shred_date": And(
        basestring,
        Use(ciso8601.parse_datetime_unaware),
        datetime.datetime,
        Use(lambda d: (d - datetime.datetime.now()).days),
        lambda d: 0 < d < 120,
        error="shred_date must be a valid future date string up to 120 days from now.")
})

data = {
    "shred_date": "2016-10-10T00:00:00Z",
}

try:
    print schema.validate(data)
except SchemaError as e:
    print(e)
operations = {"upload": "PUT", "delete": "DELETE", "replace": "POST"}
schema = Schema(And(Use(json.loads, error="Invalid JSON"), {
        "operation": And(lambda s: s in operations.keys(), Use(operations.get), error="Valid operations are: %s" % ", ".join(operations.keys())),
        "files": {And(basestring, lambda s: len(s) > 5, error="Filename must be a string longer than 5 characters."): {
            Optional("size"): And(int, lambda i: i > 0, error="Size must be a positive integer."),
            Optional("shred_date"): And(
                basestring,  # Make sure it's a string.
                Use(ciso8601.parse_datetime_unaware),  # Parse it into a date.
                datetime.datetime,  # Make sure it's a date now.
                lambda d: 0 < (d - datetime.datetime.now()).days < 120,  # Make sure it's in the future, up to 120 days.
                error="shred_date must be a valid future date string up to 120 days from now.")
            }}}))
data = """{
    "operation": "repklace",
    "files": {
        "file.nam": {
            "size": 100,
            "shred_date": "2016-01-01T00:00:00Z"
        }}}}"""
try:
    print schema.validate(data)
except SchemaError as e:
    print e<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14970368461736666, 'fraction_numerical': 0.028085544962638494, 'mean_word_length': 3.11228813559322, 'pattern_counts': {'":': 27, '<': 22, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '368845', 'license': 'MIT', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1296, 'n_words': 387, 'path': '006 - Frank Underwood/schema-presentation/Schema presentation.ipynb', 'repo_name': 'orfeasa/pythess-files', 'size': '3881'}