<jupyter_start><jupyter_text>📇 Schemas  Avro<jupyter_code># Avro schema from Pandas Dataframe

import json

import pandas as pd
import pandavro as pda
from sklearn import datasets

# load iris dataset from sklearn
iris = datasets.load_iris()

# load iris dataset as pandas dataframe
df = pd.DataFrame(iris.data, columns=iris.feature_names)
df.head()
# use pandavro schema inference for quick schema outline
schema = pda.__schema_infer(df=df, times_as_micros=True)

# indent from json dumps helps make formatting more legible
print(json.dumps(schema, indent=4))<jupyter_output>{
    "type": "record",
    "name": "Root",
    "fields": [
        {
            "name": "sepal length (cm)",
            "type": [
                "null",
                "double"
            ]
        },
        {
            "name": "sepal width (cm)",
            "type": [
                "null",
                "double"
            ]
        },
        {
            "name": "petal length (cm)",
            "type": [
                "null",
                "double"
            ]
        },
        {
            "name": "petal width (cm)",
            "type": [
                "null",
                "double"
            ]
        }
    ]
}<jupyter_text>Arrow and Pandas<jupyter_code># Arrow Schema from Pandas Dataframe
import json

import pandas as pd
import pyarrow as pa
from sklearn import datasets

# load iris dataset from sklearn
iris = datasets.load_iris()

# load iris dataset as pandas dataframe
df = pd.DataFrame(iris.data, columns=iris.feature_names)
df.head()
table = pa.Table.from_pandas(df)
table.schema
table.schema.metadata<jupyter_output><empty_output><jupyter_text>Arrow and Parquet<jupyter_code># Parquet Schema from Arrow and to Arrow
import json

import pandas as pd
import pyarrow as pa
from sklearn import datasets

# load iris dataset from sklearn
iris = datasets.load_iris()

# load iris dataset as pandas dataframe
df = pd.DataFrame(iris.data, columns=iris.feature_names)
df.head()
df.to_parquet("iris.parquet")
parquet_file = pa.parquet.ParquetFile("iris.parquet")
parquet_file.metadata
parquet_file.schema
parquet_file.schema.to_arrow_schema()<jupyter_output><empty_output><jupyter_text>JSON Schema<jupyter_code>from jsonschema import validate

schema = {
  "type" : "object",
  "properties" : {
    "a_number" : {"type" : "number"},
    "a_name" : {"type" : "string"},
  },
}

schema_correct_sample = {"a_number": 123,
                         "a_name": "Green Eggs and Ham",
                         }
schema_incorrect_sample = {"a_number": "123",
                         "a_name": "Green Eggs and Ham",
                         }
validate(instance=schema_correct_sample, schema=schema)
validate(instance=schema_incorrect_sample, schema=schema)
schema = {
    "type" : "array",
    "items" :  {"type": "number"},
    "maxItems" : 5,
}

schema_correct_sample = [1,2,3,4,5]
schema_incorrect_sample = [1,2,3,4,5,6]
validate(instance=schema_correct_sample, schema=schema)
validate(instance=schema_incorrect_sample, schema=schema)
schema = {
    "type" : "array",
    "contains": {
     "type": "number"
    },
    "minItems" : 5,
}

schema_correct_sample = [1,2,3,4,"cat"]
schema_incorrect_sample = ["1","2","3","4","5"]
validate(instance=schema_correct_sample, schema=schema)
validate(instance=schema_incorrect_sample, schema=schema)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1233362910381544, 'fraction_numerical': 0.00857734398107069, 'mean_word_length': 2.918887601390498, 'pattern_counts': {'":': 18, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '126773', 'license': 'MIT', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1110, 'n_words': 249, 'path': 'schemas.ipynb', 'repo_name': 'd33bs/notebook_letterpress', 'size': '3381'}