<jupyter_start><jupyter_text>Scikit learn comes with several[datasets](https://scikit-learn.org/stable/datasets/index.html).This notebook specifies JSON [schemas](https://json-schema.org/)for a few representative ones.Eventually these schemas will probably move to Lale's [datasets](https://github.com/IBM/lale/tree/master/lale/datasets).<jupyter_code>import sklearn.datasets
import lale.helpers
import sklearn.utils
import jsonschema<jupyter_output><empty_output><jupyter_text>digits (image classification)<jupyter_code>digits_data = sklearn.datasets.load_digits()
assert isinstance(digits_data, sklearn.utils.Bunch)
digits_data = lale.helpers.data_to_json({**digits_data}, subsample_array = False)
print(digits_data.keys())
print('n_samples: {}'.format(len(digits_data['data'])))
digits_schema = {
  '$schema': 'http://json-schema.org/draft-04/schema#',
  'type': 'object',
  'properties': {
    'data': {
      'type': 'array',
      'minItems': 1797, 'maxItems': 1797,
      'items': {
        'type': 'array',
        'minItems': 64, 'maxItems': 64,
        'items': {
          'type': 'number',
          'minimum': 0.0}}},
    'target': {
      'type': 'array',
      'minItems': 1797, 'maxItems': 1797,
      'items': {
        'type': 'integer',
        'minimum': 0,
        'maximum': 9}},
    'target_names': {
      'type': 'array',
      'minItems': 10, 'maxItems': 10,
      'items': [
        {'enum': [0]}, {'enum': [1]}, {'enum': [2]}, {'enum': [3]},
        {'enum': [4]}, {'enum': [5]}, {'enum': [6]}, {'enum': [7]},
        {'enum': [8]}, {'enum': [9]}]},
    'images': {
      'type': 'array',
      'minItems': 1797, 'maxItems': 1797,
      'items': {
        'type': 'array',
        'minItems': 8, 'maxItems': 8,
        'items': {
          'type': 'array',
          'minItems': 8, 'maxItems': 8,
          'items': {
            'type': 'number'},
            'minimum': 0.0}}},
    'DESCR': {
      'type': 'string'}}}
lale.helpers.validate_is_schema(digits_schema)
jsonschema.validate(digits_data, digits_schema)<jupyter_output><empty_output><jupyter_text>news (text classification)<jupyter_code>news_data = sklearn.datasets.fetch_20newsgroups()
assert isinstance(news_data, sklearn.utils.Bunch)
news_data = lale.helpers.data_to_json({**news_data}, subsample_array = False)
print(news_data.keys())
print('n_samples: {}'.format(len(news_data['data'])))
news_schema = {
  '$schema': 'http://json-schema.org/draft-04/schema#',
  'type': 'object',
  'properties': {
    'data': {
      'type': 'array',
      'minItems': 11314, 'maxItems': 11314,
      'items': {
        'type': 'string'}},
    'filenames': {
      'type': 'array',
      'minItems': 11314, 'maxItems': 11314,
      'items': {
        'type': 'string'}},
    'target_names': {
      'type': 'array',
      'minItems': 20, 'maxItems': 20,
      'items': [
        {'enum': ['alt.atheism']},
        {'enum': ['comp.graphics']},
        {'enum': ['comp.os.ms-windows.misc']},
        {'enum': ['comp.sys.ibm.pc.hardware']},
        {'enum': ['comp.sys.mac.hardware']},
        {'enum': ['comp.windows.x']},
        {'enum': ['misc.forsale']},
        {'enum': ['rec.autos']},
        {'enum': ['rec.motorcycles']},
        {'enum': ['rec.sport.baseball']},
        {'enum': ['rec.sport.hockey']},
        {'enum': ['sci.crypt']},
        {'enum': ['sci.electronics']},
        {'enum': ['sci.med']},
        {'enum': ['sci.space']},
        {'enum': ['soc.religion.christian']},
        {'enum': ['talk.politics.guns']},
        {'enum': ['talk.politics.mideast']},
        {'enum': ['talk.politics.misc']},
        {'enum': ['talk.religion.misc']}]},
    'target': {
      'type': 'array',
      'minItems': 11314, 'maxItems': 11314,
      'items': {
        'type': 'integer',
        'minimum': 0,
        'maximum': 19}},
    'DESCR': {
      'type': 'string'}}}
lale.helpers.validate_is_schema(news_schema)
jsonschema.validate(news_data, news_schema)<jupyter_output><empty_output><jupyter_text>boston (structured data, no categorical features, regression)<jupyter_code>boston_data = sklearn.datasets.load_boston()
assert isinstance(boston_data, sklearn.utils.Bunch)
boston_data = lale.helpers.data_to_json({**boston_data}, subsample_array = False)
print(boston_data.keys())
print('n_samples: {}'.format(len(boston_data['data'])))
boston_schema = {
  '$schema': 'http://json-schema.org/draft-04/schema#',
  'type': 'object',
  'properties': {
    'data': {
      'type': 'array',
      'minItems': 506, 'maxItems': 506,
      'items': {
        'type': 'array',
        'minItems': 13, 'maxItems': 13,
        'items': [
          { 'description': 'CRIM, per capita crime rate by town',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'ZN, proportion of residential land zoned for lots over '
              '25,000 sq.ft.',
            'type': 'number', 'minimum': 0.0, 'maximum': 100.0},
          { 'description':
              'INDUS, proportion of non-retail business acres per town',
            'type': 'number', 'minimum': 0.0, 'maximum': 100.0},
          { 'description':
              'CHAS, Charles River dummy variable '
              '(= 1 if tract bounds river; 0 otherwise)',
            'enum': [0, 1]},
          { 'description':
              'NOX, nitric oxides concentration (parts per 10 million)',
            'type': 'number', 'minimum': 0.0},
          { 'description': 'RM, average number of rooms per dwelling',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'AGE, proportion of owner-occupied units built prior to 1940',
            'type': 'number', 'minimum': 0.0, 'maximum': 100.0},
          { 'description':
              'DIS, weighted distances to five Boston employment centres',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'RAD, index of accessibility to radial highways',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'TAX, full-value property-tax rate per $10,000',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'PTRATIO, pupil-teacher ratio by town',
            'type': 'number', 'minimum': 0.0},
          { 'description':
              'B, 1000(Bk - 0.63)^2 where Bk is the proportion of '
              'blacks by town',
            'type': 'number', 'minimum': 0.0},
          { 'description': 'LSTAT, % lower status of the population',
            'type': 'number', 'minimum': 0.0, 'maximum': 100.0},
          { 'description':
              "MEDV, Median value of owner-occupied homes in $1000's",
            'type': 'number', 'minimum': 0.0}]}},
    'target': {
      'type': 'array',
      'minItems': 506, 'maxItems': 506,
      'items': {
        'type': 'number',
        'minimum': 5.0,
        'maximum': 50.0}},
    'feature_names': {
      'type': 'array',
      'minItems': 13, 'maxItems': 13,
      'items': [
        {'enum': ['CRIM']},
        {'enum': ['ZN']},
        {'enum': ['INDUS']},
        {'enum': ['CHAS']},
        {'enum': ['NOX']},
        {'enum': ['RM']},
        {'enum': ['AGE']},
        {'enum': ['DIS']},
        {'enum': ['RAD']},
        {'enum': ['TAX']},
        {'enum': ['PTRATIO']},
        {'enum': ['B']},
        {'enum': ['LSTAT']}]},
    'DESCR': {
      'type': 'string'},
    'filename': {
      'type': 'string'}}}
lale.helpers.validate_is_schema(boston_schema)
jsonschema.validate(boston_data, boston_schema)<jupyter_output><empty_output><jupyter_text>mnist (image, from tensorflow datasets)<jupyter_code>import tensorflow_datasets as tfds
#the downloaded files get stored in ~/tensorflow_datasets/mnist/1.0.0/
mnist_data = tfds.load("mnist", with_info=True)
print(tfds.list_builders())
import json
mnist_data = {
  'data': mnist_data[0],
  'info': json.loads(mnist_data[1].as_json)
}
mnist_data
mnist_schema = {
  '$schema': 'http://json-schema.org/draft-04/schema#',
  'type': 'object',
  'properties': {
    'data': {
      'type': 'object',
      'properties': {
        'test': {
          'type': 'object',
          'properties': {
            'image': {
              'type': 'array',
              'minItems': 10000, 'maxItems': 10000,
              'items': {
                'type': 'array',
                'minItems': 28, 'maxItems': 28,
                'items': {
                  'type': 'array',
                  'minItems': 28, 'maxItems': 28,
                  'items': {
                    'type': 'number',
                    'minimum': 0.0, 'maximum': 255.0}}}},
            'label': {
              'type': 'integer',
              'minimum': 0, 'maximum': 9}}},
        'train': {
          'type': 'object',
          'properties': {
            'image': {
              'type': 'array',
              'minItems': 60000, 'maxItems': 60000,
              'items': {
                'type': 'array',
                'minItems': 28, 'maxItems': 28,
                'items': {
                  'type': 'array',
                  'minItems': 28, 'maxItems': 28,
                  'items': {
                    'type': 'number',
                    'minimum': 0.0, 'maximum': 255.0}}}},
            'label': {
              'type': 'integer',
              'minimum': 0, 'maximum': 9}}}}},
    'info': {
      'type': 'object',
      'properties': {
        'citation': {'enum': [mnist_data['info']['citation']]}}}}}
lale.helpers.validate_is_schema(mnist_schema)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2014917533354344, 'fraction_numerical': 0.026788528206744405, 'mean_word_length': 2.3711048158640224, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '575072', 'license': 'Apache-2.0', 'n_tokens_mistral': 3360, 'n_tokens_neox': 3087, 'n_words': 655, 'path': 'examples/demo_schemas_datasets.ipynb', 'repo_name': 'kant/lale', 'size': '9519'}