<jupyter_start><jupyter_text>create index and document<jupyter_code>import sys, os, json
sys.path.append(sys.path.append(os.path.join(os.getcwd(), '..')))
from datamart.query_manager import QueryManager
from datamart.index_builder import IndexBuilder

es_index = "datamart_tmp"   # es index for your metadata, make sure you change to your own

index_builder = IndexBuilder()
tmp_description_dir = "tmp"  # dataset schema json file
tmp_description = "tmp/tmp.json"  # dir of all dataset schema json files
tmp_out = "tmp/tmp.metadata"  # output of metadata<jupyter_output><empty_output><jupyter_text>Indexing single dataset<jupyter_code>this_metadata = index_builder.indexing(description_path=tmp_description,
                                       es_index=es_index,
                                       data_path=None,
                                       query_data_for_indexing=True,
                                       save_to_file=tmp_out,
                                       save_to_file_mode="w",
                                       delete_old_es_index=True)<jupyter_output><empty_output><jupyter_text>Take a look at the last metadata generated<jupyter_code>print(json.dumps(this_metadata, indent=2))<jupyter_output>{
  "datamart_id": 10000,
  "title": "TAVG",
  "description": "Average temperature (tenths of degrees C)[Note that TAVG from source 'S' corresponds to an average for the period ending at 2400 UTC rather than local midnight]",
  "url": "https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt",
  "keywords": [
    "Average Temperature."
  ],
  "date_updated": "2018-09-28T00:00:00",
  "provenance": "noaa.org",
  "materialization": {
    "python_path": "noaa_materializer",
    "arguments": {
      "type": "TAVG"
    }
  },
  "variables": [
    {
      "datamart_id": 10001,
      "name": "date",
      "description": "the date of data",
      "semantic_type": [
        "https://metadata.datadrivendiscovery.org/types/Time"
      ],
      "temporal_coverage": {
        "start": "1874-10-13T00:00:00",
        "end": "2018-10-01T00:00:00"
      }
    },
    {
      "datamart_id": 10002,
      "name": "stationId",
      "description": "the id of station which has this data",
      "semantic_typ[...]<jupyter_text>Bulk indexing multiple dataset<jupyter_code>index_builder.bulk_indexing(description_dir=tmp_description_dir,
                            es_index=es_index,
                            data_dir=None,
                            query_data_for_indexing=True,
                            save_to_file=tmp_out,
                            save_to_file_mode="w",
                            delete_old_es_index=True)<jupyter_output>==== Creating metadata and indexing for tmp.json
{'fraction_non_alphanumeric': 0.1102968120190546, 'fraction_numerical': 0.023085379259802124, 'mean_word_length': 2.464467005076142, 'pattern_counts': {'":': 22, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '7231', 'license': 'MIT', 'n_tokens_mistral': 907, 'n_tokens_neox': 848, 'n_words': 161, 'path': 'test/indexing.ipynb', 'repo_name': 'BhavyaGulati/datamart', 'size': '2729'}