<jupyter_start><jupyter_text>Downloading Datamart Snapshot<jupyter_code>%%bash

curl -LO https://auctus.vida-nyu.org/snapshot/index.tar.gz
if [ -e index.snapshot ]; then
    rm -rf index.snapshot;
fi
mkdir index.snapshot
tar xfC index.tar.gz index.snapshot<jupyter_output>% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  1 71.2M    1  799k    0     0   708k      0  0:01:42  0:00:01  0:01:41  707k
 20 71.2M   20 14.2M    0     0  6949k      0  0:00:10  0:00:02  0:00:08 6946k
 44 71.2M   44 31.6M    0     0  10.1M      0  0:00:07  0:00:03  0:00:04 10.1M
 66 71.2M   66 47.5M    0     0  11.6M      0  0:00:06  0:00:04  0:00:02 11.6M
 92 71.2M   92 65.7M    0     0  12.9M      0  0:00:05  0:00:05 --:--:-- 13.2M
100 71.2M  100 71.2M    0     0  13.3M      0  0:00:05  0:00:05 --:--:-- 16.7M<jupyter_text>Choosing Datasets from Snapshot<jupyter_code>import json
import os
import shutil

input_path = 'index.snapshot'
output_path = 'sample.snapshot'

if os.path.exists(output_path):
    shutil.rmtree(output_path)
os.mkdir(output_path)

input_path_datasets = os.listdir(input_path)
datasets = [[d] for d in input_path_datasets if d.startswith('datamart.')]
for i in range(len(datasets)):
    for d in input_path_datasets:
        if d.startswith('lazo.%s'%datasets[i][0]):
            datasets[i].append(d)

datasets_to_upload = list()
for dataset in datasets:
    dataset_path = os.path.join(input_path, dataset[0])
    metadata = json.load(open(dataset_path))

    # ignoring datasets bigger than 50 MB
    if metadata["size"] >= 50000000:
        continue

    datasets_to_upload.append(dataset)

print("Size:", len(datasets_to_upload))

for dataset in datasets_to_upload:
    for d in dataset:
        shutil.copyfile(
            os.path.join(input_path, d),
            os.path.join(output_path, d)
        )
%%bash

rm -rf index.snapshot<jupyter_output><empty_output><jupyter_text>Importing to Datamart (Local Deployment)<jupyter_code>%%bash

DATAMART_DIRECTORY=/Users/fchirigati/projects/d3m/datamart
SNAPSHOT=$(pwd)/sample.snapshot
AMQP_USER=datamart
AMQP_PASSWORD=<PASSWORD>

cd $DATAMART_DIRECTORY
PROJ="$(basename "$(pwd)")"
docker run --rm --network ${PROJ}_default -v $PWD/scripts:/scripts -v $SNAPSHOT:/snapshot -e ELASTICSEARCH_HOSTS=elasticsearch:9200 -e AMQP_HOST=rabbitmq -e AMQP_USER=$AMQP_USER -e AMQP_PASSWORD=$AMQP_PASSWORD -w /tmp ${PROJ}_coordinator sh -c 'python /scripts/import_all.py /snapshot'<jupyter_output>Importing Elasticsearch data............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................[...]<jupyter_text>Adding companion datasets to Datamart<jupyter_code>import requests
import os
def upload_dataset(data_path, name, description):
    url = 'http://localhost:8001/upload'
    with open(data_path, 'rb') as data_p:
        response = requests.post(
            url,
            files={
                'file': data_p
            },
            data={
                'name': name,
                'description': description
            }
        )
    if response.status_code == 400:
        print('Error: %s' % response.json()['error'])
    print(response.status_code)
files_to_upload = [
    (
        '../data/taxi-vehicle-collision/weather-v2.csv',
        'Daily Weather Data for NY',
        'This dataset contains daily weather data for NYC.'
    ),
    (
        '../data/ny-taxi-demand/weather-v2.csv',
        'Hourly Weather Data for NY',
        'This dataset contains hourly weather data for NYC.'
    ),
    (
        '../data/college-debt/college-scorecard-v2.csv',
        'College Scorecard Data - Most Recent',
        'The College Scorecard has data for consumers to compare the cost and value of higher education institutions in the United States.'
    ),
    (
        '../data/poverty-estimation/unemployment-v2.csv',
        'Unemployment in the US',
        'This dataset contains unemployment rates for different years in the US.'
    )
]
for file in files_to_upload:
    upload_dataset(file[0], file[1], file[2])<jupyter_output>200
200
200
200
{'fraction_non_alphanumeric': 0.28772403160531895, 'fraction_numerical': 0.04952784736943534, 'mean_word_length': 1.3677007299270072, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '664355', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1655, 'n_tokens_neox': 1499, 'n_words': 358, 'path': 'use-cases/datamart-data/import-datasets-to-datamart.ipynb', 'repo_name': 'VIDA-NYU/prida', 'size': '5187'}