<jupyter_start><jupyter_text>Data Loader for World Modeler Dec 2020 Datasets<jupyter_code>from pathlib import Path
import papermill as pm

WM_DATAMART_URL = 'https://datamart:<EMAIL>:8888/datamart-api-wm'
WM_TEST_URL = 'https://datamart:<EMAIL>:8888/datamart-api-d3m'
WM_TEST_INTERNAL_URL = 'http://dsbox01.isi.edu:12543'
LOCAL_URL = 'http://localhost:12543'

username = None
password = <PASSWORD>
# Datamart URL
# datamart_api_url = LOCAL_URL
datamart_api_url = WM_TEST_INTERNAL_URL

# World Modeler shared data directory
shared_data_dir = Path('/lfs1/ktyao/Shared/kgtk-private-data/world-modelers')

# Directory where the git repositories are cloned. 
datamart_repos_dir = Path('/home/ktyao/dev/dsbox')
# datamart-api repository
datamart_dir = datamart_repos_dir / 'datamart-api'
data_refresh_dir = datamart_dir / 'data-refresh'

# Directory to store KGTK edge files
kgtk_dir = shared_data_dir / 'kgtk'

executed_notebook_dir = shared_data_dir / 'executed_notebook'

mitre_dir = shared_data_dir / 'MITRE Annotated Datasets for Dec 2020'<jupyter_output><empty_output><jupyter_text>Upload datasets  ACLED dataset<jupyter_code>dataset_id = 'ACLED'
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / f'{dataset_id}.ipynb',
    parameters=dict(
        DEBUG = True,
        dataset_id = dataset_id,
        datamart_api_url=datamart_api_url,
        annotated_path = str(mitre_dir / 'ACLED' / 'ACLED_1997-01-01-2020-11-12-Ethiopia_ANNOTATED_MODIFIED.xlsx'),
        save_tsv_path = str(kgtk_dir / f'{dataset_id}.tar.gz')));<jupyter_output><empty_output><jupyter_text>ETH-CROP dataset<jupyter_code>dataset_id = 'ETH-CROP'
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / f'{dataset_id}.ipynb',
    parameters=dict(
        DEBUG = True,
        dataset_id = dataset_id,
        datamart_api_url=datamart_api_url,
        annotated_path = str(shared_data_dir / 'Three Priority Datasets' / 'Jemal-Crop land Area- Production and Yield of Major Crops-2012-2018_Annotated_JS00.xlsx'),
        save_tsv_path = str(kgtk_dir / f'{dataset_id}.tar.gz')));<jupyter_output><empty_output><jupyter_text>ETH-CENSUS dataset<jupyter_code>dataset_id = 'ETH-CENSUS'
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / f'{dataset_id}.ipynb',
    parameters=dict(
        DEBUG = True,
        dataset_id = dataset_id,
        datamart_api_url = datamart_api_url,
        save_tsv_path = str(kgtk_dir / f'{dataset_id}.tar.gz'),
        annotated_path = str(shared_data_dir / 'Three Priority Datasets' / 'Population 2007.xlsx'),
        yamlfile_path = str(shared_data_dir / 'Three Priority Datasets' / 'Population 2007.yaml')));<jupyter_output><empty_output><jupyter_text>Kimetrica_Ethiopia_Crop_Production<jupyter_code>dataset_id = 'Kimetrica_Ethiopia_Crop_Production'
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / f'{dataset_id}.ipynb',
    parameters=dict(
        DEBUG = True,
        dataset_id = dataset_id,
        datamart_api_url = datamart_api_url,
        save_tsv_path = str(kgtk_dir / f'{dataset_id}.tar.gz'),
        annotated_path = str(mitre_dir / 'Crop production' / 'ethiopia-crop-production-statistics-meher-and-belg-admin-2-level-2004-2017_edit_ANNOTATED-modified.xlsx')));<jupyter_output><empty_output><jupyter_text>CHIRPS dataset`2 files`: Sequential upload required<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'CHIRPS.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'CHIRPSEthiopiaDailyPrecip_ANNOTATED-modified.xlsx'),
        put_data=True, DEBUG=False));
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'CHIRPS.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'CHIRPS', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'CHIRPSOromiaDailyPrecip_1981-01-01_2020-08-31_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'CHIRPS.tar.gz'), 
        put_data=False, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>MERRA2 Re-analysis dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'MERRA2 Re-analysis.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'MERRA2 Re-analysis', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'MERRA2OromiaDailyPrecip_1981-01-01_2020-09-30_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'MERRA2 Re-analysis.tar.gz'),
        put_data=True, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>ERA5 dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'ERA5.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'ERA5', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'ERA5OromiaDailyMeanTemp_1981-01-01_2020-07-09_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'ERA5.tar.gz'),
        put_data=True, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>TerraClimate (11 datasets)`2 files`: Sequential upload required for `5 variables`<jupyter_code>#### Actual Evaporation
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateET.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateET', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlyActualEvapotranspiration_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateET.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateET', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateOromiaMonthlyActualEvapotranspiraton_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimateET.tar.gz'),
        put_data=False, DEBUG=True, TEST_ALL=True));
#### Max Temperature
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateMaxTemp.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateMaxTemp', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlyMaxTemp_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateMaxTemp.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateMaxTemp', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateOromiaMonthlyMaxTemp_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimateMaxTemp.tar.gz'),
        put_data=False, DEBUG=True, TEST_ALL=True));
#### Min Temperature
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateMinTemp.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateMinTemp', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlyMinTemp_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateMinTemp.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateMinTemp', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateOromiaMonthlyMinTemp_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimateMinTemp.tar.gz'),
        put_data=False, DEBUG=True, TEST_ALL=True));
#### TerraClimatePrecipitation
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimatePrecipitation.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimatePrecipitation', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlyPrecip_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimatePrecipitation.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimatePrecipitation', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateOromiaMonthlyPrecip_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimatePrecipitation.tar.gz'),
        put_data=False, DEBUG=True, TEST_ALL=True));
#### PDSI
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimatePDSI.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimatePDSI', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlyPalmerDroughtSeverityIndex_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimatePDSI.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimatePDSI', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateOromiaMonthlyPalmerDroughtSeverityIndex_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimatePDSI.tar.gz'),
        put_data=False, DEBUG=True, TEST_ALL=True));
#### Soil Moisture
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'TerraClimateSoilMoisture.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'TerraClimateSoilMoisture', username=username, password=password,
        annotated_path = str(mitre_dir / 'Climate and Soil Indicators' / 'TerraClimateEthiopiaMonthlySoilMoisture_ANNOTATED.xlsx'),
        save_tsv_path = str(kgtk_dir / 'TerraClimateSoilMoisture.tar.gz'),
        put_data=True, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>DHS_Nutrition`2 files`: Sequential upload required<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'DHS_Nutrition.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, username=username, password=password,
        annotated_path = str(mitre_dir / 'DHS Nutrition Data' / 'DHS Nutrition STATcompilerExport2020721_161028_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));

pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'DHS_Nutrition.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'DHS_Nutrition', username=username, password=password,
        annotated_path = str(mitre_dir / 'DHS Nutrition Data' / 'DHS Nutrition STATcompilerExport2020721_161028_Totals_ANNOTATED.xlsx'),
        save_tsv_path = str(kgtk_dir / 'DHS_Nutrition.tar.gz'), 
        put_data=False, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>IHME_Wasting_Data dataset`2 files`: Sequential upload required<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'IHME_Wasting_Data.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, username=username, password=password,
        annotated_path = str(mitre_dir / 'IHME' / 'IHME_LMIC_CGF_2000_2017_WASTING_PREV_ADMIN_1_Y2020M01D08_ANNOTATED.xlsx'),
        put_data=True, DEBUG=False));
pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'IHME_Wasting_Data.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, username=username, password=password,
        annotated_path = str(mitre_dir / 'IHME' / 'IHME_LMIC_CGF_2000_2017_WASTING_PREV_ADMIN_2_Y2020M01D08_ANNOTATED.xlsx'),
        save_tsv_path = str(kgtk_dir / 'IHME_Wasting_Data.tar.gz'), dataset_id = 'IHME_Wasting_Data',
        put_data=False, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>WFP_Food_Prices dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'WFP_Food_Prices.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'WFP_Food_Prices', username=username, password=password,
        annotated_path = str(mitre_dir / 'Food price data' / 'wfp_food_prices_ethiopia _updated June 2020_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'WFP_Food_Prices.tar.gz'),
        put_data=True, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>Kimetrica_Price_Data dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'Kimetrica_Price_Data.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'Kimetrica_Price_Data', username=username, password=password,
        annotated_path = str(mitre_dir / 'Food price data' / 'Kimetrica Ethiopia Price Data with lat long_2014-2017_ANNOTATED-modified.xlsx'),
        save_tsv_path = str(kgtk_dir / 'Kimetrica_Price_Data.tar.gz'),
        put_data=True, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>FAO_Locust_Swarms dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'FAO_Locust_Swarms.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'FAO_Locust_Swarms', username=username, password=password,
        annotated_path = str(mitre_dir / 'FAO Locust Data' / 'Swarms-Simplified2-Ethiopia_ANNOTATED-modified.xlsx'),
        yamlfile_path = str(mitre_dir / 'FAO Locust Data' / 'swarms.yaml'),
        save_tsv_path = str(kgtk_dir / 'FAO_Locust_Swarms.tar.gz'),
        put_data=True, DEBUG=True, validate=False, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>FAO_Locust_Control dataset<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'FAO_Locust_Control.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'FAO_Locust_Control', username=username, password=password,
        annotated_path = str(mitre_dir / 'FAO Locust Data' / 'Control_Operations-Simplified2-Ethiopia_ANNOTATED-modified.xlsx'),
        yamlfile_path = str(mitre_dir / 'FAO Locust Data' / 'control.yaml'),
        save_tsv_path = str(kgtk_dir / 'FAO_Locust_Control.tar.gz'),
        put_data=True, validate=False, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output><jupyter_text>FAO_Locust_Adults<jupyter_code>pm.execute_notebook(
    data_refresh_dir / 'DataLoader.ipynb',
    executed_notebook_dir / 'FAO_Locust_Adults.ipynb',
    parameters=dict(
        datamart_api_url=datamart_api_url, dataset_id = 'FAO_Locust_Adults', username=username, password=password,
        annotated_path = str(mitre_dir / 'FAO Locust Data' / 'Adults-Simplified2-Ethiopia_ANNOTATED-modified.xlsx'),
        yamlfile_path = str(mitre_dir / 'FAO Locust Data' / 'adults.yaml'),
        save_tsv_path = str(kgtk_dir / 'FAO_Locust_Adults.tar.gz'),
        put_data=True, validate=False, DEBUG=True, TEST_ALL=True));<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10619849956954865, 'fraction_numerical': 0.013098019923748616, 'mean_word_length': 5.077354260089686, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 68, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '340423', 'license': 'MIT', 'n_tokens_mistral': 6232, 'n_tokens_neox': 5983, 'n_words': 773, 'path': 'data-refresh/World Modeler Dec 2020.ipynb', 'repo_name': 'usc-isi-i2/datamart-api', 'size': '16306'}