<jupyter_start><jupyter_text>Import publications<jupyter_code>pub_files = '/Users/sophierand/RichContextMetadata/apis/all_pubs.json'
import json
with open(pub_files) as json_file:
    pubs = json.load(json_file)<jupyter_output><empty_output><jupyter_text>Import datasets<jupyter_code>dataset_path = '/Users/sophierand/RCDatasets/datasets.json'
with open(dataset_path) as json_file:
    datasets = json.load(json_file)<jupyter_output><empty_output><jupyter_text>Link datasets with pubs<jupyter_code>usda_datasets = ['FoodAPS National Household Food Acquisition and Purchase Survey'
                 ,'IRI Infoscan','IRI Consumer Network','Supplemental Nutrition Assistance Program']
usda_dataset_list = [ds['id'] for ds in datasets if ds['title'] in usda_datasets]

usda_provider_ingest  = list(set([ds['provider'] for ds in datasets if ds['title'] in usda_datasets]))

                             



usda_pubs_ingest = [p for p in pubs if p['related_dataset'][0] in usda_dataset_list]
usda_provider_ingest = [p for p in pubs if p['related_dataset'][0] in usda_dataset_list]
# usda_pubs_ingest
usda_datasets_export = [ds for ds in datasets if ds['title'] in usda_datasets]

# usda_pubs_ingest

# usda_pubs_ingest
with open('usda_datasets.json', 'w') as outfile:
    json.dump(usda_datasets_export, outfile,indent = 2)
with open('usda_publications.json', 'w') as outfile:
    json.dump(usda_pubs_ingest, outfile,indent = 2)
with open('usda_providers.json', 'w') as outfile:
    json.dump(usda_provider_ingest, outfile,indent = 2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10904425914047466, 'fraction_numerical': 0.003207184092366902, 'mean_word_length': 5.964285714285714, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '460549', 'license': 'CC0-1.0', 'n_tokens_mistral': 569, 'n_tokens_neox': 552, 'n_words': 114, 'path': 'views/usda_adrf_export.ipynb', 'repo_name': 'Coleridge-Initiative/RichContextMetadata', 'size': '1559'}