<jupyter_start><jupyter_text>Read<jupyter_code>import json

with open('data.txt') as json_file:
    data = json.load(json_file)<jupyter_output><empty_output><jupyter_text>Read in datasets<jupyter_code>dataset_json_path = os.path.join(os.getcwd(),'datasets.json')

with open(dataset_json_path) as json_file:
    dataset_json = json.load(json_file)
dataset_names =[{'dataset_name':d['title'],'dataset_id':d['dataset_id']} for d in dataset_json]<jupyter_output><empty_output><jupyter_text>Read in manually curated linkages<jupyter_code>manual_linkages = os.path.join(os.getcwd(),'dataset_metadata/curated_linkages.csv')
manual_df = pd.read_csv(manual_linkages)
# manual_df_doi.head()<jupyter_output><empty_output><jupyter_text>Filter out publications without a doi from manually curated linkages. DEAL WITH others later**<jupyter_code>manual_df_doi = manual_df.loc[manual_df.doi.notnull()]
manual_df_doi_dict = manual_df_doi.to_dict('records')<jupyter_output><empty_output><jupyter_text>Connect to dimensions api<jupyter_code>user = input()
password = <PASSWORD>()
api_client = metadata_funs.connect_ds_api(username = user,password = password)<jupyter_output>API credentials have been set<jupyter_text>Link dois to dataset id and get publication metadata based on doi from dimensions api<jupyter_code>def fetch_curated_metadata(manual_df_doi_dict,dataset_names):
    manual_pub_dataset_list = []
    for i in manual_df_doi_dict:
        time.sleep( 6 )
        doi_id = i['doi']
        dataset_name = i['dataset_name']
        try:
            dataset_id = [d for d in dataset_names if d['dataset_name'] == dataset_name][0]['dataset_id']
            pub_metadata = metadata_funs.run_doi_search(doi_id,api_client)
            if pub_metadata:
                pub_metadata.update({'related_dataset': dataset_id})
                manual_pub_dataset_list.append(pub_metadata)
            elif pub_metadata == None:
                pass
        except:
            pass
    return manual_pub_dataset_list
def main():                   
    dataset_names_list = metadata_funs.read_datasets()
#     a_dataset_names_listing = [d for d in dataset_names_list if d['dataset_id'] in ['dataset-f442e418ac191ac60f7f','dataset-01bf466ee1063265fc2c']]
    user = input()
    password = <PASSWORD>()
    my_api_client = metadata_funs.connect_ds_api(username=user,password=password)
    stringsearch_pubs = gen_stringsearch_pub_metadata(api_client = my_api_client, dataset_names_listing=a_dataset_names_listing)
    json.dump(stringsearch_pubs, open('./stringsearch_pubs.json', 'w'), indent=2)
    return stringsearch_pubs

if __name__ == "__main__":
    main()

def main():
    dataset_names = metadata_funs.read_datasets()
    user = input()
    password = <PASSWORD>()
    api_client = metadata_funs.connect_ds_api(username=user,password=password)
    manual_df_doi_dict =  read_curated_linkages()
    manual_pub_dataset_list = fetch_curated_metadata(manual_df_doi_dict,dataset_names,api_client)
    json.dump(stringsearch_pubs, open('./stringsearch_pubs.json', 'w'), indent=2)
    return stringsearch_pubs


def fetch_curated_metadata(manual_df_doi_dict,dataset_names,api_client):
    manual_pub_dataset_list = []
    for i in manual_df_doi_dict:
        time.sleep( 6 )
        doi_id = i['doi']
        dataset_name = i['dataset_name']
        try:
            dataset_id = [d for d in dataset_names if d['dataset_name'] == dataset_name][0]['dataset_id']
            pub_metadata = metadata_funs.run_doi_search(doi_id,api_client)
            if pub_metadata:
                pub_metadata.update({'related_dataset': dataset_id})
                manual_pub_dataset_list.append(pub_metadata)
            elif pub_metadata == None:
                pass
        except:
            pass
    return manual_pub_dataset_list
# manual_pub_dataset_list[0]
len(manual_pub_dataset_list)
len(manual_df_doi_dict)
i = manual_df_doi_dict[0]
i
# doi_id = i['doi']
doi_id = '10.1080/10454446.2018.1428259'
dataset_name = i['dataset_name']
# dataset_id = [d for d in a_dataset_names_listing if d['dataset_name'] == dataset_name][0]['dataset_id']
    
pub_metadata = metadata_funs.run_doi_search(doi_id,api_client)
# pub_metadata
manual_pub_dataset_list[0]<jupyter_output><empty_output><jupyter_text>String search for dataset names in dimensions api and return publication metadata<jupyter_code>import stringsearch_publications
import importlib
importlib.reload(stringsearch_publications)
a = stringsearch_publications.main()
import manually_curated_publications
import importlib
importlib.reload(manually_curated_publications)
b = manually_curated_publications.main()<jupyter_output><EMAIL>
········
API credentials have been set
{'fraction_non_alphanumeric': 0.09859454855195911, 'fraction_numerical': 0.012776831345826235, 'mean_word_length': 4.900753768844221, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '382707', 'license': 'CC0-1.0', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1538, 'n_words': 259, 'path': 'OOD/scrap/gen_publications.ipynb', 'repo_name': 'Coleridge-Initiative/RichContextMetadata', 'size': '4718'}