<jupyter_start><jupyter_text>Finding journal info per paper Use crossref API to get journal infoNB: Not all papers are registered with crossref, but it seems to be the vast majorityUseful reference:https://github.com/CrossRef/rest-api-docfilter-names1) Split data into sets of n_per_req (entries per request)2) send a query to crossref api. concatenate the DOIs of n_per_req papers. request the doi and container-title (journal name) in results3) match the container-titles returned to the DOI to fill in the journal column4) Save<jupyter_code># go through all DOIs in neuro_data, access journal information using the crossref API
n_per_req = 100 # number of entries per request
# initialize query
query = {'filter':'doi:10.3389/fnsys.2016.00088',
         'rows':n_per_req,
         'select':'DOI,container-title',
         'mailto':'<EMAIL>'}
# loop through articles
for x in np.arange(0,neuro_data.shape[0]+n_per_req,n_per_req): #
    #build query string with dois of n_per_req papers
    t = time.time()
    doi_str = ""
    for xx in np.arange(n_per_req):
        if (x + xx) < neuro_data.shape[0]:
            doi_str += "doi:" + neuro_data.published_doi.iloc[x+xx] + ','
    doi_str = doi_str[:-1] # get rid of last comma
    query['filter'] = doi_str
    
    #access crossref api
    response = requests.get(crossref_url, params=query)
    r = response.json()
    print(f"{x} of {neuro_data.shape[0]}. {time.time()-t}s elapsed for query. {r['message']['total-results']} results found")
    t = time.time()
    for xx in np.arange(r['message']['total-results']):
        ind = r['message']['items'][xx]['DOI']
        neuro_data.loc[ind,'journal'] = r['message']['items'][xx]['container-title'][0]
    print(f"{time.time()-t}s elapsed for pandas bookkeepping")  
#save df
neuro_data.to_csv('data/neuro_journals_doi_2.csv')<jupyter_output><empty_output><jupyter_text>Alternative (original) way of ac<jupyter_code># use crossref api to get journal title from DOI
# ****very slow bc it gets one entry per request****
journals = neuro_data.published_doi.copy().rename_axis('journal_name')
crossref_url = 'http://api.crossref.org/works' 
query = {'filter':'doi:10.3389/fnsys.2016.00088','rows':1,'select':'DOI,container-title','mailto':'<EMAIL>'}
for x, doi in enumerate(neuro_data.published_doi.iteritems()):
    print(str(x) + ' out of ' + str(journals.shape[0]) + ': ' + doi[1])
    query['filter'] = 'doi:' + doi[1]
    response = requests.get(crossref_url, params=query)
    r = response.json()
    if r['message']['total-results'] > 0:
        journals[doi[0]] = r['message']['items'][0]['container-title'][0]
    else:
        journals[doi[0]] = ''

pd.concat([journals,neuro_data.published_doi],axis=1).to_csv('neuro_journals_doi_2.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15190784737221022, 'fraction_numerical': 0.019798416126709864, 'mean_word_length': 4.54690618762475, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '140239', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1008, 'n_tokens_neox': 941, 'n_words': 227, 'path': 'biorxiv data and scripts/extract_journals.ipynb', 'repo_name': 'marinafridman/delays', 'size': '2806'}