<jupyter_start><jupyter_text>PMID to Year Map -- Pubmed Central<jupyter_code>import pandas as pd
from tqdm import tqdm
!wget -O ../data/PMC-ids-csv.gz ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/PMC-ids.csv.gz
pmids = []
with open('../data/pmid_list_ver43.txt', 'r') as fin:
    for line in fin.readlines():
        pmids.append(line.strip())
print('{:,}'.format(len(pmids)))
df = pd.read_csv('../data/PMC-ids-csv.gz')
df.head()
df.dtypes
print('{:,}'.format(len(df)))
print('{:,}'.format(df['PMID'].count()))
df = df.dropna(subset=['PMID'])
df['PMID'] = df['PMID'].astype(int).astype(str)

df.head(2)
pmid_to_year = df.set_index('PMID')['Year'].to_dict()
mapped = set(pmid_to_year.keys())
no_map = set(pmids) - mapped
print('{:,}'.format(len(no_map)))
import pickle
pickle.dump(pmid_to_year, open('../data/pmid_to_year_PMC.pkl', 'wb'))
pickle.dump(no_map, open('../data/no_map_PMC.pkl', 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2360655737704918, 'fraction_numerical': 0.003278688524590164, 'mean_word_length': 5.361111111111111, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '79002', 'license': 'PostgreSQL', 'n_tokens_mistral': 411, 'n_tokens_neox': 399, 'n_words': 56, 'path': '0_prepare/02A-id_to_publication_year-PMC.ipynb', 'repo_name': 'turoger/semmed', 'size': '915'}