<jupyter_start><jupyter_text>Process PubMed journal catalogDownload and process PubMed/NLM [journal catalog](http://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.journal_lists/).<jupyter_code>import os
import re

import pandas
# Download PubMed Journals
url = 'ftp://ftp.ncbi.nih.gov/pubmed/J_Medline.txt'
! wget --no-verbose --directory-prefix download --timestamping {url}
# Read PubMed journals
path = os.path.join('download', 'J_Medline.txt')
with open(path) as read_file:
    text = read_file.read()
# Create a dataframe of PubMed journals
rows = list()
pattern = re.compile('^-+$', re.MULTILINE)
for stanza in re.split(pattern, text):
    stanza = stanza.strip()
    if not stanza:
        continue
    row = dict()
    for line in stanza.split('\n'):
        key, value = line.split(': ', 1)
        row[key] = value or None
    rows.append(row)

journal_df = pandas.DataFrame(rows)
journal_df = journal_df.sort_values(by='NlmId')
# Order columns by percent missing
missing_pct = journal_df.isnull().mean().sort_values()
journal_df = journal_df[missing_pct.index]
missing_pct
# Save journal dataframe as a TSV
path = 'data/pubmed-journals.tsv'
journal_df.to_csv(path, sep='\t', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13522267206477734, 'fraction_numerical': 0.004048582995951417, 'mean_word_length': 4.493333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '356467', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 445, 'n_tokens_neox': 421, 'n_words': 99, 'path': 'process-nlm-catalog.ipynb', 'repo_name': 'marinafridman/delays', 'size': '1235'}