<jupyter_start><jupyter_text>Extract a dataframe of PubMed articles from efetch & esummary XML results<jupyter_code>import collections
import itertools
import pathlib
import pandas
from pubmedpy.xml import iter_extract_elems
from pubmedpy.efetch import extract_all
from pubmedpy.esummary import parse_esummary_article_info<jupyter_output><empty_output><jupyter_text>Load incoming citation counts from esummary records<jupyter_code>docsums = list()
# generator of XML PubmedArticle elements
docsum_elems = iter_extract_elems('data/pubmed/esummary/compbio-english.xml.xz', tag='DocSum')
for elem in docsum_elems:
    # Example esummary XML for <DocSum> at https://github.com/dhimmel/pubmedpy/blob/f554a06e13e24d661dc5ff93ad07179fb3d7f0af/pubmedpy/data/esummary.xml
    docsums.append(parse_esummary_article_info(elem))
len(docsums)
citation_count_df = (
    pandas.DataFrame(docsums)
    .rename(columns={"pubmed_id": "pmid"})
    [["pmid", "pmc_cited_by_count"]]
)
citation_count_df.head(2)<jupyter_output><empty_output><jupyter_text>Load article information from efetch records<jupyter_code>articles = list()
# generator of XML PubmedArticle elements
article_elems = iter_extract_elems('data/pubmed/efetch/compbio-english.xml.xz', tag='PubmedArticle')
for elem in article_elems:
    # Example efetch XML for <PubmedArticle> at https://github.com/dhimmel/pubmedpy/blob/f554a06e13e24d661dc5ff93ad07179fb3d7f0af/pubmedpy/data/efetch.xml
    articles.append(extract_all(elem))
len(articles)
article_df = pandas.DataFrame(articles)
article_df.pmid = article_df.pmid.astype(int)
article_df = article_df.merge(citation_count_df, how="left")
article_df = article_df.sort_values('pmid')
article_df.sample(n=20, random_state=0).sort_values('pmid')
author_rows = list()
affiliation_rows = list()
for pmid, authors in zip(article_df.pmid, article_df.authors):
    for i, author in enumerate(authors):
        row_template = {'pmid': pmid, 'position': i + 1}
        row = row_template.copy()
        row.update(author)
        row['reverse_position'] = len(authors) - i
        for affiliation in row.pop('affiliations'):
            affiliation_rows.append(
                {**row_template, "affiliation": affiliation}
            )
        author_rows.append(row)
author_df = pandas.DataFrame(author_rows)
author_df = author_df.sort_values(['pmid', 'position'])
author_df.head()    
affiliation_df = pandas.DataFrame(affiliation_rows)
affiliation_df = affiliation_df.sort_values(['pmid', 'position'])
affiliation_df.head()    
# Number of unique affiliations
affiliation_df.affiliation.nunique()
# Write article dataframe to TSV
(
    article_df
    [['pmid', 'pmcid', 'doi', 'journal', 'publication_date', 'pmc_cited_by_count', 'title']]
    .to_csv('data/pubmed/articles.tsv.xz', sep='\t', index=False)
)
# Write author dataframe to TSV
author_df.to_csv('data/pubmed/authors.tsv.xz', sep='\t', index=False)

# Write affiliation dataframe to TSV
affiliation_df.to_csv('data/pubmed/affiliations.tsv.xz', sep='\t', index=False)
# PubMed articles in PMC by journal
article_df.journal.value_counts()
# PubMed articles in PMC by journal
pandas.crosstab(article_df.pmcid.notna(), article_df.journal)
# articles without a publication date
article_df[article_df.publication_date.isna()]
# article proportions by date precision
article_df.publication_date.str.len().value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11497641509433962, 'fraction_numerical': 0.015035377358490566, 'mean_word_length': 5.924489795918367, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '463963', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1169, 'n_words': 215, 'path': '02.process-pubmed.ipynb', 'repo_name': 'greenelab/iscb-diversity', 'size': '3392'}