<jupyter_start><jupyter_text>analysis> API details.<jupyter_code>#hide
from nbdev.showdoc import *
#export
from pymongo import MongoClient
import pandas as pd
from collections import Counter
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import re

%matplotlib inline
MONGO_HOST = 'mongodb://localhost:27017'
client = MongoClient(MONGO_HOST)
# create or load pubmed database
db = client.PubMed
db.list_collection_names()
pubmed = db.pubmed_meta
pubmed.estimated_document_count()
docs = [doc for doc in tqdm(pubmed.find())]
docs[0]
df = pd.DataFrame(docs)<jupyter_output><empty_output><jupyter_text>Filter out `magnetic resonance`<jupyter_code>df = df[df['keyword'] != 'magnetic resonance']<jupyter_output><empty_output><jupyter_text>Authors<jupyter_code>paper_authors = df['authors'].apply(lambda x: x.split(', '))
len(paper_authors)
_authors = paper_authors.explode(); len(_authors)
_authors = _authors[(_authors.str.len() > 3) & 
                    (_authors != '[No authors listed]')]; len(_authors)
Counter(_authors).most_common(10)
authors = set(_authors)
len(authors)<jupyter_output><empty_output><jupyter_text>Organizations<jupyter_code>_orgs = df['organizations'].explode(); len(_orgs)
_orgs
orgs_common = Counter(_orgs).most_common(100);
x, height = zip(*orgs_common[2:])
plt.figure(figsize=(20, 10))
plt.bar(x, height)
plt.xticks(rotation=80);
# orgs_common
orgs = set(_orgs); len(orgs)<jupyter_output><empty_output><jupyter_text>Citations => year<jupyter_code>ym_pat = re.compile(r'(([0-9])\d{3}\s([A-Z])\w{2})')
y_pat = re.compile(r'(([0-9])\d{3})')
for c in df['citation'][:1]:
    print(c)
    print(y_pat.search(c).group(0))<jupyter_output>Epilepsy Behav. 2020 Feb 21;105:106964. doi: 10.1016/j.yebeh.2020.106964. [Epub ahead of print]
2020<jupyter_text>Yearly growth<jupyter_code>authors = df['authors'].apply(lambda x: x.split(', ')); len(authors)
def parse_year(cit, match='y'):
    if isinstance(cit, str): 
        if match == 'ym':
            ym = ym_pat.search(cit)
            if ym: return ym.group(0)
        if match == 'y':
            y = y_pat.search(cit)
            if y:
                y = int(y.group(0))
                if y > 1900:
                    return y
    return False
years = df['citation'].apply(parse_year)
years_count = Counter(years).most_common()
years_count[:50]
x, height = zip(*years_count[:20])
plt.figure(figsize=(20, 10))
plt.bar(x, height)
plt.xticks(rotation=80);
len(authors)
len(years)
# researchers = pd.DataFrame([authors, years])
years
authors
df['citation'] = df['citation'].apply(parse_year)
df['authors'] = df['authors'].apply(lambda x: x.split(', '))
last_10 = df[df['citation'] > 2009]
len(last_10)
researchers = []
for y in range(2010, 2021):
    rs = last_10[last_10['citation'] == y]['authors']
    rs = len(set(rs.explode()))
    researchers.append((y, rs))
researchers
y, n = zip(*researchers)
plt.figure(figsize=(20, 10))
plt.bar(y, n)
plt.xticks(rotation=80);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16526986255447537, 'fraction_numerical': 0.03519946362722092, 'mean_word_length': 5.152577319587629, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '462068', 'license': 'Apache-2.0', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1100, 'n_words': 206, 'path': '02_analysis.ipynb', 'repo_name': 'markdyousef/research-crawler', 'size': '2983'}