<jupyter_start><jupyter_text>Load the data Meta-data for the full article corpus<jupyter_code>df = pd.read_csv("metadata.csv", encoding="latin-1")
df = df.dropna(subset=["PMID"])
df["PMID"] = df["PMID"].astype(int)
print("The corpus has {} articles".format(len(df)))
df.head(3)<jupyter_output>The corpus has 18155 articles<jupyter_text>Subset with manually curated study and sample information<jupyter_code>sub = pd.read_csv("sample_ehb.csv", encoding="latin-1")
sub = sub[:250]
print("The subset has {} articles".format(len(sub)))
sub.head()<jupyter_output>The subset has 250 articles<jupyter_text>Set the plotting parameters<jupyter_code>import sys
sys.path.append("..")
from style.style import *

from matplotlib import font_manager, rcParams
font_prop = font_manager.FontProperties(fname=font, size=18)
rcParams["axes.linewidth"] = 1.5
def plot_ordinal_bar(x, y, filename, height=1.8, width=4.5, xlim=[]):
    
    fig = plt.figure(figsize=(width, height))
    ax = fig.add_axes([0,0,1,1])

    plt.bar(x, y, color="gray", edgecolor="black", alpha=0.65, width=1)
    
    if len(xlim) == 2:
        plt.xlim(xlim)

    plt.xticks(fontproperties=font_manager.FontProperties(fname=font, size=16))
    plt.yticks(fontproperties=font_manager.FontProperties(fname=font, size=16))

    ax.xaxis.set_tick_params(width=1.5, length=7)
    ax.yaxis.set_tick_params(width=1.5, length=7)

    for side in ["right", "top"]:
        ax.spines[side].set_visible(False)

    plt.savefig("figures/{}.png".format(filename), dpi=250, bbox_inches="tight")
    plt.show()
def plot_categorical_bar(x, y, filename, height=1.8, width=4.5, xlabsize=13):
    
    fig = plt.figure(figsize=(width, height))
    ax = fig.add_axes([0,0,1,1])

    plt.bar(x, y, color="gray", edgecolor="black", alpha=0.65, width=1)

    plt.xticks(fontproperties=font_manager.FontProperties(fname=font, size=xlabsize))
    plt.yticks(fontproperties=font_manager.FontProperties(fname=font, size=16))
    
    ax.set_xticklabels(x, rotation=55, ha="right")
    ax.xaxis.set_tick_params(width=1.5, length=7)
    ax.yaxis.set_tick_params(width=1.5, length=7)

    for side in ["right", "top"]:
        ax.spines[side].set_visible(False)

    plt.savefig("figures/{}.png".format(filename), dpi=250, bbox_inches="tight")
    plt.show()
def plot_hist(y, filename, width=4.5, barwidth=1):
    
    fig = plt.figure(figsize=(width, 1.8))
    ax = fig.add_axes([0,0,1,1])

    plt.hist(y, color="gray", edgecolor="black", alpha=0.65, width=barwidth)

    plt.xticks(fontproperties=font_manager.FontProperties(fname=font, size=16))
    plt.yticks(fontproperties=font_manager.FontProperties(fname=font, size=16))

    ax.xaxis.set_tick_params(width=1.5, length=7)
    ax.yaxis.set_tick_params(width=1.5, length=7)

    for side in ["right", "top"]:
        ax.spines[side].set_visible(False)

    plt.savefig("figures/{}.png".format(filename), dpi=250, bbox_inches="tight")
    plt.show()<jupyter_output><empty_output><jupyter_text>Plot full cohort information Publication year<jupyter_code>df["YEAR"] = df["YEAR"].astype(int)
years = sorted(list(set(df["YEAR"])))
n_per_year = [sum(df["YEAR"] == yr) for yr in years]
plot_ordinal_bar(years, n_per_year, "publication_year", xlim=[1985,2019], height=1.6, width=4.25)<jupyter_output><empty_output><jupyter_text>Journal<jupyter_code>df["JOURNAL"] = [j.strip().title() for j in df["JOURNAL"]]
journals = list(set(df["JOURNAL"]))
n_per_journal = [sum(df["JOURNAL"] == j) for j in journals]
j_ser = pd.Series(n_per_journal, index=journals).sort_values(ascending=False)[:15]
j_ser
names = {"Neuroimage": "NeuroImage",
         "Human Brain Mapping": "Human Brain Mapping",
         "Plos One": "PloS One",
         "The Journal Of Neuroscience : The Official Journal Of The Society For Neuroscience": "Journal of Neuro.",
         "Neuropsychologia": "Neuropsychologia",
         "Cerebral Cortex (New York, N.Y. : 1991)": "Cerebral Cortex",
         "Journal Of Cognitive Neuroscience": "Journal of Cog. Neuro.",
         "Frontiers In Human Neuroscience": "Frontiers in Hum. Neuro.",
         "Social Cognitive And Affective Neuroscience": "SCAN",
         "Brain Research": "Brain Research",
         "Biological Psychiatry": "Biological Psychiatry",
         "Psychiatry Research": "Psychiatry Research",
         "Neuroscience Letters": "Neuroscience Letters",
         "Cortex; A Journal Devoted To The Study Of The Nervous System And Behavior": "Cortex",
         "Brain And Language": "Brain & Language"}
j_ser.index = [names[j] for j in j_ser.index]
j_ser
plot_categorical_bar(j_ser.index, j_ser.values, "journal", xlabsize=15, height=1.6, width=4.25)<jupyter_output><empty_output><jupyter_text>Plot subset information Imaging modality<jupyter_code># sub["MODALITY"] = [m if m in ["fMRI", "PET"] else "Other" for m in sub["MODALITY"]]
# mod = ["fMRI", "Other", "PET"]
mod = ["Functional MRI", "Structural MRI", "PET", "Other"]
n_per_mod = [sum(sub["MODALITY_CATEGORY"] == m) for m in mod]
plot_categorical_bar(mod, n_per_mod, "modality", height=1.9, width=1.5, xlabsize=17)<jupyter_output><empty_output><jupyter_text>Number of subjects<jupyter_code>plot_hist(sub["N"].values, "n_subjects")
np.mean(sub["N"])
np.std(sub["N"])<jupyter_output><empty_output><jupyter_text>Gender<jupyter_code>np.mean(sub["N_FEMALE"])
np.std(sub["N_FEMALE"])
np.mean(sub["N_MALE"])
np.std(sub["N_MALE"])<jupyter_output><empty_output><jupyter_text>Age<jupyter_code>np.mean(sub["AGE_MEAN"])
np.std(sub["AGE_MEAN"])
plot_hist(sub["AGE_MEAN"].values, "age_mean", barwidth=6.6, width=2.25)
np.mean(sub["AGE_MIN"])
np.std(sub["AGE_MIN"])
np.mean(sub["AGE_MAX"])
np.std(sub["AGE_MAX"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16783954961294864, 'fraction_numerical': 0.023399014778325122, 'mean_word_length': 5.519495412844036, 'pattern_counts': {'":': 15, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '399541', 'license': 'MIT', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2119, 'n_words': 379, 'path': 'data/data_table.ipynb', 'repo_name': 'ehbeam/neuro-knowledge-engine', 'size': '5684'}