<jupyter_start><jupyter_text>Look at supply and demand of available datasets and ML models<jupyter_code>import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
dataset_regular = '../datasets/Dataset_evolution_regular.csv'
dataset_consortia = '../datasets/Dataset_evolution_consortia.csv'

freesurfer_papers = '../datasets/FreeSurfer_papers.csv'
DL_papers = '../datasets/DL_papers.csv'
datasets_regular_df = pd.read_csv(dataset_regular)
datasets_regular_df['Number of datasets'] = 1
datasets_regular_df['Dataset Type'] = 'regular'

datasets_consortia_df = pd.read_csv(dataset_consortia)
datasets_consortia_df['Dataset'] = datasets_consortia_df['Working group']
datasets_consortia_df['Dataset Type'] = 'consortia'

useful_cols = ['Dataset','Sample size', 'Year', 'Number of datasets', 'Dataset Type']
datasets_df = datasets_regular_df[useful_cols].append(datasets_consortia_df[useful_cols])
datasets_df.head()
plot_df = datasets_df.copy()
plot_df['Year'] = plot_df['Year'].astype(int)
plot_df['Sample size'] = plot_df['Sample size'].astype(int)

sns.set(font_scale = 6)
palette = ['deepskyblue','navy'] #['tomato','firebrick'] # sns.color_palette("husl", 2)
with sns.axes_style("whitegrid"):
    fig, ax1 = plt.subplots(figsize=(40,25),sharex=True,sharey=True)
    g = sns.scatterplot(x='Year',y='Sample size', hue='Dataset Type', size='Sample size', sizes=(1000,5000), data=plot_df, palette=palette,ax=ax1)
    
    g.grid(True,which="both",ls="--",c='lightgray') 
    plt.title('Dataset Sizes Over the Years')

    g.set(yscale='log')
    # g.set(xlim=(1e6, 1e8))

    # EXTRACT CURRENT HANDLES AND LABELS
    h,l = ax1.get_legend_handles_labels() 
    col_lgd = plt.legend(h[:3], l[:3], loc='upper left')  
    col_lgd.legendHandles[1]._sizes = [1000]
    col_lgd.legendHandles[2]._sizes = [1000]
    
    # add model names as bubble labels
    def label_point(x, y, val, ax, x_shift=0, y_shift=0):
        a = pd.concat({'x': x, 'y': y, 'val': val}, axis=1)
        for i, point in a.iterrows():
            x_shift, y_shift = 0.1*np.random.randint(-3,3, 2)
            ax.text(point['x']+x_shift, point['y']+y_shift, str(point['val']), fontsize=32)

    label_point(plot_df['Year'], plot_df['Sample size'], plot_df['Dataset'], plt.gca())
freesurfer_papers_df = pd.read_csv(freesurfer_papers)
DL_papers_df = pd.read_csv(DL_papers)

citation_df = pd.merge(freesurfer_papers_df, DL_papers_df, on='Year', how='left')

citation_df.head()
plot_df = citation_df[citation_df['Year']!=2021].copy()

pal = sns.color_palette("husl", 2)
with sns.axes_style("whitegrid"):
    fig, ax1 = plt.subplots(figsize=(40,10),sharex=True,sharey=True)
    sns.lineplot(x='Year',y='Total', marker='d', markersize=40, data=plot_df, linewidth = 20, color=pal[1], label='FreeSurfer')
    sns.lineplot(x='Year',y='N_AI-papers',marker='d', markersize=40, data=plot_df, linewidth = 20, color=pal[0], label='Machine-learning')

plt.title('Number of Citations in Neuroimaging Studies', fontsize=80)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16354679802955666, 'fraction_numerical': 0.02167487684729064, 'mean_word_length': 5.97025171624714, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '5957', 'license': 'MIT', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1150, 'n_words': 201, 'path': 'lit_review/notebooks/datasets_and_pipelines.ipynb', 'repo_name': 'neurodatascience/watts_up_compute', 'size': '3045'}