<jupyter_start><jupyter_text>Start-upIn this notebook, we provide an overview of publication medatata and create the data frames for further analyses.<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt

# magics and warnings
%load_ext autoreload
%autoreload 2
import warnings; warnings.simplefilter('ignore')

import os, random, codecs, json
import pandas as pd
import numpy as np

seed = 99
random.seed(seed)
np.random.seed(seed)

import nltk, sklearn
import matplotlib.pyplot as plt
from  matplotlib.ticker import FuncFormatter
import seaborn as sns
sns.set(style="white")
sns.set_context("paper", font_scale=1.2, rc={"lines.linewidth": 2.5})
# load publication metadata

df_meta = pd.read_csv("../cwts_covid/datasets_output/df_pub.csv",compression="gzip")
df_meta.head()
df_meta.columns
df_meta.shape<jupyter_output><empty_output><jupyter_text>Publication years<jupyter_code>import re

def clean_year(s):
    if pd.isna(s):
        return np.nan
    if not (s>1900):
        return np.nan
    elif s>2020:
        return 2020
    return s

df_meta["publication_year"] = df_meta["publication_year"].apply(clean_year)
df_meta.publication_year.describe()
df_meta[df_meta.publication_year>2019].shape
b = sns.distplot(df_meta.publication_year, bins=60, kde=False)
b.set_xlabel("Publication year",fontsize=15)
b.set_ylabel("Publication count",fontsize=15)
b.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/meta_pub_year_all.pdf")
b = sns.distplot([int(y) for y in df_meta[df_meta.publication_year>1999].publication_year.tolist() if y], bins=21, kde=False)
b.set_xlabel("Publication year",fontsize=15)
b.set_ylabel("Publication count",fontsize=15)
b.tick_params(labelsize=12)
plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))
plt.tight_layout()
plt.savefig("figures/meta_pub_year_2000.pdf")<jupyter_output><empty_output><jupyter_text>Detect language<jupyter_code>from langdetect import detect
from langdetect import DetectorFactory
DetectorFactory.seed = seed
df_meta["title"] = df_meta["title"].fillna('')
df_meta["abstract"] = df_meta["abstract"].fillna('')
df_meta["title_abstract"] = df_meta.title +" "+df_meta.abstract
df_meta["lang"] = None
def detect_lang(txt):
    try:
        return detect(txt)
    except:
        return None
%%time

df_meta["lang"] = df_meta["title_abstract"].apply(detect_lang)
df_meta.drop(columns="title_abstract", inplace=True)
# languages

df_sub = df_meta[df_meta.lang.isin(df_meta.lang.value_counts()[:10].index.tolist())]
b = sns.countplot(y="lang", data=df_sub, order=df_sub['lang'].value_counts().index)
#b.axes.set_title("Title",fontsize=50)
b.set_xlabel("Count",fontsize=15)
b.set_ylabel("Language",fontsize=15)
b.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/meta_lang.pdf")<jupyter_output><empty_output><jupyter_text>Dimensions<jupyter_code># load Dimensions data (you will need to download it on your own)

directory_name = "../cwts_covid/datasets_output/json_dimensions_cwts"

all_dimensions = list()
for root, dirs, files in os.walk(directory_name):
    for file in files:
        if ".json" in file:
            all_data = codecs.open(os.path.join(root,file)).read()
            for record in all_data.split("\n"):
                if record:
                    all_dimensions.append(json.loads(record))
#all_dimensions[0]
df_dimensions = pd.DataFrame.from_dict({
    "id":[r["id"] for r in all_dimensions],
    "altmetric_id":[r["altmetric_id"] for r in all_dimensions],
    "year":[r["year"] for r in all_dimensions],
    "month":[int(r["publication_date"][5:7]) for r in all_dimensions],
    "publication_date":[r["publication_date"] for r in all_dimensions],
    "publication_type":[r["publication_type"] for r in all_dimensions],
    "doi":[r["doi"] for r in all_dimensions],
    "pmid":[r["pmid"] for r in all_dimensions],
    "issn":[r["journal"]["issn"] for r in all_dimensions],
    "times_cited":[r["times_cited"] for r in all_dimensions],
    "relative_citation_ratio":[r["relative_citation_ratio"] for r in all_dimensions],
    "for_top":[r["for"][0]["first_level"]["name"] if len(r["for"])>0 else "" for r in all_dimensions],
    "for_bottom":[r["for"][0]["second_level"]["name"] if len(r["for"])>0 else "" for r in all_dimensions]
})
df_dimensions.head()
df_dimensions.dtypes
# We only focus on publications with DOI or PMID since Altmetrics data is limited to those
# to check:
# df_dimensions[pd.isnull(df_dimensions.doi) & pd.isnull(df_dimensions.pmid) & pd.notnull(df_dimensions.altmetric_id)]
df_dimensions.pmid = df_dimensions.pmid.astype(float)
df_dimensions.shape
df_joined_doi = df_meta[pd.notnull(df_meta.doi)].merge(df_dimensions[pd.notnull(df_dimensions.doi)], how="inner", suffixes=('','_y'), left_on="doi", right_on="doi", copy=False)
df_joined_doi.shape
df_joined_pmid = df_meta[pd.isnull(df_meta.doi) & pd.notnull(df_meta.pmid)].merge(df_dimensions[pd.isnull(df_dimensions.doi) & pd.notnull(df_dimensions.pmid)], how="inner", suffixes=('','_y'), left_on="pmid", right_on="pmid", copy=False)
df_joined_pmid.shape
df_joined = pd.concat([df_joined_doi,df_joined_pmid])
df_joined = df_joined[df_joined.columns.drop(["doi_y","pmid_y"])]
df_joined.head()
df_joined.shape
df_joined = df_joined.groupby("pub_id").first().reset_index()
df_joined[df_joined.pub_id==29534]
# Most publications are in Dimensions via DOI or PMID: we focus on these
df_joined.shape
df_meta.shape
# publication typology

df_sub = df_joined[df_joined.publication_type.isin(df_joined.publication_type.value_counts()[:10].index.tolist())]
b = sns.countplot(y="publication_type", data=df_sub, order=df_sub['publication_type'].value_counts().index)
#b.axes.set_title("Title",fontsize=50)
b.set_xlabel("Count",fontsize=15)
b.set_ylabel("Publication typology",fontsize=15)
b.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/dim_pub_type.pdf")<jupyter_output><empty_output><jupyter_text>Citation counts<jupyter_code># scatter of citations vs time of publication

sns.scatterplot(df_joined.publication_year.to_list(),df_joined.times_cited.to_list())
plt.xlabel("Publication year", fontsize=15)
plt.ylabel("Number of citations", fontsize=15)
b.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/dim_citations_year.pdf")
# most cited papers

df_joined[["title","times_cited","relative_citation_ratio","journal","publication_year","doi"]].sort_values("times_cited",ascending=False).head(10)
# same but in 2020; note that duplicates are due to SI or pre-prints with different PMIDs

df_joined[df_joined.publication_year>2019][["title","times_cited","relative_citation_ratio","journal","publication_year","doi"]].sort_values("times_cited",ascending=False).head(10)
# most cited journals

df_joined[['journal','times_cited']].groupby('journal').sum().sort_values('times_cited',ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Altmetrics<jupyter_code># load Altmetrics data (you will need to download it on your own)

directory_name = "../cwts_covid/datasets_output/json_altmetrics_cwts"

all_altmetrics = list()
for root, dirs, files in os.walk(directory_name):
    for file in files:
        if ".json" in file:
            all_altmetrics.append(json.loads(codecs.open(os.path.join(root,file)).read()))
#all_altmetrics[1]
len(all_altmetrics)
df_altmetrics = pd.DataFrame.from_dict({
    "altmetric_id":[r["altmetric_id"] if "altmetric_id" in r.keys() else None for r in all_altmetrics],
    "counts_mendeley":[r["counts"]["readers"]["mendeley"] if "readers" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_news":[r["counts"]["news"]["posts_count"] if "news" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_blogs":[r["counts"]["blogs"]["posts_count"] if "blogs" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_facebook":[r["counts"]["facebook"]["posts_count"] if "facebook" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_policy":[r["counts"]["policy"]["posts_count"] if "policy" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_wikipedia":[r["counts"]["wikipedia"]["posts_count"] if "wikipedia" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_twitter_count":[r["counts"]["twitter"]["posts_count"] if "twitter" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_twitter_unique":[r["counts"]["twitter"]["unique_users_count"] if "twitter" in r["counts"].keys() else 0 for r in all_altmetrics],
    "counts_total":[r["counts"]["total"]["posts_count"] if "total" in r["counts"].keys() else 0 for r in all_altmetrics],
    "doi":[r["citation"]["doi"] if "doi" in r["citation"] else None for r in all_altmetrics],
    "pmid":[r["citation"]["pmid"] if "pmid" in r["citation"] else None for r in all_altmetrics],
    "year":[int(r["citation"]["pubdate"][:4]) if "pubdate" in r["citation"] else None for r in all_altmetrics],
    "title":[r["citation"]["title"] if "title" in r["citation"] else None for r in all_altmetrics],
    #"abstract":[r["citation"]["abstract"] if "abstract" in r["citation"] else None for r in all_altmetrics],
    "altmetric_score":[r["altmetric_score"]["score"] for r in all_altmetrics],
    "member_of_the_public":[r["demographics"]["poster_types"]["member_of_the_public"] if "demographics" in r.keys() and isinstance(r["demographics"],dict) and "poster_types" in r["demographics"].keys() and "member_of_the_public" in r["demographics"]["poster_types"].keys() else 0 for r in all_altmetrics],
    "researcher":[r["demographics"]["poster_types"]["researcher"] if "demographics" in r.keys() and isinstance(r["demographics"],dict) and "poster_types" in r["demographics"].keys() and "researcher" in r["demographics"]["poster_types"].keys() else 0 for r in all_altmetrics],
    "practitioner":[r["demographics"]["poster_types"]["practitioner"] if "demographics" in r.keys() and isinstance(r["demographics"],dict) and "poster_types" in r["demographics"].keys() and "practitioner" in r["demographics"]["poster_types"].keys() else 0 for r in all_altmetrics],
    "science_communicator":[r["demographics"]["poster_types"]["science_communicator"] if "demographics" in r.keys() and isinstance(r["demographics"],dict) and "poster_types" in r["demographics"].keys() and "science_communicator" in r["demographics"]["poster_types"].keys() else 0 for r in all_altmetrics]
})
df_altmetrics.shape
df_altmetrics['counts_mendeley'] = df_altmetrics['counts_mendeley'].astype(float)
# aggregate possible duplicates by summing their values up
# this is the recommended approach by Altmetric

df_altmetrics_aggr = df_altmetrics[['altmetric_id', 'counts_mendeley', 'counts_news', 'counts_blogs',
       'counts_facebook', 'counts_policy', 'counts_wikipedia',
       'counts_twitter_count', 'counts_twitter_unique', 'counts_total', 'altmetric_score', 'member_of_the_public',
       'researcher', 'practitioner', 'science_communicator']].groupby("altmetric_id").agg('sum')
df_altmetrics = df_altmetrics[['altmetric_id', 'doi',
       'pmid', 'year', 'title']].merge(df_altmetrics_aggr, how="inner", left_on="altmetric_id", right_on="altmetric_id")
df_altmetrics.drop_duplicates('altmetric_id',inplace=True)
df_altmetrics.shape
df_altmetrics[df_altmetrics.doi=="10.1086/511159"]
#all_altmetrics[26239]
df_altmetrics.head()
# Extract Wikipedia data frames
from datetime import date
#>>> date.fromisoformat('2019-12-04')

dois = list()
altmetric_ids = list()
wikipedia_ids = list()
wikipedia_titles = list()
wikipedia_urls = list()
wikipedia_langs = list()
wikipedia_summaries = list()
wikipedia_postedons = list()

for a in all_altmetrics:
    if "counts" in a.keys():
        if "wikipedia" in a["counts"].keys():
            for w in a["posts"]["wikipedia"]:
                try:
                    if "doi" in a["citation"].keys():
                        dois.append(a["citation"]["doi"])
                    else:
                        dois.append(None)
                    altmetric_ids.append(a["altmetric_id"])
                    wiki_id = w["page_url"].split("curid=")[-1]
                    wikipedia_ids.append(wiki_id)
                    wikipedia_titles.append(w["title"])
                    wikipedia_urls.append(w["page_url"])
                    wikipedia_langs.append(w["wiki_lang"])
                    if "summary" in w.keys():
                        wikipedia_summaries.append(w["summary"])
                    else:
                        wikipedia_summaries.append("")
                    wikipedia_postedons.append(date.fromisoformat(w["posted_on"].split("T")[0]))
                except:
                    print(w)
                    continue
len(wikipedia_ids)
df_wikipedia = pd.DataFrame.from_dict({
    "wikipedia_id": wikipedia_ids,
    "title": wikipedia_titles,
    "page_url": wikipedia_urls,
    "lang": wikipedia_langs,
    "summary": wikipedia_summaries
})
df_wikipedia = df_wikipedia.groupby('wikipedia_id').first()
df_wikipedia.reset_index(inplace=True)
df_wikipedia.shape
df_wikipedia.head()
def to_ym(d):
    if d.month < 10:
        return str(d.year)+"-0"+str(d.month)
    return str(d.year)+"-"+str(d.month)

df_alt_to_wiki = pd.DataFrame.from_dict({
    "doi": dois,
    "altmetric_id": altmetric_ids,
    "wikipedia_id": wikipedia_ids,
    "posted_on": wikipedia_postedons,
    "posted_on_year": [d.year for d in wikipedia_postedons],
    "posted_on_month": [d.month for d in wikipedia_postedons],
    "posted_on_ym": [to_ym(d) for d in wikipedia_postedons]
})
df_alt_to_wiki.shape
df_alt_to_wiki.drop_duplicates(inplace=True)
df_alt_to_wiki.shape
df_alt_to_wiki.head()
# year posted_on actions histogram

#plt.figure(figsize=(10,5))
b = sns.countplot(x="posted_on_year", data=df_alt_to_wiki, color="skyblue")
b.set_xticklabels(b.get_xticklabels(), rotation=45)
b.set_xlabel("Year",fontsize=15)
b.set_ylabel("Number of citations",fontsize=15)
b.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/wiki_postedon_year.pdf")
# month-year posted_on actions histogram

since_when = 2018
plt.figure(figsize=(10,5))
b = sns.countplot(x="posted_on_ym", data=df_alt_to_wiki[(df_alt_to_wiki.posted_on_year>=since_when)&(df_alt_to_wiki.posted_on_ym!="2020-07")], order=sorted(df_alt_to_wiki[(df_alt_to_wiki.posted_on_year>=since_when)&(df_alt_to_wiki.posted_on_ym!="2020-07")].posted_on_ym.unique()), color="skyblue")
b.set_xticklabels(b.get_xticklabels(), rotation=45)
b.set_xlabel("Year-Month",fontsize=15)
b.set_ylabel("Number of citations",fontsize=15)
b.tick_params(labelsize=12)
#plt.xlim(("2018-01","2020-06"))
plt.tight_layout()
plt.savefig("figures/wiki_postedon_year_month.pdf")
# publication years of articles cited by Wikipedia by posted_on action 
#(i.e., with duplicates for multiple posted_on actions on different Wikipedia pages)

since_when = 2020
from_when = 2000

df_alt_to_wiki_joined_altmetrics = df_alt_to_wiki.merge(df_altmetrics, how="left", left_on="altmetric_id", right_on="altmetric_id")
plt.figure(figsize=(10,5))
#sns.distplot([int(l) for l in df_alt_to_wiki_joined_altmetrics[(df_alt_to_wiki_joined_altmetrics.year>=from_when)].year.to_list()], bins=20, color="skyblue", kde=False, label="All")
sns.distplot([int(l) for l in df_alt_to_wiki_joined_altmetrics[(df_alt_to_wiki_joined_altmetrics.posted_on_year<since_when) & (df_alt_to_wiki_joined_altmetrics.year>=from_when)].year.to_list()], bins=20, color="green", kde=False, label="Before %d"%since_when)
b = sns.distplot([int(l) for l in df_alt_to_wiki_joined_altmetrics[(df_alt_to_wiki_joined_altmetrics.posted_on_year>=since_when) & (df_alt_to_wiki_joined_altmetrics.year>=from_when)].year.to_list()], bins=20, color="blue", kde=False, label="On %d"%since_when)
b.set_xlabel("Publication year",fontsize=15)
b.set_ylabel("Number of citations",fontsize=15)
b.tick_params(labelsize=12)
plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))
plt.legend(fontsize=15)
plt.tight_layout()
plt.savefig("figures/wiki_year_postedon.pdf")
# publication years of articles cited by Wikipedia

#plt.figure(figsize=(10,5))
#sns.distplot(df_meta[df_meta.publication_year>1979].publication_year, color="skyblue", bins=39, label="All", norm_hist=False, kde=False)
b = sns.distplot(df_alt_to_wiki_joined_altmetrics[df_alt_to_wiki_joined_altmetrics.year>1979].groupby("altmetric_id").first().reset_index().year, bins=39, label="In Wikipedia", norm_hist=False, kde=False)
plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))
#b.set_xticklabels(b.get_xticklabels(), rotation=45)
b.set_xlabel("Publication year",fontsize=15)
b.set_ylabel("Publication count",fontsize=15)
b.tick_params(labelsize=12)
#plt.legend(fontsize=15)
plt.tight_layout()
#plt.savefig("figures/wiki_year_both.pdf")
plt.savefig("figures/wiki_year.pdf")
# Fraction of cited papers per year
coverage_by_year = list()
since_when = 2003
for y in range(since_when,2021):
    cov = df_alt_to_wiki_joined_altmetrics[df_alt_to_wiki_joined_altmetrics.year==y].shape[0]/df_meta[df_meta.publication_year==y].shape[0]
    coverage_by_year.append(cov)
    print(y,cov)
b = sns.scatterplot([y for y in range(since_when,2021)],coverage_by_year,s=70)
b.set_xlabel("Year",fontsize=15)
b.set_ylabel("Fraction of COVID-19 literature\ncited from Wikipedia",fontsize=15)
b.set_xticklabels(range(since_when,2021),rotation=45)
b.tick_params(labelsize=12)
plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))
plt.ylim((0,0.1))
plt.tight_layout()
plt.savefig("figures/wiki_fraction_year.pdf")
# most Wiki cited papers
df_top_cited = df_alt_to_wiki.groupby("doi").agg('count').sort_values('wikipedia_id',ascending=False)[:20]
df_top_cited.reset_index(inplace=True)
df_top_cited[["doi",'wikipedia_id']].merge(df_joined, how="left", right_on="doi", left_on="doi")[["doi",
    "wikipedia_id","title","publication_year","journal","times_cited"]]
#print(df_top_cited[["doi",'wikipedia_id']].merge(df_joined, how="left", right_on="doi", left_on="doi")[[
#    "wikipedia_id","doi","title","publication_year","journal","times_cited"]].to_latex(index=False,bold_rows=True,caption="",label=""))
# most citing Wiki articles
df_top_citing = df_alt_to_wiki.groupby("wikipedia_id").agg('count').sort_values('doi',ascending=False)[:20]
df_top_citing.reset_index(inplace=True)
df_top_citing[["altmetric_id",'wikipedia_id']].merge(df_wikipedia, how="left", right_on="wikipedia_id", left_on="wikipedia_id")[["altmetric_id",
    "wikipedia_id","title","page_url","lang","summary"]]
#print(df_top_citing[["altmetric_id",'wikipedia_id']].merge(df_wikipedia, how="left", right_on="wikipedia_id", left_on="wikipedia_id")[["altmetric_id",
#    "wikipedia_id","title","lang"]].to_latex(index=False,bold_rows=True))<jupyter_output><empty_output><jupyter_text>Finish merges and save<jupyter_code>df_altmetrics.pmid = pd.to_numeric(df_altmetrics.pmid)
## join with Altmetrics

df_tmp_a = df_altmetrics[pd.isnull(df_altmetrics.doi)] 
df_tmp_j = df_joined[pd.isnull(df_joined.doi)] 
df_joined_all = df_joined[pd.notnull(df_joined.doi)].merge(df_altmetrics[pd.notnull(df_altmetrics.doi)], suffixes=('','_y'), how="left", left_on="doi", right_on="doi")
df_joined_all.drop(columns=['pmid_y', 'year_y', 'title_y'],inplace=True)
df_joined_tt = df_tmp_j.merge(df_tmp_a, how="left", suffixes=('','_y'), left_on="pmid", right_on="pmid")
df_joined_tt.drop(columns=["doi_y", 'year_y', 'title_y'],inplace=True)
df_joined_all = pd.concat([df_joined_all,df_joined_tt])
df_joined_all.drop_duplicates("pub_id",inplace=True)
df_joined_all.reset_index(inplace=True,drop=True)
df_joined_all.shape
df_joined_all.columns
df_joined_all.drop(columns=["dimensions_id","year","month","altmetric_id","timestamp"], inplace=True)
df_joined_all.rename(columns={"id":"dimensions_id","altmetric_id_y":"altmetric_id","counts_twitter_count":"counts_twitter"}, inplace=True)
df_joined_all[df_joined_all.pub_id==29534]
df_joined_all["in_wikipedia"] = df_joined_all["counts_wikipedia"]>0
df_joined_all.shape
df_posted_on_first = df_alt_to_wiki[["altmetric_id","posted_on_year"]].groupby("altmetric_id").agg(min).reset_index()
df_joined_all = df_joined_all.merge(df_posted_on_first, how="left", left_on="altmetric_id", right_on="altmetric_id")
df_joined_all[df_joined_all.in_wikipedia==True].shape[0]/df_meta.shape[0]
# save to disk

df_joined_all.to_csv("data/df_joined_all.csv", compression="gzip", index=False)
df_meta.to_csv("data/df_meta.csv", compression="gzip", index=False)<jupyter_output><empty_output><jupyter_text>Analysis of time to be cited from publication<jupyter_code># average time from publication to citations

df_posted_on_first = df_alt_to_wiki[["altmetric_id","posted_on_year","posted_on_month"]].groupby("altmetric_id").agg(min).reset_index()
df_posted_on_first = df_posted_on_first.merge(df_joined_all, how="left", right_on="altmetric_id", left_on="altmetric_id")
df_posted_on_first["time_from_pub_to_wiki"] = ((df_posted_on_first["posted_on_year_x"]*12)+df_posted_on_first["posted_on_month"]) - ((df_posted_on_first["publication_year"]*12)+df_posted_on_first["publication_month"])
sns.distplot(df_posted_on_first["time_from_pub_to_wiki"], norm_hist=False, kde=False)
plt.xlabel("Number of months from publication to first citation in Wikipedia",fontsize=15)
plt.ylabel("Count",fontsize=15)
plt.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/nb1_time_from_pub_to_wiki_hist.pdf")
sns.distplot(df_posted_on_first[df_posted_on_first.publication_year==2020]["time_from_pub_to_wiki"], color="green", kde=True, norm_hist=True, kde_kws={"shade": True}, hist=False, label="Published in 2020")
sns.distplot(df_posted_on_first[(df_posted_on_first.publication_year<2020) & (df_posted_on_first.publication_year>1999)]["time_from_pub_to_wiki"], color="blue", kde=True, norm_hist=True, kde_kws={"shade": True}, hist=False, label="Published before 2020")
plt.legend(fontsize=15)
plt.xlabel("Number of months from publication to first citation in Wikipedia",fontsize=15)
plt.ylabel("Proportion",fontsize=15)
plt.tick_params(labelsize=12)
plt.tight_layout()
plt.savefig("figures/nb1_time_from_pub_to_wiki_hist_normed.pdf")
#df_posted_on_first[["time_from_pub_to_wiki","posted_on_year_x","posted_on_month","publication_year","publication_month","in_wikipedia"]]
from_year = 2001
until_year = 2020
posted_on_limit = 2020

plt.figure(figsize=(8,5))
sns.scatterplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]].groupby("publication_year").agg(np.median).reset_index(), s=70, color="skyblue", label="median")
b = sns.scatterplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]].groupby("publication_year").agg(np.mean).reset_index(), s=70, color="red", label="mean")
b.set_xticklabels(b.get_xticklabels(), rotation=45)
plt.legend(fontsize=15)
plt.ylabel("Number of months from publication\nto a first citation in Wikipedia",fontsize=16)
plt.xlabel("Publication year",fontsize=16)
plt.xticks(range(from_year,until_year+1))
plt.tick_params(labelsize=12)
plt.axhline(y=0.0, color='black', linestyle='-', alpha=0.5)
ax = plt.gca()
ax.grid(True, axis='x')
ax.xaxis.set_major_formatter(FuncFormatter(lambda x, _: int(x)))
#plt.title("Number of months elapsed from publication to the first Wikipedia citation", fontsize=17)
plt.tight_layout()
plt.savefig("figures/nb1_time_from_pub_to_wiki_avg_per_year.pdf")
from_year = 2001
until_year = 2020
posted_on_limit = 2020

#fig, ax = plt.subplots(figsize=(8,5))
#fig, ax = plt.subplots()
#sns.scatterplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]].groupby("publication_year").agg(np.median).reset_index(), s=70, color="skyblue", label="median")
#sns.violinplot(x="day", y="total_bill", hue="sex",
#                    data=tips, palette="Set2", split=True,
#                    scale="count", inner="stick")
#a = sns.scatterplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]].groupby("publication_year").agg(np.mean).reset_index(), s=70, color="red", ax=ax)
#ax2 = ax.twinx()
#b = sns.catplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]], color="gray", s=3)
b = sns.violinplot(x="publication_year", y="time_from_pub_to_wiki", data=df_posted_on_first[(df_posted_on_first.publication_year>=from_year)&(df_posted_on_first.publication_year<=until_year)&(df_posted_on_first.posted_on_year_x<=posted_on_limit)][["publication_year","time_from_pub_to_wiki"]], color="skyblue", s=3)
b.set_xticklabels(range(from_year,until_year+1),rotation=45)
plt.ylabel("Number of months",fontsize=16)
plt.xlabel("Publication year",fontsize=16)
plt.tick_params(labelsize=12)
plt.axhline(y=0.0, color='black', linestyle='-', alpha=0.3)
plt.tight_layout()
plt.savefig("figures/nb1_time_from_pub_to_wiki_avg_per_year_violin.pdf")
df_meta[df_meta.publication_year==2020].shape
df_posted_on_first[["publication_year","time_from_pub_to_wiki"]].groupby("publication_year").agg(np.mean).reset_index()
df_posted_on_first[(df_posted_on_first.publication_year==2020)&(df_posted_on_first.time_from_pub_to_wiki < 0)]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15605071071840185, 'fraction_numerical': 0.015751056473300037, 'mean_word_length': 7.860108917631042, 'pattern_counts': {'":': 51, '<': 49, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '115729', 'license': 'MIT', 'n_tokens_mistral': 10081, 'n_tokens_neox': 9493, 'n_words': 1366, 'path': 'Notebook_1_startup.ipynb', 'repo_name': 'Giovanni1085/covid-19_wikipedia', 'size': '26030'}