<jupyter_start><jupyter_text>Read paper-meta files<jupyter_code>inpath = "/home/vicp/data/2021-08-02/masters/"
psych = pd.read_csv(f"{inpath}psychology_paper_meta.csv")
econ = pd.read_csv(f"{inpath}economics_paper_meta.csv")
pol = pd.read_csv(f"{inpath}politicalscience_paper_meta.csv")
soc = pd.read_csv(f"{inpath}sociology_paper_meta.csv")
# to datetime 
def to_datetime(df): 
    df["Date"] = pd.to_datetime(df["Date"]).dt.date
    return df
psych = to_datetime(psych)
econ = to_datetime(econ)
pol = to_datetime(pol)
soc = to_datetime(soc)<jupyter_output><empty_output><jupyter_text>Quick Check<jupyter_code>def quick_check(df): 
    doctype = df.groupby('DocType').size().to_frame('size').reset_index().sort_values('size', ascending=False)
    field = df.groupby('NormalizedName').size().to_frame('size').reset_index().sort_values('size', ascending=False)
    print(f"*rows*: \n{len(df)} \n")
    print(f"*DocType*: \n{doctype.head(5)} \n")
    print(f"*field*: \n{field.head(5)}")
# economics
quick_check(econ)
# psychology
quick_check(psych)
# pol
quick_check(pol)
# sociology
quick_check(soc)
## overall looks good, but one caveat: 
# (1) we are not seing "None" for DocType here because it is somehow "hidden".<jupyter_output><empty_output><jupyter_text>How many are part of original "focus" articles<jupyter_code>psych_sub = pd.read_csv(f"{inpath}psychology_subset.csv")
econ_sub = pd.read_csv(f"{inpath}economics_subset.csv")
pol_sub = pd.read_csv(f"{inpath}politicalscience_subset.csv")
soc_sub = pd.read_csv(f"{inpath}sociology_subset.csv")
def sanity_check(df, df_sub, fos): 
    df_paperids = df[["PaperId"]].drop_duplicates()
    #df_sub_paperids = df_sub[["PaperId"]].drop_duplicates()
    # overlap should be all of subset 
    overlap = df_sub.merge(df_paperids, on = "PaperId", how = "inner").drop_duplicates()
    # overlap (& subset) should be the same as filtering
    # should become more or less equal to "subset" if we also do by age...
    filtering = len(df[(df["NormalizedName"] == f'{fos}') & 
                       (df["DocType"] == 'Journal') & 
                       (df["Date"] >= datetime.date(2010, 1, 1)) & 
                       (df["Date"] <= datetime.date(2021, 1, 1))])
    print(f"total papers {fos}: \n{len(df)} \n")
    print(f"subset papers {fos}: \n{len(df_sub)} \n")
    print(f"overlap: \n{len(overlap)} \n")
    print(f"by filtering: \n{filtering}")
sanity_check(psych, psych_sub, "psychology")
sanity_check(econ, econ_sub, "economics")
sanity_check(soc, soc_sub, "sociology")
sanity_check(pol, pol_sub, "political science")<jupyter_output>total papers political science: 
1794879 

subset papers political science: 
768704 

overlap: 
768704 

by filtering: 
768704<jupyter_text>authorships<jupyter_code>psych_paa = pd.read_csv(f"{inpath}psychology_paper_author.csv")
econ_paa = pd.read_csv(f"{inpath}economics_paper_author.csv")
pol_paa = pd.read_csv(f"{inpath}politicalscience_paper_author.csv")
soc_paa = pd.read_csv(f"{inpath}sociology_paper_author.csv")
# psychology slightly larger studies on avg. driven by outliers?
print(f"psychology: \nauthorships: {len(psych_paa)} \nstudies: {len(psych)} \navg: {len(psych_paa)/len(psych)} \n")
print(f"economy: \nauthorships: {len(econ_paa)} \nstudies: {len(econ)} \navg: {len(econ_paa)/len(econ)} \n")
print(f"political science: \nauthorships: {len(pol_paa)} \nstudies: {len(pol)} \navg: {len(pol_paa)/len(pol)} \n")
print(f"sociology: \nauthorships: {len(soc_paa)} \nstudies: {len(soc)} \navg: {len(soc_paa)/len(soc)} \n")<jupyter_output>psychology: 
authorships: 17881799 
studies: 4757436 
avg: 3.758705109222699 

economy: 
authorships: 5228483 
studies: 1970876 
avg: 2.652872631256355 

political science: 
authorships: 4651242 
studies: 1794879 
avg: 2.591395854539498 

sociology: 
authorships: 5493290 
studies: 2414270 
avg: 2.275342028853442
{'fraction_non_alphanumeric': 0.15814074266424305, 'fraction_numerical': 0.04388470527135809, 'mean_word_length': 5.652849740932642, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '317264', 'license': 'MIT', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1495, 'n_words': 261, 'path': 'check_preprocessing.ipynb', 'repo_name': 'victor-m-p/reform-psychology-MAG', 'size': '3851'}