<jupyter_start><jupyter_text>Load data<jupyter_code># load our documents 
psych = mag.getDataframe('psychology_total')
econ = mag.getDataframe('economics_total')
pol = mag.getDataframe('politicalscience_total')
soc = mag.getDataframe('sociology_total')
# load other data-sets 
FoS = mag.getDataframe('FoS')
pfs = mag.getDataframe('PaperFieldsOfStudy')<jupyter_output><empty_output><jupyter_text>Gather by search terms (open science, reproducibility, replication)<jupyter_code>open_science = FoS.filter(F.col('NormalizedName') == 'open science') \
    .select('FieldOfStudyId', 'NormalizedName')
reproducibility = FoS.filter(F.col('NormalizedName') == 'reproducibility') \
    .select('FieldOfStudyId', 'NormalizedName')
replication = FoS.filter((F.col('NormalizedName') == 'replication') & (F.col('Level') == 2)) \
    .select('FieldOfStudyId', 'NormalizedName')<jupyter_output><empty_output><jupyter_text>Get overlap between main fields & sub-categories of interest<jupyter_code>def get_overlap(fos, sub_fos): 
    '''
    fos: <str> main field of study (e.g. "psychology" or "economics")
    sub_fos: <str> sub-field, e.g. "open science" or "reproducibility"
    '''
    
    # load stuff
    fos_total = mag.getDataframe(f'{fos}_total')
    FoS = mag.getDataframe('FoS')
    pfs = mag.getDataframe('PaperFieldsOfStudy')
    subfos_df = FoS.filter((F.col('NormalizedName') == sub_fos) & (F.col('Level') == 2)) \
        .select('FieldOfStudyId', 'NormalizedName') 
    
    # for output
    sub_fos_name = sub_fos.replace(" ", "")
    
    # join everything to get the articles 
    subfos_df.join(pfs, ['FieldOfStudyId'], "inner") \
        .join(fos_total, ['PaperId'], "inner") \
        .select('PaperId', 'NormalizedName') \
        .distinct() \
        .toPandas() \
        .to_csv(f"/home/vicp/data/2021-08-02/masters/{fos}_{sub_fos_name}.csv", index = False)
    
    # print 
    print(f"--- finished computing overlap between {fos} and {sub_fos} \n")
for fos in ["psychology", "economics", "politicalscience", "sociology"]: 
    for sub_fos in ["open science", "replication", "reproducibility"]: 
        get_overlap(fos, sub_fos)<jupyter_output>--- finished computing overlap between psychology and open science 

--- finished computing overlap between psychology and replication 

--- finished computing overlap between psychology and reproducibility 

--- finished computing overlap between economics and open science 

--- finished computing overlap between economics and replication 

--- finished computing overlap between economics and reproducibility 

--- finished computing overlap between politicalscience and open science<jupyter_text>Rough check on results<jupyter_code># just run this loop: 
for fos in ["psychology", "economics", "politicalscience", "sociology"]: 
    for sub_fos in ["openscience", "replication", "reproducibility"]: 
        case = pd.read_csv(f"/home/vicp/data/2021-08-02/masters/{fos}_{sub_fos}.csv")
        n_records = len(case) 
        print(f"--- overlap between {fos} and {sub_fos} is {n_records} \n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14156138645934566, 'fraction_numerical': 0.0058309037900874635, 'mean_word_length': 4.436619718309859, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '257717', 'license': 'MIT', 'n_tokens_mistral': 1055, 'n_tokens_neox': 997, 'n_words': 220, 'path': 'get_subfields.ipynb', 'repo_name': 'victor-m-p/reform-psychology-MAG', 'size': '3087'}