<jupyter_start><jupyter_text>Opening file<jupyter_code>import re
import pandas as pd
import numpy as np
import json
import os
import math

os.chdir("../data/")
df=pd.read_csv("Satellite Data.csv")
df.rename(columns={'Id':'id','Ti':'ti','CC':'cc','AA':'aa'}, inplace=True)
df=df.loc[df['cc']>2]
def get_names(string):
    if pd.isna(string):
        return ""
    return re.findall('(?<=|)([A-Za-z ]+)(?=|)', string)
def get_nums(string):
    if pd.isna(string):
        return ""
    return re.findall('(?<=|)([0-9]{4,})(?=|)', string)
df['aanames']=df['aa'].apply(get_names)
df['aanums']=df['aa'].apply(get_nums)

#Print CSV of selected columns in paper data
df.to_csv('PaperPoints.csv',columns=['id', 'ti', 'cc', 'aanames', 'aanums'], index=False)
audf=pd.DataFrame()
audf=audf.reindex(columns=['name','num','citations','numpapers'])
for i in df.index:
    numarr=df['aanums'][i]
    namarr=df['aanames'][i]
    for j in range(0, len(numarr)):
#         print(numarr[j])
        if int(numarr[j]) not in audf.num.values:
            audf=audf.append({'name':namarr[j], 'num':int(numarr[j]), 'citations':int(df['cc'][i]), 'numpapers':1}, ignore_index=True)
        else:
            audf.loc[audf['num']==int(numarr[j]), 'citations']+=int(df['cc'][i])
            audf.loc[audf['num']==int(numarr[j]), 'numpapers']+=1
audf.to_csv('Authors.csv')
linksdf=pd.DataFrame()
aulinksdf=pd.DataFrame()
aulinksdf=aulinksdf.reindex(columns=['source','target','mag'])
needtograb=[]
for row in df.itertuples():
    #If this paper did not cite other papers, move on
    if pd.isnull(row.RId):
        continue
#     For each cited paper, get paper ID
    for ref in row.RId.split(','):
        ref=int(ref)
        #If cited paper is in existing database of papers, add the link to the list of links
        if ref in df.id.values:
            linksdf=linksdf.append({'source': str(row.id), 'target':str(ref)}, ignore_index=True)

            #For each author of the citing paper, connect to each paper they cited
            for src in row.aanums:
                #Print all authors of the paper they cited
                trgarr=df.loc[df['id']==int(ref)]['aanums'].values[0]
                for trg in trgarr:
                    if (len(aulinksdf.loc[(aulinksdf['source']==int(src))&(aulinksdf['target']==int(trg))])>0):
                        aulinksdf.loc[(aulinksdf['source']==int(src))&(aulinksdf['target']==int(trg)), 'mag']+=1
                    else:
                        aulinksdf=aulinksdf.append({'source':int(src), 'target':int(trg), 'mag':1},ignore_index=True)
        else:
            needtograb.append(ref)
linksdf.to_csv('PaperLinks.csv',index=False)
aulinksdf.to_csv('AuthorLinks.csv',index=False)
#dont use
audf=audf.loc[audf['citations']>20]
audf.to_csv('Authors.csv',index=False)
#Get top journals
df['jnames']=df['J'].apply(get_names)
df['jnums']=df['J'].apply(get_nums)

jdf=pd.DataFrame()
jdf=jdf.reindex(columns=['name','num','citations','numpapers'])
for i in df.index:
    numarr=df['jnums'][i]
    namarr=df['jnames'][i]
    for j in range(0, len(numarr)):
#         print(numarr[j])
        if int(numarr[j]) not in jdf.num.values:
            jdf=jdf.append({'name':namarr[j], 'num':int(numarr[j]), 'citations':int(df['cc'][i]), 'numpapers':1}, ignore_index=True)
        else:
            jdf.loc[jdf['num']==int(numarr[j]), 'citations']+=int(df['cc'][i])
            jdf.loc[jdf['num']==int(numarr[j]), 'numpapers']+=1
print(jdf)
jdf.to_csv('Journals.csv')
#Papers needed
papersdf=pd.read_csv("papersneeded.csv")
papersdf['ID'].value_counts().to_csv('papersneededcounts.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20401982378854625, 'fraction_numerical': 0.004405286343612335, 'mean_word_length': 3.9903846153846154, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '395438', 'license': 'MIT', 'n_tokens_mistral': 1389, 'n_tokens_neox': 1346, 'n_words': 229, 'path': 'src/.ipynb_checkpoints/DataProcessing-checkpoint.ipynb', 'repo_name': 'anlandu/insurgent-artifacts', 'size': '3632'}