<jupyter_start><jupyter_text>Join arxiv papers and  segmented tables<jupyter_code>arxiv_papers = read_arxiv_papers(ARXIV_PAPERS_URL)
print(f'Number of papers:           {len(arxiv_papers):8}')
print(f'└── with LaTeX source:      {(~arxiv_papers.status.isin(["no-tex", "withdrawn"])).sum():8}')
print(f'Number of extracted tables: {arxiv_papers.tables.sum():8}')
arxiv_papers.head()
arxiv_papers.shape
arxiv_papers.dtypes
#change dtype of arxiv_id to string
arxiv_papers['arxiv_id'] = arxiv_papers.arxiv_id.apply(str)
#segmented_tables
segmented_tables_annotations.head()
segmented_tables_annotations.dtypes
#change dtype of segmented tables to string
segmented_tables_annotations['arxiv_id'] = segmented_tables_annotations.arxiv_id.apply(str)
#merge arxiv_papers and seg tables
arxiv_papers_seg_tables = arxiv_papers.merge(segmented_tables_annotations, how='inner',on='arxiv_id')
#arxiv ids in segmented_tables
seg_arxiv_ids = set(segmented_tables_annotations['arxiv_id'].values)
#arxiv ids in arxiv_papers
arxiv_arxiv_ids = set(arxiv_papers['arxiv_id'].values)
#checking for uncommon arxiv_ids
seg_ids_not_in_arxiv_ids = [id for id in seg_arxiv_ids if id not in arxiv_arxiv_ids]
len(seg_ids_not_in_arxiv_ids)

seg_ids_not_in_arxiv_ids

#removing "v1","v2" etc from end of arxiv_id in both arxiv_papers and segmented_tables
arxiv_papers['arxiv_id'] = arxiv_papers['arxiv_id'].apply(lambda x: x[:-2])
segmented_tables_annotations['arxiv_id'] = segmented_tables_annotations['arxiv_id'].apply(lambda x : x[:-2])
arxiv_papers_seg_tables = arxiv_papers.merge(segmented_tables_annotations, how='inner',on='arxiv_id')
arxiv_papers_seg_tables.shape
arxiv_papers_seg_tables.head()
#create records column
def return_records(row):
  return_records = []
  for table in row['tables_y']:
    table_records = table['records']
    if(len(table_records)>0):
      for record in table_records:
        return_records.append(record)
  return return_records

arxiv_papers_seg_tables['records'] = arxiv_papers_seg_tables.apply(return_records,axis=1)

arxiv_papers_seg_tables.head()
arxiv_papers_seg_tables_with_atleast_one_record = arxiv_papers_seg_tables[arxiv_papers_seg_tables.records.map(len) > 0]
arxiv_papers_seg_tables_with_atleast_one_record

#expand records into each row
arxiv_papers_seg_with_records_expanded  = pd.DataFrame()
for index,row in arxiv_papers_seg_tables_with_atleast_one_record.iterrows():
  temp_df = pd.DataFrame(columns = ['arxiv_id','title','fold','record_id','record'])
  for index,record in enumerate(row['records']):
    # print(record)
    # print(record['task'])
    df = pd.Series({'arxiv_id': row['arxiv_id'], 'title': row['title'],'fold':row['fold'],'record_id':index+1,'record':record})
    # print(df)
    temp_df = temp_df.append(df,ignore_index=True)
  arxiv_papers_seg_with_records_expanded = arxiv_papers_seg_with_records_expanded.append(temp_df,ignore_index=True)

arxiv_papers_seg_with_records_expanded

arxiv_papers_seg_with_records_expanded['fold'].value_counts()

arxiv_papers_seg_with_records_expanded_final = pd.concat([arxiv_papers_seg_with_records_expanded.drop(['record'], axis=1), arxiv_papers_seg_with_records_expanded['record'].apply(pd.Series)], axis=1)
arxiv_papers_seg_with_records_expanded_final.head()
arxiv_papers_seg_with_records_expanded_final['arxiv_id'] = arxiv_papers_seg_with_records_expanded_final['arxiv_id'].astype('str')
def extract_year(row):
  return '20' + row['arxiv_id'][0:2]
df = arxiv_papers_seg_with_records_expanded_final
df['year'] = df.apply(lambda row: extract_year(row), axis=1)
def extract_month(row):
  return row['arxiv_id'][2:4]
df['month'] = df.apply(lambda row: extract_month(row), axis=1)
df
df['year'] = df['year'].astype(str).astype(int)
df['month'] = df['month'].astype(str).astype(int)
#df['value'] = df['value'].astype(float, errors = 'raise')
df['value'] = pd.to_numeric(df['value'],errors = 'coerce')
#df['value'] = df['value'].astype(str).astype(float)
df.dtypes
dir_path
#arxiv_papers_seg_with_records_expanded_final.to_csv(dir_path + 'arxiv_papers_seg_with_records_expanded_final.csv',index=False)
df.to_csv(dir_path + 'df_final_value_float.csv',index=False)
df_year = df['year'].value_counts().rename_axis('year').reset_index(name='counts')
df_year



pwc_leaderboards['arxiv_id'] = pwc_leaderboards.arxiv_id.apply(str)
# 

arxiv_papers_pwc_leaderboards.shape
arxiv_papers_pwc_leaderboards.tables_y.isna().sum()
arxiv_papers_pwc_leaderboards_segmented_tables = arxiv_papers_pwc_leaderboards.merge(segmented_tables_annotations, how='left',on='arxiv_id')
arxiv_papers_pwc_leaderboards_segmented_tables.head()
arxiv_papers_pwc_leaderboards_segmented_tables['fold'].isna().sum()

arxiv_papers_pwc_leaderboards_segmented_tables.fold.value_counts()
arxiv_papers_pwc_leaderboards_segmented_tables_filtered = arxiv_papers_pwc_leaderboards_segmented_tables[arxiv_papers_pwc_leaderboards_segmented_tables['fold'].notnull() | arxiv_papers_pwc_leaderboards_segmented_tables['tables_y'].notnull()]
arxiv_papers_pwc_leaderboards_segmented_tables_filtered.shape
arxiv_papers_pwc_leaderboards_segmented_tables_filtered.head()
arxiv_papers_pwc_leaderboards_segmented_tables_filtered.fold.value_counts()
arxiv_papers_pwc_leaderboards_segmented_tables_filtered.isna().sum()
arxiv_papers_pwc_leaderboards_segmented_tables_filtered['tables'].isna().sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12048643592142189, 'fraction_numerical': 0.0039289055191768, 'mean_word_length': 10.749450549450549, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '586753', 'license': '0BSD', 'n_tokens_mistral': 2130, 'n_tokens_neox': 1856, 'n_words': 236, 'path': 'preprocess_dataset_shao.ipynb', 'repo_name': 'GODZAOZAO/Literature-Visualization-Tool', 'size': '5345'}