<jupyter_start><jupyter_text>Define Classes to Be Aggregated<jupyter_code>material_classes = ['ceramic','polymer','semiconductor','metal','organometallic']
experiment_classes = ['experiment','simulation']
material_base = ["material"]
all_classes = material_classes + experiment_classes
class_combinations = list(product(material_classes, experiment_classes, material_base))
class_combinations<jupyter_output><empty_output><jupyter_text>Define Helper Functions<jupyter_code>def clean_results(r):
    return r.get('summary').replace('\n', ' ')

def get_arxiv_results(query, start, max_results):
    res = arxiv.query(' AND '.join(c), 
                      start = start, 
                      max_results = max_results)
    return [clean_results(r) for r in res]<jupyter_output><empty_output><jupyter_text>Fetch Results from ArXiv<jupyter_code>total_results = 100
max_results = 20
query_wait = 0.1

results = []

for c in all_classes:
    print(c)
    n_results = 0
    tmp_array = []
    while n_results < total_results:
        r = get_arxiv_results(' AND '.join(c), 
                          start = n_results, 
                          max_results = max_results)
        if len(r) == 0: break
        n_results = n_results + len(r)
        tmp_array = tmp_array + r
        time.sleep(query_wait)
    results.append({"label":c, "data":tmp_array})<jupyter_output>('ceramic', 'experiment', 'material')
('ceramic', 'simulation', 'material')
('polymer', 'experiment', 'material')
('polymer', 'simulation', 'material')
('semiconductor', 'experiment', 'material')
('semiconductor', 'simulation', 'material')
('metal', 'experiment', 'material')
('metal', 'simulation', 'material')
('organometallic', 'experiment', 'material')
('organometallic', 'simulation', 'material')<jupyter_text>Build DataFrame and Output to File<jupyter_code>output_file = "./data/arxiv_results.pkl"
df = pd.DataFrame(results)

# One-hot encoded columns
for c in all_classes:
    df[c] = df['label'].apply(lambda x: 1 if c in x else 0)
df.drop('labels', axis=0)

df.to_pickle(output_file)
df<jupyter_output><empty_output><jupyter_text>Add One-hot Encoded Columns<jupyter_code>df<jupyter_output><empty_output><jupyter_text>Load Previous Results<jupyter_code>input_file = "./data/arxiv_results.pkl"
df = pd.read_pickle(input_file)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14016288041148736, 'fraction_numerical': 0.005143591941705958, 'mean_word_length': 5.015463917525773, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '491370', 'license': 'Apache-2.0', 'n_tokens_mistral': 815, 'n_tokens_neox': 770, 'n_words': 149, 'path': 'arxiv_pull.ipynb', 'repo_name': 'blaiszik/arxiv_pull', 'size': '2333'}