<jupyter_start><jupyter_text>> Using stopwords we can extract summary and text separately.. but only half of them? is it worth a work?<jupyter_code>master_file = []

for i in range(len(pdf_files)):

    parsed_file = parser.from_file(pdf_files[i])
    extracted_text = parsed_file["content"].strip()
    
    newinput ={}
    
    newinput["title"] = bik_df['Title'][i]
    newinput['warc_date'] = '20220328'
    newinput["text"]= extracted_text #+ ' [CLS]'
    newinput["inst_index"] = str(random.randint(9999,100000)) #radom 5 figure number
    newinput["domain"] = bik_df['Citation'][i]
    newinput["url"] = bik_df['URL'][i]
    newinput["summary"] = "" #Blank until we figure out the abstract stuff
    newinput['authors'] = [bik_df['Authors'][i]]
    newinput['publish_date'] =  '03-28-' + str(bik_df['Year'][i])  #Needs actual published date in form of 'M-D-2022' but try with today + publushed year
    newinput["split"] = "gen"
    newinput["status"] = "succcess"
    
    master_file.append(newinput)

with open('Bik_text.jsonl','w', encoding='utf-8') as f:
    for item in master_file:
        f.write(json.dumps(item) + "\n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16281062553556128, 'fraction_numerical': 0.023993144815766924, 'mean_word_length': 4.3577981651376145, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '564969', 'license': 'Apache-2.0', 'n_tokens_mistral': 434, 'n_tokens_neox': 414, 'n_words': 90, 'path': 'text_extraction/grover_preparation_test.ipynb', 'repo_name': 'alexdseo/Falsified-Scientific-Literature-Generation', 'size': '1167'}