<jupyter_start><jupyter_text>import proc_anotated as preimport jsonimport sys<jupyter_code>#import nltk
#nltk.download('punkt')
f_in = sys.argv[1]
f_out = f_in + ".tmp"

json_data = []
with open(f_in, 'r') as f:
    json_data = json.load(f)
tmp_json = json_data
tmp_json = pre.clean_up(tmp_json)
tmp_json = pre.construct_text(tmp_json)
len(tmp_json)
for d in tmp_json:
    text = d['text'].lower()
    text = pre.rem_stop_words(text) 
    text = pre.stem_text(text) 
    d['w_tokens'] = text
output.show()
with open(f_out, 'w') as f:
    json.dump(tmp_json, f, indent=4)
!jupyter nbconvert --to script /home/admin/text_processing/preprocess/Preprocessing.ipynb
!pwd
!ls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1402002861230329, 'fraction_numerical': 0.002861230329041488, 'mean_word_length': 5.086956521739131, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '664991', 'license': 'MIT', 'n_tokens_mistral': 294, 'n_tokens_neox': 290, 'n_words': 56, 'path': 'preprocess/Preprocessing.ipynb', 'repo_name': 'gusman/text_processing', 'size': '699'}