<jupyter_start><jupyter_text>Parallel Cleaning<jupyter_code>doclist = ['29256483.txt', '29264862.txt', '29266161.txt', '29267827.txt',
       '29267830.txt', '29267831.txt', '29267969.txt', '29268006.txt',
       '29268018.txt', '29268510.txt', '29269193.txt', '29269358.txt',
       '29270079.txt', '29270906.txt', '29272259.txt', '29273771.txt',
       '29273774.txt', '29273776.txt', '29274137.txt', '29274138.txt',
       '29274139.txt', '29274140.txt', '29274141.txt', '29274142.txt',
       '29274143.txt', '29274144.txt', '29274145.txt', '29274146.txt',
       '29274147.txt', '29274148.txt', '29274149.txt', '29274150.txt',
       '29274151.txt', '29274152.txt', '29274153.txt', '29274154.txt',
       '29274155.txt', '29274156.txt', '29274157.txt', '29274158.txt',
       '29274159.txt', '29274160.txt', '29274163.txt', '29274164.txt',
       '29274165.txt', '29274166.txt', '29274168.txt', '29274172.txt',
       '29274173.txt', '29274174.txt', '29274175.txt', '29274176.txt',
       '29274457.txt', '29274464.txt', '29274465.txt', '29274466.txt',
       '29274467.txt', '29274471.txt', '29274472.txt', '29274473.txt',
       '29274474.txt', '29274475.txt', '29274476.txt', '29274617.txt',
       '29274618.txt', '29274619.txt', '29274620.txt', '29274623.txt',
       '29274626.txt', '29274670.txt', '29275158.txt', '29275166.txt',
       '29275168.txt', '29275169.txt', '29275238.txt', '29275239.txt',
       '29275243.txt', '29275699.txt', '29275704.txt', '29275705.txt',
       '29275706.txt', '29275707.txt', '29275708.txt', '29275709.txt',
       '29275710.txt', '29275711.txt', '29275712.txt', '29275713.txt',
       '29275714.txt', '29275715.txt', '29276101.txt', '29276389.txt',
       '29276484.txt', '29276485.txt', '29276487.txt', '29276494.txt',
       '29276709.txt', '29276711.txt', '29276715.txt', '29277019.txt']
cleaner_spacy.set_input_folder('../../data/scraper/eap_files/')
cleaner_spacy.set_output_folder('../../data/scraper/cleaned_files/')
cleaned_docs = cleaner_spacy.batch_clean_docs(doclist, batch_size=10, skip_existing=False)
cleaned_docs = cleaner_spacy.batch_clean_docs(doclist, batch_size=10, skip_existing=False)
cleaned_docs = cleaner_spacy.batch_clean_docs(doclist, batch_size=10, skip_existing=False)
e = spacy.load('en')
e2 = spacy.load('en' , disable=['parser', 'ner', 'textcat']) # 'tagger'])

doc = [token.lemma_ for token in e(txt.lower())]
doc2 = [token.lemma_ for token in e2(txt.lower())]

print(len(doc), len(doc2))

q = pd.DataFrame(([(i, j, i == j) for i, j in zip(doc, doc2)]))
print(q[2].sum() == len(doc2), q[2].sum() == len(doc))
import time

class TestCacheParallel:
    cacher = {}
    
    def funcx(self, c):
        self.cacher[c] = c*2
        time.sleep(2)
        print(self.cacher)
        
    def parallel_test(self, n):
        procs = []
        for i in range(n):
            p = multiprocessing.Process(target=self.funcx, args=(i,))
            p.start()
            p.join()
#             procs.append(p)
            
#         for t in procs:
#             t.join()

tcp = TestCacheParallel()
tcp.funcx(200)
tcp.parallel_test(2)<jupyter_output>{100: 200, 200: 400, 0: 0}
{100: 200, 200: 400, 1: 2}
time: 4.05 s<jupyter_text>Spacy Server<jupyter_code>%%time
res = requests.post('http://0.0.0.0:1029/api/spacy/lemmatize', params={'raw_text': txt})
len(txt)
len(res.json())
res.content[-100:]
len([token.lemma_ for token in doc])
export XDG_RUNTIME_DIR=""
jupyter_lab_id=`pgrep jupyter-lab`
jupyter_lab_exit_status=$?
if [ $jupyter_lab_exit_status -eq 0 ]; then
    echo "jupyter-lab is running. killing pid $jupyter_lab_id ..."
    kill -15 $jupyter_lab_id
    echo "jupyter-lab killed ..."
fi
echo "running jupyter-lab ..."
source /home/wb536061/anaconda3/bin/activate
cd /R/NLP/
jupyter lab --ip="*" --port=1029 --NotebookApp.open_browser=False --LabApp.log_datefmt='%Y-%m-%d %H:%M:%S' >> jupyter.log 2>&1  & disown
echo "jupyter-lab now running detached. see jupyter.log for logs"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19337848006019565, 'fraction_numerical': 0.22021570102834212, 'mean_word_length': 3.822249093107618, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '596968', 'license': 'MIT', 'n_tokens_mistral': 2181, 'n_tokens_neox': 1670, 'n_words': 265, 'path': 'notebooks/archive/SCRIPTS/profiling/Cleaner Performance.ipynb', 'repo_name': 'worldbank/wb-nlp-apps', 'size': '3987'}