<commit_before>import pywikibot
import sys
import requests
from app import get_proposed_edits, app

def prefill_cache(max_pages=5000, starting_page=None):
    site = pywikibot.Site()
    cs1 = pywikibot.Page(site, 'Module:Citation/CS1')
    count = 0
    starting_page_seen = starting_page is None
    for p in cs1.embeddedin(namespaces=[0]):
        print(p.title().encode('utf-8'))
        if p.title() == starting_page:
           starting_page_seen = True
           continue
        if count >= max_pages:
            break
        if not starting_page_seen:
            continue
        try:
            get_proposed_edits(p.title(), False, True)
            count += 1
        except:
            print("ERROR: Something went wrong with this page!")

if __name__ == '__main__':
    if len(sys.argv) >= 2:
        prefill_cache(1000000, sys.argv[1]) 
    else:
        prefill_cache(1000000) <commit_msg>Add multithreading with threading module, run until the end<commit_after>import pywikibot
import sys
import requests
from app import get_proposed_edits, app
import threading
from time import sleep

def worker(title=None):
    try:
        get_proposed_edits(title, False, True)
    except:
        pass

def prefill_cache(max_pages=5000, starting_page=None):
    site = pywikibot.Site()
    cs1 = pywikibot.Page(site, 'Module:Citation/CS1')
    count = 0
    starting_page_seen = starting_page is None
    for p in cs1.embeddedin(namespaces=[0]):
        print(p.title().encode('utf-8'))
        if p.title() == starting_page:
           starting_page_seen = True
           continue
        if count >= max_pages:
            break
        if not starting_page_seen:
            continue
        try:
            threading.Thread(target=worker, args=[p.title()]).start()
        except:
            sleep(60)
        count += 1
        sleep(0.1)

if __name__ == '__main__':
    if len(sys.argv) >= 2:
        prefill_cache(5000000, sys.argv[1])
    else:
        prefill_cache(5000000)
{'fraction_non_alphanumeric': 0.09964947421131698, 'fraction_numerical': 0.02904356534802203, 'mean_word_length': 2.4507772020725387, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454475', 'n_tokens_mistral': 663, 'n_tokens_neox': 621, 'n_words': 152}