<commit_before>#!/usr/bin/env python3

import config
from utils.safe_schedule import SafeScheduler

import subprocess
import atexit
import sys
from time import sleep
from glob import glob


scheduler = SafeScheduler()


def make_exit_graceful():
    original_hook = sys.excepthook

    def new_hook(type, value, traceback):
        if type == KeyboardInterrupt:
            sys.exit("\nBye for now!")
        else:
            original_hook(type, value, traceback)

    sys.excepthook = new_hook


def start_server():
    cmd = 'waitress-serve --port=%s server:app' % config.SERVER_PORT
    p = subprocess.Popen(cmd.split(), cwd='server')
    return p


def load_scrapers():
    for scraper_path in glob('scrapers/*.py'):
        with open(scraper_path) as f:
            print(scraper_path)
            scraper_data = f.read()
            exec(scraper_data)


def main():
    make_exit_graceful()
    server = start_server()
    atexit.register(server.terminate)
    load_scrapers()
    scheduler.run_all()
    while True:
        scheduler.run_pending()
        sleep(1)


if __name__ == '__main__':
    main()
<commit_msg>Add docstrings to narcissa.py<commit_after>#!/usr/bin/env python3

import config
from utils.safe_schedule import SafeScheduler

import subprocess
import atexit
import sys
from time import sleep
from glob import glob


scheduler = SafeScheduler()


def make_exit_graceful():
    """
    Register a hook to display a nice message instead of a traceback on Ctrl-C.
    """
    original_hook = sys.excepthook

    def new_hook(type, value, traceback):
        if type == KeyboardInterrupt:
            sys.exit("\nBye for now!")
        else:
            original_hook(type, value, traceback)

    sys.excepthook = new_hook


def start_server():
    """Spin up the Narcissa query server using Waitress."""
    cmd = 'waitress-serve --port=%s server:app' % config.SERVER_PORT
    p = subprocess.Popen(cmd.split(), cwd='server')
    return p


def load_scrapers():
    """Load all scraper files from scrapers/*.py and exec() their contents."""
    for scraper_path in glob('scrapers/*.py'):
        with open(scraper_path) as f:
            print(scraper_path)
            scraper_data = f.read()
            exec(scraper_data)


def main():
    """
    Start the Narcissa query server, schedule all scrapers, and run all
    scrapers for the first time.
    """
    make_exit_graceful()
    server = start_server()
    atexit.register(server.terminate)
    load_scrapers()
    scheduler.run_all()
    while True:
        scheduler.run_pending()
        sleep(1)


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.10259344012204424, 'fraction_numerical': 0.0015255530129672007, 'mean_word_length': 2.868731563421829, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2036225', 'n_tokens_mistral': 857, 'n_tokens_neox': 834, 'n_words': 223}