<commit_before>import flask
from multiprocessing import Process, Queue
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from stf.spiders.juris import JurisSpider


class CrawlerWorker(Process):
    def __init__(self, flt, results):
        Process.__init__(self)
        self.flt = flt
        self.process = CrawlerProcess(get_project_settings())
        self.results = results

    def run(self):
        self.process.crawl(JurisSpider, flt=self.flt)
        self.process.start()
        self.results.put('\n'.join(self.process.spider_loader._spiders['juris'].output))


def crawl(flt):
    results = Queue()
    worker = CrawlerWorker(flt, results)
    worker.start()
    return results.get()


app = flask.Flask(__name__)


@app.route('/', methods=['GET'])
def index():
    return flask.render_template('index.html')


@app.route('/flt/', methods=['GET'])
def get_doutrinas():
    flt = flask.request.args.get('filter')
    response = flask.make_response(crawl(flt))
    response.headers['Content-Disposition'] = 'attachment; filename=doutrinas.txt'
    return response


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--debug', action='store_true', dest='debug_mode',
                        default=False)
    args = parser.parse_args()
    app_options = {
            'debug': args.debug_mode,
            'use_debugger': not args.debug_mode,
            'use_reloader': not args.debug_mode,
        } if args.debug_mode else {}

    app.run(**app_options)<commit_msg>Fetch asynchronously instead of blocking response<commit_after>import flask
from multiprocessing import Process, Queue
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from stf.spiders.juris import JurisSpider


class CrawlerWorker(Process):
    def __init__(self, flt):
        Process.__init__(self)
        self.flt = flt
        self.process = CrawlerProcess(get_project_settings())

    def run(self):
        self.process.crawl(JurisSpider, flt=self.flt)
        self.process.start()
        with open('static/%s.txt' % (self.flt,), 'w') as fp:
            fp.writelines(self.process.spider_loader._spiders['juris'].output)


def crawl(flt):
    worker = CrawlerWorker(flt)
    worker.start()


app = flask.Flask(__name__)


@app.route('/', methods=['GET'])
def index():
    return flask.render_template('index.html')


@app.route('/flt/', methods=['GET'])
def get_doutrinas():
    flt = flask.request.args.get('filter')
    crawl(flt)
    return flask.redirect('/')


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--debug', action='store_true', dest='debug_mode',
                        default=False)
    args = parser.parse_args()
    app_options = {
            'debug': args.debug_mode,
            'use_debugger': not args.debug_mode,
            'use_reloader': not args.debug_mode,
        } if args.debug_mode else {}

    app.run(**app_options)
{'fraction_non_alphanumeric': 0.13114220631304913, 'fraction_numerical': 0.0, 'mean_word_length': 3.263522884882108, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7066149', 'n_tokens_mistral': 1008, 'n_tokens_neox': 965, 'n_words': 190}