<commit_before># -*- coding: utf-8 -*-

import flask
from flask import Flask
from flask import abort
app = Flask(__name__)

import luigi
import os
from bcube_demo_pipeline import MainWorkflow
from dlib.task_helpers import clear_directory
from dlib.call_solr import pull_from_solr


@app.route('/pipe')
def trigger_pipeline():
    '''
    run the pipeline
    '''
    doc_dir = 'bcube_demo/docs'

    pull_from_solr(doc_dir)
    task = MainWorkflow(doc_dir=doc_dir, yaml_file='configs/bcube_demo.yaml')
    luigi.interface.setup_interface_logging()
    sch = luigi.scheduler.CentralPlannerScheduler()
    w = luigi.worker.Worker(scheduler=sch)
    w.add(task)
    w.run()

    return ''<commit_msg>add a fake log generator<commit_after># -*- coding: utf-8 -*-

import flask
from flask import Flask
from flask import Response
app = Flask(__name__)

import luigi
import os
from bcube_demo_pipeline import MainWorkflow
from dlib.task_helpers import clear_directory
from dlib.call_solr import pull_from_solr

import sys
import StringIO


@app.route('/pipe')
def trigger_pipeline():
    '''
    run the pipeline
    '''
    doc_dir = 'bcube_demo/docs'

    std = sys.stderr
    sys.stderr = pipeline_output = StringIO.StringIO()

    pull_from_solr(doc_dir)
    task = MainWorkflow(doc_dir=doc_dir, yaml_file='configs/bcube_demo.yaml')
    luigi.interface.setup_interface_logging()
    sch = luigi.scheduler.CentralPlannerScheduler()
    w = luigi.worker.Worker(scheduler=sch)
    w.add(task)
    w.run()

    piped = pipeline_output.getvalue()
    sys.stderr = std

    # fake the generator
    def generate():
        for pipe in piped.split('\n'):
            yield pipe + '\n'

    return Response(generate(), mimetype='text/plain')
{'fraction_non_alphanumeric': 0.10404624277456648, 'fraction_numerical': 0.0011560693641618498, 'mean_word_length': 3.768595041322314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1243630', 'n_tokens_mistral': 582, 'n_tokens_neox': 568, 'n_words': 133}