<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import json

import beanstalkc
from flask import Blueprint, current_app, jsonify, g, request

import config


api = Blueprint('api', __name__)


@api.before_request
def before_request():
    g.beanstalk = beanstalkc.Connection(
        host=config.BEANSTALKD_HOST, port=config.BEANSTALKD_PORT)


@api.route('/push', methods=['POST'])
def push_jobs():
    jobs = request.json
    for job in jobs:
        if job['app_name'] not in config.APPS:
            ret = dict(
                error='unknown_app_name',
                detail='Unknown app name %s' % job['app_name'])
            return jsonify(ret), 400

    if len(jobs) < 5:
        for job in jobs:
            if job['app_name'] not in config.APPS:
                continue
            g.beanstalk.use(config.PUSH_TUBE % job['app_name'])
            priority = config.PRIORITIES.get(job.get('priority', 'low'))
            delay = job.get('delay', 0)
            g.beanstalk.put(json.dumps(job), priority=priority, delay=delay)
    else:
        g.beanstalk.use(config.BATCH_PUSH_TUBE)
        g.beanstalk.put(json.dumps(jobs))
    current_app.logger.info(jobs)
    return jsonify(dict())


@api.route('/push_stats', methods=['GET'])
def push_stats():
    ret = g.beanstalk.stats()
    ret['tubes'] = []
    for app_name in config.APPS.keys():
        ret['tubes'].append(
            g.beanstalk.stats_tube(config.PUSH_TUBE % app_name))
    return jsonify(ret)<commit_msg>add CommandFailed exception in push_stats<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import json

import beanstalkc
from flask import Blueprint, current_app, jsonify, g, request

import config


api = Blueprint('api', __name__)


@api.before_request
def before_request():
    g.beanstalk = beanstalkc.Connection(
        host=config.BEANSTALKD_HOST, port=config.BEANSTALKD_PORT)


@api.route('/push', methods=['POST'])
def push_jobs():
    jobs = request.json
    for job in jobs:
        if job['app_name'] not in config.APPS:
            ret = dict(
                error='unknown_app_name',
                detail='Unknown app name %s' % job['app_name'])
            return jsonify(ret), 400

    if len(jobs) < 5:
        for job in jobs:
            if job['app_name'] not in config.APPS:
                continue
            g.beanstalk.use(config.PUSH_TUBE % job['app_name'])
            priority = config.PRIORITIES.get(job.get('priority', 'low'))
            delay = job.get('delay', 0)
            g.beanstalk.put(json.dumps(job), priority=priority, delay=delay)
    else:
        g.beanstalk.use(config.BATCH_PUSH_TUBE)
        g.beanstalk.put(json.dumps(jobs))
    current_app.logger.info(jobs)
    return jsonify(dict())


@api.route('/push_stats', methods=['GET'])
def push_stats():
    ret = g.beanstalk.stats()
    ret['tubes'] = []
    for app_name in config.APPS.keys():
        try:
            ret['tubes'].append(
                g.beanstalk.stats_tube(config.PUSH_TUBE % app_name))
        except beanstalkc.CommandFailed:
            continue
    return jsonify(ret)
{'fraction_non_alphanumeric': 0.13482056256062075, 'fraction_numerical': 0.0038797284190106693, 'mean_word_length': 2.589327146171694, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1937612', 'n_tokens_mistral': 1069, 'n_tokens_neox': 993, 'n_words': 205}