<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gevent import monkey
monkey.patch_all()

from os import sys, path
import schedule
from time import sleep

from bottle.ext.mongo import MongoPlugin

sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
from bin.mining import run
from utils import conf, log_it


log_it("START", "bin-scheduler")


def job(slug):
    log_it("START JOB: {}".format(slug), "bin-scheduler")
    run(slug)
    log_it("END JOB: {}".format(slug), "bin-scheduler")


def rules(cube, scheduler_type=None, scheduler_interval=None):
    if scheduler_type:
        scheduler_type = cube.get('scheduler_type', 'minutes')
    if scheduler_interval:
        scheduler_interval = cube.get('scheduler_interval', 60)

    log_it("START REGISTER", "bin-scheduler")
    log_it("cube: {}".format(cube.get('slug')), "bin-scheduler")
    log_it("type: {}".format(scheduler_type), "bin-scheduler")
    log_it("interval: {}".format(scheduler_interval), "bin-scheduler")
    log_it("END REGISTER", "bin-scheduler")

    t = {}
    if scheduler_type == 'minutes':
        t = schedule.every(int(scheduler_interval)).minutes
    elif scheduler_type == 'hour':
        t = schedule.every().hour
    elif scheduler_type == 'day':
        t = schedule.every().day

    try:
        t.do(job, slug=cube.get('slug'))
    except Exception, e:
        log_it("ERROR {}: {}".format(cube.get('slug'), e))


mongo = MongoPlugin(
    uri=conf("mongodb")["uri"],
    db=conf("mongodb")["db"],
    json_mongo=True).get_mongo()

register = []
for cube in mongo['cube'].find({'scheduler_status': True}):
    rules(cube)
    register.append(cube['slug'])

for dashboard in mongo['dashboard'].find({'scheduler_status': True}):
    elements = [e['id'] for e in dashboard['element']]
    for e in elements:
        element = mongo['element'].find_one({'slug': e})
        cube = mongo['cube'].find_one({'slug': element['cube']})
        rules(cube, dashboard['scheduler_type'],
              dashboard['scheduler_interval'])
        register.append(cube['slug'])

while True:
    for cube in mongo['cube'].find({'scheduler_status': True}):
        if cube['slug'] not in register:<commit_msg>set default type/interval on scheduler bin, ref #117<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gevent import monkey
monkey.patch_all()

from os import sys, path
import schedule
from time import sleep

from bottle.ext.mongo import MongoPlugin

sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
from bin.mining import run
from utils import conf, log_it


log_it("START", "bin-scheduler")


def job(slug):
    log_it("START JOB: {}".format(slug), "bin-scheduler")
    run(slug)
    log_it("END JOB: {}".format(slug), "bin-scheduler")


def rules(cube, scheduler_type='minutes', scheduler_interval=59):
    if scheduler_type:
        scheduler_type = cube.get('scheduler_type', 'minutes')
    if scheduler_interval:
        scheduler_interval = cube.get('scheduler_interval', 59)

    log_it("START REGISTER", "bin-scheduler")
    log_it("cube: {}".format(cube.get('slug')), "bin-scheduler")
    log_it("type: {}".format(scheduler_type), "bin-scheduler")
    log_it("interval: {}".format(scheduler_interval), "bin-scheduler")
    log_it("END REGISTER", "bin-scheduler")

    t = {}
    if scheduler_type == 'minutes':
        t = schedule.every(int(scheduler_interval)).minutes
    elif scheduler_type == 'hour':
        t = schedule.every().hour
    elif scheduler_type == 'day':
        t = schedule.every().day

    try:
        t.do(job, slug=cube.get('slug'))
    except Exception, e:
        log_it("ERROR {}: {}".format(cube.get('slug'), e))

    return True


mongo = MongoPlugin(
    uri=conf("mongodb")["uri"],
    db=conf("mongodb")["db"],
    json_mongo=True).get_mongo()

register = []
for cube in mongo['cube'].find({'scheduler_status': True}):
    rules(cube)
    register.append(cube['slug'])

for dashboard in mongo['dashboard'].find({'scheduler_status': True}):
    elements = [e['id'] for e in dashboard['element']]
    for e in elements:
        element = mongo['element'].find_one({'slug': e})
        cube = mongo['cube'].find_one({'slug': element['cube']})
        rules(cube, dashboard['scheduler_type'],
              dashboard['scheduler_interval'])
        register.append(cube['slug'])

while True:
    for cube in mongo['cube'].find({'scheduler_status': True}):
        if cube['slug'] not in register:
{'fraction_non_alphanumeric': 0.1700558659217877, 'fraction_numerical': 0.002458100558659218, 'mean_word_length': 3.9513274336283186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '59894', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1506, 'n_words': 310}