<commit_before>from __future__ import absolute_import
from .celery_broker import app
import celery.signals
import os
import sys
import time
import json
import yaml
import traceback
import numpy

# import next.logging_client.LoggerHTTP as ell
from next.database_client.DatabaseAPI import DatabaseAPI
db = DatabaseAPI()
from next.logging_client.LoggerAPI import LoggerAPI
ell = LoggerAPI()
import next.utils
import next.constants
import next.apps.Butler as Butler
import next.apps.Verifier as Verifier

Butler = Butler.Butler

# Main application task
def apply(app_id, exp_uid, task_name, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	# modify args_in
	if task_name == 'processAnswer':
		args_in_dict = json.loads(args_in_json)
		args_in_dict['args']['timestamp_answer_received'] = enqueue_timestamp
		args_in_json = json.dumps(args_in_dict)
	# get stateless app
	next_app = next.utils.get_app(app_id, exp_uid, db, ell)
	# pass it to a method
	method = getattr(next_app, task_name)
	response,dt = next.utils.timeit(method)(exp_uid, args_in_json)
        args_out_json,didSucceed,message = response
        args_out_dict = json.loads(args_out_json)
	if 'args' in args_out_dict:
		return_value = (json.dumps(args_out_dict['args']),didSucceed,message)
		meta = args_out_dict.get('meta',{})

		if 'log_entry_durations' in meta:
			log_entry_durations = meta['log_entry_durations']
			log_entry_durations['app_duration'] = dt
			log_entry_durations['duration_enqueued'] = time_enqueued
			log_entry_durations['timestamp'] = next.utils.datetimeNow()
			ell.log( app_id+':ALG-DURATION', log_entry_durations  )
	else:
		return_value = (args_out_json,didSucceed,message)
	print '#### Finished %s,  time_enqueued=%s,  execution_time=%s ####' % (task_name,time_enqueued,dt)
	return return_value

def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.
        next.utils.debug_print(args_in_json)
        with open(os.path.join('next/apps', 'Apps/{}/{}.yaml'.format(app_id, app_id)),'r') as f:
                reference_dict = yaml.load(f)        
        args_dict = Verifier.verify(args_in_json, reference_dict['getStats']['values'])
        stat_id = args_dict['args'].pop('stat_id',None)
        # myApp
        app = next.utils.get_app(app_id, exp_uid, db, ell)
        butler = Butler(app_id, exp_uid, app.myApp.TargetManager, db, ell)
	# pass it to a method
        dashboard_string = 'next.apps.Apps.' + app_id + '.dashboard.Dashboard'
        dashboard_module = __import__(dashboard_string, fromlist=[''])
        dashboard = getattr(dashboard_module, app_id+'Dashboard')<commit_msg>yaml extension loading fixed in broker<commit_after>from __future__ import absolute_import
from .celery_broker import app
import celery.signals
import os
import sys
import time
import json
import yaml
import traceback
import numpy

# import next.logging_client.LoggerHTTP as ell
from next.database_client.DatabaseAPI import DatabaseAPI
db = DatabaseAPI()
from next.logging_client.LoggerAPI import LoggerAPI
ell = LoggerAPI()
import next.utils
import next.constants
import next.apps.Butler as Butler
import next.apps.Verifier as Verifier

Butler = Butler.Butler

# Main application task
def apply(app_id, exp_uid, task_name, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.

	# modify args_in
	if task_name == 'processAnswer':
		args_in_dict = json.loads(args_in_json)
		args_in_dict['args']['timestamp_answer_received'] = enqueue_timestamp
		args_in_json = json.dumps(args_in_dict)
	# get stateless app
	next_app = next.utils.get_app(app_id, exp_uid, db, ell)
	# pass it to a method
	method = getattr(next_app, task_name)
	response,dt = next.utils.timeit(method)(exp_uid, args_in_json)
        args_out_json,didSucceed,message = response
        args_out_dict = json.loads(args_out_json)
	if 'args' in args_out_dict:
		return_value = (json.dumps(args_out_dict['args']),didSucceed,message)
		meta = args_out_dict.get('meta',{})

		if 'log_entry_durations' in meta:
			log_entry_durations = meta['log_entry_durations']
			log_entry_durations['app_duration'] = dt
			log_entry_durations['duration_enqueued'] = time_enqueued
			log_entry_durations['timestamp'] = next.utils.datetimeNow()
			ell.log( app_id+':ALG-DURATION', log_entry_durations  )
	else:
		return_value = (args_out_json,didSucceed,message)
	print '#### Finished %s,  time_enqueued=%s,  execution_time=%s ####' % (task_name,time_enqueued,dt)
	return return_value

def apply_dashboard(app_id, exp_uid, args_in_json, enqueue_timestamp):
	enqueue_datetime = next.utils.str2datetime(enqueue_timestamp)
	dequeue_datetime = next.utils.datetimeNow()
	delta_datetime = dequeue_datetime - enqueue_datetime
	time_enqueued = delta_datetime.seconds + delta_datetime.microseconds/1000000.
        next.utils.debug_print(args_in_json)
        dir, _ = os.path.split(__file__)
        reference_dict = Verifier.load_doc(os.path.join('next/apps', 'Apps/{}/{}.yaml'.format(app_id, app_id)))
        args_dict = Verifier.verify(args_in_json, reference_dict['getStats']['values'])
        stat_id = args_dict['args'].pop('stat_id',None)
        # myApp
        app = next.utils.get_app(app_id, exp_uid, db, ell)
        butler = Butler(app_id, exp_uid, app.myApp.TargetManager, db, ell)
	# pass it to a method
        dashboard_string = 'next.apps.Apps.' + app_id + '.dashboard.Dashboard'
        dashboard_module = __import__(dashboard_string, fromlist=[''])
        dashboard = getattr(dashboard_module, app_id+'Dashboard')
{'fraction_non_alphanumeric': 0.10488715700600747, 'fraction_numerical': 0.005195648644260432, 'mean_word_length': 5.298568507157464, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222236', 'n_tokens_mistral': 2131, 'n_tokens_neox': 2080, 'n_words': 396}