<commit_before>"""
This file collects common code for every app. It is general across all apps and
makes a call to the specified app in the appropriate place.

Apps are specified with a YAML file with a specific structure. A given instance
of an app is verified before creation.
"""

# TODO: include docstrings (copy and paste from PoolBasedTripletsMDS.py)
import os
import sys
import numpy
import numpy.random
import json
import yaml
import traceback
import next.utils as utils
import next.apps.Verifier as Verifier
import next.constants
import next.apps.Butler as Butler

Butler = Butler.Butler
git_hash = next.constants.GIT_HASH


#TODO: App Exception logs
#TODO: move __import__ to importlib
class App(object):
    def __init__(self, app_id, exp_uid, db, ell):
        self.app_id = app_id
        self.exp_uid = exp_uid
        self.helper = Helper()
        self.myApp = __import__('next.apps.Apps.'+self.app_id, fromlist=[''])
        self.myApp = getattr(self.myApp, app_id)
        self.myApp = self.myApp(db)
        self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
        dir,_ = os.path.split(__file__)
        with open(os.path.join(dir, "Apps/{}/{}.yaml".format(app_id, app_id)),'r') as f:
            self.reference_dict = yaml.load(f)
        dashboard_string = 'next.apps.Apps.' + self.app_id + \
                           '.dashboard.Dashboard'
        dashboard_module = __import__(dashboard_string, fromlist=[''])
        self.dashboard = getattr(dashboard_module, app_id+'Dashboard')


    def initExp(self, exp_uid, args_json):
        try:
            self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
            args_dict = self.helper.convert_json(args_json)
            args_dict = Verifier.verify(args_dict, self.reference_dict['initExp']['values'])
            args_dict['exp_uid'] = exp_uid # to get doc from db
            args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
            self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())}) 
            args_dict,algs_args_dict = self.myApp.initExp(exp_uid, args_dict, self.butler)
            # Set doc in algorithms bucket. These objects are used by the algorithms to store data.
            for algorithm in args_dict['args']['alg_list']:
                algorithm['exp_uid'] = exp_uid
                self.butler.algorithms.set(uid=algorithm['alg_label'], value=algorithm)
            args_dict['git_hash'] = git_hash
            self.butler.experiment.set(value=args_dict)
            for algorithm in args_dict['args']['alg_list']:
                params = algorithm.get('params',None)
                butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id'])
                alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])

                # utils.debug_print("App.py:66, algs_args_dict = {}".format(algs_args_dict))
                # I got rid of a timeit function here; it wasn't handling the
                # argument unpacking correctly? --Scott, 2016-3-7
                # TODO: put dt back in and change log_entry to relfect that
                alg.initExp(butler,params=params, **algs_args_dict)
                if not True:
                    raise Exception('Algorithm {} failed to initialize.'.format(algorithm['alg_label']))

                log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()}
                self.butler.log('ALG-DURATION', log_entry)
            return '{}', True, ''
        except Exception, error:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            full_error = str(traceback.format_exc())+'\n'+str(error)
            print "initExp Exception: " + full_error<commit_msg>exception raised if alg.initExp returns False<commit_after>"""
This file collects common code for every app. It is general across all apps and
makes a call to the specified app in the appropriate place.

Apps are specified with a YAML file with a specific structure. A given instance
of an app is verified before creation.
"""

# TODO: include docstrings (copy and paste from PoolBasedTripletsMDS.py)
import os
import sys
import numpy
import numpy.random
import json
import yaml
import traceback
import next.utils as utils
import next.apps.Verifier as Verifier
import next.constants
import next.apps.Butler as Butler

Butler = Butler.Butler
git_hash = next.constants.GIT_HASH


#TODO: App Exception logs
#TODO: move __import__ to importlib
class App(object):
    def __init__(self, app_id, exp_uid, db, ell):
        self.app_id = app_id
        self.exp_uid = exp_uid
        self.helper = Helper()
        self.myApp = __import__('next.apps.Apps.'+self.app_id, fromlist=[''])
        self.myApp = getattr(self.myApp, app_id)
        self.myApp = self.myApp(db)
        self.butler = Butler(self.app_id, self.exp_uid, self.myApp.TargetManager, db, ell)
        dir,_ = os.path.split(__file__)
        with open(os.path.join(dir, "Apps/{}/{}.yaml".format(app_id, app_id)),'r') as f:
            self.reference_dict = yaml.load(f)
        dashboard_string = 'next.apps.Apps.' + self.app_id + \
                           '.dashboard.Dashboard'
        dashboard_module = __import__(dashboard_string, fromlist=[''])
        self.dashboard = getattr(dashboard_module, app_id+'Dashboard')


    def initExp(self, exp_uid, args_json):
        try:
            self.helper.ensure_indices(self.app_id,self.butler.db, self.butler.ell)
            args_dict = self.helper.convert_json(args_json)
            args_dict = Verifier.verify(args_dict, self.reference_dict['initExp']['values'])
            args_dict['exp_uid'] = exp_uid # to get doc from db
            args_dict['start_date'] = utils.datetime2str(utils.datetimeNow())
            self.butler.admin.set(uid=exp_uid,value={'exp_uid': exp_uid, 'app_id':self.app_id, 'start_date':str(utils.datetimeNow())}) 
            args_dict,algs_args_dict = self.myApp.initExp(exp_uid, args_dict, self.butler)
            # Set doc in algorithms bucket. These objects are used by the algorithms to store data.
            for algorithm in args_dict['args']['alg_list']:
                algorithm['exp_uid'] = exp_uid
                self.butler.algorithms.set(uid=algorithm['alg_label'], value=algorithm)
            args_dict['git_hash'] = git_hash
            self.butler.experiment.set(value=args_dict)
            for algorithm in args_dict['args']['alg_list']:
                params = algorithm.get('params',None)
                butler = Butler(self.app_id, exp_uid, self.myApp.TargetManager, self.butler.db, self.butler.ell, algorithm['alg_label'], algorithm['alg_id'])
                alg = utils.get_app_alg(self.app_id, algorithm['alg_id'])

                # utils.debug_print("App.py:66, algs_args_dict = {}".format(algs_args_dict))
                # I got rid of a timeit function here; it wasn't handling the
                # argument unpacking correctly? --Scott, 2016-3-7
                # TODO: put dt back in and change log_entry to relfect that
                alg_succeed = alg.initExp(butler, params=params, **algs_args_dict)
                if not alg_succeed:
                    raise Exception('Algorithm {} failed to initialize.'.format(algorithm['alg_label']))

                log_entry = {'exp_uid':exp_uid, 'alg_label':algorithm['alg_label'], 'task':'initExp', 'duration':-1, 'timestamp':utils.datetimeNow()}
                self.butler.log('ALG-DURATION', log_entry)
            return '{}', True, ''
        except Exception, error:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            full_error = str(traceback.format_exc())+'\n'+str(error)
            print "initExp Exception: " + full_error
{'fraction_non_alphanumeric': 0.11629667003027246, 'fraction_numerical': 0.0025227043390514633, 'mean_word_length': 2.8678048780487804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3049536', 'n_tokens_mistral': 2451, 'n_tokens_neox': 2408, 'n_words': 568}