<commit_before>"""
Generic service method calls
Usage example from narrative UI:
In method 'bindRunButton' of 'src/notebook/ipython_profiles/profile_narrative/kbase_templates/static/kbase/js/widgets/kbaseNarrativeWorkspace.js'
instead of line:
                    //self.runCell()(cell, method.service, method.title, paramList);
you can use:
                    var methodSpec = {parameters: [{id: "param0", field_type : "text"}, {id : "param1", field_type : "text"}], 
                            behavior: {kb_service_url: "https://kbase.us/services/trees", 
                                kb_service_name: "KBaseTrees", kb_service_method: "construct_tree_for_alignment", 
                                kb_service_parameters_mapping: {
                                    param0: {target_property: "msa_ref", target_type_transform: "ref"}, 
                                    param1: {target_property: "out_tree_id"}
                                }, kb_service_workspace_name_mapping: {target_property: "out_workspace"}}};
                    self.runCell()(cell, "generic_service", "method_call", [JSON.stringify(methodSpec), JSON.stringify(paramList)]);
This is hardcoded example for calling gene tree building method. For generic case just use method descriptor for methodSpec variable.
"""
__author__ = '<NAME> <<EMAIL>>'
__date__ = '10/16/14'

## Imports

import urllib2
import httplib
import urlparse
import random
import base64
import httplib2
from urllib2 import URLError, HTTPError
from ConfigParser import ConfigParser
import json
import os
import numbers
import uuid
import hashlib
import re
import sys
# Third-party
import IPython.utils.traitlets as trt
from IPython.core.application import Application
from biokbase.njs_mock.Client import NJSMock
from biokbase.NarrativeJobService.Client import NarrativeJobService
from biokbase.workspace.client import Workspace as workspaceService

## Globals

def prepare_generic_method_input(token, workspace, methodSpec, paramValues, input):
    rpcArgs = []
    narrSysProps = {'workspace': workspace, 'token': token}
    parameters = methodSpec['parameters']
    inputMapping = methodSpec['behavior']['kb_service_input_mapping']
    
    for paramPos in range(0, len(parameters)):
        param = parameters[paramPos]
        paramId = param['id']
        paramValue = paramValues[paramPos]
        input[paramId] = paramValue
    
    for mapping in inputMapping:
        paramValue = None
        paramId = None
        if 'input_parameter' in mapping:
            paramId = mapping['input_parameter']
            paramValue = input[paramId]
        elif 'narrative_system_variable' in mapping:
            sysProp = mapping['narrative_system_variable']
            paramValue = narrSysProps[sysProp]
        if 'constant_value' in mapping and not_defined(paramValue):
            paramValue = mapping['constant_value']
        if 'generated_value' in mapping and not_defined(paramValue):
            paramValue = generate_value(mapping['generated_value'])
            if paramId is not None:
                input[paramId] = paramValue
        if paramValue is None:
            raise ValueError("Value is not defined in input mapping: " + json.dumps(mapping))
        build_args(paramValue, mapping, workspace, rpcArgs)
    return rpcArgs

def prepare_generic_method_output(token, workspace, methodSpec, input, output):
    narrSysProps = {'workspace': workspace, 'token': token}
    outArgs = []
    isScript = False
    if 'kb_service_output_mapping' in methodSpec['behavior']:
        outputMapping = methodSpec['behavior']['kb_service_output_mapping']
    elif 'output_mapping' in methodSpec['behavior']:
        outputMapping = methodSpec['behavior']['output_mapping']
    else:
        outputMapping = methodSpec['behavior']['script_output_mapping']
        isScript = True
    for mapping in outputMapping:
        paramValue = None
        if 'input_parameter' in mapping:
            paramId = mapping['input_parameter']
            paramValue = input[paramId]
        elif 'constant_value' in mapping:
            paramValue = mapping['constant_value']
        elif 'narrative_system_variable' in mapping:
            sysProp = mapping['narrative_system_variable']
            paramValue = narrSysProps[sysProp]
        elif (not isScript) and 'service_method_output_path' in mapping:
            paramValue = get_sub_path(output, mapping['service_method_output_path'], 0)
        elif isScript and 'script_output_path' in mapping:
            paramValue = get_sub_path(output, mapping['script_output_path'], 0)
        if paramValue is None:
            raise ValueError("Value is not defined in output mapping [" + json.dumps(mapping) + "], actual output is: " + json.dumps(output))
        build_args(paramValue, mapping, workspace, outArgs)
    if len(outArgs) < 1:
        return {}
    return outArgs[0]

def correct_method_specs_json(method_specs_json):
    method_specs_json = method_specs_json.replace('\n', '\\n')
    method_specs_json = method_specs_json.replace('\t', '\\t')
    return method_specs_json

def _app_get_state(workspace, token, URLS, job_manager, app_spec_json, method_specs_json, param_values_json, app_job_id):
    appSpec = json.loads(app_spec_json)
    paramValues = json.loads(param_values_json)
    methIdToSpec = json.loads(correct_method_specs_json(method_specs_json))
    
    njsClient = NarrativeJobService(URLS.job_service, token = token)
    wsClient = workspaceService(URLS.workspace, token = token)
    if app_job_id.startswith("njs:"):
        app_job_id = app_job_id[4:]
    appState = njsClient.check_app_state(app_job_id)
    appState['widget_outputs'] = {}
    for stepSpec in appSpec['steps']:
        stepId = stepSpec['step_id']
        if not stepId in appState['step_outputs']:
            continue
        rpcOut = appState['step_outputs'][stepId]
        methodId = stepSpec['method_id']
        methodSpec = methIdToSpec[methodId]
        methodOut = None
        if 'kb_service_input_mapping' in methodSpec['behavior'] or 'script_input_mapping' in methodSpec['behavior']:
            input = {}
            tempArgs = []
            methodInputValues = extract_param_values(paramValues, stepId)
            prepare_njs_method_input(token, wsClient, workspace, methodSpec, methodInputValues, input);
            methodOut = prepare_generic_method_output(token, workspace, methodSpec, input, rpcOut)
        else:
            methodOut = rpcOut
        appState['widget_outputs'][stepId] = methodOut
    appState['job_id'] = "njs:" + appState['job_id']
    return appState

def _method_get_state(workspace, token, URLS, job_manager, method_spec_json, param_values_json, method_job_id):
    methodSpec = json.loads(method_spec_json)
    paramValues = json.loads(param_values_json)
    
    njsClient = NarrativeJobService(URLS.job_service, token = token)
    wsClient = workspaceService(URLS.workspace, token = token)
    if mathod_job_id.startswith("method:"):
        method_job_id = method_job_id[7:]
    appState = njsClient.check_app_state(method_job_id)
    for stepId in appState['step_outputs']:
        rpcOut = appState['step_outputs'][stepId]
        methodOut = None
        if 'kb_service_input_mapping' in methodSpec['behavior'] or 'script_input_mapping' in methodSpec['behavior']:
            input = {}
            tempArgs = []
            prepare_njs_method_input(token, wsClient, workspace, methodSpec, paramValues, input);
            methodOut = prepare_generic_method_output(token, workspace, methodSpec, input, rpcOut)
        else:
            methodOut = rpcOut
        appState['widget_output'] = methodOut
    appState['job_id'] = "method:" + appState['job_id']
    return appState

def extract_param_values(paramValues, stepId):
    ret = None
    for paramVal in paramValues:
        if paramVal['stepId'] == stepId:
            ret = []
            for keyVal in paramVal['values']:
                ret.append(keyVal['value'])
    if ret is None:
        raise ValueError("Step [" + stepId + "] wasn't found in input values: " + json.dumps(paramValues))
    return ret

def not_defined(paramValue):
    return paramValue is None or len(str(paramValue).strip()) == 0

def generate_value(generProps):
    symbols = 8
    if 'symbols' in generProps:
        symbols = int(generProps['symbols'])
    ret = ''.join([chr(random.randrange(0, 26) + ord('A')) for _ in xrange(symbols)])
    if 'prefix' in generProps:
        ret = str(generProps['prefix']) + ret;
    if 'suffix' in generProps:
        ret = ret + str(generProps['suffix']);
    return ret
<commit_msg>Fix for JSON returning from NJS for every app step.<commit_after>"""
Generic service method calls
Usage example from narrative UI:
In method 'bindRunButton' of 'src/notebook/ipython_profiles/profile_narrative/kbase_templates/static/kbase/js/widgets/kbaseNarrativeWorkspace.js'
instead of line:
                    //self.runCell()(cell, method.service, method.title, paramList);
you can use:
                    var methodSpec = {parameters: [{id: "param0", field_type : "text"}, {id : "param1", field_type : "text"}], 
                            behavior: {kb_service_url: "https://kbase.us/services/trees", 
                                kb_service_name: "KBaseTrees", kb_service_method: "construct_tree_for_alignment", 
                                kb_service_parameters_mapping: {
                                    param0: {target_property: "msa_ref", target_type_transform: "ref"}, 
                                    param1: {target_property: "out_tree_id"}
                                }, kb_service_workspace_name_mapping: {target_property: "out_workspace"}}};
                    self.runCell()(cell, "generic_service", "method_call", [JSON.stringify(methodSpec), JSON.stringify(paramList)]);
This is hardcoded example for calling gene tree building method. For generic case just use method descriptor for methodSpec variable.
"""
__author__ = '<NAME> <<EMAIL>>'
__date__ = '10/16/14'

## Imports

import urllib2
import httplib
import urlparse
import random
import base64
import httplib2
from urllib2 import URLError, HTTPError
from ConfigParser import ConfigParser
import json
import os
import numbers
import uuid
import hashlib
import re
import sys
# Third-party
import IPython.utils.traitlets as trt
from IPython.core.application import Application
from biokbase.njs_mock.Client import NJSMock
from biokbase.NarrativeJobService.Client import NarrativeJobService
from biokbase.workspace.client import Workspace as workspaceService

## Globals

def prepare_generic_method_input(token, workspace, methodSpec, paramValues, input):
    rpcArgs = []
    narrSysProps = {'workspace': workspace, 'token': token}
    parameters = methodSpec['parameters']
    inputMapping = methodSpec['behavior']['kb_service_input_mapping']
    
    for paramPos in range(0, len(parameters)):
        param = parameters[paramPos]
        paramId = param['id']
        paramValue = paramValues[paramPos]
        input[paramId] = paramValue
    
    for mapping in inputMapping:
        paramValue = None
        paramId = None
        if 'input_parameter' in mapping:
            paramId = mapping['input_parameter']
            paramValue = input[paramId]
        elif 'narrative_system_variable' in mapping:
            sysProp = mapping['narrative_system_variable']
            paramValue = narrSysProps[sysProp]
        if 'constant_value' in mapping and not_defined(paramValue):
            paramValue = mapping['constant_value']
        if 'generated_value' in mapping and not_defined(paramValue):
            paramValue = generate_value(mapping['generated_value'])
            if paramId is not None:
                input[paramId] = paramValue
        if paramValue is None:
            raise ValueError("Value is not defined in input mapping: " + json.dumps(mapping))
        build_args(paramValue, mapping, workspace, rpcArgs)
    return rpcArgs

def prepare_generic_method_output(token, workspace, methodSpec, input, output):
    narrSysProps = {'workspace': workspace, 'token': token}
    outArgs = []
    isScript = False
    if 'kb_service_output_mapping' in methodSpec['behavior']:
        outputMapping = methodSpec['behavior']['kb_service_output_mapping']
    elif 'output_mapping' in methodSpec['behavior']:
        outputMapping = methodSpec['behavior']['output_mapping']
    else:
        outputMapping = methodSpec['behavior']['script_output_mapping']
        isScript = True
    for mapping in outputMapping:
        paramValue = None
        if 'input_parameter' in mapping:
            paramId = mapping['input_parameter']
            paramValue = input[paramId]
        elif 'constant_value' in mapping:
            paramValue = mapping['constant_value']
        elif 'narrative_system_variable' in mapping:
            sysProp = mapping['narrative_system_variable']
            paramValue = narrSysProps[sysProp]
        elif (not isScript) and 'service_method_output_path' in mapping:
            paramValue = get_sub_path(output, mapping['service_method_output_path'], 0)
        elif isScript and 'script_output_path' in mapping:
            paramValue = get_sub_path(output, mapping['script_output_path'], 0)
        if paramValue is None:
            raise ValueError("Value is not defined in output mapping [" + json.dumps(mapping) + "], actual output is: " + json.dumps(output))
        build_args(paramValue, mapping, workspace, outArgs)
    if len(outArgs) < 1:
        return {}
    return outArgs[0]

def correct_method_specs_json(method_specs_json):
    method_specs_json = method_specs_json.replace('\n', '\\n')
    method_specs_json = method_specs_json.replace('\t', '\\t')
    return method_specs_json

def app_state_output_into_method_output(workspace, token, wsClient, methodSpec, methodInputValues, rpcOut):
    methodOut = None
    if 'kb_service_input_mapping' in methodSpec['behavior'] or 'script_input_mapping' in methodSpec['behavior']:
        if 'kb_service_input_mapping' in methodSpec['behavior']:
            rpcOut = json.loads(rpcOut)
        input = {}
        tempArgs = []
        prepare_njs_method_input(token, wsClient, workspace, methodSpec, methodInputValues, input);
        methodOut = prepare_generic_method_output(token, workspace, methodSpec, input, rpcOut)
    else:
        methodOut = rpcOut
    return methodOut

def _app_get_state(workspace, token, URLS, job_manager, app_spec_json, method_specs_json, param_values_json, app_job_id):
    appSpec = json.loads(app_spec_json)
    paramValues = json.loads(param_values_json)
    methIdToSpec = json.loads(correct_method_specs_json(method_specs_json))
    njsClient = NarrativeJobService(URLS.job_service, token = token)
    wsClient = workspaceService(URLS.workspace, token = token)
    if app_job_id.startswith("njs:"):
        app_job_id = app_job_id[4:]
    appState = njsClient.check_app_state(app_job_id)
    appState['widget_outputs'] = {}
    for stepSpec in appSpec['steps']:
        stepId = stepSpec['step_id']
        if not stepId in appState['step_outputs']:
            continue
        rpcOut = appState['step_outputs'][stepId]
        methodId = stepSpec['method_id']
        methodSpec = methIdToSpec[methodId]
        methodInputValues = extract_param_values(paramValues, stepId)
        appState['widget_outputs'][stepId] = app_state_output_into_method_output(workspace, token, wsClient, methodSpec, methodInputValues, rpcOut)
    appState['job_id'] = "njs:" + appState['job_id']
    return appState

def _method_get_state(workspace, token, URLS, job_manager, method_spec_json, param_values_json, method_job_id):
    methodSpec = json.loads(method_spec_json)
    methodInputValues = json.loads(param_values_json)
    njsClient = NarrativeJobService(URLS.job_service, token = token)
    wsClient = workspaceService(URLS.workspace, token = token)
    if mathod_job_id.startswith("method:"):
        method_job_id = method_job_id[7:]
    appState = njsClient.check_app_state(method_job_id)
    for stepId in appState['step_outputs']:
        rpcOut = appState['step_outputs'][stepId]
        appState['widget_output'] = app_state_output_into_method_output(workspace, token, wsClient, methodSpec, methodInputValues, rpcOut)
    appState['job_id'] = "method:" + appState['job_id']
    return appState

def extract_param_values(paramValues, stepId):
    ret = None
    for paramVal in paramValues:
        if paramVal['stepId'] == stepId:
            ret = []
            for keyVal in paramVal['values']:
                ret.append(keyVal['value'])
    if ret is None:
        raise ValueError("Step [" + stepId + "] wasn't found in input values: " + json.dumps(paramValues))
    return ret

def not_defined(paramValue):
    return paramValue is None or len(str(paramValue).strip()) == 0

def generate_value(generProps):
    symbols = 8
    if 'symbols' in generProps:
        symbols = int(generProps['symbols'])
    ret = ''.join([chr(random.randrange(0, 26) + ord('A')) for _ in xrange(symbols)])
    if 'prefix' in generProps:
        ret = str(generProps['prefix']) + ret;
    if 'suffix' in generProps:
        ret = ret + str(generProps['suffix']);
    return ret

{'fraction_non_alphanumeric': 0.09818836372082221, 'fraction_numerical': 0.003135524329346185, 'mean_word_length': 3.363567266278186, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2976993', 'n_tokens_mistral': 5074, 'n_tokens_neox': 5000, 'n_words': 1173}