<commit_before>import datetime
import json
import os
from string import Template

import numpy as np
import tensorflow as tf
from sqlalchemy.orm import sessionmaker
from sqlalchemy import or_
from tensorflow.contrib import learn
from daphne_API import data_helpers

import daphne_API.historian.models as models
import daphne_API.data_extractors as extractors
import daphne_API.data_processors as processors
import daphne_API.runnable_functions as run_func
from daphne_API.errors import ParameterMissingError


def classify(question, module_name):
    cleaned_question = data_helpers.clean_str(question)

    # Map data into vocabulary
    vocab_path = os.path.join("./daphne_API/models/" + module_name + "/vocab")
    vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
    x_test = np.array(list(vocab_processor.transform([cleaned_question])))

    print("\nEvaluating...\n")

    # Evaluation
    # ==================================================
    checkpoint_file = tf.train.latest_checkpoint("./daphne_API/models/" + module_name + "/")
    graph = tf.Graph()
    with graph.as_default():
        session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
        sess = tf.Session(config=session_conf)
        with sess.as_default():
            # Load the saved meta graph and restore variables
            saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
            saver.restore(sess, checkpoint_file)

            # Get the placeholders from the graph by name
            input_x = graph.get_operation_by_name("input_x").outputs[0]
            # input_y = graph.get_operation_by_name("input_y").outputs[0]
            dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]

            # Tensors we want to evaluate
            logits = graph.get_operation_by_name("output/logits").outputs[0]

            # get the prediction
            result_logits = sess.run(logits, {input_x: x_test, dropout_keep_prob: 1.0})
            prediction = data_helpers.get_label_using_logits(result_logits, top_number=1)

    named_labels = set()
    for filename in os.listdir("./daphne_API/command_types/" + module_name):
        specific_label = int(filename.split('.', 1)[0])
        named_labels.add(specific_label)
    return list(named_labels)[prediction[0][0]]


def load_type_info(question_type, module_name):
    with open('./daphne_API/command_types/' + module_name + '/' + str(question_type) + '.json', 'r') as file:
        type_info = json.load(file)
    information = []
    information.append(type_info["params"])
    if type_info["type"] == "db_query":
        information.append(type_info["query"])
    elif type_info["type"] == "run_function":
        information.append(type_info["function"])
    information.append(type_info["voice_response"])
    information.append(type_info["visual_response"])
    return information


extract_function = {}
extract_function["mission"] = extractors.extract_mission
extract_function["measurement"] = extractors.extract_measurement
extract_function["technology"] = extractors.extract_technology
extract_function["year"] = extractors.extract_date
extract_function["design_id"] = extractors.extract_design_id
extract_function["agent"] = extractors.extract_agent
extract_function["instrument_parameter"] = extractors.extract_instrument_parameter
extract_function["vassar_instrument"] = extractors.extract_vassar_instrument
extract_function["vassar_measurement"] = extractors.extract_vassar_measurement
extract_function["vassar_stakeholder"] = extractors.extract_vassar_stakeholder


process_function = {}
process_function["mission"] = processors.process_mission
process_function["measurement"] = processors.not_processed
process_function["technology"] = processors.not_processed
process_function["year"] = processors.process_date
process_function["design_id"] = processors.not_processed
process_function["agent"] = processors.not_processed
process_function["instrument_parameter"] = processors.not_processed
process_function["vassar_instrument"] = processors.not_processed
process_function["vassar_measurement"] = processors.not_processed
process_function["vassar_stakeholder"] = processors.not_processed


def extract_data(processed_question, params, context):
    """ Extract the features from the processed question, with a correcting factor """
    number_of_features = {}
    extracted_raw_data = {}
    extracted_data = {}
    # Count how many non-context params of each type are needed
    for param in params:
        if not param["from_context"]:
            if param["type"] in number_of_features:
                number_of_features[param["type"]] += 1
            else:
                number_of_features[param["type"]] = 1
    # Try to extract the required number of parameters
    for type, num in number_of_features.items():
        extracted_raw_data[type] = extract_function[type](processed_question, num, context)
    # For each parameter check if it's needed and apply postprocessing;
    for param in params:
        extracted_param = None
        if param["from_context"]:
            if param["name"] in context:
                extracted_param = context[param["name"]]
            elif param["mandatory"]:
                raise ParameterMissingError(param["type"])
        else:
            if len(extracted_raw_data[param["type"]]) > 0:
                extracted_param = extracted_raw_data[param["type"]].pop(0)
            elif param["mandatory"]:
                # If param is needed but not detected return error with type of parameter
                raise ParameterMissingError(param["type"])
        if extracted_param is not None:
            extracted_data[param["name"]] = process_function[param["type"]](extracted_param, param["options"], context)
    return extracted_data


def augment_data(data, context):
    data['now'] = datetime.datetime.utcnow()
    data['designs'] = context['data']<commit_msg>fixed second bug related to the same problem<commit_after>import datetime
import json
import os
from string import Template

import numpy as np
import tensorflow as tf
from sqlalchemy.orm import sessionmaker
from sqlalchemy import or_
from tensorflow.contrib import learn
from daphne_API import data_helpers

import daphne_API.historian.models as models
import daphne_API.data_extractors as extractors
import daphne_API.data_processors as processors
import daphne_API.runnable_functions as run_func
from daphne_API.errors import ParameterMissingError


def classify(question, module_name):
    cleaned_question = data_helpers.clean_str(question)

    # Map data into vocabulary
    vocab_path = os.path.join("./daphne_API/models/" + module_name + "/vocab")
    vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
    x_test = np.array(list(vocab_processor.transform([cleaned_question])))

    print("\nEvaluating...\n")

    # Evaluation
    # ==================================================
    checkpoint_file = tf.train.latest_checkpoint("./daphne_API/models/" + module_name + "/")
    graph = tf.Graph()
    with graph.as_default():
        session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
        sess = tf.Session(config=session_conf)
        with sess.as_default():
            # Load the saved meta graph and restore variables
            saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
            saver.restore(sess, checkpoint_file)

            # Get the placeholders from the graph by name
            input_x = graph.get_operation_by_name("input_x").outputs[0]
            # input_y = graph.get_operation_by_name("input_y").outputs[0]
            dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]

            # Tensors we want to evaluate
            logits = graph.get_operation_by_name("output/logits").outputs[0]

            # get the prediction
            result_logits = sess.run(logits, {input_x: x_test, dropout_keep_prob: 1.0})
            prediction = data_helpers.get_label_using_logits(result_logits, top_number=1)

    named_labels = set()
    for filename in os.listdir("./daphne_API/command_types/" + module_name):
        specific_label = int(filename.split('.', 1)[0])
        named_labels.add(specific_label)
    return list(named_labels)[prediction[0][0]]


def load_type_info(question_type, module_name):
    with open('./daphne_API/command_types/' + module_name + '/' + str(question_type) + '.json', 'r') as file:
        type_info = json.load(file)
    information = []
    information.append(type_info["params"])
    if type_info["type"] == "db_query":
        information.append(type_info["query"])
    elif type_info["type"] == "run_function":
        information.append(type_info["function"])
    information.append(type_info["voice_response"])
    information.append(type_info["visual_response"])
    return information


extract_function = {}
extract_function["mission"] = extractors.extract_mission
extract_function["measurement"] = extractors.extract_measurement
extract_function["technology"] = extractors.extract_technology
extract_function["year"] = extractors.extract_date
extract_function["design_id"] = extractors.extract_design_id
extract_function["agent"] = extractors.extract_agent
extract_function["instrument_parameter"] = extractors.extract_instrument_parameter
extract_function["vassar_instrument"] = extractors.extract_vassar_instrument
extract_function["vassar_measurement"] = extractors.extract_vassar_measurement
extract_function["vassar_stakeholder"] = extractors.extract_vassar_stakeholder


process_function = {}
process_function["mission"] = processors.process_mission
process_function["measurement"] = processors.not_processed
process_function["technology"] = processors.not_processed
process_function["year"] = processors.process_date
process_function["design_id"] = processors.not_processed
process_function["agent"] = processors.not_processed
process_function["instrument_parameter"] = processors.not_processed
process_function["vassar_instrument"] = processors.not_processed
process_function["vassar_measurement"] = processors.not_processed
process_function["vassar_stakeholder"] = processors.not_processed


def extract_data(processed_question, params, context):
    """ Extract the features from the processed question, with a correcting factor """
    number_of_features = {}
    extracted_raw_data = {}
    extracted_data = {}
    # Count how many non-context params of each type are needed
    for param in params:
        if not param["from_context"]:
            if param["type"] in number_of_features:
                number_of_features[param["type"]] += 1
            else:
                number_of_features[param["type"]] = 1
    # Try to extract the required number of parameters
    for type, num in number_of_features.items():
        extracted_raw_data[type] = extract_function[type](processed_question, num, context)
    # For each parameter check if it's needed and apply postprocessing;
    for param in params:
        extracted_param = None
        if param["from_context"]:
            if param["name"] in context:
                extracted_param = context[param["name"]]
            elif param["mandatory"]:
                raise ParameterMissingError(param["type"])
        else:
            if len(extracted_raw_data[param["type"]]) > 0:
                extracted_param = extracted_raw_data[param["type"]].pop(0)
            elif param["mandatory"]:
                # If param is needed but not detected return error with type of parameter
                raise ParameterMissingError(param["type"])
        if extracted_param is not None:
            extracted_data[param["name"]] = process_function[param["type"]](extracted_param, param["options"], context)
    return extracted_data


def augment_data(data, context):
    data['now'] = datetime.datetime.utcnow()

    if 'data' in context:
        data['designs'] = context['data']
{'fraction_non_alphanumeric': 0.10931005818786367, 'fraction_numerical': 0.0024937655860349127, 'mean_word_length': 4.2104807275877, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4088408', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3366, 'n_words': 698}