<commit_before>#
# routes.py
# <NAME>, 2016-02-25
# Copyright (c) 2016 Datacratic Inc. All rights reserved.
#

import json, os, datetime, math
from operator import itemgetter
import functools, urllib
import binascii

mldb2 = mldb_wrapper.wrap(mldb)

EMBEDDING_DATASET = "embedded_images"


def preProcessData():
    rp = mldb.plugin.rest_params.rest_params

    inputIndex = -1
    deploy = False
    dataset = None

    for elem_idx, elem in enumerate(rp):
        if elem[0] == "input":
            inputIndex = elem_idx
        if elem[0] == "deploy" and elem[1] == "true":
            deploy = True
        if elem[0] == "dataset":
            dataset = elem[1]

    if dataset is None:
        return ("Dataset needs to be specified", 400)

    if inputIndex == -1:
        mldb.log(rp)
        return ("Invalid input! (1)", 400)

    data = json.loads(rp[inputIndex][1])
    if "a" not in data or "b" not in data:
        return ("Data dict must contain keys a and b", 400)


    groups = [set(data["a"]), set(data["b"]), set(data["ignore"])]
    for idx, name in enumerate(("a", "b")):
        if len(groups[idx]) == 0:
            return ("Data group '%s' cannot be empty!" % name, 400)

    return data, groups, deploy, dataset



def getPrediction():
    input_data ={}

    for elem in mldb.plugin.rest_params.payload.split("&"):
        k,v = elem.split("=")
        input_data[k] = urllib.unquote(v)

    if "deploy_id" not in input_data or "image_url" not in input_data:
        return ("deploy_id and image_url must be passed to endpoint!", 400)

    mldb.log(input_data)

    score_query_rez = mldb2.query("""
            SELECT explorator_cls_%s(
                            {
                                features: {*}
                            }) as scores
            FROM (
                SELECT inception({url: '%s'}) as *
            )
        """ % (input_data["deploy_id"], input_data["image_url"]))

    mldb.log(score_query_rez)
    score = score_query_rez[1][1]

    dataset = mldb2.query("""
            select score.score from predictions_%s
            where training_labels.label = 0 and training_labels.weight = 1
            order by score.score ASC LIMIT 1
        """ % input_data["deploy_id"])
    min_score_for_pos = dataset[1][1]

    dataset = mldb2.query("""
            select score.score from predictions_%s
            order by score.score DESC LIMIT 1
        """ % input_data["deploy_id"])
    maxScore = dataset[1][1]


    rez = mldb2.query("""
        SELECT * FROM merge(
            (
                select 'A' as class, datasetName
                from predictions_%s
                where training_labels.label = 0 and training_labels.weight = 1
                order by score.score DESC LIMIT 5
            ),
            (
                select 'B' as class, datasetName
                from predictions_%s
                where training_labels.label = 1
                order by score.score ASC LIMIT 5
            )
        )
    """ % (input_data["deploy_id"], input_data["deploy_id"]))

    example_images = {"A": [], "B": []}
    for elem in rez[1:]:
        example_images[elem[1]].append([elem[0], elem[2]])


    return_val = {
            "example_image": example_images,
            "similarity": {
                "score": score,
                "threshold": min_score_for_pos,
                "max": maxScore
            },
            "prediction": "A" if score >= min_score_for_pos else "B"
        }


    return (return_val, 200)





def getSimilar(cls_func_name="explorator_cls"):

    data, groups, doDeploy, datasetName = preProcessData()

    embeddingDataset = EMBEDDING_DATASET + "_" + datasetName

    run_id = str(binascii.hexlify(os.urandom(16)))
    cls_func_name += "_" + run_id

    # keep track of ressources to delete
    to_delete = []

    dataset_config = {
        'type'    : 'sparse.mutable',
        'id'      : "training_labels_" + run_id
    }

    to_delete.append("/v1/datasets/" + dataset_config["id"])
    dataset = mldb.create_dataset(dataset_config)
    now = datetime.datetime.now()


    already_added = set()

    to_add = []
    for lbl, imgs in enumerate(groups):
        for img in imgs:
            if img in already_added:
                continue
            dataset.record_row(img, [["label", lbl, now], ["weight", 1, now]])
            already_added.add(img)

            # if it's the positive group, look at nearest neighbhours to add extra
            # labels in positive class in case we don't have enough
            if lbl == 0:
                # will return list of ["slid","596e1ca6687cd301",1.9799363613128662]
                neighbours = mldb2.get("/v1/datasets/%s/routes/rowNeighbours" % embeddingDataset,
                                    row=img)

                for nName, nId, nDist in neighbours.json():
                    to_add.append((nName, [["label", lbl, now], ["weight", 0.25, now]]))

    # add the positive nearest neighbours if they were't added as
    # explicit negative examples
    for row, cols in to_add:
        if row in already_added: continue
        dataset.record_row(row, cols)
        already_added.add(row)

    # now add all remaining examples as low weight negative examples
    query = "SELECT rowName() FROM %s WHERE NOT (rowName() IN ('%s'))" % (embeddingDataset, "','".join(list(already_added)))
    for line in mldb2.query(query)[1:]:
        dataset.record_row(line[0], [["label", 1, now], ["weight", 0.001, now]])
        already_added.add(line[0])

    dataset.commit()
<commit_msg>Adjusted treshold for RT predictor and do not use all unclassified images as negatives when deploying a model<commit_after>#
# routes.py
# <NAME>, 2016-02-25
# Copyright (c) 2016 Datacratic Inc. All rights reserved.
#

import json, os, datetime, math
from operator import itemgetter
import functools, urllib
import binascii

mldb2 = mldb_wrapper.wrap(mldb)

EMBEDDING_DATASET = "embedded_images"


def preProcessData():
    rp = mldb.plugin.rest_params.rest_params

    inputIndex = -1
    deploy = False
    dataset = None

    for elem_idx, elem in enumerate(rp):
        if elem[0] == "input":
            inputIndex = elem_idx
        if elem[0] == "deploy" and elem[1] == "true":
            deploy = True
        if elem[0] == "dataset":
            dataset = elem[1]

    if dataset is None:
        return ("Dataset needs to be specified", 400)

    if inputIndex == -1:
        mldb.log(rp)
        return ("Invalid input! (1)", 400)

    data = json.loads(rp[inputIndex][1])
    if "a" not in data or "b" not in data:
        return ("Data dict must contain keys a and b", 400)


    groups = [set(data["a"]), set(data["b"]), set(data["ignore"])]
    for idx, name in enumerate(("a", "b")):
        if len(groups[idx]) == 0:
            return ("Data group '%s' cannot be empty!" % name, 400)

    return data, groups, deploy, dataset



def getPrediction():
    input_data ={}

    for elem in mldb.plugin.rest_params.payload.split("&"):
        k,v = elem.split("=")
        input_data[k] = urllib.unquote(v)

    if "deploy_id" not in input_data or "image_url" not in input_data:
        return ("deploy_id and image_url must be passed to endpoint!", 400)

    mldb.log(input_data)

    score_query_rez = mldb2.query("""
            SELECT explorator_cls_%s(
                            {
                                features: {*}
                            }) as scores
            FROM (
                SELECT inception({url: '%s'}) as *
            )
        """ % (input_data["deploy_id"], input_data["image_url"]))

    mldb.log(score_query_rez)
    score = score_query_rez[1][1]

    dataset = mldb2.query("""
            select score.score from predictions_%s
            where training_labels.label = 0 and training_labels.weight = 1
            order by score.score ASC LIMIT 1
        """ % input_data["deploy_id"])
    min_score_for_pos = dataset[1][1]

    dataset = mldb2.query("""
            select score.score from predictions_%s
            order by score.score DESC LIMIT 1
        """ % input_data["deploy_id"])
    maxScore = dataset[1][1]


    rez = mldb2.query("""
        SELECT * FROM merge(
            (
                select 'A' as class, datasetName
                from predictions_%s
                where training_labels.label = 0 and training_labels.weight = 1
                order by score.score DESC LIMIT 5
            ),
            (
                select 'B' as class, datasetName
                from predictions_%s
                where training_labels.label = 1
                order by score.score ASC LIMIT 5
            )
        )
    """ % (input_data["deploy_id"], input_data["deploy_id"]))

    example_images = {"A": [], "B": []}
    for elem in rez[1:]:
        example_images[elem[1]].append([elem[0], elem[2]])


    return_val = {
            "example_image": example_images,
            "similarity": {
                "score": score,
                "threshold": min_score_for_pos,
                "max": maxScore
            },
            "prediction": "A" if score >= min_score_for_pos * 0.9 else "B"
        }


    return (return_val, 200)





def getSimilar(cls_func_name="explorator_cls"):

    data, groups, doDeploy, datasetName = preProcessData()

    embeddingDataset = EMBEDDING_DATASET + "_" + datasetName

    run_id = str(binascii.hexlify(os.urandom(16)))
    cls_func_name += "_" + run_id

    # keep track of ressources to delete
    to_delete = []

    dataset_config = {
        'type'    : 'sparse.mutable',
        'id'      : "training_labels_" + run_id
    }

    to_delete.append("/v1/datasets/" + dataset_config["id"])
    dataset = mldb.create_dataset(dataset_config)
    now = datetime.datetime.now()


    already_added = set()

    to_add = []
    for lbl, imgs in enumerate(groups):
        for img in imgs:
            if img in already_added:
                continue
            dataset.record_row(img, [["label", lbl, now], ["weight", 1, now]])
            already_added.add(img)

            # if it's the positive group, look at nearest neighbhours to add extra
            # labels in positive class in case we don't have enough
            if lbl == 0:
                # will return list of ["slid","596e1ca6687cd301",1.9799363613128662]
                neighbours = mldb2.get("/v1/datasets/%s/routes/rowNeighbours" % embeddingDataset,
                                    row=img)

                for nName, nId, nDist in neighbours.json():
                    to_add.append((nName, [["label", lbl, now], ["weight", 0.25, now]]))

    # add the positive nearest neighbours if they were't added as
    # explicit negative examples
    for row, cols in to_add:
        if row in already_added: continue
        dataset.record_row(row, cols)
        already_added.add(row)

    # now add all remaining examples as low weight negative examples
    if not doDeploy:
        query = "SELECT rowName() FROM %s WHERE NOT (rowName() IN ('%s'))" % (embeddingDataset, "','".join(list(already_added)))
        for line in mldb2.query(query)[1:]:
            dataset.record_row(line[0], [["label", 1, now], ["weight", 0.001, now]])
            already_added.add(line[0])

    dataset.commit()

{'fraction_non_alphanumeric': 0.12019188060762193, 'fraction_numerical': 0.01989872967931065, 'mean_word_length': 2.0911587040087865, 'pattern_counts': {'":': 22, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5118205', 'n_tokens_mistral': 3694, 'n_tokens_neox': 3413, 'n_words': 931}