<commit_before># This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.

#####
#   Test if all procedures,s functionName parameter works correctly
#   <NAME>, 22 sept 2015
#   Copyright Datacratic 2015
####

import json, random, datetime

## Create toy dataset
dataset_config = {
    'type'    : 'sparse.mutable',
    'id'      : "toy"
}


dataset = mldb.create_dataset(dataset_config)
now = datetime.datetime.now()

for i in xrange(50):
    label = random.random() < 0.2
    dataset.record_row("u%d" % i, [["feat1", random.gauss(5 if label else 15, 3), now],
                                   ["feat2", random.gauss(-5 if label else 10, 10), now],
                                   ["feat3", random.gauss(52 if label else 30, 40), now],
                                   ["label", label, now]])

dataset.commit()


def doChecks(conf):
    mldb.log(">> Checking " + conf["type"])
    rez = mldb.perform("PUT", "/v1/procedures/" + conf["type"], [], conf)
    mldb.log(rez)

    rez = mldb.perform("POST", "/v1/procedures/"+conf["type"]+"/runs")
    mldb.log(rez)
    assert rez["statusCode"] == 201

    rez = mldb.perform("GET", "/v1/functions/"+conf["params"]["functionName"])
    mldb.log(rez)
    assert rez["statusCode"] == 200



# classifier.train -> classifier
conf = {
    "type": "classifier.train",
    "params": {
        "trainingData": "select {* EXCLUDING(label)} as features, label from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya.cls",
        "algorithm": "glz",
        "mode": "boolean",
        "configuration": {
            "glz": {
                "type": "glz",
                "verbosity": 3,
                "normalize": False,
                "link": "linear",
                "ridge_regression": True
            }
        },
        "functionName": "cls_func"
    }
}
doChecks(conf)

# kmeans.train -> kmeans
conf = {
    "type": "kmeans.train",
    "params": {
        "trainingData": "select * excluding(label) from toy",
        "centroidsDataset": {"id": "kmean_out", "type": "sparse.mutable" },
        "functionName": "kmeans_func"
    }
}
doChecks(conf)


# probabilizer.train -> probabilizer
conf = {
    "type": "probabilizer.train",
    "params": {
        "trainingData": "select cls_func({{* EXCLUDING(label)} as features})[score] as score, label from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-proba.json",
        "functionName": "probabilizer_func"
    }
}
doChecks(conf)


# svd.train -> svd.embedRow
conf = {
    "type": "svd.train",
    "params": {
        "trainingData": "select * from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-svd.model",
        "functionName": "svd_func"
    }
}
doChecks(conf)



# tsne.train -> tsne.embedRow
conf = {
    "type": "tsne.train",
    "params": {
        "trainingData": "select * from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-tsne.model",
        "numOutputDimensions": 2,
        "functionName": "tsne_func"
    }
}
doChecks(conf)


mldb.script.set_return("success")

<commit_msg>[MLDB-1226] forgot to check-in one file<commit_after># This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.

#####
#   Test if all procedures,s functionName parameter works correctly
#   <NAME>, 22 sept 2015
#   Copyright Datacratic 2015
####

import json, random, datetime

## Create toy dataset
dataset_config = {
    'type'    : 'sparse.mutable',
    'id'      : "toy"
}


dataset = mldb.create_dataset(dataset_config)
now = datetime.datetime.now()

for i in xrange(50):
    label = random.random() < 0.2
    dataset.record_row("u%d" % i, [["feat1", random.gauss(5 if label else 15, 3), now],
                                   ["feat2", random.gauss(-5 if label else 10, 10), now],
                                   ["feat3", random.gauss(52 if label else 30, 40), now],
                                   ["label", label, now]])

dataset.commit()


def doChecks(conf):
    mldb.log(">> Checking " + conf["type"])
    rez = mldb.perform("PUT", "/v1/procedures/" + conf["type"], [], conf)
    mldb.log(rez)

    rez = mldb.perform("POST", "/v1/procedures/"+conf["type"]+"/runs")
    mldb.log(rez)
    assert rez["statusCode"] == 201

    rez = mldb.perform("GET", "/v1/functions/"+conf["params"]["functionName"])
    mldb.log(rez)
    assert rez["statusCode"] == 200



# classifier.train -> classifier
conf = {
    "type": "classifier.train",
    "params": {
        "trainingData": "select {* EXCLUDING(label)} as features, label from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya.cls",
        "algorithm": "glz",
        "mode": "boolean",
        "configuration": {
            "glz": {
                "type": "glz",
                "verbosity": 3,
                "normalize": False,
                "link": "linear",
                "ridge_regression": True
            }
        },
        "functionName": "cls_func"
    }
}
doChecks(conf)

# kmeans.train -> kmeans
conf = {
    "type": "kmeans.train",
    "params": {
        "trainingData": "select * excluding(label) from toy",
        "modelFileUrl": "file://tmp/MLDB-926.mks",
        "centroidsDataset": {"id": "kmean_out", "type": "sparse.mutable" },
        "functionName": "kmeans_func"
    }
}
doChecks(conf)


# probabilizer.train -> probabilizer
conf = {
    "type": "probabilizer.train",
    "params": {
        "trainingData": "select cls_func({{* EXCLUDING(label)} as features})[score] as score, label from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-proba.json",
        "functionName": "probabilizer_func"
    }
}
doChecks(conf)


# svd.train -> svd.embedRow
conf = {
    "type": "svd.train",
    "params": {
        "trainingData": "select * from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-svd.model",
        "functionName": "svd_func"
    }
}
doChecks(conf)



# tsne.train -> tsne.embedRow
conf = {
    "type": "tsne.train",
    "params": {
        "trainingData": "select * from toy",
        "modelFileUrl": "file://build/x86_64/tmp/bouya-tsne.model",
        "numOutputDimensions": 2,
        "functionName": "tsne_func"
    }
}
doChecks(conf)


mldb.script.set_return("success")


{'fraction_non_alphanumeric': 0.17217475415121716, 'fraction_numerical': 0.02144123811059165, 'mean_word_length': 2.4678591391839015, 'pattern_counts': {'":': 75, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5553963', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2106, 'n_words': 437}