<commit_before>#
# main.py
# <NAME>, 2015-03-26
# Copyright (c) 2015 Datacratic Inc. All rights reserved.
#

import csv, datetime, json

mldb.log("Pytanic Plugin Executing setup...")


cls_algos = ["glz", "dt", "bbdt"]
# cleanup in case things are already created
print "======= Cleanup"
for cls_algo in cls_algos:
    print mldb.perform("DELETE", "/v1/datasets/titanic-train", [], {})
    print mldb.perform("DELETE", "/v1/datasets/titanic-test", [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_cls_train_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_cls_test_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_prob_train_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/classifyFunction%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/apply_probabilizer%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/probabilizer%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/explainFunction%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/probabilizer%s" % cls_algo, [], {})



# load the train and test datasets
for dataset_type in ["train", "test"]:
    datasetConfig = {
            "type": "sparse.mutable",
            "id": "titanic-"+dataset_type,
        }

    dataset = mldb.create_dataset(datasetConfig)
    def featProc(k, v):
        if k=="Pclass": return "c"+v
        if k=="Cabin": return v[0]
        if k in ["Age", "SibSp", "Parch", "Fare"]: return float(v)
        return v

    ts = datetime.datetime.now()
    filename = mldb.plugin.get_plugin_dir() + "/titanic_%s.csv" % dataset_type
    for idx, csvLine in enumerate(csv.DictReader(open(filename))):
        tuples = [[k,featProc(k,v),ts] for k,v in csvLine.iteritems() if k != "PassengerId" and v!=""]
        dataset.record_row(csvLine["PassengerId"], tuples)

    # commit the dataset
    dataset.commit()


######
# train a classifier
print "======= Train"
for cls_algo in cls_algos:
    trainClassifierProcedureConfig = {
        "id": "titanic_cls_train_"+cls_algo,
        "type": "classifier.train",
        "params": {
            "trainingData": { 
                "select" : "{* EXCLUDING (Ticket, Name, label, Cabin)} as features, label = '1' as label",
                "from" : { "id": "titanic-train" },
                "where": "rowHash() % 5 != 1"
            },
            "algorithm": cls_algo,
            "configuration": {
                "bbdt": {
                    "type": "bagging",
                    "verbosity": 3,
                    "weak_learner": {
                        "type": "boosting",
                        "verbosity": 3,
                        "weak_learner": {
                            "type": "decision_tree",
                            "max_depth": 3,
                            "verbosity": 0,
                            "update_alg": "gentle",
                            "random_feature_propn": 0.5
                        },
                        "min_iter": 5,
                        "max_iter": 30
                    },
                    "num_bags": 5
                },
                "dt": {
                    "type": "decision_tree",
                    "max_depth": 8,
                    "verbosity": 3,
                    "update_alg": "prob"
                },
                "glz": {
                    "type": "glz",
                    "verbosity": 3,
                    "normalize ": " true",
                    "ridge_regression ": " true"
                }
            },
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo,
            "weight": "1.0"
        }
    }

    print mldb.perform("PUT", "/v1/procedures/titanic_cls_train_%s" % cls_algo, [], trainClassifierProcedureConfig)
    print mldb.perform("PUT", "/v1/procedures/titanic_cls_train_%s/runs/1" % cls_algo, [], {})

######
# test the classifier
print "======= Test"
for cls_algo in cls_algos:
    applyFunctionConfig = {
        "id": "classifyFunction" + cls_algo,
        "type": "classifier",
        "params": {
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo
        }
    }
    print mldb.perform("PUT", "/v1/functions/classifyFunction%s" % cls_algo, [], applyFunctionConfig)

    testClassifierProcedureConfig = {
        "id": "titanic_cls_test_%s" % cls_algo,
        "type": "classifier.test",
        "params": {
            "testingData": { 
                "select": "{*} as features, label = '1' as label",
                "from": {"id": "titanic-train" },
                "where": "rowHash() % 5 = 1"
            },            
            "outputDataset": { "id": "cls_test_results_%s" % cls_algo, "type": "sparse.mutable" },
            "score": "classifyFunction%s({ {* EXCLUDING (label)} AS features})[score]" % cls_algo,
            "weight": "1.0"
        }
    }
    print mldb.perform("PUT", "/v1/procedures/titanic_cls_test_%s" % cls_algo, [], testClassifierProcedureConfig)

    print mldb.perform("PUT", "/v1/procedures/titanic_cls_test_%s/runs/1" % cls_algo, [], {})


    explFunctionConfig = {
        "id": "explainFunction" + cls_algo,
        "type": "classifier.explain",
        "params": {
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo
        }
    }
    print mldb.perform("PUT", "/v1/functions/explainFunction%s" % cls_algo, [], explFunctionConfig)


print "====== Train probabilizer"
for cls_algo in cls_algos:
    trainProbabilizerProcedureConfig = {
        "id": "titanic_prob_train_%s" % cls_algo,
        "type": "probabilizer.train",
        "params": {
            "trainingData": { 
                # MAKES THIS FAIL!!<commit_msg>[MLDB-1104] fixed the test after the change to the classifier.test config<commit_after>#
# main.py
# <NAME>, 2015-03-26
# Copyright (c) 2015 Datacratic Inc. All rights reserved.
#

import csv, datetime, json

mldb.log("Pytanic Plugin Executing setup...")


cls_algos = ["glz", "dt", "bbdt"]
# cleanup in case things are already created
print "======= Cleanup"
for cls_algo in cls_algos:
    print mldb.perform("DELETE", "/v1/datasets/titanic-train", [], {})
    print mldb.perform("DELETE", "/v1/datasets/titanic-test", [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_cls_train_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_cls_test_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/procedures/titanic_prob_train_%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/classifyFunction%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/apply_probabilizer%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/probabilizer%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/explainFunction%s" % cls_algo, [], {})
    print mldb.perform("DELETE", "/v1/functions/probabilizer%s" % cls_algo, [], {})



# load the train and test datasets
for dataset_type in ["train", "test"]:
    datasetConfig = {
            "type": "sparse.mutable",
            "id": "titanic-"+dataset_type,
        }

    dataset = mldb.create_dataset(datasetConfig)
    def featProc(k, v):
        if k=="Pclass": return "c"+v
        if k=="Cabin": return v[0]
        if k in ["Age", "SibSp", "Parch", "Fare"]: return float(v)
        return v

    ts = datetime.datetime.now()
    filename = mldb.plugin.get_plugin_dir() + "/titanic_%s.csv" % dataset_type
    for idx, csvLine in enumerate(csv.DictReader(open(filename))):
        tuples = [[k,featProc(k,v),ts] for k,v in csvLine.iteritems() if k != "PassengerId" and v!=""]
        dataset.record_row(csvLine["PassengerId"], tuples)

    # commit the dataset
    dataset.commit()


######
# train a classifier
print "======= Train"
for cls_algo in cls_algos:
    trainClassifierProcedureConfig = {
        "id": "titanic_cls_train_"+cls_algo,
        "type": "classifier.train",
        "params": {
            "trainingData": { 
                "select" : "{* EXCLUDING (Ticket, Name, label, Cabin)} as features, label = '1' as label",
                "from" : { "id": "titanic-train" },
                "where": "rowHash() % 5 != 1"
            },
            "algorithm": cls_algo,
            "configuration": {
                "bbdt": {
                    "type": "bagging",
                    "verbosity": 3,
                    "weak_learner": {
                        "type": "boosting",
                        "verbosity": 3,
                        "weak_learner": {
                            "type": "decision_tree",
                            "max_depth": 3,
                            "verbosity": 0,
                            "update_alg": "gentle",
                            "random_feature_propn": 0.5
                        },
                        "min_iter": 5,
                        "max_iter": 30
                    },
                    "num_bags": 5
                },
                "dt": {
                    "type": "decision_tree",
                    "max_depth": 8,
                    "verbosity": 3,
                    "update_alg": "prob"
                },
                "glz": {
                    "type": "glz",
                    "verbosity": 3,
                    "normalize ": " true",
                    "ridge_regression ": " true"
                }
            },
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo
        }
    }

    print mldb.perform("PUT", "/v1/procedures/titanic_cls_train_%s" % cls_algo, [], trainClassifierProcedureConfig)
    print mldb.perform("PUT", "/v1/procedures/titanic_cls_train_%s/runs/1" % cls_algo, [], {})

######
# test the classifier
print "======= Test"
for cls_algo in cls_algos:
    applyFunctionConfig = {
        "id": "classifyFunction" + cls_algo,
        "type": "classifier",
        "params": {
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo
        }
    }
    print mldb.perform("PUT", "/v1/functions/classifyFunction%s" % cls_algo, [], applyFunctionConfig)

    testClassifierProcedureConfig = {
        "id": "titanic_cls_test_%s" % cls_algo,
        "type": "classifier.test",
        "params": {
            "testingData": { 
                "select": "label = '1' as label, classifyFunction%s({ {* EXCLUDING (label)} AS features})[score] as score" % cls_algo,
                "from": {"id": "titanic-train" },
                "where": "rowHash() % 5 = 1"
            },            
            "outputDataset": { "id": "cls_test_results_%s" % cls_algo, "type": "sparse.mutable" }
        }
    }
    print mldb.perform("PUT", "/v1/procedures/titanic_cls_test_%s" % cls_algo, [], testClassifierProcedureConfig)

    print mldb.perform("PUT", "/v1/procedures/titanic_cls_test_%s/runs/1" % cls_algo, [], {})


    explFunctionConfig = {
        "id": "explainFunction" + cls_algo,
        "type": "classifier.explain",
        "params": {
            "modelFileUrl": "file://models/titanic_%s.cls" % cls_algo
        }
    }
    print mldb.perform("PUT", "/v1/functions/explainFunction%s" % cls_algo, [], explFunctionConfig)


print "====== Train probabilizer"
for cls_algo in cls_algos:
    trainProbabilizerProcedureConfig = {
        "id": "titanic_prob_train_%s" % cls_algo,
        "type": "probabilizer.train",
        "params": {
            "trainingData": { 
                # MAKES THIS FAIL!!
{'fraction_non_alphanumeric': 0.16198102532857517, 'fraction_numerical': 0.009400295935242406, 'mean_word_length': 2.0086410054988217, 'pattern_counts': {'":': 125, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2412450', 'n_tokens_mistral': 3893, 'n_tokens_neox': 3786, 'n_words': 690}