<commit_before>/*******************************************************************************
 * Copyright (c) 2012, <NAME> (<EMAIL>).             *
 * All rights reserved.                                                        *
 *                                                                             *
 * Redistribution and use in source and binary forms, with or without          *
 * modification, are permitted provided that the following conditions are met: *
 *                                                                             *
 *     * Redistributions of source code must retain the above copyright        *
 *       notice, this list of conditions and the following disclaimer.         *
 *                                                                             *
 *     * Redistributions in binary form must reproduce the above copyright     *
 *       notice, this list of conditions and the following disclaimer in the   *
 *       documentation and/or other materials provided with the distribution.  *
 *                                                                             *
 *     * Neither the name of Carnegie Mellon University nor the                *
 *       names of the contributors may be used to endorse or promote products  *
 *       derived from this software without specific prior written permission. *
 *                                                                             *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" *
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE   *
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE  *
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE   *
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR         *
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF        *
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS    *
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN     *
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)     *
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE  *
 * POSSIBILITY OF SUCH DAMAGE.                                                 *
 ******************************************************************************/
#include "sdm/basics.hpp"
#include "sdm/sdm.hpp"
#include "sdm/kernels/linear.hpp"
#include "sdm/kernels/polynomial.hpp"
#include "sdm/kernels/gaussian.hpp"

#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <map>
#include <stdexcept>
#include <string>
#include <vector>

#include <boost/bind.hpp>
#include <boost/format.hpp>
#include <boost/program_options.hpp>
#include <boost/utility.hpp>

#include <flann/flann.hpp>

#include <np-divs/matrix_io.hpp>
#include <np-divs/div-funcs/from_str.hpp>

// TODO: support CV
// TODO: warn about dumb parameter combos, like linear kernel with distance df

using std::cerr;
using std::cin;
using std::cout;
using std::endl;
using std::ifstream;
using std::string;
using std::vector;

namespace po = boost::program_options;

// TODO: leaks memory everywhere, but whatever...
struct ProgOpts : boost::noncopyable {
    string train_bags_file;
    string test_bags_file;

    npdivs::DivFunc * div_func;
    sdm::KernelGroup * kernel_group;

    size_t k;
    size_t num_threads;
    size_t tuning_folds;
    bool prob;

    flann::IndexParams * index_params;
    flann::SearchParams * search_params;

    void parse_div_func(const string spec) {
        div_func = npdivs::div_func_from_str(spec);
    }

    void parse_kernel(const string name) {
        // TODO: support specifying cs / sigmas / degrees / etc
        if (name == "gaussian") {
            kernel_group = new sdm::GaussianKernelGroup;
        } else if (name == "linear") {
            kernel_group = new sdm::LinearKernelGroup;
        } else if (name == "polynomial") {
            kernel_group = new sdm::PolynomialKernelGroup;
        } else {
            throw std::domain_error((
                        boost::format("unknown kernel type %s") % name).str());
        }
    }

    void parse_index(const string name) {
        // TODO: more index types, support arguments
        if (name == "linear" || name == "brute") {
            index_params = new flann::LinearIndexParams;

        } else if (name == "kdtree" || name == "kd") {
            index_params = new flann::KDTreeSingleIndexParams;

        } else {
            throw std::domain_error((
                        boost::format("unknown index type %s") % name).str());
        }
    }
};

bool parse_args(int argc, char ** argv, ProgOpts& opts);

int main(int argc, char ** argv) {
    typedef flann::Matrix<double> Matrix;

    try {
        ProgOpts opts;
        if (!parse_args(argc, argv, opts))
            return 1;

        // TODO: gracefully handle nonexisting files
        // TODO: more robust input checking

        // load training bags
        size_t num_train;
        Matrix* train_bags;
        vector<string> train_labels;

        if (opts.train_bags_file == "-") {
            cout << "Enter training distributions in CSV-like "
                "format: an initial string label for each distribution, one "
                "line with comma-separated floating-point values for each "
                "point, one blank line between distributions, and an extra "
                "blank line when done.\n";
            train_bags = npdivs::labeled_matrices_from_csv(
                    std::cin, num_train, train_labels);
        } else {
            ifstream ifs(opts.train_bags_file.c_str(), ifstream::in);
            train_bags = npdivs::labeled_matrices_from_csv(
                    ifs, num_train, train_labels);
        }

        // load test
        size_t num_test;
        Matrix* test_bags;
        vector<string> test_labels;

        if (opts.test_bags_file == "-") {
            cout << "Enter testing distributions in CSV-like "
                "format: an initial string label for each distribution, one "
                "line with comma-separated floating-point values for each "
                "point, one blank line between distributions, and an extra "
                "blank line when done.\n";
            test_bags = npdivs::labeled_matrices_from_csv(
                    cin, num_test, test_labels);
        } else {
            ifstream ifs(opts.test_bags_file.c_str(), ifstream::in);
            test_bags = npdivs::labeled_matrices_from_csv(
                    ifs, num_test, test_labels);
        }

        // convert labels into integers
        // make maps to convert between int / string labels
        int this_label = -1;
        std::map<string,int> labels_to_int;
        std::map<int,string> labels_to_string;

        labels_to_int[""] = -1;
        labels_to_int["?"] = -1;
        labels_to_string[-1] = "?";

        std::vector<int> train_labels_ints;
        train_labels_ints.reserve(num_train);
        for (size_t i = 0; i < num_train; i++) {
            const string &lbl = train_labels[i];

            if (labels_to_int.count(lbl) == 0) {
                labels_to_int[lbl] = ++this_label;
                labels_to_string[this_label] = lbl;
            }
            train_labels_ints.push_back(labels_to_int[lbl]);
        }

        std::vector<int> test_labels_ints;
        test_labels_ints.reserve(num_test);
        for (size_t i = 0; i < num_train; i++) {
            const string &lbl = test_labels[i];

            if (labels_to_int.count(lbl) == 0) {
                labels_to_int[lbl] = ++this_label;
                labels_to_string[this_label] = lbl;
            }
            test_labels_ints.push_back(labels_to_int[lbl]);
        }

        // div params
        npdivs::DivParams div_params(opts.k,
                *opts.index_params, *opts.search_params,
                opts.num_threads);

        // svm params
        svm_parameter svm_params(sdm::default_svm_params);
        svm_params.probability = (int) opts.prob;

        // train the model
        sdm::SDM<double>* model = train_sdm(
                train_bags, num_train, train_labels_ints,
                *opts.div_func, *opts.kernel_group, div_params,
                sdm::default_c_vals, svm_params, opts.tuning_folds);

        // predict on test data
        const std::vector<int> &preds = model->predict(test_bags, num_test);

        // output predictions // TODO: optionally into file?
        cout << "Predicted labels:\n";
        for (size_t i = 0; i < num_test; i++) {
            cout << labels_to_string[preds[i]] << endl;
        }

        // test accuracy, if available
        size_t num_correct = 0;
        size_t total = num_test;
        for (size_t i = 0; i < num_test; i++) {
            if (test_labels_ints[i] == -1) {
                total--;
            } else if (preds[i] == test_labels_ints[i]) {
                num_correct++;
            }
        }
        if (total > 0) {
            cout << "Accuracy on " << total << " labeled test points: "
                << num_correct * 100. / total << "%\n";
        }<commit_msg>CLI: print predictions more nicely<commit_after>/*******************************************************************************
 * Copyright (c) 2012, <NAME> (<EMAIL>).             *
 * All rights reserved.                                                        *
 *                                                                             *
 * Redistribution and use in source and binary forms, with or without          *
 * modification, are permitted provided that the following conditions are met: *
 *                                                                             *
 *     * Redistributions of source code must retain the above copyright        *
 *       notice, this list of conditions and the following disclaimer.         *
 *                                                                             *
 *     * Redistributions in binary form must reproduce the above copyright     *
 *       notice, this list of conditions and the following disclaimer in the   *
 *       documentation and/or other materials provided with the distribution.  *
 *                                                                             *
 *     * Neither the name of Carnegie Mellon University nor the                *
 *       names of the contributors may be used to endorse or promote products  *
 *       derived from this software without specific prior written permission. *
 *                                                                             *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" *
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE   *
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE  *
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE   *
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR         *
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF        *
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS    *
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN     *
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)     *
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE  *
 * POSSIBILITY OF SUCH DAMAGE.                                                 *
 ******************************************************************************/
#include "sdm/basics.hpp"
#include "sdm/sdm.hpp"
#include "sdm/kernels/linear.hpp"
#include "sdm/kernels/polynomial.hpp"
#include "sdm/kernels/gaussian.hpp"

#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <map>
#include <stdexcept>
#include <string>
#include <vector>

#include <boost/bind.hpp>
#include <boost/format.hpp>
#include <boost/program_options.hpp>
#include <boost/utility.hpp>

#include <flann/flann.hpp>

#include <np-divs/matrix_io.hpp>
#include <np-divs/div-funcs/from_str.hpp>

// TODO: support CV
// TODO: warn about dumb parameter combos, like linear kernel with distance df

using std::cerr;
using std::cin;
using std::cout;
using std::endl;
using std::ifstream;
using std::string;
using std::vector;

namespace po = boost::program_options;

// TODO: leaks memory everywhere, but whatever...
struct ProgOpts : boost::noncopyable {
    string train_bags_file;
    string test_bags_file;

    npdivs::DivFunc * div_func;
    sdm::KernelGroup * kernel_group;

    size_t k;
    size_t num_threads;
    size_t tuning_folds;
    bool prob;

    flann::IndexParams * index_params;
    flann::SearchParams * search_params;

    void parse_div_func(const string spec) {
        div_func = npdivs::div_func_from_str(spec);
    }

    void parse_kernel(const string name) {
        // TODO: support specifying cs / sigmas / degrees / etc
        if (name == "gaussian") {
            kernel_group = new sdm::GaussianKernelGroup;
        } else if (name == "linear") {
            kernel_group = new sdm::LinearKernelGroup;
        } else if (name == "polynomial") {
            kernel_group = new sdm::PolynomialKernelGroup;
        } else {
            throw std::domain_error((
                        boost::format("unknown kernel type %s") % name).str());
        }
    }

    void parse_index(const string name) {
        // TODO: more index types, support arguments
        if (name == "linear" || name == "brute") {
            index_params = new flann::LinearIndexParams;

        } else if (name == "kdtree" || name == "kd") {
            index_params = new flann::KDTreeSingleIndexParams;

        } else {
            throw std::domain_error((
                        boost::format("unknown index type %s") % name).str());
        }
    }
};

bool parse_args(int argc, char ** argv, ProgOpts& opts);

int main(int argc, char ** argv) {
    typedef flann::Matrix<double> Matrix;

    try {
        ProgOpts opts;
        if (!parse_args(argc, argv, opts))
            return 1;

        // TODO: gracefully handle nonexisting files
        // TODO: more robust input checking

        // load training bags
        size_t num_train;
        Matrix* train_bags;
        vector<string> train_labels;

        if (opts.train_bags_file == "-") {
            cout << "Enter training distributions in CSV-like "
                "format: an initial string label for each distribution, one "
                "line with comma-separated floating-point values for each "
                "point, one blank line between distributions, and an extra "
                "blank line when done.\n";
            train_bags = npdivs::labeled_matrices_from_csv(
                    std::cin, num_train, train_labels);
        } else {
            ifstream ifs(opts.train_bags_file.c_str(), ifstream::in);
            train_bags = npdivs::labeled_matrices_from_csv(
                    ifs, num_train, train_labels);
        }

        // load test
        size_t num_test;
        Matrix* test_bags;
        vector<string> test_labels;

        if (opts.test_bags_file == "-") {
            cout << "Enter testing distributions in CSV-like "
                "format: an initial string label for each distribution, one "
                "line with comma-separated floating-point values for each "
                "point, one blank line between distributions, and an extra "
                "blank line when done.\n";
            test_bags = npdivs::labeled_matrices_from_csv(
                    cin, num_test, test_labels);
        } else {
            ifstream ifs(opts.test_bags_file.c_str(), ifstream::in);
            test_bags = npdivs::labeled_matrices_from_csv(
                    ifs, num_test, test_labels);
        }

        // convert labels into integers
        // make maps to convert between int / string labels
        int this_label = -1;
        std::map<string,int> labels_to_int;
        std::map<int,string> labels_to_string;

        labels_to_int[""] = -1;
        labels_to_int["?"] = -1;
        labels_to_string[-1] = "?";

        std::vector<int> train_labels_ints;
        train_labels_ints.reserve(num_train);
        for (size_t i = 0; i < num_train; i++) {
            const string &lbl = train_labels[i];

            if (labels_to_int.count(lbl) == 0) {
                labels_to_int[lbl] = ++this_label;
                labels_to_string[this_label] = lbl;
            }
            train_labels_ints.push_back(labels_to_int[lbl]);
        }

        std::vector<int> test_labels_ints;
        test_labels_ints.reserve(num_test);
        for (size_t i = 0; i < num_train; i++) {
            const string &lbl = test_labels[i];

            if (labels_to_int.count(lbl) == 0) {
                labels_to_int[lbl] = ++this_label;
                labels_to_string[this_label] = lbl;
            }
            test_labels_ints.push_back(labels_to_int[lbl]);
        }

        // div params
        npdivs::DivParams div_params(opts.k,
                *opts.index_params, *opts.search_params,
                opts.num_threads);

        // svm params
        svm_parameter svm_params(sdm::default_svm_params);
        svm_params.probability = (int) opts.prob;

        // train the model
        sdm::SDM<double>* model = train_sdm(
                train_bags, num_train, train_labels_ints,
                *opts.div_func, *opts.kernel_group, div_params,
                sdm::default_c_vals, svm_params, opts.tuning_folds);

        // predict on test data
        const std::vector<int> &preds = model->predict(test_bags, num_test);

        // output predictions // TODO: optionally into file?
        cout << "Predicted labels:\n";
        for (size_t i = 0; i < num_test; i++) {
            cout << i << ":\t" << labels_to_string[preds[i]];
            if (test_labels_ints[i] != -1)
                cout << "\t -- expected " << test_labels[i];
            cout << endl;
        }

        // test accuracy, if available
        size_t num_correct = 0;
        size_t total = num_test;
        for (size_t i = 0; i < num_test; i++) {
            if (test_labels_ints[i] == -1) {
                total--;
            } else if (preds[i] == test_labels_ints[i]) {
                num_correct++;
            }
        }
        if (total > 0) {
            cout << "Accuracy on " << total << " labeled test points: "
                << num_correct * 100. / total << "%\n";
        }
{'fraction_non_alphanumeric': 0.11413713611751461, 'fraction_numerical': 0.0023052592076341608, 'mean_word_length': 1.9344030202925908, 'pattern_counts': {'":': 1, '<': 111, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7604748', 'n_tokens_mistral': 5306, 'n_tokens_neox': 4977, 'n_words': 1456}