<commit_before>#include <fstream>
#include <iostream>
#include <string>

#include <opencv2/opencv.hpp>
#include "json/json.h"

#include "covcorr.hpp"
#include "cvutils.hpp"
#include "dists.hpp"
#include "features.hpp"
#include "filterbank.hpp"
#include "model.hpp"
#include "warco.hpp"

void foreach_img(const Json::Value& dataset, const char* traintest, std::function<void (unsigned, const cv::Mat&, std::string)> fn)
{
    auto lbls = dataset["classes"];
    for(auto ilbl = lbls.begin() ; ilbl != lbls.end() ; ++ilbl) {
        auto lbl = ilbl.index();
        auto lblname = (*ilbl).asString();

        for(Json::Value fname : dataset[traintest][lblname]) {
            cv::Mat image = cv::imread(fname.asString());
            if(! image.data) {
                std::cerr << "Skipping unreadable image " << fname << std::endl;
                continue;
            }

            fn(lbl, image, fname.asString());
        }
    }
}

int main(int argc, char** argv)
{
    if(argc != 2) {
        std::cout << "Please specify a configuration file which describes the dataset." << std::endl;
        return 0;
    }

#ifndef NDEBUG
    bool vv = getenv("WARCO_DEBUG");
#endif

    Json::Value dataset;
    {
        std::ifstream infile(argv[1]);
        if(! infile) {
            std::cerr << "Failed to open configuration file " << argv[1] << std::endl;
            return 2;
        }

        Json::Reader reader;
        if(! reader.parse(infile, dataset)) {
            std::cerr << "Failed to parse configuration file " << argv[1] << std::endl
                << reader.getFormattedErrorMessages()
                << std::endl;
            return 3;
        }
    }

    auto fb = cv::FilterBank(dataset["filterbank"].asCString());
    warco::Warco model(fb);
    foreach_img(dataset, "train", [&model](unsigned lbl, const cv::Mat& image, std::string) {
        model.add_sample(image, lbl);
    });

    std::cout << "Training model" << std::flush;
    std::vector<double> C;
    if(dataset.isMember("crossval_C"))
        for(Json::Value c : dataset["crossval_C"])
            C.push_back(c.asDouble());
    else
        C = {0.1, 1.0, 10.0};
    double avg_train = model.train(C, [](unsigned){ std::cout << "." << std::flush; });
    std::cout << std::endl << "Average training score *per patch*: " << avg_train << std::endl;

    std::cout << "Testing" << std::flush;
    std::cerr << "test,predicted,actual" << std::endl;

    Json::Value lbls = dataset["classes"];
    unsigned correct = 0, total = 0;
    foreach_img(dataset, "test", [&model, &lbls, &correct, &total](unsigned lbl, const cv::Mat& image, std::string fname) {
        std::cout << "." << std::flush;

        unsigned pred = model.predict(image);
        //unsigned pred_proba = model.predict_proba(image);

#ifndef NDEBUG
        //if(vv) {
            //std::cout << " => " << pred << " (=" << lbls[pred].asString() << ")"<< std::endl;
            //std::cout << to_s(votes) << std::endl;
            //std::cout << to_s(votes_proba) << std::endl;
            //std::cout << " => " << pred_proba << " (=" << lbls[pred_proba].asString() << ")"<< std::endl;
        //}
#endif

        std::cerr << fname << "," << lbls[pred].asString() << "," << lbls[lbl].asString() << std::endl;
        //std::cerr << fname << "," << lbls[pred_proba].asString() << "," << lbls[lbl].asString() << std::endl;<commit_msg>Switch to probabilistic predictions by default.<commit_after>#include <fstream>
#include <iostream>
#include <string>

#include <opencv2/opencv.hpp>
#include "json/json.h"

#include "covcorr.hpp"
#include "cvutils.hpp"
#include "dists.hpp"
#include "features.hpp"
#include "filterbank.hpp"
#include "model.hpp"
#include "warco.hpp"

void foreach_img(const Json::Value& dataset, const char* traintest, std::function<void (unsigned, const cv::Mat&, std::string)> fn)
{
    auto lbls = dataset["classes"];
    for(auto ilbl = lbls.begin() ; ilbl != lbls.end() ; ++ilbl) {
        auto lbl = ilbl.index();
        auto lblname = (*ilbl).asString();

        for(Json::Value fname : dataset[traintest][lblname]) {
            cv::Mat image = cv::imread(fname.asString());
            if(! image.data) {
                std::cerr << "Skipping unreadable image " << fname << std::endl;
                continue;
            }

            fn(lbl, image, fname.asString());
        }
    }
}

int main(int argc, char** argv)
{
    if(argc != 2) {
        std::cout << "Please specify a configuration file which describes the dataset." << std::endl;
        return 0;
    }

    Json::Value dataset;
    {
        std::ifstream infile(argv[1]);
        if(! infile) {
            std::cerr << "Failed to open configuration file " << argv[1] << std::endl;
            return 2;
        }

        Json::Reader reader;
        if(! reader.parse(infile, dataset)) {
            std::cerr << "Failed to parse configuration file " << argv[1] << std::endl
                << reader.getFormattedErrorMessages()
                << std::endl;
            return 3;
        }
    }

    auto fb = cv::FilterBank(dataset["filterbank"].asCString());
    warco::Warco model(fb);
    foreach_img(dataset, "train", [&model](unsigned lbl, const cv::Mat& image, std::string) {
        model.add_sample(image, lbl);
    });

    std::cout << "Training model" << std::flush;
    std::vector<double> C;
    if(dataset.isMember("crossval_C"))
        for(Json::Value c : dataset["crossval_C"])
            C.push_back(c.asDouble());
    else
        C = {0.1, 1.0, 10.0};
    double avg_train = model.train(C, [](unsigned){ std::cout << "." << std::flush; });
    std::cout << std::endl << "Average training score *per patch*: " << avg_train << std::endl;

    std::cout << "Testing" << std::flush;
    std::cerr << "test,predicted,actual" << std::endl;

    Json::Value lbls = dataset["classes"];
    unsigned correct = 0, total = 0;
    foreach_img(dataset, "test", [&model, &lbls, &correct, &total](unsigned lbl, const cv::Mat& image, std::string fname) {
        std::cout << "." << std::flush;

        //unsigned pred = model.predict(image);
        unsigned pred = model.predict_proba(image);

        std::cerr << fname << "," << lbls[pred].asString() << "," << lbls[lbl].asString() << std::endl;
{'fraction_non_alphanumeric': 0.1858831044394762, 'fraction_numerical': 0.005429575215586075, 'mean_word_length': 2.7257584770969663, 'pattern_counts': {'":': 0, '<': 191, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123715', 'n_tokens_mistral': 2090, 'n_tokens_neox': 2044, 'n_words': 449}