<commit_before>/** moses-exec.h ---
 *
 * Copyright (C) 2010 OpenCog Foundation
 *
 * Author: <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifndef _OPENCOG_MOSES_EXEC_H
#define _OPENCOG_MOSES_EXEC_H

#include <iostream>
#include <fstream>
#include <memory>
#include <stdio.h>

#include <boost/program_options.hpp>
#include <boost/lexical_cast.hpp>

#include <opencog/util/mt19937ar.h>
#include <opencog/util/Logger.h>
#include <opencog/util/lru_cache.h>
#include <opencog/util/algorithm.h>
#include <opencog/util/iostreamContainer.h>
#include <opencog/util/oc_omp.h>

#include <opencog/comboreduct/combo/combo.h>

// for operator>> to combo
#include <opencog/comboreduct/ant_combo_vocabulary/ant_combo_vocabulary.h>

#include "../moses/moses.h"
#include "../moses/distributed_moses.h"
#include "../optimization/optimization.h"
#include "../moses/metapopulation.h"
#include "../moses/scoring_functions.h"
#include "../moses/scoring.h"
#include "moses_exec_def.h"

// enable a cache on bscore (takes more memory)
#define ENABLE_BCACHE

namespace opencog { namespace moses {

using namespace boost::program_options;
using boost::lexical_cast;
using namespace std;
using namespace reduct;
using namespace ant_combo;

// default number of samples to describe a problem
static const unsigned int default_nsamples = 20;

// problems
static const string it="it"; // regression based on input table
static const string pre="pre"; // regression based on input table by
                               // maximizing precision (or negative
                               // predictive value)
static const string ip="ip"; // find interesting patterns
static const string cp="cp"; // regression based on combo program to fit
static const string pa="pa"; // even parity
static const string dj="dj"; // disjunction
static const string mux="mux"; // multiplex
static const string sr="sr"; // simple regression of f(x)_o = sum_{i={1,o}} x^i
static const string ann_it="ann-it"; // regression based on input table using ann
static const string ann_cp="ann-cp"; // regression based on combo program using ann
static const string ann_xor="ann-xor"; // binary-xor problem using ann
static const string ann_pole1="ann-pole1"; // pole balancing problem using ann
static const string ann_pole2="ann-pole2"; // double pole balancing problem ann

// optimization algorithms
static const string un="un"; // univariate
static const string sa="sa"; // star-shaped search
static const string hc="hc"; // local search

// used by the main function, it is included in the library for its
// convenience
int moses_exec(int argc, char** argv);
// helper for the function above, note that the first still represents
// the name of the supposed executable
int moses_exec(const vector<string>& argv);

// Parameters controlling output printing and display.
struct metapop_moses_results_parameters
{
    metapop_moses_results_parameters(const variables_map& _vm,
                                     long _result_count,
                                     bool _output_score,
                                     bool _output_complexity,
                                     bool _output_bscore,
                                     bool _output_dominated,
                                     bool _output_eval_number,
                                     bool _output_with_labels,
                                     const string& _opt_algo,
                                     bool _enable_cache,
                                     const vector<string>& _labels,
                                     const string& _output_file,
                                     const jobs_t& _jobs,
                                     bool _only_local,
                                     bool _output_python,
                                     bool _hc_widen_search,
                                     bool _hc_single_step,
                                     bool _hc_crossover) :
        vm(_vm), result_count(_result_count), output_score(_output_score),
        output_complexity(_output_complexity),
        output_bscore(_output_bscore),
        output_dominated(_output_dominated),
        output_eval_number(_output_eval_number),
        output_with_labels(_output_with_labels),
        opt_algo(_opt_algo),
        enable_cache(_enable_cache), labels(_labels),
        output_file(_output_file),
        jobs(_jobs), only_local(_only_local),
        output_python(_output_python),
        hc_widen_search(_hc_widen_search),
        hc_single_step(_hc_single_step),
        hc_crossover(_hc_crossover) {}

    const variables_map& vm;
    long result_count;
    bool output_score;
    bool output_complexity;
    bool output_bscore;
    bool output_dominated;
    bool output_eval_number;
    bool output_with_labels;
    string opt_algo;
    bool enable_cache;
    const vector<string>& labels;
    string output_file;
    const jobs_t& jobs;
    bool only_local;
    bool output_python;

    // Not really results, but we have nowhere else to put these...
    bool hc_widen_search;
    bool hc_single_step;
    bool hc_crossover;
};

/**
 * 1) create a metapopulation
 * 2) run moses
 * 3) print the results
 */
template<typename Score, typename BScore, typename Optimization>
void metapop_moses_results_a(RandGen& rng,
                           Optimization& opt,
                           const std::vector<combo_tree>& bases,
                           const opencog::combo::type_tree& tt,
                           const reduct::rule& si_ca,
                           const reduct::rule& si_kb,
                           const Score& sc,
                           const BScore& bsc,
                           const metapop_parameters& meta_params,
                           const moses_parameters& moses_params,
                           const metapop_moses_results_parameters& pa)
{
    // instantiate metapop
    metapopulation<Score, BScore, Optimization>
        metapop(rng, bases, tt, si_ca, si_kb, sc, bsc, opt, meta_params);

    // run moses, either on localhost, or distributed.
    if (pa.only_local)
        moses::moses(metapop, moses_params);
    else
        moses::distributed_moses(metapop, pa.vm, pa.jobs, moses_params);

    // output result
    {
        stringstream ss;
        metapop.ostream(ss,
                        pa.result_count, pa.output_score,
                        pa.output_complexity,
                        pa.output_bscore,
                        pa.output_dominated,
                        pa.output_python); 

        if (pa.output_eval_number)
            ss << number_of_evals_str << ": " << metapop.n_evals() << std::endl;;
        string res = (pa.output_with_labels && !pa.labels.empty()?
                      ph2l(ss.str(), pa.labels) : ss.str());
        if (pa.output_file.empty())
            std::cout << res;
        else {
            ofstream of(pa.output_file.c_str());
            of << res;
            of.close();
        }
    }

    // Log the best candidate
    {
        stringstream ss;
        metapop.ostream(ss, 1, true, true);
        string res = (pa.output_with_labels && !pa.labels.empty()?
                      ph2l(ss.str(), pa.labels) : ss.str());
        if (res.empty())
            logger().info("No candidate is good enough to be returned. Yeah that's bad!");
        else
            logger().info("Best candidate (preceded by its score and complexity): %s", res.c_str());
    }
}

/**
 * like above but takes the algo type instead of the algo template
 */
template<typename Score, typename BScore>
void metapop_moses_results_b(RandGen& rng,
                           const std::vector<combo_tree>& bases,
                           const opencog::combo::type_tree& tt,
                           const reduct::rule& si_ca,
                           const reduct::rule& si_kb,
                           const Score& sc,
                           const BScore& bsc,
                           const optim_parameters& opt_params,
                           const metapop_parameters& meta_params,
                           const moses_parameters& moses_params,
                           const metapop_moses_results_parameters& pa)
{
    if (pa.opt_algo == un) { // univariate
        univariate_optimization unopt(rng, opt_params);
        metapop_moses_results_a(rng, unopt,
                              bases, tt, si_ca, si_kb, sc, bsc,
                              meta_params, moses_params, pa);
    }<commit_msg>minor identation fix<commit_after>/** moses-exec.h ---
 *
 * Copyright (C) 2010 OpenCog Foundation
 *
 * Author: <NAME> <<EMAIL>>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License v3 as
 * published by the Free Software Foundation and including the exceptions
 * at http://opencog.org/wiki/Licenses
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program; if not, write to:
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifndef _OPENCOG_MOSES_EXEC_H
#define _OPENCOG_MOSES_EXEC_H

#include <iostream>
#include <fstream>
#include <memory>
#include <stdio.h>

#include <boost/program_options.hpp>
#include <boost/lexical_cast.hpp>

#include <opencog/util/mt19937ar.h>
#include <opencog/util/Logger.h>
#include <opencog/util/lru_cache.h>
#include <opencog/util/algorithm.h>
#include <opencog/util/iostreamContainer.h>
#include <opencog/util/oc_omp.h>

#include <opencog/comboreduct/combo/combo.h>

// for operator>> to combo
#include <opencog/comboreduct/ant_combo_vocabulary/ant_combo_vocabulary.h>

#include "../moses/moses.h"
#include "../moses/distributed_moses.h"
#include "../optimization/optimization.h"
#include "../moses/metapopulation.h"
#include "../moses/scoring_functions.h"
#include "../moses/scoring.h"
#include "moses_exec_def.h"

// enable a cache on bscore (takes more memory)
#define ENABLE_BCACHE

namespace opencog { namespace moses {

using namespace boost::program_options;
using boost::lexical_cast;
using namespace std;
using namespace reduct;
using namespace ant_combo;

// default number of samples to describe a problem
static const unsigned int default_nsamples = 20;

// problems
static const string it="it"; // regression based on input table
static const string pre="pre"; // regression based on input table by
                               // maximizing precision (or negative
                               // predictive value)
static const string ip="ip"; // find interesting patterns
static const string cp="cp"; // regression based on combo program to fit
static const string pa="pa"; // even parity
static const string dj="dj"; // disjunction
static const string mux="mux"; // multiplex
static const string sr="sr"; // simple regression of f(x)_o = sum_{i={1,o}} x^i
static const string ann_it="ann-it"; // regression based on input table using ann
static const string ann_cp="ann-cp"; // regression based on combo program using ann
static const string ann_xor="ann-xor"; // binary-xor problem using ann
static const string ann_pole1="ann-pole1"; // pole balancing problem using ann
static const string ann_pole2="ann-pole2"; // double pole balancing problem ann

// optimization algorithms
static const string un="un"; // univariate
static const string sa="sa"; // star-shaped search
static const string hc="hc"; // local search

// used by the main function, it is included in the library for its
// convenience
int moses_exec(int argc, char** argv);
// helper for the function above, note that the first still represents
// the name of the supposed executable
int moses_exec(const vector<string>& argv);

// Parameters controlling output printing and display.
struct metapop_moses_results_parameters
{
    metapop_moses_results_parameters(const variables_map& _vm,
                                     long _result_count,
                                     bool _output_score,
                                     bool _output_complexity,
                                     bool _output_bscore,
                                     bool _output_dominated,
                                     bool _output_eval_number,
                                     bool _output_with_labels,
                                     const string& _opt_algo,
                                     bool _enable_cache,
                                     const vector<string>& _labels,
                                     const string& _output_file,
                                     const jobs_t& _jobs,
                                     bool _only_local,
                                     bool _output_python,
                                     bool _hc_widen_search,
                                     bool _hc_single_step,
                                     bool _hc_crossover) :
        vm(_vm), result_count(_result_count), output_score(_output_score),
        output_complexity(_output_complexity),
        output_bscore(_output_bscore),
        output_dominated(_output_dominated),
        output_eval_number(_output_eval_number),
        output_with_labels(_output_with_labels),
        opt_algo(_opt_algo),
        enable_cache(_enable_cache), labels(_labels),
        output_file(_output_file),
        jobs(_jobs), only_local(_only_local),
        output_python(_output_python),
        hc_widen_search(_hc_widen_search),
        hc_single_step(_hc_single_step),
        hc_crossover(_hc_crossover) {}

    const variables_map& vm;
    long result_count;
    bool output_score;
    bool output_complexity;
    bool output_bscore;
    bool output_dominated;
    bool output_eval_number;
    bool output_with_labels;
    string opt_algo;
    bool enable_cache;
    const vector<string>& labels;
    string output_file;
    const jobs_t& jobs;
    bool only_local;
    bool output_python;

    // Not really results, but we have nowhere else to put these...
    bool hc_widen_search;
    bool hc_single_step;
    bool hc_crossover;
};

/**
 * 1) create a metapopulation
 * 2) run moses
 * 3) print the results
 */
template<typename Score, typename BScore, typename Optimization>
void metapop_moses_results_a(RandGen& rng,
                             Optimization& opt,
                             const std::vector<combo_tree>& bases,
                             const opencog::combo::type_tree& tt,
                             const reduct::rule& si_ca,
                             const reduct::rule& si_kb,
                             const Score& sc,
                             const BScore& bsc,
                             const metapop_parameters& meta_params,
                             const moses_parameters& moses_params,
                             const metapop_moses_results_parameters& pa)
{
    // instantiate metapop
    metapopulation<Score, BScore, Optimization>
        metapop(rng, bases, tt, si_ca, si_kb, sc, bsc, opt, meta_params);

    // run moses, either on localhost, or distributed.
    if (pa.only_local)
        moses::moses(metapop, moses_params);
    else
        moses::distributed_moses(metapop, pa.vm, pa.jobs, moses_params);

    // output result
    {
        stringstream ss;
        metapop.ostream(ss,
                        pa.result_count, pa.output_score,
                        pa.output_complexity,
                        pa.output_bscore,
                        pa.output_dominated,
                        pa.output_python); 

        if (pa.output_eval_number)
            ss << number_of_evals_str << ": " << metapop.n_evals() << std::endl;;
        string res = (pa.output_with_labels && !pa.labels.empty()?
                      ph2l(ss.str(), pa.labels) : ss.str());
        if (pa.output_file.empty())
            std::cout << res;
        else {
            ofstream of(pa.output_file.c_str());
            of << res;
            of.close();
        }
    }

    // Log the best candidate
    {
        stringstream ss;
        metapop.ostream(ss, 1, true, true);
        string res = (pa.output_with_labels && !pa.labels.empty()?
                      ph2l(ss.str(), pa.labels) : ss.str());
        if (res.empty())
            logger().info("No candidate is good enough to be returned. Yeah that's bad!");
        else
            logger().info("Best candidate (preceded by its score and complexity): %s", res.c_str());
    }
}

/**
 * like above but takes the algo type instead of the algo template
 */
template<typename Score, typename BScore>
void metapop_moses_results_b(RandGen& rng,
                             const std::vector<combo_tree>& bases,
                             const opencog::combo::type_tree& tt,
                             const reduct::rule& si_ca,
                             const reduct::rule& si_kb,
                             const Score& sc,
                             const BScore& bsc,
                             const optim_parameters& opt_params,
                             const metapop_parameters& meta_params,
                             const moses_parameters& moses_params,
                             const metapop_moses_results_parameters& pa)
{
    if (pa.opt_algo == un) { // univariate
        univariate_optimization unopt(rng, opt_params);
        metapop_moses_results_a(rng, unopt,
                              bases, tt, si_ca, si_kb, sc, bsc,
                              meta_params, moses_params, pa);
    }
{'fraction_non_alphanumeric': 0.08697066637748739, 'fraction_numerical': 0.0036870357317139294, 'mean_word_length': 2.0612448132780083, 'pattern_counts': {'":': 2, '<': 77, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4215911', 'n_tokens_mistral': 5255, 'n_tokens_neox': 4998, 'n_words': 1452}