<commit_before>/*
 *        Learn command for CRFsuite frontend.
 *
 * Copyright (c) 2007-2010, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the names of the authors nor the names of its contributors
 *       may be used to endorse or promote products derived from this
 *       software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* $Id: learn.c 176 2010-07-14 09:31:04Z naoaki $ */

#include <os.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <crfsuite.h>
#include "option.h"
#include "readdata.h"

#define    SAFE_RELEASE(obj)    if ((obj) != NULL) { (obj)->release(obj); (obj) = NULL; }
#define    MAX(a, b)    ((a) < (b) ? (b) : (a))


typedef struct {
    char* model;
    char* training;
	char* features;
    char* evaluation;

    int help;

    int num_params;
    char** params;
} learn_option_t;

static char* mystrdup(const char *src)
{
    char* dst = (char*)malloc(strlen(src)+1);
    if (dst != NULL) {
        strcpy(dst, src);
    }
    return dst;
}

static void learn_option_init(learn_option_t* opt)
{
    memset(opt, 0, sizeof(*opt));
    opt->num_params = 0;
    opt->model = mystrdup("crfsuite.model");
}

static void learn_option_finish(learn_option_t* opt)
{
    int i;

    free(opt->model);
    free(opt->training);
    free(opt->evaluation);

    for (i = 0;i < opt->num_params;++i) {
        free(opt->params[i]);
    }
    free(opt->params);
}

BEGIN_OPTION_MAP(parse_learn_options, learn_option_t)

    ON_OPTION_WITH_ARG(SHORTOPT('m') || LONGOPT("model"))
        free(opt->model);
        opt->model = mystrdup(arg);

    ON_OPTION_WITH_ARG(SHORTOPT('t') || LONGOPT("test"))
        free(opt->evaluation);
        opt->evaluation = mystrdup(arg);

    ON_OPTION_WITH_ARG(SHORTOPT('f') || LONGOPT("features"))
        free(opt->features);
        opt->features = mystrdup(arg);

    ON_OPTION(SHORTOPT('h') || LONGOPT("help"))
        opt->help = 1;

    ON_OPTION_WITH_ARG(SHORTOPT('p') || LONGOPT("param"))
        opt->params = (char **)realloc(opt->params, sizeof(char*) * (opt->num_params + 1));
        opt->params[opt->num_params] = mystrdup(arg);
        ++opt->num_params;

END_OPTION_MAP()

static void show_usage(FILE *fp, const char *argv0, const char *command)
{
    fprintf(fp, "USAGE: %s %s [OPTIONS] [DATA]\n", argv0, command);
    fprintf(fp, "Obtain a model from a training set of instances given by a file (DATA).\n");
    fprintf(fp, "If argument DATA is omitted or '-', this utility reads a data from STDIN.\n");
    fprintf(fp, "\n");
    fprintf(fp, "OPTIONS:\n");
    fprintf(fp, "    -m, --model=MODEL   Store the obtained model in a file (MODEL)\n");
    fprintf(fp, "    -t, --test=TEST     Report the performance of the model on a data (TEST)\n");
    fprintf(fp, "    -p, --param=NAME=VALUE  Set the parameter NAME to VALUE\n");
    fprintf(fp, "    -h, --help          Show the usage of this command and exit\n");
}



typedef struct {
    FILE *fpo;
    crf_data_t* data;
    crf_evaluation_t* eval;
    crf_dictionary_t* attrs;
    crf_dictionary_t* labels;
} callback_data_t;

static int message_callback(void *instance, const char *format, va_list args)
{
    callback_data_t* cd = (callback_data_t*)instance;
    FILE *fpo = cd->fpo;
    vfprintf(fpo, format, args);
    fflush(fpo);
    return 0;
}

static int evaluate_callback(void *instance, crf_tagger_t* tagger)
{
    int i, ret = 0;
    crf_output_t output;
    callback_data_t* cd = (callback_data_t*)instance;
    FILE *fpo = cd->fpo;
    crf_data_t* data = cd->data;
    crf_dictionary_t* labels = cd->labels;

    /* Do nothing if no test data was given. */
    if (data->num_instances == 0) {
        return 0;
    }

    /* Clear the evaluation table. */
    crf_evaluation_clear(cd->eval);

    /* Tag the evaluation instances and accumulate the performance. */
    for (i = 0;i < data->num_instances;++i) {
        /* An instance to be tagged. */
        crf_sequence_t* instance = &data->instances[i];

        crf_output_init(&output);

        /* Tag an instance (ignoring any error occurrence). */
        ret = tagger->tag(tagger, instance, &output);

        /* Accumulate the tagging performance. */
        crf_evaluation_accmulate(cd->eval, instance, &output);
    }

    /* Compute the performance. */
    crf_evaluation_compute(cd->eval);

    /* Report the performance. */
    crf_evaluation_output(cd->eval, labels, fpo);

    return 0;
}

int main_learn(int argc, char *argv[], const char *argv0)
{
    int i, ret = 0, arg_used = 0;
    time_t ts;
    char timestamp[80];
    clock_t clk_begin, clk_current;
    learn_option_t opt;
    const char *command = argv[0];
    FILE *fp = NULL, *fpi = stdin, *fpo = stdout, *fpe = stderr;
    callback_data_t cd;
    crf_data_t data_train, data_test;
    crf_evaluation_t eval;
    crf_trainer_t *trainer = NULL;
    crf_dictionary_t *attrs = NULL, *labels = NULL;
	int num_features;

    /* Initializations. */
    learn_option_init(&opt);
    crf_data_init(&data_train);
    crf_data_init(&data_test);
    crf_evaluation_init(&eval, 0);

    /* Parse the command-line option. */
    arg_used = option_parse(++argv, --argc, parse_learn_options, &opt);
    if (arg_used < 0) {
        ret = 1;
        goto force_exit;
    }

    /* Show the help message for this command if specified. */
    if (opt.help) {
        show_usage(fpo, argv0, command);
        goto force_exit;
    }

    /* Set a training file. */
    if (arg_used < argc) {
        opt.training = mystrdup(argv[arg_used]);
    } else {
        opt.training = mystrdup("-");    /* STDIN. */
    }

    /* Create dictionaries for attributes and labels. */
    ret = crf_create_instance("dictionary", (void**)&attrs);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a dictionary instance.\n");
        ret = 1;
        goto force_exit;
    }
    ret = crf_create_instance("dictionary", (void**)&labels);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a dictionary instance.\n");
        ret = 1;
        goto force_exit;
    }

    /* Create a trainer instance. */
    ret = crf_create_instance("trainer.crfvo", (void**)&trainer);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a trainer instance.\n");
        ret = 1;
        goto force_exit;
    }

    /* Set parameters. */<commit_msg>Now we make sure that features file is designated when we learn.<commit_after>/*
 *        Learn command for CRFsuite frontend.
 *
 * Copyright (c) 2007-2010, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the names of the authors nor the names of its contributors
 *       may be used to endorse or promote products derived from this
 *       software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* $Id: learn.c 176 2010-07-14 09:31:04Z naoaki $ */

#include <os.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <crfsuite.h>
#include "option.h"
#include "readdata.h"

#define    SAFE_RELEASE(obj)    if ((obj) != NULL) { (obj)->release(obj); (obj) = NULL; }
#define    MAX(a, b)    ((a) < (b) ? (b) : (a))


typedef struct {
    char* model;
    char* training;
	char* features;
    char* evaluation;

    int help;

    int num_params;
    char** params;
} learn_option_t;

static char* mystrdup(const char *src)
{
    char* dst = (char*)malloc(strlen(src)+1);
    if (dst != NULL) {
        strcpy(dst, src);
    }
    return dst;
}

static void learn_option_init(learn_option_t* opt)
{
    memset(opt, 0, sizeof(*opt));
    opt->num_params = 0;
    opt->model = mystrdup("crfsuite.model");
}

static void learn_option_finish(learn_option_t* opt)
{
    int i;

    free(opt->model);
	free(opt->features);
    free(opt->training);
    free(opt->evaluation);

    for (i = 0;i < opt->num_params;++i) {
        free(opt->params[i]);
    }
    free(opt->params);
}

BEGIN_OPTION_MAP(parse_learn_options, learn_option_t)

    ON_OPTION_WITH_ARG(SHORTOPT('m') || LONGOPT("model"))
        free(opt->model);
        opt->model = mystrdup(arg);

    ON_OPTION_WITH_ARG(SHORTOPT('t') || LONGOPT("test"))
        free(opt->evaluation);
        opt->evaluation = mystrdup(arg);

    ON_OPTION_WITH_ARG(SHORTOPT('f') || LONGOPT("features"))
        free(opt->features);
        opt->features = mystrdup(arg);

    ON_OPTION(SHORTOPT('h') || LONGOPT("help"))
        opt->help = 1;

    ON_OPTION_WITH_ARG(SHORTOPT('p') || LONGOPT("param"))
        opt->params = (char **)realloc(opt->params, sizeof(char*) * (opt->num_params + 1));
        opt->params[opt->num_params] = mystrdup(arg);
        ++opt->num_params;

END_OPTION_MAP()

static void show_usage(FILE *fp, const char *argv0, const char *command)
{
    fprintf(fp, "USAGE: %s %s [OPTIONS] [DATA]\n", argv0, command);
    fprintf(fp, "Obtain a model from a training set of instances given by a file (DATA).\n");
    fprintf(fp, "If argument DATA is omitted or '-', this utility reads a data from STDIN.\n");
    fprintf(fp, "\n");
    fprintf(fp, "OPTIONS:\n");
    fprintf(fp, "    -f, --features=FEATURES   Designate a file to read features from [Required] (FEATURES)\n");
    fprintf(fp, "    -m, --model=MODEL   Store the obtained model in a file (MODEL)\n");
    fprintf(fp, "    -t, --test=TEST     Report the performance of the model on a data (TEST)\n");
    fprintf(fp, "    -p, --param=NAME=VALUE  Set the parameter NAME to VALUE\n");
    fprintf(fp, "    -h, --help          Show the usage of this command and exit\n");
}



typedef struct {
    FILE *fpo;
    crf_data_t* data;
    crf_evaluation_t* eval;
    crf_dictionary_t* attrs;
    crf_dictionary_t* labels;
} callback_data_t;

static int message_callback(void *instance, const char *format, va_list args)
{
    callback_data_t* cd = (callback_data_t*)instance;
    FILE *fpo = cd->fpo;
    vfprintf(fpo, format, args);
    fflush(fpo);
    return 0;
}

static int evaluate_callback(void *instance, crf_tagger_t* tagger)
{
    int i, ret = 0;
    crf_output_t output;
    callback_data_t* cd = (callback_data_t*)instance;
    FILE *fpo = cd->fpo;
    crf_data_t* data = cd->data;
    crf_dictionary_t* labels = cd->labels;

    /* Do nothing if no test data was given. */
    if (data->num_instances == 0) {
        return 0;
    }

    /* Clear the evaluation table. */
    crf_evaluation_clear(cd->eval);

    /* Tag the evaluation instances and accumulate the performance. */
    for (i = 0;i < data->num_instances;++i) {
        /* An instance to be tagged. */
        crf_sequence_t* instance = &data->instances[i];

        crf_output_init(&output);

        /* Tag an instance (ignoring any error occurrence). */
        ret = tagger->tag(tagger, instance, &output);

        /* Accumulate the tagging performance. */
        crf_evaluation_accmulate(cd->eval, instance, &output);
    }

    /* Compute the performance. */
    crf_evaluation_compute(cd->eval);

    /* Report the performance. */
    crf_evaluation_output(cd->eval, labels, fpo);

    return 0;
}

int main_learn(int argc, char *argv[], const char *argv0)
{
    int i, ret = 0, arg_used = 0;
    time_t ts;
    char timestamp[80];
    clock_t clk_begin, clk_current;
    learn_option_t opt;
    const char *command = argv[0];
    FILE *fp = NULL, *fpi = stdin, *fpo = stdout, *fpe = stderr;
    callback_data_t cd;
    crf_data_t data_train, data_test;
    crf_evaluation_t eval;
    crf_trainer_t *trainer = NULL;
    crf_dictionary_t *attrs = NULL, *labels = NULL;
	int num_features;

    /* Initializations. */
    learn_option_init(&opt);
    crf_data_init(&data_train);
    crf_data_init(&data_test);
    crf_evaluation_init(&eval, 0);

    /* Parse the command-line option. */
    arg_used = option_parse(++argv, --argc, parse_learn_options, &opt);
    if (arg_used < 0) {
        ret = 1;
        goto force_exit;
    }

    /* Show the help message for this command if specified. */
    if (opt.help) {
        show_usage(fpo, argv0, command);
        goto force_exit;
    }

	/* Make sure that -f or --features option is provided. */
	if (opt.features == NULL) {
        fprintf(fpe, "ERROR: You have to designate a file to read features from.\n");
        show_usage(fpo, argv0, command);
        goto force_exit;
	}

    /* Set a training file. */
    if (arg_used < argc) {
        opt.training = mystrdup(argv[arg_used]);
    } else {
        opt.training = mystrdup("-");    /* STDIN. */
    }

    /* Create dictionaries for attributes and labels. */
    ret = crf_create_instance("dictionary", (void**)&attrs);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a dictionary instance.\n");
        ret = 1;
        goto force_exit;
    }
    ret = crf_create_instance("dictionary", (void**)&labels);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a dictionary instance.\n");
        ret = 1;
        goto force_exit;
    }

    /* Create a trainer instance. */
    ret = crf_create_instance("trainer.crfvo", (void**)&trainer);
    if (!ret) {
        fprintf(fpe, "ERROR: Failed to create a trainer instance.\n");
        ret = 1;
        goto force_exit;
    }

    /* Set parameters. */
{'fraction_non_alphanumeric': 0.12595322826639552, 'fraction_numerical': 0.0066725978647686835, 'mean_word_length': 2.9931489469677746, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2174696', 'n_tokens_mistral': 5302, 'n_tokens_neox': 5045, 'n_words': 1537}