<commit_before>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>

#include "hmm.h"
#include "docinfo.h"
#include "utils.h"
#include "random.h"

#define EPS 1e-10

void hmm_reset(hmm *h)
{
	h->ss = NULL;
	h->ss2 = NULL;
	h->sw = NULL;
	h->sw2 = NULL;

	h->dps = NULL;
	h->dpe = NULL;
	h->dps_s = NULL;
	h->dpe_s = NULL;

	h->opt_ss = NULL;
	h->opt_ss_i = NULL;
	h->opt_sw = NULL;
	h->opt_sw_i = NULL;
	h->tmp_i = NULL;
	h->tmp_d = NULL;
}

int hmm_initialize(hmm *h)
{
	hmm_reset(h);
	return TRUE;
}

static
void hmm_cleanup_tables(hmm *h)
{
	if (h->ss) {
		free(h->ss);
		h->ss = NULL;
	}
	if (h->ss2) {
		free(h->ss2);
		h->ss2 = NULL;
	}
	if (h->sw) {
		free(h->sw);
		h->sw = NULL;
	}
	if (h->sw2) {
		free(h->sw2);
		h->sw2 = NULL;
	}
}

static
void hmm_cleanup_dp_tables(hmm *h)
{
	if (h->dps) {
		free(h->dps);
		h->dps = NULL;
	}
	if (h->dpe) {
		free(h->dpe);
		h->dpe = NULL;
	}
	if (h->dps_s) {
		free(h->dps_s);
		h->dps_s = NULL;
	}
	if (h->dpe_s) {
		free(h->dpe_s);
		h->dpe_s = NULL;
	}
}


static
void hmm_cleanup_optimization_tables(hmm *h)
{
	if (h->opt_ss) {
		free(h->opt_ss);
		h->opt_ss = NULL;
	}
	if (h->opt_ss_i) {
		free(h->opt_ss_i);
		h->opt_ss_i = NULL;
	}
	if (h->opt_sw) {
		free(h->opt_sw);
		h->opt_sw = NULL;
	}
	if (h->opt_sw_i) {
		free(h->opt_sw_i);
		h->opt_sw_i = NULL;
	}
	if (h->tmp_i) {
		free(h->tmp_i);
		h->tmp_i = NULL;
	}
	if (h->tmp_d) {
		free(h->tmp_d);
		h->tmp_d = NULL;
	}
}

void hmm_cleanup(hmm *h)
{
	hmm_cleanup_tables(h);
	hmm_cleanup_dp_tables(h);
	hmm_cleanup_optimization_tables(h);
}

static
void hmm_initialize_random(hmm *h)
{
	unsigned int i, j, k, pos;
	double sum;

	for (i = 0; i < h->num_states; i++) {
		sum = 0;

		pos = i * h->num_states;
		h->ss[pos] = 0.0;
		if (i != 1) {
			for (j = 1; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] = -log(genrand_real1());
				sum += h->ss[pos];
			}
			for (j = 1; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] /= sum;
			}
		} else {
			pos = h->num_states + 1;
			h->ss[pos] = 1.0;
			for (j = 2; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] = 0.0;
			}
		}

		if (i <= 1) continue;
		sum = 0;
		for (k = 0; k < h->num_words; k++) {
			pos = i * h->num_words + k;
			h->sw[pos] = -log(genrand_real1());
			sum += h->sw[pos];
		}
		for (k = 0; k < h->num_words; k++) {
			pos = i * h->num_words + k;
			h->sw[pos] /= sum;
		}
	}
}

static
int hmm_allocate_tables(hmm *h, unsigned int num_words,
                        unsigned int num_documents, unsigned int num_states)
{
	size_t size;

	hmm_cleanup_tables(h);
	h->num_documents = num_documents;
	h->num_words = num_words;
	h->num_states = num_states;

	size = num_states * num_states * sizeof(double);
	h->ss = (double *) xmalloc(size);
	if (!h->ss) return FALSE;

	h->ss2 = (double *) xmalloc(size);
	if (!h->ss2) return FALSE;

	size = num_states * h->num_words * sizeof(double);
	h->sw = (double *) xmalloc(size);
	if (!h->sw) return FALSE;

	h->sw2 = (double *) xmalloc(size);
	if (!h->sw2) return FALSE;

	return TRUE;
}

static
int hmm_allocate_dp_tables(hmm *h, unsigned int max_document_length)
{
	size_t size;

	hmm_cleanup_dp_tables(h);
	size = (max_document_length + 1) * sizeof(double);
	h->dps = (double *) xmalloc(size);
	if (!h->dps) return FALSE;

	h->dpe = (double *) xmalloc(size);
	if (!h->dpe) return FALSE;

	size = h->num_states * (max_document_length + 1) * sizeof(double);
	h->dps_s = (double *) xmalloc(size);
	if (!h->dps_s) return FALSE;

	h->dpe_s = (double *) xmalloc(size);
	if (!h->dpe_s) return FALSE;

	return TRUE;
}

static
double hmm_iteration(hmm *h, docinfo *doc)
{
	return 0;
}

int hmm_train(hmm *h, docinfo *doc, unsigned int num_states,
              unsigned int max_iterations, double tol)
{
	double likelihood, old_likelihood;
	unsigned int iter;

	if (!hmm_allocate_tables(h, docinfo_num_different_words(doc),
	                         docinfo_num_documents(doc), num_states))
		return FALSE;

	if (!hmm_allocate_dp_tables(h, docinfo_get_max_document_length(doc)))
		return FALSE;

	hmm_initialize_random(h);<commit_msg>Computing the dynamic programming tables<commit_after>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>

#include "hmm.h"
#include "docinfo.h"
#include "utils.h"
#include "random.h"

#define EPS 1e-10

void hmm_reset(hmm *h)
{
	h->ss = NULL;
	h->ss2 = NULL;
	h->sw = NULL;
	h->sw2 = NULL;

	h->dps = NULL;
	h->dpe = NULL;
	h->dps_s = NULL;
	h->dpe_s = NULL;

	h->opt_ss = NULL;
	h->opt_ss_i = NULL;
	h->opt_sw = NULL;
	h->opt_sw_i = NULL;
	h->tmp_i = NULL;
	h->tmp_d = NULL;
}

int hmm_initialize(hmm *h)
{
	hmm_reset(h);
	return TRUE;
}

static
void hmm_cleanup_tables(hmm *h)
{
	if (h->ss) {
		free(h->ss);
		h->ss = NULL;
	}
	if (h->ss2) {
		free(h->ss2);
		h->ss2 = NULL;
	}
	if (h->sw) {
		free(h->sw);
		h->sw = NULL;
	}
	if (h->sw2) {
		free(h->sw2);
		h->sw2 = NULL;
	}
}

static
void hmm_cleanup_dp_tables(hmm *h)
{
	if (h->dps) {
		free(h->dps);
		h->dps = NULL;
	}
	if (h->dpe) {
		free(h->dpe);
		h->dpe = NULL;
	}
	if (h->dps_s) {
		free(h->dps_s);
		h->dps_s = NULL;
	}
	if (h->dpe_s) {
		free(h->dpe_s);
		h->dpe_s = NULL;
	}
}


static
void hmm_cleanup_optimization_tables(hmm *h)
{
	if (h->opt_ss) {
		free(h->opt_ss);
		h->opt_ss = NULL;
	}
	if (h->opt_ss_i) {
		free(h->opt_ss_i);
		h->opt_ss_i = NULL;
	}
	if (h->opt_sw) {
		free(h->opt_sw);
		h->opt_sw = NULL;
	}
	if (h->opt_sw_i) {
		free(h->opt_sw_i);
		h->opt_sw_i = NULL;
	}
	if (h->tmp_i) {
		free(h->tmp_i);
		h->tmp_i = NULL;
	}
	if (h->tmp_d) {
		free(h->tmp_d);
		h->tmp_d = NULL;
	}
}

void hmm_cleanup(hmm *h)
{
	hmm_cleanup_tables(h);
	hmm_cleanup_dp_tables(h);
	hmm_cleanup_optimization_tables(h);
}

static
void hmm_initialize_random(hmm *h)
{
	unsigned int i, j, k, pos;
	double sum;

	for (i = 0; i < h->num_states; i++) {
		sum = 0;

		pos = i * h->num_states;
		h->ss[pos] = 0.0;
		if (i != 1) {
			for (j = 1; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] = -log(genrand_real1());
				sum += h->ss[pos];
			}
			for (j = 1; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] /= sum;
			}
		} else {
			pos = h->num_states + 1;
			h->ss[pos] = 1.0;
			for (j = 2; j < h->num_states; j++) {
				pos = i * h->num_states + j;
				h->ss[pos] = 0.0;
			}
		}

		if (i <= 1) continue;
		sum = 0;
		for (k = 0; k < h->num_words; k++) {
			pos = i * h->num_words + k;
			h->sw[pos] = -log(genrand_real1());
			sum += h->sw[pos];
		}
		for (k = 0; k < h->num_words; k++) {
			pos = i * h->num_words + k;
			h->sw[pos] /= sum;
		}
	}
}

static
int hmm_allocate_tables(hmm *h, unsigned int num_words,
                        unsigned int num_documents, unsigned int num_states)
{
	size_t size;

	hmm_cleanup_tables(h);
	h->num_documents = num_documents;
	h->num_words = num_words;
	h->num_states = num_states;

	size = num_states * num_states * sizeof(double);
	h->ss = (double *) xmalloc(size);
	if (!h->ss) return FALSE;

	h->ss2 = (double *) xmalloc(size);
	if (!h->ss2) return FALSE;

	size = num_states * h->num_words * sizeof(double);
	h->sw = (double *) xmalloc(size);
	if (!h->sw) return FALSE;

	h->sw2 = (double *) xmalloc(size);
	if (!h->sw2) return FALSE;

	return TRUE;
}

static
int hmm_allocate_dp_tables(hmm *h, unsigned int max_document_length)
{
	size_t size;

	hmm_cleanup_dp_tables(h);
	size = (max_document_length + 2) * sizeof(double);
	h->dps = (double *) xmalloc(size);
	if (!h->dps) return FALSE;

	h->dpe = (double *) xmalloc(size);
	if (!h->dpe) return FALSE;

	size = h->num_states * (max_document_length + 2) * sizeof(double);
	h->dps_s = (double *) xmalloc(size);
	if (!h->dps_s) return FALSE;

	h->dpe_s = (double *) xmalloc(size);
	if (!h->dpe_s) return FALSE;

	return TRUE;
}

static
void hmm_compute_dp_tables(hmm *h, docinfo *doc, unsigned int doc_idx)
{
	docinfo_document *document;
	unsigned int i, j, k, l;
	unsigned int pos, pos2, pos3, pos4;

	document = docinfo_get_document(doc, doc_idx);

	h->dps[0] = 1;
	memset(h->dps_s, 0, h->num_states * sizeof(double));
	h->dps_s[0] = 1;
	for (i = 1; i <= document->word_count; i++) {
		h->dps[i] = 0;
		l = doc->words[document->words + i - 2] - 1;
		for (j = 2; j < h->num_states; j++) {
			pos = i * h->num_states + j;
			h->dps_s[pos] = 0;
			for (k = 0; k < h->num_states; k++) {
				pos2 = (i - 1) * h->num_states + k;
				pos3 = k * h->num_states + j;
				h->dps_s[pos] += h->ss[pos3] * h->dps_s[pos2];
			}
			pos4 = j * h->num_words + l;
			h->dps_s[pos] *= h->sw[pos4];
			h->dps[i] += h->dps_s[pos];
		}
	}
	pos = i * h->num_states + 1;
	h->dps_s[pos] = 0;
	for (k = 0; k < h->num_states; k++) {
		pos2 = (i - 1) * h->num_states + k;
		pos3 = k * h->num_states + 1;
		h->dps_s[pos] += h->ss[pos3] * h->dps_s[pos2];
	}
	h->dps[i] = h->dps_s[pos];

}

static
double hmm_iteration(hmm *h, docinfo *doc)
{
	unsigned int i;
	for (i = 0; i < docinfo_num_documents(doc); i++) {
		hmm_compute_dp_tables(h, doc, i + 1);
	}
	return 0;
}

int hmm_train(hmm *h, docinfo *doc, unsigned int num_states,
              unsigned int max_iterations, double tol)
{
	double likelihood, old_likelihood;
	unsigned int iter;

	if (!hmm_allocate_tables(h, docinfo_num_different_words(doc),
	                         docinfo_num_documents(doc), num_states))
		return FALSE;

	if (!hmm_allocate_dp_tables(h, docinfo_get_max_document_length(doc)))
		return FALSE;

	hmm_initialize_random(h);
{'fraction_non_alphanumeric': 0.19408835681745948, 'fraction_numerical': 0.01133594660451319, 'mean_word_length': 3.058469475494411, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 240, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5332033', 'n_tokens_mistral': 4718, 'n_tokens_neox': 4313, 'n_words': 921}