<commit_before>#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "ccmpred.h"
#include "io.h"
#include "conjugrad.h"

#ifdef MSGPACK
#include <msgpack.h>

#ifdef JANSSON
#include <jansson.h>
#include "meta.h"
#endif

#endif

void write_matrix(FILE *out, conjugrad_float_t *mat, int ncol, int nrow) {
	for(int i = 0; i < nrow; i++) {
		for(int j = 0; j < ncol; j++) {
			fprintf(out, "%.20e\t", mat[i * ncol + j]);
		}

		fprintf(out, "\n");
	}
}


void write_raw(FILE *out, conjugrad_float_t *x, int ncol) {

	int nsingle = ncol * (N_ALPHA - 1);
	int nsingle_padded = nsingle + N_ALPHA_PAD - (nsingle % N_ALPHA_PAD);

	conjugrad_float_t *x1 = x;
	//conjugrad_float_t *x2 = &x[nsingle];
	conjugrad_float_t *x2 = &x[nsingle_padded];
	(void)x2;

	for(int i = 0; i < ncol; i++) {
		for(int a = 0; a < N_ALPHA - 1; a++) {
			fprintf(out, "%g\t", V(i, a));
		}
		fprintf(out, "\n");
	}


	for(int i = 0; i < ncol; i++) {
		for(int j = i+1; j < ncol; j++) {
			fprintf(out, "# %d %d\n", i, j);
			for(int a = 0; a < N_ALPHA; a++) {
				for(int b = 0; b < N_ALPHA; b++) {
					fprintf(out, "%.20e\t", W(b,j,a,i));
				}
				fprintf(out, "\n");
			}
		}
	}
}

void read_raw(char *filename, userdata *ud, conjugrad_float_t *x) {
	FILE *f = fopen(filename, "r");
	char *line = (char*) malloc(sizeof(char) * 8192);
	char* expected = malloc(1024);
	int ncol = ud->ncol;

	conjugrad_float_t *x1 = x;
	conjugrad_float_t *x2 = x + ud->nsingle;

	for(int i = 0; i < ncol; i++) {
		fgets(line, 8192, f);
		char *token = line;
		for(int a = 0; a < N_ALPHA-1; a++) {
			V(i, a) = strtod(token, &token);
		}
	}

	for(int i = 0; i < ncol; i++) {
		for(int j = i+1; j < ncol; j++) {
			fgets(line, 8192, f);

			snprintf(expected, 1024, "# %d %d\n", i, j);<commit_msg>read_raw: corrects start index for x2<commit_after>#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "ccmpred.h"
#include "io.h"
#include "conjugrad.h"

#ifdef MSGPACK
#include <msgpack.h>

#ifdef JANSSON
#include <jansson.h>
#include "meta.h"
#endif

#endif

void write_matrix(FILE *out, conjugrad_float_t *mat, int ncol, int nrow) {
	for(int i = 0; i < nrow; i++) {
		for(int j = 0; j < ncol; j++) {
			fprintf(out, "%.20e\t", mat[i * ncol + j]);
		}

		fprintf(out, "\n");
	}
}


void write_raw(FILE *out, conjugrad_float_t *x, int ncol) {

	int nsingle = ncol * (N_ALPHA - 1);
	int nsingle_padded = nsingle + N_ALPHA_PAD - (nsingle % N_ALPHA_PAD);

	conjugrad_float_t *x1 = x;
	//conjugrad_float_t *x2 = &x[nsingle];
	conjugrad_float_t *x2 = &x[nsingle_padded];
	(void)x2;

	for(int i = 0; i < ncol; i++) {
		for(int a = 0; a < N_ALPHA - 1; a++) {
			fprintf(out, "%g\t", V(i, a));
		}
		fprintf(out, "\n");
	}


	for(int i = 0; i < ncol; i++) {
		for(int j = i+1; j < ncol; j++) {
			fprintf(out, "# %d %d\n", i, j);
			for(int a = 0; a < N_ALPHA; a++) {
				for(int b = 0; b < N_ALPHA; b++) {
					fprintf(out, "%.20e\t", W(b,j,a,i));
				}
				fprintf(out, "\n");
			}
		}
	}
}

void read_raw(char *filename, userdata *ud, conjugrad_float_t *x) {
	FILE *f = fopen(filename, "r");
	char *line = (char*) malloc(sizeof(char) * 8192);
	char* expected = malloc(1024);
	int ncol = ud->ncol;
	int nsingle_padded = ud->nsingle + N_ALPHA_PAD - (ud->nsingle % N_ALPHA_PAD);

	conjugrad_float_t *x1 = x;
	//conjugrad_float_t *x2 = x + ud->nsingle;
	conjugrad_float_t *x2 = &x[nsingle_padded];

	for(int i = 0; i < ncol; i++) {
		fgets(line, 8192, f);
		char *token = line;
		for(int a = 0; a < N_ALPHA-1; a++) {
			V(i, a) = strtod(token, &token);
		}
	}

	for(int i = 0; i < ncol; i++) {
		for(int j = i+1; j < ncol; j++) {
			fgets(line, 8192, f);

			snprintf(expected, 1024, "# %d %d\n", i, j);
{'fraction_non_alphanumeric': 0.20613562970936491, 'fraction_numerical': 0.024757804090419805, 'mean_word_length': 3.3271245634458673, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6084887', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1747, 'n_words': 427}