<commit_before>/*
 * token.c - Source code for tokens.
 *
 * Copyright (C) 2014 <NAME>
 *
 * This file released under the AGPLv3.
 *
 */

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <libgen.h>

#include "token.h"
#include "logger.h"
#include "parser.tab.h"

static const int TEXT_CHUNK_SIZE = 128;

static size_t token_sval_size;

static char *print_category(int t);
static void token_realloc_sval(struct token *t);
static void token_realloc_text(struct token *t, const char *s);

/* malloc token and assign values */
struct token *token_new(int category, int lineno,
                        const char *text, const char* filename)
{
	struct token *t = malloc(sizeof(*t));
	if (t == NULL)
		log_error("token_new(): could not malloc token");

	t->category = category;
	t->lineno = lineno;
	t->text = strdup(text);
	t->filename = strdup(filename);

	switch(category) {
	case INTEGER:
		t->ival = atoi(text);
		break;
	case FALSE:
		t->ival = 0;
		break;
	case TRUE:
		t->ival = 1;
		break;
	case FLOATING:
		t->fval = atof(text);
		break;
	case STRING:
		t->ssize = 0; /* append null later */
		token_sval_size = TEXT_CHUNK_SIZE;
		t->sval = calloc(token_sval_size, sizeof(char));
		break;
	default:
		break;
	}

	return t;
}

/* free internal values */
void token_free(struct token *t)
{
	log_assert(t);
	free(t->text);
	free(t->filename);
	if (t->category == STRING)
		free(t->sval);
	free(t);
}

/* print a token */
void token_print(struct token *t)
{
		char *filename = strdup(t->filename);
		log_assert(filename);

		printf("%-5d%-12s%-12s%s ",
		       t->lineno,
		       basename(filename),
		       print_category(t->category),
		       t->text);

		free(filename);

		if (t->category == INTEGER)
			printf("-> %d", t->ival);
		else if (t->category == FLOATING)
			printf("-> %f", t->fval);
		else if (t->category == CHARACTER)
			printf("-> %c", t->ival);
		else if (t->category == STRING)
			printf("-> %s", t->sval);

		printf("\n");
}

/*
 * append a single char to sval (used for processing escape codes)
 */
void token_push_sval_char(struct token *t, char c)
{
	++t->ssize;
	token_realloc_sval(t);
	t->sval[t->ssize-1] = c; /* 0 indexed */
}

/*
 * append a string (without escape codes) to sval
 *
 * memcpy is used over strcat because the current value of sval is not
 * guaranteed to be a null-terminated string, and may very well have
 * embedded null characters
 */
void token_push_sval_string(struct token *t, const char *s)
{
	size_t end = t->ssize;
	t->ssize += strlen(s);
	token_realloc_sval(t);
	memcpy(t->sval + end, s, strlen(s));
}

/*
 * called at end of string pattern
 *
 * Resets token_sval_size to 0. Appends terminating null
 * character. Shrinks string with realloc to recorded length of
 * string. 'strlen' cannot be used because of potentially embedded
 * null charaters.
 */
void token_finish_sval(struct token *t)
{
	token_sval_size = 0;
	++t->ssize;
	token_push_sval_char(t, '\0');
	t->sval = realloc(t->sval, t->ssize);
	log_assert(t->sval);
}

/*
 * append string to text field (these are actual null terminated
 * strings, so the issues with sval do not apply)<commit_msg>Fixing bug in token.c when appending null<commit_after>/*
 * token.c - Source code for tokens.
 *
 * Copyright (C) 2014 <NAME>
 *
 * This file released under the AGPLv3.
 *
 */

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <libgen.h>

#include "token.h"
#include "logger.h"
#include "parser.tab.h"

static const int TEXT_CHUNK_SIZE = 128;

static size_t token_sval_size;

static char *print_category(int t);
static void token_realloc_sval(struct token *t);
static void token_realloc_text(struct token *t, const char *s);

/* malloc token and assign values */
struct token *token_new(int category, int lineno,
                        const char *text, const char* filename)
{
	struct token *t = malloc(sizeof(*t));
	if (t == NULL)
		log_error("token_new(): could not malloc token");

	t->category = category;
	t->lineno = lineno;
	t->text = strdup(text);
	t->filename = strdup(filename);

	switch(category) {
	case INTEGER:
		t->ival = atoi(text);
		break;
	case FALSE:
		t->ival = 0;
		break;
	case TRUE:
		t->ival = 1;
		break;
	case FLOATING:
		t->fval = atof(text);
		break;
	case STRING:
		t->ssize = 0; /* append null later */
		token_sval_size = TEXT_CHUNK_SIZE;
		t->sval = calloc(token_sval_size, sizeof(char));
		break;
	default:
		break;
	}

	return t;
}

/* free internal values */
void token_free(struct token *t)
{
	log_assert(t);
	free(t->text);
	free(t->filename);
	if (t->category == STRING)
		free(t->sval);
	free(t);
}

/* print a token */
void token_print(struct token *t)
{
		char *filename = strdup(t->filename);
		log_assert(filename);

		printf("%-5d%-12s%-12s%s ",
		       t->lineno,
		       basename(filename),
		       print_category(t->category),
		       t->text);

		free(filename);

		if (t->category == INTEGER)
			printf("-> %d", t->ival);
		else if (t->category == FLOATING)
			printf("-> %f", t->fval);
		else if (t->category == CHARACTER)
			printf("-> %c", t->ival);
		else if (t->category == STRING)
			printf("-> %s", t->sval);

		printf("\n");
}

/*
 * append a single char to sval (used for processing escape codes)
 */
void token_push_sval_char(struct token *t, char c)
{
	++t->ssize;
	token_realloc_sval(t);
	t->sval[t->ssize-1] = c; /* 0 indexed */
}

/*
 * append a string (without escape codes) to sval
 *
 * memcpy is used over strcat because the current value of sval is not
 * guaranteed to be a null-terminated string, and may very well have
 * embedded null characters
 */
void token_push_sval_string(struct token *t, const char *s)
{
	size_t end = t->ssize;
	t->ssize += strlen(s);
	token_realloc_sval(t);
	memcpy(t->sval + end, s, strlen(s));
}

/*
 * called at end of string pattern
 *
 * Resets token_sval_size to 0. Appends terminating null
 * character. Shrinks string with realloc to recorded length of
 * string. 'strlen' cannot be used because of potentially embedded
 * null characters.
 */
void token_finish_sval(struct token *t)
{
	token_sval_size = 0;
	token_push_sval_char(t, '\0');
	t->sval = realloc(t->sval, t->ssize);
	log_assert(t->sval);
}

/*
 * append string to text field (these are actual null terminated
 * strings, so the issues with sval do not apply)
{'fraction_non_alphanumeric': 0.1491255961844197, 'fraction_numerical': 0.006677265500794912, 'mean_word_length': 3.5620014503263233, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1927036', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2280, 'n_words': 666}