<commit_before>/*
 * Copyright 2008-2009 Katholieke Universite<NAME>
 *
 * Use of this software is governed by the MIT license
 *
 * Written by <NAME>, K.U.Leuven, Departement
 * Computerwetenschappen, Celestijnenlaan 200A, B-3001 Leuven, Belgium
 */

#include <ctype.h>
#include <string.h>
#include <strings.h>
#include <isl/ctx.h>
#include <isl_stream_private.h>
#include <isl/map.h>
#include <isl/aff.h>

struct isl_keyword {
	char			*name;
	enum isl_token_type	type;
};

static int same_name(const void *entry, const void *val)
{
	const struct isl_keyword *keyword = (const struct isl_keyword *)entry;

	return !strcmp(keyword->name, val);
}

enum isl_token_type isl_stream_register_keyword(struct isl_stream *s,
	const char *name)
{
	struct isl_hash_table_entry *entry;
	struct isl_keyword *keyword;
	uint32_t name_hash;

	if (!s->keywords) {
		s->keywords = isl_hash_table_alloc(s->ctx, 10);
		if (!s->keywords)
			return ISL_TOKEN_ERROR;
		s->next_type = ISL_TOKEN_LAST;
	}

	name_hash = isl_hash_string(isl_hash_init(), name);

	entry = isl_hash_table_find(s->ctx, s->keywords, name_hash,
					same_name, name, 1);
	if (!entry)
		return ISL_TOKEN_ERROR;
	if (entry->data) {
		keyword = entry->data;
		return keyword->type;
	}

	keyword = isl_calloc_type(s->ctx, struct isl_keyword);
	if (!keyword)
		return ISL_TOKEN_ERROR;
	keyword->type = s->next_type++;
	keyword->name = strdup(name);
	if (!keyword->name) {
		free(keyword);
		return ISL_TOKEN_ERROR;
	}
	entry->data = keyword;

	return keyword->type;
}

struct isl_token *isl_token_new(isl_ctx *ctx,
	int line, int col, unsigned on_new_line)
{
	struct isl_token *tok = isl_alloc_type(ctx, struct isl_token);
	if (!tok)
		return NULL;
	tok->line = line;
	tok->col = col;
	tok->on_new_line = on_new_line;
	tok->is_keyword = 0;
	tok->u.s = NULL;
	return tok;
}

void isl_token_free(struct isl_token *tok)
{
	if (!tok)
		return;
	if (tok->type == ISL_TOKEN_VALUE)
		isl_int_clear(tok->u.v);
	else if (tok->type == ISL_TOKEN_MAP)
		isl_map_free(tok->u.map);
	else if (tok->type == ISL_TOKEN_AFF)
		isl_pw_aff_free(tok->u.pwaff);
	else
		free(tok->u.s);
	free(tok);
}

void isl_stream_error(struct isl_stream *s, struct isl_token *tok, char *msg)
{
	int line = tok ? tok->line : s->line;
	int col = tok ? tok->col : s->col;
	fprintf(stderr, "syntax error (%d, %d): %s\n", line, col, msg);
	if (tok) {
		if (tok->type < 256)
			fprintf(stderr, "got '%c'\n", tok->type);
		else if (tok->type == ISL_TOKEN_IDENT)
			fprintf(stderr, "got ident '%s'\n", tok->u.s);
		else if (tok->is_keyword)
			fprintf(stderr, "got keyword '%s'\n", tok->u.s);
		else if (tok->type == ISL_TOKEN_VALUE) {
			fprintf(stderr, "got value '");
			isl_int_print(stderr, tok->u.v, 0);
			fprintf(stderr, "'\n");
		} else if (tok->type == ISL_TOKEN_MAP) {
			isl_printer *p;
			fprintf(stderr, "got map '");
			p = isl_printer_to_file(s->ctx, stderr);
			p = isl_printer_print_map(p, tok->u.map);
			isl_printer_free(p);
			fprintf(stderr, "'\n");
		} else if (tok->type == ISL_TOKEN_AFF) {
			isl_printer *p;
			fprintf(stderr, "got affine expression '");
			p = isl_printer_to_file(s->ctx, stderr);
			p = isl_printer_print_pw_aff(p, tok->u.pwaff);
			isl_printer_free(p);
			fprintf(stderr, "'\n");
		} else if (tok->u.s)
			fprintf(stderr, "got token '%s'\n", tok->u.s);
		else
			fprintf(stderr, "got token type %d\n", tok->type);
	}
}

static struct isl_stream* isl_stream_new(struct isl_ctx *ctx)
{
	int i;
	struct isl_stream *s = isl_alloc_type(ctx, struct isl_stream);
	if (!s)
		return NULL;
	s->ctx = ctx;
	isl_ctx_ref(s->ctx);
	s->file = NULL;
	s->str = NULL;
	s->len = 0;
	s->line = 1;
	s->col = 0;
	s->eof = 0;
	s->c = -1;
	s->n_un = 0;
	for (i = 0; i < 5; ++i)
		s->tokens[i] = NULL;
	s->n_token = 0;
	s->keywords = NULL;
	s->size = 256;
	s->buffer = isl_alloc_array(ctx, char, s->size);
	if (!s->buffer)
		goto error;
	return s;
error:
	isl_stream_free(s);
	return NULL;
}

struct isl_stream* isl_stream_new_file(struct isl_ctx *ctx, FILE *file)
{
	struct isl_stream *s = isl_stream_new(ctx);
	if (!s)
		return NULL;
	s->file = file;
	return s;
}

struct isl_stream* isl_stream_new_str(struct isl_ctx *ctx, const char *str)
{
	struct isl_stream *s = isl_stream_new(ctx);
	if (!s)
		return NULL;
	s->str = str;
	return s;
}

static int stream_getc(struct isl_stream *s)
{
	int c;
	if (s->eof)
		return -1;
	if (s->n_un)
		return s->c = s->un[--s->n_un];
	if (s->file)
		c = fgetc(s->file);
	else {<commit_msg>isl_stream_new_str: fail on NULL input<commit_after>/*
 * Copyright 2008-2009 Katholieke Universiteit Leuven
 *
 * Use of this software is governed by the MIT license
 *
 * Written by <NAME>, K.U.Leuven, Departement
 * Computerwetenschappen, Celestijnenlaan 200A, B-3001 Leuven, Belgium
 */

#include <ctype.h>
#include <string.h>
#include <strings.h>
#include <isl/ctx.h>
#include <isl_stream_private.h>
#include <isl/map.h>
#include <isl/aff.h>

struct isl_keyword {
	char			*name;
	enum isl_token_type	type;
};

static int same_name(const void *entry, const void *val)
{
	const struct isl_keyword *keyword = (const struct isl_keyword *)entry;

	return !strcmp(keyword->name, val);
}

enum isl_token_type isl_stream_register_keyword(struct isl_stream *s,
	const char *name)
{
	struct isl_hash_table_entry *entry;
	struct isl_keyword *keyword;
	uint32_t name_hash;

	if (!s->keywords) {
		s->keywords = isl_hash_table_alloc(s->ctx, 10);
		if (!s->keywords)
			return ISL_TOKEN_ERROR;
		s->next_type = ISL_TOKEN_LAST;
	}

	name_hash = isl_hash_string(isl_hash_init(), name);

	entry = isl_hash_table_find(s->ctx, s->keywords, name_hash,
					same_name, name, 1);
	if (!entry)
		return ISL_TOKEN_ERROR;
	if (entry->data) {
		keyword = entry->data;
		return keyword->type;
	}

	keyword = isl_calloc_type(s->ctx, struct isl_keyword);
	if (!keyword)
		return ISL_TOKEN_ERROR;
	keyword->type = s->next_type++;
	keyword->name = strdup(name);
	if (!keyword->name) {
		free(keyword);
		return ISL_TOKEN_ERROR;
	}
	entry->data = keyword;

	return keyword->type;
}

struct isl_token *isl_token_new(isl_ctx *ctx,
	int line, int col, unsigned on_new_line)
{
	struct isl_token *tok = isl_alloc_type(ctx, struct isl_token);
	if (!tok)
		return NULL;
	tok->line = line;
	tok->col = col;
	tok->on_new_line = on_new_line;
	tok->is_keyword = 0;
	tok->u.s = NULL;
	return tok;
}

void isl_token_free(struct isl_token *tok)
{
	if (!tok)
		return;
	if (tok->type == ISL_TOKEN_VALUE)
		isl_int_clear(tok->u.v);
	else if (tok->type == ISL_TOKEN_MAP)
		isl_map_free(tok->u.map);
	else if (tok->type == ISL_TOKEN_AFF)
		isl_pw_aff_free(tok->u.pwaff);
	else
		free(tok->u.s);
	free(tok);
}

void isl_stream_error(struct isl_stream *s, struct isl_token *tok, char *msg)
{
	int line = tok ? tok->line : s->line;
	int col = tok ? tok->col : s->col;
	fprintf(stderr, "syntax error (%d, %d): %s\n", line, col, msg);
	if (tok) {
		if (tok->type < 256)
			fprintf(stderr, "got '%c'\n", tok->type);
		else if (tok->type == ISL_TOKEN_IDENT)
			fprintf(stderr, "got ident '%s'\n", tok->u.s);
		else if (tok->is_keyword)
			fprintf(stderr, "got keyword '%s'\n", tok->u.s);
		else if (tok->type == ISL_TOKEN_VALUE) {
			fprintf(stderr, "got value '");
			isl_int_print(stderr, tok->u.v, 0);
			fprintf(stderr, "'\n");
		} else if (tok->type == ISL_TOKEN_MAP) {
			isl_printer *p;
			fprintf(stderr, "got map '");
			p = isl_printer_to_file(s->ctx, stderr);
			p = isl_printer_print_map(p, tok->u.map);
			isl_printer_free(p);
			fprintf(stderr, "'\n");
		} else if (tok->type == ISL_TOKEN_AFF) {
			isl_printer *p;
			fprintf(stderr, "got affine expression '");
			p = isl_printer_to_file(s->ctx, stderr);
			p = isl_printer_print_pw_aff(p, tok->u.pwaff);
			isl_printer_free(p);
			fprintf(stderr, "'\n");
		} else if (tok->u.s)
			fprintf(stderr, "got token '%s'\n", tok->u.s);
		else
			fprintf(stderr, "got token type %d\n", tok->type);
	}
}

static struct isl_stream* isl_stream_new(struct isl_ctx *ctx)
{
	int i;
	struct isl_stream *s = isl_alloc_type(ctx, struct isl_stream);
	if (!s)
		return NULL;
	s->ctx = ctx;
	isl_ctx_ref(s->ctx);
	s->file = NULL;
	s->str = NULL;
	s->len = 0;
	s->line = 1;
	s->col = 0;
	s->eof = 0;
	s->c = -1;
	s->n_un = 0;
	for (i = 0; i < 5; ++i)
		s->tokens[i] = NULL;
	s->n_token = 0;
	s->keywords = NULL;
	s->size = 256;
	s->buffer = isl_alloc_array(ctx, char, s->size);
	if (!s->buffer)
		goto error;
	return s;
error:
	isl_stream_free(s);
	return NULL;
}

struct isl_stream* isl_stream_new_file(struct isl_ctx *ctx, FILE *file)
{
	struct isl_stream *s = isl_stream_new(ctx);
	if (!s)
		return NULL;
	s->file = file;
	return s;
}

struct isl_stream* isl_stream_new_str(struct isl_ctx *ctx, const char *str)
{
	struct isl_stream *s;
	if (!str)
		return NULL;
	s = isl_stream_new(ctx);
	if (!s)
		return NULL;
	s->str = str;
	return s;
}

static int stream_getc(struct isl_stream *s)
{
	int c;
	if (s->eof)
		return -1;
	if (s->n_un)
		return s->c = s->un[--s->n_un];
	if (s->file)
		c = fgetc(s->file);
	else {
{'fraction_non_alphanumeric': 0.16428571428571428, 'fraction_numerical': 0.008482142857142856, 'mean_word_length': 4.020168067226891, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 174, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6832914', 'n_tokens_mistral': 4033, 'n_tokens_neox': 3833, 'n_words': 876}