<commit_before>#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <errno.h>
/* dirname */
#include <libgen.h>
/* chdir, getcwd */
#include <unistd.h>
/* open */
#include <fcntl.h>

#include "pp.h"
#include "../util/util.h"
#include "../util/alloc.h"
#include "../util/platform.h"

struct def
{
	char *name, *val;
	struct def *next;
};


static struct def *defs = NULL;
static const char **dirs = NULL;
static int    ndirs = 0;
static FILE  *devnull;
static int    pp_verbose = 0;

int current_line = 0, current_chr = 0;
const char *current_fname = NULL;

static void ppdie(struct pp *p, const char *fmt, ...)
{
	va_list l;
	struct where where;

	if(p){
		where.fname = p->fname;
		where.line  = p->nline;
		where.chr   = -1;
	}

	va_start(l, fmt);
	vdie(&where, l, fmt);
}

void adddir(const char *d)
{
	if(!dirs){
		dirs = umalloc(2 * sizeof(*dirs));
		dirs[0] = d;
		dirs[1] = NULL;
		ndirs = 1;
	}else{
		dirs = realloc(dirs, (ndirs + 2) * sizeof(*dirs));
		if(!dirs)
			die(NULL, "realloc: %s\n", strerror(errno));

		dirs[ndirs++] = d;
		dirs[ndirs  ] = NULL;
	}

	if(pp_verbose)
		fprintf(stderr, "adddir(\"%s\")\n", d);
}

void adddef(const char *n, const char *v)
{
	/* FIXME: check for substring v in n */
	char *n2, *v2;
	struct def *d = umalloc(sizeof *d);

	n2 = ustrdup(n);
	v2 = ustrdup(v);

	strcpy(n2, n);
	strcpy(v2, v);

	d->name = n2;
	d->val  = v2;

	d->next = defs;
	defs = d;

	if(pp_verbose)
		fprintf(stderr, "adddef(\"%s\", \"%s\")\n", n, v);
}

static char *strdup_printf(const char *fmt, ...)
{
	va_list l;
	char *buf = NULL;
	int len = 8, ret;

	do{
		len *= 2;
		buf = urealloc(buf, len);
		va_start(l, fmt);
		ret = vsnprintf(buf, len, fmt, l);
		va_end(l);
	}while(ret >= len);

	return buf;
}

static struct def *getdef(const char *s)
{
	struct def *d;

	for(d = defs; d; d = d->next)
		if(!strcmp(d->name, s))
			return d;
	return NULL;
}

static void substitutedef(struct pp *p, char **line)
{
	struct def *d;
	char *pos;

	for(d = defs; d; d = d->next){
		while((pos = strstr(*line, d->name))){
			char *const post = pos + strlen(d->name);
			char *new;
			const char *val;

			*pos = post[-1] = '\0';

			if(!strcmp(d->name, "__LINE__")){
				static char linebuf[16];
				snprintf(linebuf, sizeof linebuf, "%d", p->nline);
				val = linebuf;
			}else if(!strcmp(d->name, "__FILE__")){
				val = p->fname; /* FIXME: quote */
			}else{
				val = d->val;
			}

			new = strdup_printf("%s%s%s", *line, val, post);
			free(*line);
			*line = new;
			d = defs;
			/*
			 * recursive defs - could be infinite loop, but oh well,
			 * run out of memory eventually
			 */
		}
	}
}

static void freedefs()
{
	struct def *d;
	while(defs){
		d = defs;
		defs = defs->next;
		free(d->name);
		free(d->val);
		free(d);
	}
}

static int pp(struct pp *p, int skip)
{
#define RET(x) do{ ret = x; goto fin; }while(0)
	int curwdfd;
	int ret;
	char *line, *nl;
	char *wd;

	/* save for "cd -" */
	curwdfd = open(".", O_RDONLY | O_DIRECTORY);
	if(curwdfd == -1)
		ppdie(p, "open(\".\"): %s", strerror(errno));

	/* make sure everything is relative to the file */
	wd = udirname(p->fname);
	if(chdir(wd))
		ppdie(p, "chdir(\"%s\"): %s", wd, strerror(errno));


	do{
		line = fline(p->in);
		p->nline++;

		if(!line){<commit_msg>open() and fchdir() fixes<commit_after>#define _POSIX_C_SOURCE 200809L
#include <stdio.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <errno.h>
/* dirname */
#include <libgen.h>
/* chdir, getcwd */
#include <unistd.h>
/* open */
#include <fcntl.h>

#include "pp.h"
#include "../util/util.h"
#include "../util/alloc.h"
#include "../util/platform.h"

struct def
{
	char *name, *val;
	struct def *next;
};


static struct def *defs = NULL;
static const char **dirs = NULL;
static int    ndirs = 0;
static FILE  *devnull;
static int    pp_verbose = 0;

int current_line = 0, current_chr = 0;
const char *current_fname = NULL;

static void ppdie(struct pp *p, const char *fmt, ...)
{
	va_list l;
	struct where where;

	if(p){
		where.fname = p->fname;
		where.line  = p->nline;
		where.chr   = -1;
	}

	va_start(l, fmt);
	vdie(&where, l, fmt);
}

void adddir(const char *d)
{
	if(!dirs){
		dirs = umalloc(2 * sizeof(*dirs));
		dirs[0] = d;
		dirs[1] = NULL;
		ndirs = 1;
	}else{
		dirs = realloc(dirs, (ndirs + 2) * sizeof(*dirs));
		if(!dirs)
			die(NULL, "realloc: %s\n", strerror(errno));

		dirs[ndirs++] = d;
		dirs[ndirs  ] = NULL;
	}

	if(pp_verbose)
		fprintf(stderr, "adddir(\"%s\")\n", d);
}

void adddef(const char *n, const char *v)
{
	/* FIXME: check for substring v in n */
	char *n2, *v2;
	struct def *d = umalloc(sizeof *d);

	n2 = ustrdup(n);
	v2 = ustrdup(v);

	strcpy(n2, n);
	strcpy(v2, v);

	d->name = n2;
	d->val  = v2;

	d->next = defs;
	defs = d;

	if(pp_verbose)
		fprintf(stderr, "adddef(\"%s\", \"%s\")\n", n, v);
}

static char *strdup_printf(const char *fmt, ...)
{
	va_list l;
	char *buf = NULL;
	int len = 8, ret;

	do{
		len *= 2;
		buf = urealloc(buf, len);
		va_start(l, fmt);
		ret = vsnprintf(buf, len, fmt, l);
		va_end(l);
	}while(ret >= len);

	return buf;
}

static struct def *getdef(const char *s)
{
	struct def *d;

	for(d = defs; d; d = d->next)
		if(!strcmp(d->name, s))
			return d;
	return NULL;
}

static void substitutedef(struct pp *p, char **line)
{
	struct def *d;
	char *pos;

	for(d = defs; d; d = d->next){
		while((pos = strstr(*line, d->name))){
			char *const post = pos + strlen(d->name);
			char *new;
			const char *val;

			*pos = post[-1] = '\0';

			if(!strcmp(d->name, "__LINE__")){
				static char linebuf[16];
				snprintf(linebuf, sizeof linebuf, "%d", p->nline);
				val = linebuf;
			}else if(!strcmp(d->name, "__FILE__")){
				val = p->fname; /* FIXME: quote */
			}else{
				val = d->val;
			}

			new = strdup_printf("%s%s%s", *line, val, post);
			free(*line);
			*line = new;
			d = defs;
			/*
			 * recursive defs - could be infinite loop, but oh well,
			 * run out of memory eventually
			 */
		}
	}
}

static void freedefs()
{
	struct def *d;
	while(defs){
		d = defs;
		defs = defs->next;
		free(d->name);
		free(d->val);
		free(d);
	}
}

static int pp(struct pp *p, int skip)
{
#define RET(x) do{ ret = x; goto fin; }while(0)
	int curwdfd;
	int ret;
	char *line, *nl;
	char *wd;

	/* save for "cd -" */
	curwdfd = open(".", O_RDONLY);
	if(curwdfd == -1)
		ppdie(p, "open(\".\"): %s", strerror(errno));

	/* make sure everything is relative to the file */
	wd = udirname(p->fname);
	if(chdir(wd))
		ppdie(p, "chdir(\"%s\"): %s", wd, strerror(errno));


	do{
		line = fline(p->in);
		p->nline++;

		if(!line){
{'fraction_non_alphanumeric': 0.201629942650166, 'fraction_numerical': 0.008753395713854513, 'mean_word_length': 3.1187072715972652, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591038', 'n_tokens_mistral': 3109, 'n_tokens_neox': 2877, 'n_words': 750}