<commit_before>char rcsid_lex[] = "$Id$";

#include <ctype.h>
#include <stdio.h>
#include <string.h>
#include "b.h"
#include "fe.h"
#include "gram.h"

static char buf[BUFSIZ];

static int yyline = 1;

typedef int (*ReadFn) ARGS((void));

static char *StrCopy ARGS((char *));
static int code_get ARGS((void));
static int simple_get ARGS((void));
static void ReadCharString ARGS((ReadFn, int));
static void ReadCodeBlock ARGS((void));
static void ReadOldComment ARGS((ReadFn));

extern char *inFileName;

static char *
StrCopy(s) char *s;
{
	char *t = (char *)zalloc(strlen(s) + 1);
	strcpy(t,s);
	return t;
}

static int
simple_get()
{
	int ch;
	if ((ch = getchar()) == '\n') {
		yyline++;
	}
	return ch;
}

static int
code_get()
{
	int ch;
	if ((ch = getchar()) == '\n') {
		yyline++;
	}
	if (ch != EOF) {
		fputc(ch, outfile);
	}
	return ch;
}

void
yypurge()
{
        char linetag[sizeof ("#line ????? ''\n") + FILENAME_MAX];

        /* print `#line' tag */
        if (inFileName) {
            sprintf (linetag, "#line %i \"%s\"\n", yyline, inFileName);
            fputs (linetag, outfile);
        }

	while (code_get() != EOF) ;
}


static void
ReadCharString(rdfn, which) ReadFn rdfn; int which;
{
	int ch;
	int backslash = 0;
	int firstline = yyline;

	while ((ch = rdfn()) != EOF) {
		if (ch == which && !backslash) {
			return;
		}
		if (ch == '\\' && !backslash) {
			backslash = 1;
		} else {
			backslash = 0;
		}
	}
	yyerror1("Unexpected EOF in string on line ");
	fprintf(stderr, "%d\n", firstline);
	exit(1);
}

static void
ReadOldComment(rdfn) ReadFn rdfn;
{
	/* will not work for comments delimiter in string */

	int ch;
	int starred = 0;
	int firstline = yyline;

	while ((ch = rdfn()) != EOF) {
		if (ch == '*') {
			starred = 1;
		} else if (ch == '/' && starred) {
			return;
		} else {
			starred = 0;
		}
	}
	yyerror1("Unexpected EOF in comment on line ");
	fprintf(stderr, "%d\n", firstline);
	exit(1);
}

static void
ReadCodeBlock()
{
	int ch;
	int firstline = yyline;
        char linetag[sizeof ("#line ????? ''\n") + FILENAME_MAX];

        /* print `#line' tag */
        if (inFileName) {
            sprintf (linetag, "#line %i \"%s\"\n", firstline, inFileName);
            fputs (linetag, outfile);
        }

	while ((ch = getchar()) != EOF) {
		if (ch == '%') {
			ch = getchar();
			if (ch != '}') {
				yyerror("bad %%");
			}
			return;
		}
		fputc(ch, outfile);
		if (ch == '\n') {
			yyline++;
		}<commit_msg>Though <NAME>'s BURG code has proven amazingly solid, in fact I found a bug in there over the weekend.<commit_after>char rcsid_lex[] = "$Id$";

#include <ctype.h>
#include <stdio.h>
#include <string.h>
#include "b.h"
#include "fe.h"
#include "gram.h"

static char buf[BUFSIZ];

static int yyline = 1;

typedef int (*ReadFn) ARGS((void));

static char *StrCopy ARGS((char *));
static int code_get ARGS((void));
static int simple_get ARGS((void));
static void ReadCharString ARGS((ReadFn, int));
static void ReadCodeBlock ARGS((void));
static void ReadOldComment ARGS((ReadFn));

extern char *inFileName;

static char *
StrCopy(s) char *s;
{
	char *t = (char *)zalloc(strlen(s) + 1);
	strcpy(t,s);
	return t;
}

static int
simple_get()
{
	int ch;
	if ((ch = getchar()) == '\n') {
		yyline++;
	}
	return ch;
}

static int
code_get()
{
	int ch;
	if ((ch = getchar()) == '\n') {
		yyline++;
	}
	if (ch != EOF) {
		fputc(ch, outfile);
	}
	return ch;
}

void
yypurge()
{
        char linetag[sizeof ("#line ????? ''\n") + FILENAME_MAX];

        /* print `#line' tag */
        if (inFileName) {
            sprintf (linetag, "#line %i \"%s\"\n", yyline, inFileName);
            fputs (linetag, outfile);
        }

	while (code_get() != EOF) ;
}


static void
ReadCharString(rdfn, which) ReadFn rdfn; int which;
{
	int ch;
	int backslash = 0;
	int firstline = yyline;

	while ((ch = rdfn()) != EOF) {
		if (ch == which && !backslash) {
			return;
		}
		if (ch == '\\' && !backslash) {
			backslash = 1;
		} else {
			backslash = 0;
		}
	}
	yyerror1("Unexpected EOF in string on line ");
	fprintf(stderr, "%d\n", firstline);
	exit(1);
}

static void
ReadOldComment(rdfn) ReadFn rdfn;
{
	/* will not work for comments delimiter in string */

	int ch;
	int starred = 0;
	int firstline = yyline;

	while ((ch = rdfn()) != EOF) {
		if (ch == '*') {
			starred = 1;
		} else if (ch == '/' && starred) {
			return;
		} else {
			starred = 0;
		}
	}
	yyerror1("Unexpected EOF in comment on line ");
	fprintf(stderr, "%d\n", firstline);
	exit(1);
}

static void
ReadCodeBlock()
{
	int ch;
	int firstline = yyline;
        char linetag[sizeof ("#line ????? ''\n") + FILENAME_MAX];

        /* print `#line' tag */
        if (inFileName) {
            sprintf (linetag, "#line %i \"%s\"\n", firstline, inFileName);
            fputs (linetag, outfile);
        }

	while ((ch = getchar()) != EOF) {
		if (ch == '%') {
			ch = getchar();
			if (ch != '}')
                            fputc ('%', outfile);
                        else
                            return;
		}
		fputc(ch, outfile);
		if (ch == '\n') {
			yyline++;
		}
{'fraction_non_alphanumeric': 0.1855649773220272, 'fraction_numerical': 0.004732794320646815, 'mean_word_length': 2.7850746268656716, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '472782', 'n_tokens_mistral': 2102, 'n_tokens_neox': 1950, 'n_words': 534}