<commit_before>/*
 * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
 * See the copyright notice in the ACK home directory, in the file "Copyright".
 */
/* $Header$ */
/* PREPROCESSOR: INPUT SKIP FUNCTIONS */

#include	"arith.h"
#include	"LLlex.h"
#include	"class.h"
#include	"input.h"

extern int InputLevel;

int
skipspaces(ch, skipnl)
	register int ch;
{
	/*	skipspaces() skips any white space and returns the first
		non-space character.
	*/
	register int nlseen = 0;

	for (;;) {
		while (class(ch) == STSKIP)
			ch = GetChar();
		if (skipnl && class(ch) == STNL) {
			ch = GetChar();
			LineNumber++;
			nlseen++;
			continue;
		}
		if (ch == TOKSEP && InputLevel) {
			ch = GetChar();
			continue;
		}

		/* \\\n are handled by trigraph */

		if (ch == '/') {
			ch = GetChar();
			if (ch == '*' && !InputLevel) {
				skipcomment();
				ch = GetChar();
			}
			else	{
				UnGetChar();
				return '/';
			}
		}
		else if (nlseen && ch == '#') {
			domacro();
			ch = GetChar();
		} else
			return ch;
	}
}

SkipToNewLine()
{
	register int ch;
	register int garbage = 0;
	register int delim = 0;

	while ((ch = GetChar()) != '\n') {
		if (delim) {
			if (ch == '\\') {
				if (GetChar() == '\n') break;
			} else if (ch == delim) {
				delim = 0;
			}
			continue;
		} else if (ch == '\'' || ch == '\"') {
			delim = ch;
			garbage = 1;
		} else if (ch == '/') {
			if ((ch = GetChar()) == '*' && !InputLevel) {
				skipcomment();
				continue;
			}
		}
		if (!is_wsp(ch))
			garbage = 1;
	}<commit_msg>in SkipToNewLine(), check for TOKSEP<commit_after>/*
 * (c) copyright 1987 by the Vrije Universiteit, Amsterdam, The Netherlands.
 * See the copyright notice in the ACK home directory, in the file "Copyright".
 */
/* $Header$ */
/* PREPROCESSOR: INPUT SKIP FUNCTIONS */

#include	"arith.h"
#include	"LLlex.h"
#include	"class.h"
#include	"input.h"

extern int InputLevel;

int
skipspaces(ch, skipnl)
	register int ch;
{
	/*	skipspaces() skips any white space and returns the first
		non-space character.
	*/
	register int nlseen = 0;

	for (;;) {
		while (class(ch) == STSKIP)
			ch = GetChar();
		if (skipnl && class(ch) == STNL) {
			ch = GetChar();
			LineNumber++;
			nlseen++;
			continue;
		}
		if (ch == TOKSEP && InputLevel) {
			ch = GetChar();
			continue;
		}

		/* \\\n are handled by trigraph */

		if (ch == '/') {
			ch = GetChar();
			if (ch == '*' && !InputLevel) {
				skipcomment();
				ch = GetChar();
			}
			else	{
				UnGetChar();
				return '/';
			}
		}
		else if (nlseen && ch == '#') {
			domacro();
			ch = GetChar();
		} else
			return ch;
	}
}

SkipToNewLine()
{
	register int ch;
	register int garbage = 0;
	register int delim = 0;

	while ((ch = GetChar()) != '\n') {
		if (delim) {
			if (ch == '\\') {
				if (GetChar() == '\n') break;
			} else if (ch == delim) {
				delim = 0;
			}
			continue;
		} else if (ch == '\'' || ch == '\"') {
			delim = ch;
			garbage = 1;
		} else if (ch == '/') {
			if ((ch = GetChar()) == '*' && !InputLevel) {
				skipcomment();
				continue;
			}
		}
		else if (ch == TOKSEP && InputLevel) {
			continue;
		}
		if (!is_wsp(ch))
			garbage = 1;
	}
{'fraction_non_alphanumeric': 0.18990155604953954, 'fraction_numerical': 0.006351222610352493, 'mean_word_length': 2.515625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3361087', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1199, 'n_words': 336}