<commit_before>/*
 * SourceStream:
 * The SourceStream object implements the first two phases of translation.
 * It supplies methods for accessing a stream of characters from the source file.
 *
 * options -> Options object containing global information
 * string  -> Source code to stream from
 */

/*
 * 5.1.1.2 Translation Phases:
 *
 *   1. Physical source file multibyte characters are mapped, in an implementation-
 *      defined manner, to the source character set (introducing new-line characters for
 *      end-of-line indicators) if necessary. Trigraph sequences are replaced by
 *      corresponding single-character internal representations.
 *
 *   2. Each instance of a backslash character (\) immediately followed by a new-line
 *      character is deleted, splicing physical source lines to form logical source lines.
 *      Only the last backslash on any physical source line shall be eligible for being part
 *      of such a splice. A source file that is not empty shall end in a new-line character,
 *      which shall not be immediately preceded by a backslash character before any such
 *      splicing takes place.
 */

var SourceStream = function(options, string) {
	this.options = options;
	this.source = string;
	this.cursor = 0;
	this.line = 1;
	this.column = 1;
};

/*
 * Gets the character code of the source at the current cursor
 * and increments the cursor.
 */
SourceStream.prototype.nextch = function() {
	var ch;

	ch = this.peekch ();
	this.cursor++;

	/* If character is a newline character */
	if (ch === 10) {
		this.line++;
		this.column = 1;
	} else {
		this.column++;
	}

	return ch;
};

/*
 * Gets the character code of the source at the current cursor,
 * or zero if out-of-bounds
 */
SourceStream.prototype.peekch = function() {
	var ch;

	ch = this.source.charCodeAt (this.cursor) | 0;
	return ch;
};<commit_msg>Join lines with an ending backslash<commit_after>/*
 * SourceStream:
 * The SourceStream object implements the first two phases of translation.
 * It supplies methods for accessing a stream of characters from the source file.
 *
 * options -> Options object containing global information
 * string  -> Source code to stream from
 */

/*
 * 5.1.1.2 Translation Phases:
 *
 *   1. Physical source file multibyte characters are mapped, in an implementation-
 *      defined manner, to the source character set (introducing new-line characters for
 *      end-of-line indicators) if necessary. Trigraph sequences are replaced by
 *      corresponding single-character internal representations.
 *
 *   2. Each instance of a backslash character (\) immediately followed by a new-line
 *      character is deleted, splicing physical source lines to form logical source lines.
 *      Only the last backslash on any physical source line shall be eligible for being part
 *      of such a splice. A source file that is not empty shall end in a new-line character,
 *      which shall not be immediately preceded by a backslash character before any such
 *      splicing takes place.
 */

var SourceStream = function(options, string) {
	this.options = options;
	this.source = string;
	this.cursor = 0;
	this.line = 1;
	this.column = 1;
};

/*
 * Gets the character code of the source at the current cursor
 * and increments the cursor.
 */
SourceStream.prototype.nextch = function() {
	var ch;

	ch = this.peekch ();
	this.cursor++;

	switch (ch) {
		case 92: /* backslash */
			if (this.peekch () == 10) {
				/* backslash and newline - join lines */
				this.line++;
				this.cursor++;
				return this.nextch ();
			}
			break;
		case 10: /* newline */
			this.line++;
			this.column = 1;
			break;
		default:
			this.column++;
			break;
	}

	return ch;
};

/*
 * Gets the character code of the source at the current cursor,
 * or zero if out-of-bounds
 */
SourceStream.prototype.peekch = function() {
	var ch;

	ch = this.source.charCodeAt (this.cursor) | 0;
	return ch;
};
{'fraction_non_alphanumeric': 0.09088594704684318, 'fraction_numerical': 0.007637474541751527, 'mean_word_length': 3.153276955602537, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2525018', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1081, 'n_words': 467}