<commit_before>"use strict";

var fs = require("fs");
var expect = require("chai").expect;
var PrismLoader = require("./prism-loader");

module.exports = {

	/**
	 * Runs the given test case file and asserts the result
	 *
	 * @param {string} language
	 * @param {string} filePath
	 */
	runTestCase: function (language, filePath) {
		var testCase = this.parseTestCaseFile(filePath);

		if (null === testCase) {
			throw new Error("Test case file has invalid format, please read the docs.");
		}

		var Prism = PrismLoader.createInstance(language);
		var compiledTokenStream = Prism.tokenize(testCase.testSource, Prism.languages[language]);
		var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream);

		expect(simplifiedTokenStream).to.eql(testCase.expectedTokenStream);
	},


	/**
	 * Simplifies the token stream to ease the matching with the expected token stream
	 *
	 * @param {string} tokenStream
	 * @returns {Array.<string[]>}
	 */
	transformCompiledTokenStream: function (tokenStream) {
		return tokenStream.filter(
			function (token) {
				// only support objects
				return (typeof token === "object");
			}
		).map(
			function (entry)
			{
				return [entry.type, entry.content];
			}
		);
	},


	/**
	 * Parses the test case from the given test case file
	 *
	 * @private
	 * @param {string} filePath
	 * @returns {{testSource: string, expectedTokenStream: *}|null}
	 */
	parseTestCaseFile: function (filePath) {
		var testCaseSource = fs.readFileSync(filePath, "utf8");
		var testCase = testCaseSource.split(/^----*\w*$/m);

		if (2 === testCase.length) {
			return {
				testSource: testCase[0].trim(),
				expectedTokenStream: JSON.parse(testCase[1])
			};
		}

		return null;
	}
};<commit_msg>Added support for test case comments<commit_after>"use strict";

var fs = require("fs");
var assert = require("chai").assert;
var PrismLoader = require("./prism-loader");

/**
 * Handles parsing of a test case file.
 *
 *
 * A test case file consists of at least two parts, separated by a line of dashes.
 * This separation line must start at the beginning of the line and consist of at least three dashes.
 *
 * The test case file can either consist of two parts:
 *
 *     {source code}
 *     ----
 *     {expected token stream}
 *
 *
 * or of three parts:
 *
 *     {source code}
 *     ----
 *     {expected token stream}
 *     ----
 *     {text comment explaining the test case}
 *
 * If the file contains more than three parts, the remaining parts are just ignored.
 * If the file however does not contain at least two parts (so no expected token stream),
 * the test case will later be marked as failed.
 *
 *
 * @type {{runTestCase: Function, transformCompiledTokenStream: Function, parseTestCaseFile: Function}}
 */
module.exports = {

	/**
	 * Runs the given test case file and asserts the result
	 *
	 * @param {string} language
	 * @param {string} filePath
	 */
	runTestCase: function (language, filePath) {
		var testCase = this.parseTestCaseFile(filePath);

		if (null === testCase) {
			throw new Error("Test case file has invalid format, please read the docs.");
		}

		var Prism = PrismLoader.createInstance(language);
		var compiledTokenStream = Prism.tokenize(testCase.testSource, Prism.languages[language]);
		var simplifiedTokenStream = this.transformCompiledTokenStream(compiledTokenStream);

		assert.deepEqual(simplifiedTokenStream, testCase.expectedTokenStream, testCase.comment);
	},


	/**
	 * Simplifies the token stream to ease the matching with the expected token stream
	 *
	 * @param {string} tokenStream
	 * @returns {Array.<string[]>}
	 */
	transformCompiledTokenStream: function (tokenStream) {
		return tokenStream.filter(
			function (token) {
				// only support objects
				return (typeof token === "object");
			}
		).map(
			function (entry)
			{
				return [entry.type, entry.content];
			}
		);
	},


	/**
	 * Parses the test case from the given test case file
	 *
	 * @private
	 * @param {string} filePath
	 * @returns {{testSource: string, expectedTokenStream: Array.<Array.<string>>, comment:string?}|null}
	 */
	parseTestCaseFile: function (filePath) {
		var testCaseSource = fs.readFileSync(filePath, "utf8");
		var testCaseParts = testCaseSource.split(/^----*\w*$/m);

		// No expected token stream found
		if (2 > testCaseParts.length) {
			return null;
		}

		var testCase = {
			testSource: testCaseParts[0].trim(),
			expectedTokenStream: JSON.parse(testCaseParts[1]),
			comment: null
		};

		// if there are three parts, the third one is the comment
		// explaining the test case
		if (testCaseParts[2]) {
			testCase.comment = testCaseParts[2].trim();
		}

		return testCase;
	}
};
{'fraction_non_alphanumeric': 0.12425149700598802, 'fraction_numerical': 0.00213857998289136, 'mean_word_length': 3.6215415019762847, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4861061', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1354, 'n_words': 430}