<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

//@ts-check

const path = require('path');
const glob = require('glob');
const events = require('events');
const mocha = require('mocha');
const createStatsCollector = require('../../../node_modules/mocha/lib/stats-collector');
const MochaJUnitReporter = require('mocha-junit-reporter');
const url = require('url');
const minimatch = require('minimatch');
const playwright = require('playwright');

// opts
const defaultReporterName = process.platform === 'win32' ? 'list' : 'spec';
const optimist = require('optimist')
	// .describe('grep', 'only run tests matching <pattern>').alias('grep', 'g').alias('grep', 'f').string('grep')
	.describe('build', 'run with build output (out-build)').boolean('build')
	.describe('run', 'only run tests matching <relative_file_path>').string('run')
	.describe('glob', 'only run tests matching <glob_pattern>').string('glob')
	.describe('debug', 'do not run browsers headless').boolean('debug')
	.describe('browser', 'browsers in which tests should run').string('browser').default('browser', ['chromium', 'firefox', 'webkit'])
	.describe('reporter', 'the mocha reporter').string('reporter').default('reporter', defaultReporterName)
	.describe('reporter-options', 'the mocha reporter options').string('reporter-options').default('reporter-options', '')
	.describe('tfs', 'tfs').string('tfs')
	.describe('help', 'show the help').alias('help', 'h');

// logic
const argv = optimist.argv;

if (argv.help) {
	optimist.showHelp();
	process.exit(0);
}

const withReporter = (function () {
	if (argv.tfs) {
		{
			return (browserType, runner) => {
				new mocha.reporters.Spec(runner);
				new MochaJUnitReporter(runner, {
					reporterOptions: {
						testsuitesTitle: `${argv.tfs} ${process.platform}`,
						mochaFile: process.env.BUILD_ARTIFACTSTAGINGDIRECTORY ? path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${browserType}-${argv.tfs.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`) : undefined
					}
				});
			}
		}
	} else {
		const reporterPath = path.join(path.dirname(require.resolve('mocha')), 'lib', 'reporters', argv.reporter);
		let ctor;

		try {
			ctor = require(reporterPath);
		} catch (err) {
			try {
				ctor = require(argv.reporter);
			} catch (err) {
				ctor = process.platform === 'win32' ? mocha.reporters.List : mocha.reporters.Spec;
				console.warn(`could not load reporter: ${argv.reporter}, using ${ctor.name}`);
			}
		}

		function parseReporterOption(value) {
			let r = /^([^=]+)=(.*)$/.exec(value);
			return r ? { [r[1]]: r[2] } : {};
		}

		let reporterOptions = argv['reporter-options'];
		reporterOptions = typeof reporterOptions === 'string' ? [reporterOptions] : reporterOptions;
		reporterOptions = reporterOptions.reduce((r, o) => Object.assign(r, parseReporterOption(o)), {});

		return (_, runner) => new ctor(runner, { reporterOptions })
	}
})()

const outdir = argv.build ? 'out-build' : 'out';
const out = path.join(__dirname, `../../../${outdir}`);

function ensureIsArray(a) {
	return Array.isArray(a) ? a : [a];
}

const testModules = (async function () {

	const excludeGlob = '**/{node,electron-sandbox,electron-browser,electron-main}/**/*.test.js';
	let isDefaultModules = true;
	let promise;

	if (argv.run) {
		// use file list (--run)
		isDefaultModules = false;
		promise = Promise.resolve(ensureIsArray(argv.run).map(file => {
			file = file.replace(/^src/, 'out');
			file = file.replace(/\.ts$/, '.js');
			return path.relative(out, file);
		}));

	} else {
		// glob patterns (--glob)
		const defaultGlob = '**/*.test.js';
		const pattern = argv.glob || defaultGlob
		isDefaultModules = pattern === defaultGlob;

		promise = new Promise((resolve, reject) => {
			glob(pattern, { cwd: out }, (err, files) => {
				if (err) {
					reject(err);
				} else {
					resolve(files)
				}
			});
		});
	}

	return promise.then(files => {
		const modules = [];
		for (let file of files) {
			if (!minimatch(file, excludeGlob)) {
				modules.push(file.replace(/\.js$/, ''));

			} else if (!isDefaultModules) {
				console.warn(`DROPPONG ${file} because it cannot be run inside a browser`);
			}
		}
		return modules;
	})
})();


async function runTestsInBrowser(testModules, browserType) {
	const args = process.platform === 'linux' && browserType === 'chromium' ? ['--no-sandbox'] : undefined; // disable sandbox to run chrome on certain Linux distros
	const browser = await playwright[browserType].launch({ headless: !Boolean(argv.debug), args });
	const context = await browser.newContext();
	const page = await context.newPage();
	const target = url.pathToFileURL(path.join(__dirname, 'renderer.html'));
	if (argv.build) {
		target.search = `?build=true`;
	}
	await page.goto(target.href);

	const emitter = new events.EventEmitter();
	await page.exposeFunction('mocha_report', (type, data1, data2) => {
		emitter.emit(type, data1, data2)
	});

	page.on('console', async msg => {
		console[msg.type()](msg.text(), await Promise.all(msg.args().map(async arg => await arg.jsonValue())));
	});

	withReporter(browserType, new EchoRunner(emitter, browserType.toUpperCase()));
<commit_msg>browser tests - fix regression from updating playwright<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

//@ts-check

const path = require('path');
const glob = require('glob');
const events = require('events');
const mocha = require('mocha');
const createStatsCollector = require('../../../node_modules/mocha/lib/stats-collector');
const MochaJUnitReporter = require('mocha-junit-reporter');
const url = require('url');
const minimatch = require('minimatch');
const playwright = require('playwright');

// opts
const defaultReporterName = process.platform === 'win32' ? 'list' : 'spec';
const optimist = require('optimist')
	// .describe('grep', 'only run tests matching <pattern>').alias('grep', 'g').alias('grep', 'f').string('grep')
	.describe('build', 'run with build output (out-build)').boolean('build')
	.describe('run', 'only run tests matching <relative_file_path>').string('run')
	.describe('glob', 'only run tests matching <glob_pattern>').string('glob')
	.describe('debug', 'do not run browsers headless').boolean('debug')
	.describe('browser', 'browsers in which tests should run').string('browser').default('browser', ['chromium', 'firefox', 'webkit'])
	.describe('reporter', 'the mocha reporter').string('reporter').default('reporter', defaultReporterName)
	.describe('reporter-options', 'the mocha reporter options').string('reporter-options').default('reporter-options', '')
	.describe('tfs', 'tfs').string('tfs')
	.describe('help', 'show the help').alias('help', 'h');

// logic
const argv = optimist.argv;

if (argv.help) {
	optimist.showHelp();
	process.exit(0);
}

const withReporter = (function () {
	if (argv.tfs) {
		{
			return (browserType, runner) => {
				new mocha.reporters.Spec(runner);
				new MochaJUnitReporter(runner, {
					reporterOptions: {
						testsuitesTitle: `${argv.tfs} ${process.platform}`,
						mochaFile: process.env.BUILD_ARTIFACTSTAGINGDIRECTORY ? path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${browserType}-${argv.tfs.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`) : undefined
					}
				});
			}
		}
	} else {
		const reporterPath = path.join(path.dirname(require.resolve('mocha')), 'lib', 'reporters', argv.reporter);
		let ctor;

		try {
			ctor = require(reporterPath);
		} catch (err) {
			try {
				ctor = require(argv.reporter);
			} catch (err) {
				ctor = process.platform === 'win32' ? mocha.reporters.List : mocha.reporters.Spec;
				console.warn(`could not load reporter: ${argv.reporter}, using ${ctor.name}`);
			}
		}

		function parseReporterOption(value) {
			let r = /^([^=]+)=(.*)$/.exec(value);
			return r ? { [r[1]]: r[2] } : {};
		}

		let reporterOptions = argv['reporter-options'];
		reporterOptions = typeof reporterOptions === 'string' ? [reporterOptions] : reporterOptions;
		reporterOptions = reporterOptions.reduce((r, o) => Object.assign(r, parseReporterOption(o)), {});

		return (_, runner) => new ctor(runner, { reporterOptions })
	}
})()

const outdir = argv.build ? 'out-build' : 'out';
const out = path.join(__dirname, `../../../${outdir}`);

function ensureIsArray(a) {
	return Array.isArray(a) ? a : [a];
}

const testModules = (async function () {

	const excludeGlob = '**/{node,electron-sandbox,electron-browser,electron-main}/**/*.test.js';
	let isDefaultModules = true;
	let promise;

	if (argv.run) {
		// use file list (--run)
		isDefaultModules = false;
		promise = Promise.resolve(ensureIsArray(argv.run).map(file => {
			file = file.replace(/^src/, 'out');
			file = file.replace(/\.ts$/, '.js');
			return path.relative(out, file);
		}));

	} else {
		// glob patterns (--glob)
		const defaultGlob = '**/*.test.js';
		const pattern = argv.glob || defaultGlob
		isDefaultModules = pattern === defaultGlob;

		promise = new Promise((resolve, reject) => {
			glob(pattern, { cwd: out }, (err, files) => {
				if (err) {
					reject(err);
				} else {
					resolve(files)
				}
			});
		});
	}

	return promise.then(files => {
		const modules = [];
		for (let file of files) {
			if (!minimatch(file, excludeGlob)) {
				modules.push(file.replace(/\.js$/, ''));

			} else if (!isDefaultModules) {
				console.warn(`DROPPONG ${file} because it cannot be run inside a browser`);
			}
		}
		return modules;
	})
})();

function consoleLogFn(msg) {
	const type = msg.type();
	const candidate = console[type];
	if (candidate) {
		return candidate;
	}

	if (type === 'warning') {
		return console.warn;
	}

	return console.log;
}

async function runTestsInBrowser(testModules, browserType) {
	const args = process.platform === 'linux' && browserType === 'chromium' ? ['--no-sandbox'] : undefined; // disable sandbox to run chrome on certain Linux distros
	const browser = await playwright[browserType].launch({ headless: !Boolean(argv.debug), args });
	const context = await browser.newContext();
	const page = await context.newPage();
	const target = url.pathToFileURL(path.join(__dirname, 'renderer.html'));
	if (argv.build) {
		target.search = `?build=true`;
	}
	await page.goto(target.href);

	const emitter = new events.EventEmitter();
	await page.exposeFunction('mocha_report', (type, data1, data2) => {
		emitter.emit(type, data1, data2)
	});

	page.on('console', async msg => {
		consoleLogFn(msg)(msg.text(), await Promise.all(msg.args().map(async arg => await arg.jsonValue())));
	});

	withReporter(browserType, new EchoRunner(emitter, browserType.toUpperCase()));

{'fraction_non_alphanumeric': 0.21174693442331616, 'fraction_numerical': 0.001954860494046561, 'mean_word_length': 3.705267558528428, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '487791', 'n_tokens_mistral': 3859, 'n_tokens_neox': 3533, 'n_words': 789}