<commit_before>// Properties starting with _ are meant for the higher-level Nick browser
// Properties starting with __ are private to the driver
// Read-only properties should be configured as such

const TabDriver = require("./TabDriver")
//const chromeLauncher = require('chrome-launcher')
const CDP = require('chrome-remote-interface')
const _ = require("lodash")

class BrowserDriver {

	constructor(options) {
		this._options = options
	}

	exit(code) {
		process.exit(code)
	}

	_initialize(callback) {
		//console.log("init")

		//const options = {
		//	port: 9222,
		//	logLevel: "verbose",
		//	//chromePath: 'google-chrome',
		//	chromeFlags: [
		//		`--window-size=${this._options.width},${this._options.height}`,
		//		"--disable-gpu",
		//		"--headless",
		//		"--disable-web-security",
		//		"--allow-insecure-localhost",
		//		"--allow-running-insecure-content",
		//		"--allow-file-access-from-files",
		//		"--hide-scrollbars",
		//	]
		//}
		//chromeLauncher.launch(options).then(() => {
		//	console.log("launch ok")
		//	callback(null)
		//}).catch((err) => {
		//	console.log("launch NOT ok")
		//	callback(err)
		//})

		const execFile = require("child_process").execFile
		const chromePath = process.env.CHROME_PATH || "google-chrome-unstable"
		const child = execFile(chromePath, [
			"--remote-debugging-port=9222",
			// headless flags
			"--disable-gpu",
			"--headless",
			"--hide-scrollbars",
			// allow ugly things because we want to scrape without being bothered
			"--disable-web-security",
			"--allow-insecure-localhost",
			"--allow-running-insecure-content",
			"--allow-file-access-from-files",
			// set window size
			`--window-size=${this._options.width},${this._options.height}`,
			// flags taken from chrome-launcher
			"--disable-translate", // built-in Google Translate stuff
			"--disable-extensions",
			"--disable-sync", // google account sync
			"--disable-background-networking",
			"--safebrowsing-disable-auto-update",
			"--metrics-recording-only",
			"--disable-default-apps",
			"--no-first-run",
		])
		process.on("exit", () => {
			try {
				child.kill()
			} catch (e) {}
		})
		child.on("error", (err) => {
			callback(`could not start chrome: ${err}`) // it's alright, _initialize() is wrapped with once()
		})
		const cleanSocket = (socket) => {
			socket.removeAllListeners()
			socket.end()
			socket.destroy()
			socket.unref()
		}
		const net = require("net")
		const checkStart = Date.now()
		let nbChecks = 0
		const checkDebuggerPort = () => {
			setTimeout(() => {<commit_msg>add no-sandbox flag as an easy fix for running inside docker containers<commit_after>// Properties starting with _ are meant for the higher-level Nick browser
// Properties starting with __ are private to the driver
// Read-only properties should be configured as such

const TabDriver = require("./TabDriver")
//const chromeLauncher = require('chrome-launcher')
const CDP = require('chrome-remote-interface')
const _ = require("lodash")

class BrowserDriver {

	constructor(options) {
		this._options = options
	}

	exit(code) {
		process.exit(code)
	}

	_initialize(callback) {
		//console.log("init")

		//const options = {
		//	port: 9222,
		//	logLevel: "verbose",
		//	//chromePath: 'google-chrome',
		//	chromeFlags: [
		//		`--window-size=${this._options.width},${this._options.height}`,
		//		"--disable-gpu",
		//		"--headless",
		//		"--disable-web-security",
		//		"--allow-insecure-localhost",
		//		"--allow-running-insecure-content",
		//		"--allow-file-access-from-files",
		//		"--hide-scrollbars",
		//	]
		//}
		//chromeLauncher.launch(options).then(() => {
		//	console.log("launch ok")
		//	callback(null)
		//}).catch((err) => {
		//	console.log("launch NOT ok")
		//	callback(err)
		//})

		const execFile = require("child_process").execFile
		const chromePath = process.env.CHROME_PATH || "google-chrome-unstable"
		const child = execFile(chromePath, [
			"--remote-debugging-port=9222",
			// headless flags
			"--disable-gpu",
			"--headless",
			"--hide-scrollbars",
			// allow ugly things because we want to scrape without being bothered
			"--disable-web-security",
			"--allow-insecure-localhost",
			"--allow-running-insecure-content",
			"--allow-file-access-from-files",
			// set window size
			`--window-size=${this._options.width},${this._options.height}`,
			// flags taken from chrome-launcher
			"--disable-translate", // built-in Google Translate stuff
			"--disable-extensions",
			"--disable-sync", // google account sync
			"--disable-background-networking",
			"--safebrowsing-disable-auto-update",
			"--metrics-recording-only",
			"--disable-default-apps",
			"--no-first-run",
			"--no-sandbox", // TODO remove this
		])
		process.on("exit", () => {
			try {
				child.kill()
			} catch (e) {}
		})
		child.on("error", (err) => {
			callback(`could not start chrome: ${err}`) // it's alright, _initialize() is wrapped with once()
		})
		const cleanSocket = (socket) => {
			socket.removeAllListeners()
			socket.end()
			socket.destroy()
			socket.unref()
		}
		const net = require("net")
		const checkStart = Date.now()
		let nbChecks = 0
		const checkDebuggerPort = () => {
			setTimeout(() => {
{'fraction_non_alphanumeric': 0.18621220874253802, 'fraction_numerical': 0.0034662045060658577, 'mean_word_length': 3.0139103554868623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '795038', 'n_tokens_mistral': 1969, 'n_tokens_neox': 1664, 'n_words': 343}