<commit_before>shellHelper.grep = (function () {

return function (cmd) {
	var hlStart = "\033[01;31m\033[K";   // red
	var hlEnd = "\033[m\033[K";          // clear

	var flags = "g";
	var negate = false;

	// parse options/pattern

	try {
		// do this properly with https://github.com/substack/node-shell-quote
		// FIXME: still doesn't work in common cases, because the parser output ends up like:
		// [ ": false", "db.getParameter", { "op" : "(" }, { "op" : ")" } ]
		var parse = require('shell-quote').parse;
		var parser = function (str) {
			return parse(str);
		};
	} catch (e) {
		// crappy fallback
		print("grep: Warning: Unable to load `shell-quote` module");
		var parser = function (str) {
			return str.split(" ");
		};
	}
	var words = parser(cmd);

	var i = 0;
	while (i < words.length) {
		if (words[i] == "-i") {
			flags += "i";
		} else if (words[i] == "-v") {
			negate = true;
		// FIXME: -c
		// FIXME: --color/--nocolor
		// FIXME: -m
		// FIXME: -n
		// FIXME: -A/-B/-C would be awesome
		} else if (words[i] == "--") {
			i++; // eat the "--"
			break;
		} else if (words[i][0] != "-") {
			break;
		}
		i++;
	}
	// next word is the pattern
	var patternStr = words[i];
	i++;

	// get rid of the first i words, what's left is the actual cmd to run
	var prefix = "";
	for (var j = 0; j < i; j++) {
		prefix += words[j] + "  *";
	}
	cmd = cmd.replace(new RegExp(prefix), "");

	var pattern = new RegExp(patternStr, flags);

	var lines = print.captureAllOutput(function () { shellPrint(eval(cmd)) }).output;
	for (var l in lines) {<commit_msg>grep now parses properly, and can handle quoting/escaping in the pattern<commit_after>shellHelper.grep = (function () {

var parser = function (s) {
	var res = {
		options: [],
		pattern: "",
		cmd: "",
	};

	// this stup hand-constructed parser (including the mess in parsing the pattern) is actually the best/easiest option... >:(

	var i = 0;

	// optional: leading whitespace
	while (i < s.length && (s[i] == ' ' || s[i] == '\t') ) {
		i++;
	}

	// optional: one or more options (starts with -)
	while (i < s.length && s[i] == '-') {
		var word = '';
		while (i < s.length && (s[i] != ' ' && s[i] != '\t') ) {
			word += s[i];
			i++;
		}
		while (i < s.length && (s[i] == ' ' || s[i] == '\t') ) {
			i++;
		}
		// -- is special
		if (word == "--") {
			break;
		} else {
			res.options.push(word);
		}
	}

	// required: pattern
	const normal = 0;
	const singleQuote = 1;
	const doubleQuote = 2;
	var context = normal;
	while (true) {
		if (i >= s.length) {
			if (context == normal) {
				break;
			} else if (context == singleQuote) {
				throw "Unterminated single quote";
			} else if (context == doubleQuote) {
				throw "Unterminated double quote";
			}
		} else if (s[i] == ' ' || s[i] == '\t') {
			if (context == normal) {
				break;
			} else if (context == singleQuote || context == doubleQuote) {
				res.pattern += s[i];
			}
		} else if (s[i] == '\\') {
			i++;
			if (i >= s.length) {
				res.pattern += '\\';
				break;
			} else if (s[i] == '"' || s[i] == "'" || s[i] == '\\' || s[i] == ' ') {
				res.pattern += s[i];
			} else if (s[i] == 'a') {
				res.pattern += '\a';
			} else if (s[i] == 'b') {
				res.pattern += '\b';
			} else if (s[i] == 't') {
				res.pattern += '\t';
			} else if (s[i] == 'n') {
				res.pattern += '\n';
			} else if (s[i] == 'v') {
				res.pattern += '\v';
			} else if (s[i] == 'f') {
				res.pattern += '\f';
			} else if (s[i] == 'r') {
				res.pattern += '\r';
			} else {
				res.pattern += '\\' + s[i];
			}
		} else if (s[i] == "'") {
			if (context == normal) {
				context = singleQuote;
			} else if (context == singleQuote) {
				context = normal;
			} else if (context == doubleQuote) {
				res.pattern += s[i];
			}
		} else if (s[i] == '"') {
			if (context == normal) {
				context = doubleQuote;
			} else if (context == doubleQuote) {
				context = normal;
			} else if (context == singleQuote) {
				res.pattern += s[i];
			}
		} else {
			res.pattern += s[i];
		}
		i++;
	}

	// required: whitespace
	while (i < s.length && (s[i] == ' ' || s[i] == '\t') ) {
		i++;
	}

	// required: rest (cmd)
	res.cmd = s.substr(i);

	return res;
};


return function (str) {
	var hlStart = "\033[01;31m\033[K";   // red
	var hlEnd = "\033[m\033[K";          // clear

	var flags = "g";
	var negate = false;

	// parse options/pattern
	//print("str = " + tojson(str));
	var parsed = parser(str);
	//print("parsed = " + tojson(parsed));

	if (parsed.pattern == "") {
		print("grep: no cmd specified");
		return;
	}
	if (parsed.cmd == "") {
		print("grep: no cmd specified");
		return;
	}

	var i = 0;
	while (i < parsed.options.length) {
		if (parsed.options[i] == "-i") {
			flags += "i";
		} else if (parsed.options[i] == "-v") {
			negate = true;
		// FIXME: -c
		// FIXME: --color/--nocolor
		// FIXME: -m
		// FIXME: -n
		// FIXME: -A/-B/-C would be awesome
		}
		i++;
	}

	var pattern = new RegExp(parsed.pattern, flags);

	var lines = print.captureAllOutput(function () { shellPrint(eval(parsed.cmd)) }).output;
	for (var l in lines) {
{'fraction_non_alphanumeric': 0.21919431279620852, 'fraction_numerical': 0.007898894154818325, 'mean_word_length': 2.4038978494623655, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '313364', 'n_tokens_mistral': 2207, 'n_tokens_neox': 1945, 'n_words': 519}