<commit_before>
function removeLineComments(line){
	var pos = line.indexOf('//');
	if(pos !== -1){
		//console.log('removing line comments: ' + line)
		return line.substring(0, pos);
	}else{
		//console.log('no comments: ' + line)
		return line;
	}
}

function removeMultilineComments(lines){
	
	var inComment = false;
	for(var j=0;j<lines.length;++j){
		var line = lines[j];
		if(inComment){
			var ei = line.indexOf('*/');
			if(ei !== -1){
				inComment = false;
				lines[j] = line.substr(ei+2);
			}else{
				lines.splice(j, 1);
			}
			--j;
		}else{
			var i = line.indexOf('/*');
			if(i !== -1){
				var ei = line.indexOf('*/', i);
				if(ei !== -1){
					lines[j] = line.substr(0, i) + line.substr(ei+2);
					--j;
				}else{
					lines[j] = line.substr(0, i);
					inComment = true;
				}
			}
		}
	}
}

function countIndents(line){
	var k=0;
	while(k < line.length && line.charAt(k) === '\t'){
		++k;
	}
	return k;
}

function splitOn(str, delim){
	var r = [];
	var cur = '';
	var rb = 0;
	var sb = 0;
	var cb = 0;
	for(var i=0;i<str.length;++i){
		var c = str.charAt(i);
		if(c === '[') ++sb;
		else if(c === ']') --sb;
		else if(c === '(') ++sb;
		else if(c === ')') --sb;
		else if(c === '{') ++sb;
		else if(c === '}') --sb;
		else if(c === '<') ++sb;
		else if(c === '>') --sb;

		if(c === delim && rb === 0 && sb === 0 && cb === 0){
			if(cur.length > 0){
				r.push(cur);
				cur = '';
			}
		}else{
			cur += c;			
		}
	}
	if(cur.length > 0) r.push(cur);
	//console.log('[' + str + ']');
	//console.log(r);
	return r;
}

function parseTokens(line){
	var tempTokens = splitOn(line, ' ');//line.split(' ');
	//console.log(tempTokens);
	var tokens = [];
	for(var k=0;k<tempTokens.length;++k){
		var token = tempTokens[k];
		token = token.trim();
		if(token.length > 0) tokens.push(token);
	}
	return tokens;
}

//var sys = require('sys');

exports.parse = function(str){

	if(typeof(str) !== 'string'){
		throw new Error('parse str is not of type string');
	}

	var actualLines = str.split('\n');
	for(var i=0;i<actualLines.length;++i){
		actualLines[i] = removeLineComments(actualLines[i]);
	}
	removeMultilineComments(actualLines);
	var reStr = ''
	for(var i=0;i<actualLines.length;++i){
		reStr += actualLines[i]+'\n'
	}
	var lines = splitOn(reStr, '\n')//str.split('\n');
	
	//console.log(lines);
	//removeMultilineComments(lines);
	///sys.debug(lines);

	var stack = [{tokens: [], children: []}];
	var depth = 0;
	for(var i=0;i<lines.length;++i){
	
		var line = lines[i]//removeLineComments(lines[i]);
		
		if(line.trim().length === 0) continue;
		
		var indent = countIndents(line);
		line = line.substr(indent);
		if(indent < depth){
			var many = depth - indent;
			for(var j=0;j<many;++j){stack.pop();}
		}else if(indent > depth){
			console.log('invalid indentation on line ' + (i+1));
			console.log('line: ' + lines[i]);
			throw 'parse error';
		}
			
		var node = {tokens: parseTokens(line), children: []};
		stack[stack.length-1].children.push(node);
		stack.push(node);
		depth = indent + 1;
	}
	
	return stack[0];
}

exports.printAst = function(ast, tabs){
	var str = '';
	str += (tabs || '') + ast.tokens.join(' ') + '\n';
	for(var i=0;i<ast.children.length;++i){
		str += exports.printAst(ast.children[i], tabs !== undefined ? tabs + '\t' : '');
	}
	return str;<commit_msg>added pass-through of original line in string variable<commit_after>
function removeLineComments(line){
	var pos = line.indexOf('//');
	if(pos !== -1){
		//console.log('removing line comments: ' + line)
		return line.substring(0, pos);
	}else{
		//console.log('no comments: ' + line)
		return line;
	}
}

function removeMultilineComments(lines){
	
	var inComment = false;
	for(var j=0;j<lines.length;++j){
		var line = lines[j];
		if(inComment){
			var ei = line.indexOf('*/');
			if(ei !== -1){
				inComment = false;
				lines[j] = line.substr(ei+2);
			}else{
				lines.splice(j, 1);
			}
			--j;
		}else{
			var i = line.indexOf('/*');
			if(i !== -1){
				var ei = line.indexOf('*/', i);
				if(ei !== -1){
					lines[j] = line.substr(0, i) + line.substr(ei+2);
					--j;
				}else{
					lines[j] = line.substr(0, i);
					inComment = true;
				}
			}
		}
	}
}

function countIndents(line){
	var k=0;
	while(k < line.length && line.charAt(k) === '\t'){
		++k;
	}
	return k;
}

function splitOn(str, delim){
	var r = [];
	var cur = '';
	var rb = 0;
	var sb = 0;
	var cb = 0;
	for(var i=0;i<str.length;++i){
		var c = str.charAt(i);
		if(c === '[') ++sb;
		else if(c === ']') --sb;
		else if(c === '(') ++sb;
		else if(c === ')') --sb;
		else if(c === '{') ++sb;
		else if(c === '}') --sb;
		else if(c === '<') ++sb;
		else if(c === '>') --sb;

		if(c === delim && rb === 0 && sb === 0 && cb === 0){
			if(cur.length > 0){
				r.push(cur);
				cur = '';
			}
		}else{
			cur += c;			
		}
	}
	if(cur.length > 0) r.push(cur);
	//console.log('[' + str + ']');
	//console.log(r);
	return r;
}

function parseTokens(line){
	var tempTokens = splitOn(line, ' ');//line.split(' ');
	//console.log(tempTokens);
	var tokens = [];
	for(var k=0;k<tempTokens.length;++k){
		var token = tempTokens[k];
		token = token.trim();
		if(token.length > 0) tokens.push(token);
	}
	return tokens;
}

//var sys = require('sys');

exports.parse = function(str){

	if(typeof(str) !== 'string'){
		throw new Error('parse str is not of type string');
	}

	var actualLines = str.split('\n');
	for(var i=0;i<actualLines.length;++i){
		actualLines[i] = removeLineComments(actualLines[i]);
	}
	removeMultilineComments(actualLines);
	var reStr = ''
	for(var i=0;i<actualLines.length;++i){
		reStr += actualLines[i]+'\n'
	}
	var lines = splitOn(reStr, '\n')//str.split('\n');
	
	//console.log(lines);
	//removeMultilineComments(lines);
	///sys.debug(lines);

	var stack = [{tokens: [], children: []}];
	var depth = 0;
	for(var i=0;i<lines.length;++i){
	
		var line = lines[i]//removeLineComments(lines[i]);
		
		if(line.trim().length === 0) continue;
		
		var indent = countIndents(line);
		line = line.substr(indent);
		if(indent < depth){
			var many = depth - indent;
			for(var j=0;j<many;++j){stack.pop();}
		}else if(indent > depth){
			console.log('invalid indentation on line ' + (i+1));
			console.log('line: ' + lines[i]);
			throw 'parse error';
		}
			
		var node = {tokens: parseTokens(line), children: []};
		node.string = line
		stack[stack.length-1].children.push(node);
		stack.push(node);
		depth = indent + 1;
	}
	
	return stack[0];
}

exports.printAst = function(ast, tabs){
	var str = '';
	str += (tabs || '') + ast.tokens.join(' ') + '\n';
	for(var i=0;i<ast.children.length;++i){
		str += exports.printAst(ast.children[i], tabs !== undefined ? tabs + '\t' : '');
	}
	return str;
{'fraction_non_alphanumeric': 0.23105720275531597, 'fraction_numerical': 0.010182689427972447, 'mean_word_length': 3.2085696282293634, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2907782', 'n_tokens_mistral': 2976, 'n_tokens_neox': 2653, 'n_words': 560}