<commit_before>module.exports = (function(root) {

	// include the fs mmodule
	var fs = require("fs"),
	// include the path module
	path = require("path");

	function trim(str) {
		return str.replace(/^\s*/, "").replace(/\s*$/, "");
	}

	function parseDocs(source) {
		var docRegexp = /\s*\/\*\*(?:[\S\s]+?)\*\//mg;
		var fnRegexp = /\bfunction(\s*|[^\(]+)?\(/;
		var commentRegexp = /^\/\*\*[\S\s]+?\s*(\*\s+@|\*\/$)/;
		var returnsRegexp = /@returns\s+([\S\s]+?)\s*(\*\s+@|\*\/$)/;
		var paramsRegexp = /@param\s+([\S\s]+?)\s*(?:\*\s*@|\*\/$)/;
		var exampleRegexp = /@example\n[\S\s]+$/g;
		var staticRegexp = /@static\b/;
		var privateRegexp = /@private\b/;
		var constructorRegexp = /@constructor\b/;
		var memberOfRegexp = /@memberOf\s+.+$/m;
		var nameRegexp = /@name\s+.+$/m;
		var superRegexp = /@super\s+.+$/m;
		var augmentsRegexp = /@augments\s+.+$/m;
		var typeRegexp = /@type\s+.+$/m;
		var docs = [];
		// get all docblocks and the line right after it
		source.replace(docRegexp, function(match, index) {
			var docblock = trim(match);
			var definition = source.substr(match.length + index).split(/\n/g)[1];
			docs.push({
				doc : docblock,
				def : definition
			});
		});
		var docblocks = [];
		docs.forEach(function(block) {
			var doc = {};
			// pull out the docblock comment
			var commentMatch = commentRegexp.exec(block.doc);
			if (commentMatch && commentMatch[0]) {
				doc.comment = trim(commentMatch[0].replace(/^\/\*\*/, "").replace(/^\s*\*\s*/gm, "").replace(/^\s*\*\s*@\s*$/gm, "").split(/\n/g).join(" ").replace(/\s+/g, " "));
			}
			// parse example if any
			var exampleMatch = exampleRegexp.exec(block.doc);
			if (exampleMatch && exampleMatch[0]) {
				doc.example = trim(exampleMatch[0].replace(/\s*\*\/$/, "").replace(/^\s*\*\s*/gm, "").replace(/^\s*\*\s*@\s*$/gm, "").replace(/^\s*@example/, ""));
			}

			var memberOfMatch = memberOfRegexp.exec(block.doc);
			if (memberOfMatch && memberOfMatch[0]) {
				doc.memberOf = trim(memberOfMatch[0].replace(/@memberOf\s+/, "")).replace(/^{|}$/g, "");
			}
			var typeMatch = typeRegexp.exec(block.doc);
			if (typeMatch && typeMatch[0]) {
				doc.type = trim(typeMatch[0].replace(/@type\s+/, "")).replace(/^{|}$/g, "");
			}

			doc.isStatic = staticRegexp.test(block.doc);
			doc.isPrivate = privateRegexp.test(block.doc);

			// check if it is a function or not
			if (fnRegexp.test(block.def)) {
				doc.varType = "function";
				doc.isConstructor = constructorRegexp.test(block.doc);
				var returnsMatch = returnsRegexp.exec(block.doc);
				if (returnsMatch && returnsMatch[0]) {
					var returnsMatchParts = {};
					var returnsMatchClean = trim(returnsMatch[1].replace(/^\s*\*\s*/gm, "").split(/\n/g).join(" ").replace(/\s+/g, " "));
					returnsMatchParts.type = (/^{?([^\s]+)}/.exec(returnsMatchClean) || [])[1] || null;
					returnsMatchParts.comment = (/^{?[^\s]+}(.+)$/.exec(returnsMatchClean) || [])[1] || null;
					doc.returns = returnsMatchParts;
				}
				var params = [], paramMatcher, paramBlock = block.doc;
				// strip out all the params from block
				while ((paramMatcher = paramsRegexp.exec(paramBlock)) !== null) {
					var paramsMatchParts = {};<commit_msg>Fix parsing example tags in doc<commit_after>module.exports = (function(root) {

	// include the fs mmodule
	var fs = require("fs"),
	// include the path module
	path = require("path");

	function trim(str) {
		return str.replace(/^\s*/, "").replace(/\s*$/, "");
	}

	function parseDocs(source) {
		var docRegexp = /\s*\/\*\*(?:[\S\s]+?)\*\//mg;
		var fnRegexp = /\bfunction(\s*|[^\(]+)?\(/;
		var commentRegexp = /^\/\*\*[\S\s]+?\s*(\*\s+@|\*\/$)/;
		var returnsRegexp = /@returns\s+([\S\s]+?)\s*(\*\s+@|\*\/$)/;
		var paramsRegexp = /@param\s+([\S\s]+?)\s*(?:\*\s*@|\*\/$)/;
		var exampleRegexp = /@example\s+[\S\s]+$/g;
		var staticRegexp = /@static\b/;
		var privateRegexp = /@private\b/;
		var constructorRegexp = /@constructor\b/;
		var memberOfRegexp = /@memberOf\s+.+$/m;
		var nameRegexp = /@name\s+.+$/m;
		var superRegexp = /@super\s+.+$/m;
		var augmentsRegexp = /@augments\s+.+$/m;
		var typeRegexp = /@type\s+.+$/m;
		var docs = [];
		// get all docblocks and the line right after it
		source.replace(docRegexp, function(match, index) {
			var docblock = trim(match);
			var definition = source.substr(match.length + index).split(/\n/g)[1];
			docs.push({
				doc : docblock,
				def : definition
			});
		});
		var docblocks = [];
		docs.forEach(function(block) {
			var doc = {};
			// pull out the docblock comment
			var commentMatch = commentRegexp.exec(block.doc);
			if (commentMatch && commentMatch[0]) {
				doc.comment = trim(commentMatch[0].replace(/^\/\*\*/, "").replace(/^\s*\*\s*/gm, "").replace(/^\s*\*\s*@\s*$/gm, "").split(/\n/g).join(" ").replace(/\s+/g, " "));
			}
			// parse example if any
			var exampleMatch = exampleRegexp.exec(block.doc);
			if (exampleMatch && exampleMatch[0]) {
				doc.example = trim(trim(exampleMatch[0].replace(/\s*\*\/$/, "").replace(/^\s*\*\s*/gm, "").replace(/^\s*\*\s*@\s*$/gm, "").replace(/^\s*@example/, "")).replace(/^\<code\>/, "").replace(/\<\/code\>$/, ""));
			}

			var memberOfMatch = memberOfRegexp.exec(block.doc);
			if (memberOfMatch && memberOfMatch[0]) {
				doc.memberOf = trim(memberOfMatch[0].replace(/@memberOf\s+/, "")).replace(/^{|}$/g, "");
			}
			var typeMatch = typeRegexp.exec(block.doc);
			if (typeMatch && typeMatch[0]) {
				doc.type = trim(typeMatch[0].replace(/@type\s+/, "")).replace(/^{|}$/g, "");
			}

			doc.isStatic = staticRegexp.test(block.doc);
			doc.isPrivate = privateRegexp.test(block.doc);

			// check if it is a function or not
			if (fnRegexp.test(block.def)) {
				doc.varType = "function";
				doc.isConstructor = constructorRegexp.test(block.doc);
				var returnsMatch = returnsRegexp.exec(block.doc);
				if (returnsMatch && returnsMatch[0]) {
					var returnsMatchParts = {};
					var returnsMatchClean = trim(returnsMatch[1].replace(/^\s*\*\s*/gm, "").split(/\n/g).join(" ").replace(/\s+/g, " "));
					returnsMatchParts.type = (/^{?([^\s]+)}/.exec(returnsMatchClean) || [])[1] || null;
					returnsMatchParts.comment = (/^{?[^\s]+}(.+)$/.exec(returnsMatchClean) || [])[1] || null;
					doc.returns = returnsMatchParts;
				}
				var params = [], paramMatcher, paramBlock = block.doc;
				// strip out all the params from block
				while ((paramMatcher = paramsRegexp.exec(paramBlock)) !== null) {
					var paramsMatchParts = {};
{'fraction_non_alphanumeric': 0.24863770823602677, 'fraction_numerical': 0.0040479526700918575, 'mean_word_length': 3.2096985583224114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3698608', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2347, 'n_words': 415}