<commit_before>enyo.kind({
	name: "Documentor",
	kind: null,
	group: "public",
	constructor: function(inTokens) {
		this.comment = [];
		return this.parse(inTokens);
	},
	parse: function(inTokens) {
		var it = new Iterator(inTokens);
		return this.walk(it);
	},
	walk: function(it, inState) {
		var objects = [], node, obj;
		while (it.next()) {
			var node = it.value;
			if (node.kind == "comment") {
				this.cook_comment(node.token);
			} 
			else if (node.token == "enyo.kind" && it.future.kind == "association") {
				obj = this.cook_kind(it);
			}
			else if (node.kind == "assignment") {
				obj = this.cook_assignment(it);
			}
			else if (node.kind == "association") {
				// closure? [( [function] [()] [{...}] [()]) ]
				if (node.children && node.children.length == 1 && node.children[0].kind == "function") {
					// closure
					var fn = node.children[0];
					if (fn.children && fn.children.length == 2) {
						var body = fn.children[1];
						var objs = this.walk(new Iterator(body.children));
						// add whatever was in the closure to the main object list
						objects = objects.concat(objs);
					}
					// skip the closure invocation [()]
					it.next();
				}
			}
			if (obj) {
				objects.push(obj);
				obj = null;
			}
		}
		return objects;
	},
	cook_kind: function(it) {
		// Get inProps[name].value[0].token
		var val = function(inProps, inName) {
			var i = Documentor.indexByName(inProps, inName);
			if (i >= 0) {
				var p = inProps[i];
				inProps.splice(p, 1);
			}
			return p && p.value && p.value.length && p.value[0].token;
		};
		/*
		// Set inProps[name].properties to inProps[name].value[0].properties
		var flatten = function(inProps, inName) {
			var p = Documentor.findByName(inProps, inName);
			var v = p && p.value && p.value.length && p.value[0];
			if (v) {
				p.properties = p.value[0].properties;
			}
		};
		*/
		//
		var obj = this.make("kind", it.value);
		// arguments
		it.next();
		// kind takes one argument
		var args = it.value.children;
		// if it's a block
		if (args && args[0] && args[0].kind == "block") {
			// these are the properties
			obj.properties = this.cook_block(args[0].children);
			// process special properties
			obj.name = Documentor.stripQuotes(val(obj.properties, "name") || "");
			obj.superkind = Documentor.stripQuotes(val(obj.properties, "kind") || "enyo.Control");
			if (obj.superkind == "null") {
				obj.superkind = null;
			}
			// remove excess value nodes
			//flatten(obj.properties, "published");
		}
		return obj;
	},
	cook_block: function(inNodes) {
		var props = [];
		for (var i=0, n; n=inNodes[i]; i++) {
			if (n.kind == "comment") {
				this.cook_comment(n.token);
			}
			else if (n.kind == "assignment") {
				var prop = this.make("property", n);
				if (n.children) {
					prop.value = [this.walkValue(new Iterator(n.children))];
				}
				props.push(prop);
			}
		}
		return props;
	},
	walkValue: function(it, inState) {
		while (it.next()) {
			var node = it.value;
			if (node.kind == "comment") {
				this.cook_comment(node.token);
			}
			else if (node.kind == "block") {
				var obj = this.make("block", node);
				obj.properties = this.cook_block(node.children);
				return obj;
			}
			else if (node.kind == "array") {
				return this.cook_array(it);
			}
			else {
				var obj = this.make("expression", node);
				var t = node.token;
				while (it.next()) {
					t += it.value.token;
				};
				obj.token = t;
				return obj;
			}
		}
	},
	cook_array: function(it) {
		var node = it.value;
		var obj = this.make("array", node);
		var nodes = node.children;
		if (nodes) {
			var elts = [];
			for (var i=0, n, v; n=nodes[i]; i++) {
				v = this.walkValue(new Iterator(n.children));
				if (v) {
					elts.push(v);
				}
			}
			obj.properties = elts;
		}
		return obj;
	},<commit_msg>add function arguments to Documentor<commit_after>enyo.kind({
	name: "Documentor",
	kind: null,
	group: "public",
	constructor: function(inTokens) {
		this.comment = [];
		return this.parse(inTokens);
	},
	parse: function(inTokens) {
		var it = new Iterator(inTokens);
		return this.walk(it);
	},
	walk: function(it, inState) {
		var objects = [], node, obj;
		while (it.next()) {
			var node = it.value;
			if (node.kind == "comment") {
				this.cook_comment(node.token);
			} 
			else if (node.token == "enyo.kind" && it.future.kind == "association") {
				obj = this.cook_kind(it);
			}
			else if (node.kind == "assignment") {
				obj = this.cook_assignment(it);
			}
			else if (node.kind == "association") {
				// closure? [( [function] [()] [{...}] [()]) ]
				if (node.children && node.children.length == 1 && node.children[0].kind == "function") {
					// closure
					var fn = node.children[0];
					if (fn.children && fn.children.length == 2) {
						var args = fn.children[0];
						var body = fn.children[1];
						var objs = this.walk(new Iterator(body.children));
						// add whatever was in the closure to the main object list
						objects = objects.concat(objs);
					}
					// skip the closure invocation [()]
					it.next();
				}
			}
			if (obj) {
				objects.push(obj);
				obj = null;
			}
		}
		return objects;
	},
	cook_kind: function(it) {
		// Get inProps[name].value[0].token
		var val = function(inProps, inName) {
			var i = Documentor.indexByName(inProps, inName);
			if (i >= 0) {
				var p = inProps[i];
				inProps.splice(p, 1);
			}
			return p && p.value && p.value.length && p.value[0].token;
		};
		/*
		// Set inProps[name].properties to inProps[name].value[0].properties
		var flatten = function(inProps, inName) {
			var p = Documentor.findByName(inProps, inName);
			var v = p && p.value && p.value.length && p.value[0];
			if (v) {
				p.properties = p.value[0].properties;
			}
		};
		*/
		//
		var obj = this.make("kind", it.value);
		// arguments
		it.next();
		// kind takes one argument
		var args = it.value.children;
		// if it's a block
		if (args && args[0] && args[0].kind == "block") {
			// these are the properties
			obj.properties = this.cook_block(args[0].children);
			// process special properties
			obj.name = Documentor.stripQuotes(val(obj.properties, "name") || "");
			obj.superkind = Documentor.stripQuotes(val(obj.properties, "kind") || "enyo.Control");
			if (obj.superkind == "null") {
				obj.superkind = null;
			}
			// remove excess value nodes
			//flatten(obj.properties, "published");
		}
		return obj;
	},
	cook_block: function(inNodes) {
		var props = [];
		for (var i=0, n; n=inNodes[i]; i++) {
			if (n.kind == "comment") {
				this.cook_comment(n.token);
			}
			else if (n.kind == "assignment") {
				var prop = this.make("property", n);
				if (n.children) {
					prop.value = [this.walkValue(new Iterator(n.children))];
				}
				props.push(prop);
			}
		}
		return props;
	},
	walkValue: function(it, inState) {
		while (it.next()) {
			var node = it.value;
			if (node.kind == "comment") {
				this.cook_comment(node.token);
			}
			else if (node.kind == "block") {
				var obj = this.make("block", node);
				obj.properties = this.cook_block(node.children);
				return obj;
			}
			else if (node.kind == "array") {
				return this.cook_array(it);
			}
			else if (node.kind == "function") {
				return this.cook_function(it);
			}
			else {
				var obj = this.make("expression", node);
				var t = node.token;
				while (it.next()) {
					t += it.value.token;
				};
				obj.token = t;
				return obj;
			}
		}
	},
	cook_function: function(it) {
		var node = it.value;
		var obj = this.make("expression", node);
		obj.arguments = enyo.map(node.children[0].children, function(n) { return n.token });
		return obj;
	},
	cook_array: function(it) {
		var node = it.value;
		var obj = this.make("array", node);
		var nodes = node.children;
		if (nodes) {
			var elts = [];
			for (var i=0, n, v; n=nodes[i]; i++) {
				v = this.walkValue(new Iterator(n.children));
				if (v) {
					elts.push(v);
				}
			}
			obj.properties = elts;
		}
		return obj;
	},
{'fraction_non_alphanumeric': 0.18023914411579608, 'fraction_numerical': 0.004531151667715544, 'mean_word_length': 2.7078861409239385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637772', 'n_tokens_mistral': 3446, 'n_tokens_neox': 2879, 'n_words': 647}