<commit_before>'use strict'

var exports = module.exports = { }

const token = require('../app/token'),
	tokenizer = require('../app/tokenizer'),
	tokenType = require('../app/tokenType').Type,
	astType = require('../app/astType').AstType

const stringify = (root) => {
	
	if (has(root.children)) {
		if (root.type === astType.UNARY) {
			if (root.children[0].type === astType.PROP) {
				return root.token.value + root.children[0].token.value 
			}
		} else {
			const a = root.children[0]
			const b = root.children[1]
			return `(${stringify(a)}${root.token.value}${stringify(b)})`
		}
	} else {
		return root.token.value
	}
}

const has = (children) => !!children && children.length > 0

const get = (expression) => {
	const tokens = tokenizer.get(expression)
	let _current = 0
	let last

	while (_current < tokens.length) {
		last = _identify(tokens[_current])
		_current++
	}

	return last

	function _identify(tokenItem) {
		switch(tokenItem.type) {<commit_msg>Improve code<commit_after>'use strict'

var exports = module.exports = { }

const token = require('../app/token'),
	tokenizer = require('../app/tokenizer'),
	tokenType = require('../app/tokenType').Type,
	astType = require('../app/astType').AstType

const stringify = (root) => {
	
	if (has(root.children)) {
		const a = root.children[0]
		if (root.type === astType.UNARY) {
			if (a.type === astType.PROP) {
				return root.token.value + a.token.value 
			}
		} else {
			const b = root.children[1]
			return `(${stringify(a)}${root.token.value}${stringify(b)})`
		}
	} else {
		return root.token.value
	}
}

const has = (children) => !!children && children.length > 0

const get = (expression) => {
	const tokens = tokenizer.get(expression)
	let _current = 0
	let last

	while (_current < tokens.length) {
		last = _identify(tokens[_current])
		_current++
	}

	return last

	function _identify(tokenItem) {
		switch(tokenItem.type) {
{'fraction_non_alphanumeric': 0.17455310199789695, 'fraction_numerical': 0.005257623554153523, 'mean_word_length': 3.630170316301703, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753052', 'n_tokens_mistral': 715, 'n_tokens_neox': 656, 'n_words': 137}