<commit_before>import re
from collections import OrderedDict
import compiler.lang as lang

doc_next = None
doc_prev_component = None
doc_root_component = None

class CustomParser(object):
	def match(self, next):
		raise Exception("Expression should implement match method")

escape_re = re.compile(r"[\0\n\r\v\t\b\f]")
escape_map = {
	'\0': '\\0',
	'\n': '\\n',
	'\r': '\\r',
	'\v': '\\v',
	'\t': '\\t',
	'\b': '\\b',
	'\f': '\\f'
}
def escape(str):
	return escape_re.sub(lambda m: escape_map[m.group(0)], str)

class StringParser(CustomParser):
	def match(self, next):
		n = len(next)
		if n < 2:
			return

		quote = next[0]
		if quote != "'" and quote != "\"":
			return
		pos = 1
		while next[pos] != quote:
			if next[pos] == "\\":
				pos += 2
			else:
				pos += 1
			if pos >= n:
				raise Exception("Unexpected EOF while parsing string")
		return next[:pos + 1]

skip_re = re.compile(r'(?:\s+|/\*.*?\*/|//[^\n]*(?:$|\n))', re.DOTALL)
COMPONENT_NAME = r'(?:[a-z][a-zA-Z0-9._]*\.)?[A-Z][A-Za-z0-9]*'
component_name = re.compile(COMPONENT_NAME)
component_name_lookahead = re.compile(COMPONENT_NAME + r'\s*{')
identifier_re = re.compile(r'[a-z_][A-Za-z0-9_]*')
property_type_re = re.compile(r'[a-z][a-z0-9]*', re.IGNORECASE)
nested_identifier_re = re.compile(r'[a-z_][A-Za-z0-9_\.]*')
function_name_re = re.compile(r'[a-z_][a-z0-9_\.]*', re.IGNORECASE)
string_re = StringParser()
kw_re = re.compile(r'(?:true|false|null)')
NUMBER_RE = r"(?:(?:(?:\d*\.\d+)|(?:\d+\.?))(?:e[+-]?\d+)?|(?:0x)?[0-9]+)"
number_re = re.compile(NUMBER_RE, re.IGNORECASE)
percent_number_re = re.compile(NUMBER_RE + r'%', re.IGNORECASE)
scale_number_re = re.compile(NUMBER_RE + r's', re.IGNORECASE)
rest_of_the_line_re = re.compile(r".*$", re.MULTILINE)
json_object_value_delimiter_re = re.compile(r"[,;]")
dep_var = re.compile(r"\${(.*?)}")

class Expression(object):
	__slots__ = ('op', 'args')
	def __init__(self, op, *args):<commit_msg>allow underscore in propery_type_re<commit_after>import re
from collections import OrderedDict
import compiler.lang as lang

doc_next = None
doc_prev_component = None
doc_root_component = None

class CustomParser(object):
	def match(self, next):
		raise Exception("Expression should implement match method")

escape_re = re.compile(r"[\0\n\r\v\t\b\f]")
escape_map = {
	'\0': '\\0',
	'\n': '\\n',
	'\r': '\\r',
	'\v': '\\v',
	'\t': '\\t',
	'\b': '\\b',
	'\f': '\\f'
}
def escape(str):
	return escape_re.sub(lambda m: escape_map[m.group(0)], str)

class StringParser(CustomParser):
	def match(self, next):
		n = len(next)
		if n < 2:
			return

		quote = next[0]
		if quote != "'" and quote != "\"":
			return
		pos = 1
		while next[pos] != quote:
			if next[pos] == "\\":
				pos += 2
			else:
				pos += 1
			if pos >= n:
				raise Exception("Unexpected EOF while parsing string")
		return next[:pos + 1]

skip_re = re.compile(r'(?:\s+|/\*.*?\*/|//[^\n]*(?:$|\n))', re.DOTALL)
COMPONENT_NAME = r'(?:[a-z][a-zA-Z0-9._]*\.)?[A-Z][A-Za-z0-9]*'
component_name = re.compile(COMPONENT_NAME)
component_name_lookahead = re.compile(COMPONENT_NAME + r'\s*{')
identifier_re = re.compile(r'[a-z_][A-Za-z0-9_]*')
property_type_re = re.compile(r'[a-z_][a-z0-9_]*', re.IGNORECASE)
nested_identifier_re = re.compile(r'[a-z_][A-Za-z0-9_\.]*')
function_name_re = re.compile(r'[a-z_][a-z0-9_\.]*', re.IGNORECASE)
string_re = StringParser()
kw_re = re.compile(r'(?:true|false|null)')
NUMBER_RE = r"(?:(?:(?:\d*\.\d+)|(?:\d+\.?))(?:e[+-]?\d+)?|(?:0x)?[0-9]+)"
number_re = re.compile(NUMBER_RE, re.IGNORECASE)
percent_number_re = re.compile(NUMBER_RE + r'%', re.IGNORECASE)
scale_number_re = re.compile(NUMBER_RE + r's', re.IGNORECASE)
rest_of_the_line_re = re.compile(r".*$", re.MULTILINE)
json_object_value_delimiter_re = re.compile(r"[,;]")
dep_var = re.compile(r"\${(.*?)}")

class Expression(object):
	__slots__ = ('op', 'args')
	def __init__(self, op, *args):
{'fraction_non_alphanumeric': 0.23977213878819265, 'fraction_numerical': 0.01294665976178146, 'mean_word_length': 4.350415512465374, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3668421', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1657, 'n_words': 294}