<commit_before>from __future__ import print_function
from builtins import object

import json
import re
from compiler.js import split_name, escape_package, get_package, mangle_package, escape
from compiler.js.component import component_generator
from collections import OrderedDict

root_type_package = 'core'
root_type_name = 'CoreObject'
root_type = root_type_package + '.' + root_type_name

class generator(object):
	def __init__(self, ns, bid):
		self.module = False
		self.ns, self.bid = ns, bid
		self.components = OrderedDict()
		self.used_packages = OrderedDict()
		self.used_components = OrderedDict()
		self.imports = OrderedDict()
		self.packages = OrderedDict()
		self.startup = []
		self.l10n = OrderedDict()
		self.id_set = set(['context', 'model'])

	def add_component(self, name, component, declaration):
		if name in self.components:
			raise Exception("duplicate component " + name)

		package, component_name = split_name(name)
		package = escape_package(package)

		if not declaration:
			name = "%s.Ui%s" %(package, component_name[0].upper() + escape(component_name[1:]))
			self.used_components[name] = None
			self.used_packages[package] = None
			self.startup.append("\tcontext.start(new qml.%s(context))" %name)
			self.startup.append("\tcontext.run()")
		else:
			name = package + '.' + component_name

		if package not in self.packages:
			self.packages[package] = set()
		self.packages[package].add(component_name)

		gen = component_generator(self.ns, None, name, component, True)
		self.components[name] = gen

	def add_js(self, name, data):
		if name in self.imports:
			raise Exception("duplicate js name " + name)
		self.imports[name] = data

	def wrap(self, code, use_globals = False):
		return "(function() {/** @const */\nvar exports = %s;\n%s\nreturn exports;\n} )" %("_globals" if use_globals else "{}", code)

	def find_component(self, package, name, register_used = True, mangle = False, use_globals = False):
		if name == root_type_name:
			package = root_type_package
			#fixme: copypasted
			if mangle:
				package = mangle_package(root_type_package)
			if use_globals:
				package = "_globals." + package
			return package + "." + root_type_name

		original_name = name
		name_package, name = split_name(name)

		candidates = []
		for package_name, components in self.packages.items():
			if name in components:
				if name_package:
					#match package/subpackage
					if package_name != name_package and not package_name.endswith('.' + name_package):
						continue
				candidates.append(package_name)

		if not candidates:
			raise Exception("component %s was not found" %(original_name))

		if len(candidates) > 1:
			if name_package in candidates: #specified in name, e.g. core.Text
				package_name = name_package
			if package in candidates: #local to current package
				package_name = package
			elif 'core' in candidates: #implicit core lookup
				package_name = 'core'
			else:
				raise Exception("ambiguous component %s, you have to specify one of the packages explicitly: %s" \
					%(name, " ".join(["%s.%s" %(p, name) for p in candidates])))
		else:
			package_name = candidates[0]

		if register_used:
			self.used_components[package_name + '.' + name] = None

		if mangle:
			package_name = mangle_package(package_name)
		if use_globals:
			package_name = "_globals." + package_name
		return "%s.%s" %(package_name, name)

	def generate_component(self, gen):
		name = gen.name

		self.used_packages[gen.package] = None

		code = ''
		code += "\n\n//=====[component %s]=====================\n\n" %name
		code += gen.generate(self)

		code += gen.generate_prototype(self)
		return code

	used_re = re.compile(r'@using\s*{(.*?)}')

	def scan_using(self, code):
		for m in generator.used_re.finditer(code):
			name = m.group(1).strip()
			package, _ = split_name(name)
			package = escape_package(package)
			self.used_components[name] = None
			self.used_packages[package] = None

	def generate_components(self):
		#finding explicit @using declarations in code
		for name, code in self.imports.items():
			self.scan_using(code)

		context_type = self.find_component('core', 'Context')
		context_gen = self.components[context_type]
		for i, pi in enumerate(context_gen.properties):
			for j, nv in enumerate(pi.properties):
				if nv[0] == 'buildIdentifier':
					pi.properties[j] = (nv[0], self.bid)
					break

		queue = ['core.Context']
		code, base_class = {}, {}
		code[root_type] = ''

		for gen in self.components.values():
			gen.pregenerate(self)

		while queue or self.used_components:
			for component in self.used_components.keys():
				if component not in code:
					queue.append(component)
			self.used_components = OrderedDict()

			if queue:
				name = queue.pop(0)
				component = self.components[name]
				base_type = self.find_component(component.package, component.component.name)
				base_class[name] = base_type

				if name not in code:
					try:
						code[name] = self.generate_component(component)
					except Exception as ex:
						ex.filename = name
						raise

		r = ''
		order = []
		visited = set([root_type])
		def visit(type):
			if type in visited:
				return
			visit(base_class[type])
			order.append(type)<commit_msg>add exception for non-existing used components<commit_after>from __future__ import print_function
from builtins import object

import json
import re
from compiler.js import split_name, escape_package, get_package, mangle_package, escape
from compiler.js.component import component_generator
from collections import OrderedDict

root_type_package = 'core'
root_type_name = 'CoreObject'
root_type = root_type_package + '.' + root_type_name

class generator(object):
	def __init__(self, ns, bid):
		self.module = False
		self.ns, self.bid = ns, bid
		self.components = OrderedDict()
		self.used_packages = OrderedDict()
		self.used_components = OrderedDict()
		self.imports = OrderedDict()
		self.packages = OrderedDict()
		self.startup = []
		self.l10n = OrderedDict()
		self.id_set = set(['context', 'model'])

	def add_component(self, name, component, declaration):
		if name in self.components:
			raise Exception("duplicate component " + name)

		package, component_name = split_name(name)
		package = escape_package(package)

		if not declaration:
			name = "%s.Ui%s" %(package, component_name[0].upper() + escape(component_name[1:]))
			self.used_components[name] = None
			self.used_packages[package] = None
			self.startup.append("\tcontext.start(new qml.%s(context))" %name)
			self.startup.append("\tcontext.run()")
		else:
			name = package + '.' + component_name

		if package not in self.packages:
			self.packages[package] = set()
		self.packages[package].add(component_name)

		gen = component_generator(self.ns, None, name, component, True)
		self.components[name] = gen

	def add_js(self, name, data):
		if name in self.imports:
			raise Exception("duplicate js name " + name)
		self.imports[name] = data

	def wrap(self, code, use_globals = False):
		return "(function() {/** @const */\nvar exports = %s;\n%s\nreturn exports;\n} )" %("_globals" if use_globals else "{}", code)

	def find_component(self, package, name, register_used = True, mangle = False, use_globals = False):
		if name == root_type_name:
			package = root_type_package
			#fixme: copypasted
			if mangle:
				package = mangle_package(root_type_package)
			if use_globals:
				package = "_globals." + package
			return package + "." + root_type_name

		original_name = name
		name_package, name = split_name(name)

		candidates = []
		for package_name, components in self.packages.items():
			if name in components:
				if name_package:
					#match package/subpackage
					if package_name != name_package and not package_name.endswith('.' + name_package):
						continue
				candidates.append(package_name)

		if not candidates:
			raise Exception("component %s was not found" %(original_name))

		if len(candidates) > 1:
			if name_package in candidates: #specified in name, e.g. core.Text
				package_name = name_package
			if package in candidates: #local to current package
				package_name = package
			elif 'core' in candidates: #implicit core lookup
				package_name = 'core'
			else:
				raise Exception("ambiguous component %s, you have to specify one of the packages explicitly: %s" \
					%(name, " ".join(["%s.%s" %(p, name) for p in candidates])))
		else:
			package_name = candidates[0]

		if register_used:
			self.used_components[package_name + '.' + name] = None

		if mangle:
			package_name = mangle_package(package_name)
		if use_globals:
			package_name = "_globals." + package_name
		return "%s.%s" %(package_name, name)

	def generate_component(self, gen):
		name = gen.name

		self.used_packages[gen.package] = None

		code = ''
		code += "\n\n//=====[component %s]=====================\n\n" %name
		code += gen.generate(self)

		code += gen.generate_prototype(self)
		return code

	used_re = re.compile(r'@using\s*{(.*?)}')

	def scan_using(self, code):
		for m in generator.used_re.finditer(code):
			name = m.group(1).strip()
			package, _ = split_name(name)
			package = escape_package(package)
			self.used_components[name] = None
			self.used_packages[package] = None

	def generate_components(self):
		#finding explicit @using declarations in code
		for name, code in self.imports.items():
			self.scan_using(code)

		context_type = self.find_component('core', 'Context')
		context_gen = self.components[context_type]
		for i, pi in enumerate(context_gen.properties):
			for j, nv in enumerate(pi.properties):
				if nv[0] == 'buildIdentifier':
					pi.properties[j] = (nv[0], self.bid)
					break

		queue = ['core.Context']
		code, base_class = {}, {}
		code[root_type] = ''

		for gen in self.components.values():
			gen.pregenerate(self)

		while queue or self.used_components:
			for component in self.used_components.keys():
				if component not in code:
					queue.append(component)
			self.used_components = OrderedDict()

			if queue:
				name = queue.pop(0)
				if name not in self.components:
					raise Exception("Component '%s' is marked as used, but no declaration found. Maybe you have @using declaration with wrong component name." %name)
				component = self.components[name]
				base_type = self.find_component(component.package, component.component.name)
				base_class[name] = base_type

				if name not in code:
					try:
						code[name] = self.generate_component(component)
					except Exception as ex:
						ex.filename = name
						raise

		r = ''
		order = []
		visited = set([root_type])
		def visit(type):
			if type in visited:
				return
			visit(base_class[type])
			order.append(type)
{'fraction_non_alphanumeric': 0.12429537767756482, 'fraction_numerical': 0.0018789928598271326, 'mean_word_length': 4.009411764705883, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '887963', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3369, 'n_words': 892}