<commit_before>#-*- coding: utf-8 -*-
# Copyright 2017 ibelie, <NAME>, Joungtao. All rights reserved.
# Use of this source code is governed by The MIT License
# that can be found in the LICENSE file.

# [joungtao] The field-number/tag in protobuf is auto generated by this sorting function.
#	The sorted field sequence of the encoding and decoding sides must match.
def SortedMessage(d, sequence):
	sequence = sequence or ()
	return [(k, d[k]) for k in sequence] + sorted([(k, v) for k, v in d.iteritems() if k not in sequence], key = lambda (k, v): k)


def CompareWrite(path, content):
	import os
	import codecs
	if os.path.isfile(path):
		with codecs.open(path, 'r', 'utf-8') as f:
			if f.read() == content:
				return
	with codecs.open(path, 'w', 'utf-8') as f:
		f.write(content)


def ShortName(prefix, name):
	import hashlib
	import base64
	if len(name) > 25:
		name = prefix + base64.b64encode(hashlib.md5(name).digest())[:-2].replace('+', '__').replace('/', '_')
	return name


def RecordNesting(prefix, types):
	from Type import toType, isEnum, List, Dict, Instance
	shortName = {
		'Integer': 'i',
		'Float': 'f',
		'Double': 'd',
		'FixedPoint': 'fp',
		'Boolean': 'b',
		'String': 's',
		'Bytes': 'by',
		'Enum': 'e',
	}
	properties = {'Enum' if isEnum(p) else p.__name__: toType(p) for p in types if p is not None}
	names = sorted([shortName[k] for k in properties if k in shortName])
	for name in sorted([k for k in properties if k not in shortName]):
		prop = properties[name]
		if isinstance(prop, Instance):
			if len(prop.pyType) == 1 and not isinstance(prop.pyType[0], List):
				names.append(prop.pyType[0].__name__)
			else:
				names.append(RecordNesting('V', prop.pyType)[0])
		elif isinstance(prop, List):
			names.append(RecordNesting('L', [prop.elementType])[0])
		elif isinstance(prop, Dict):
			names.append(RecordNesting('D', [prop.keyType, prop.valueType])[0])
		else:
			names.append(name)
	name = ShortName(prefix, '%s%s' % (prefix, ''.join(names)))
	return name, properties


def ClearTypes():
	from Object import MetaObject
	MetaObject.Objects = {}
	from Enum import MetaEnum
	MetaEnum.Enums = {}
	import Type
	Type.PythonTypes = {}


class PyObject(object): pass


def _ProtoFormat(t, *args):
	args = args + tuple(('%s = %s' % (k, repr(v)) for k, v in sorted(t.metadata.iteritems())))
	return '.'.join(['typy'] + sorted([k.__name__ for k in t.____keywords__]) + [t.__class__.__name__]) + ('(%s)' % ', '.join(args))


def _GetProtoFromTypy(p, codes, types):
	from Object import MetaObject
	import Type
	from Type import pb, toType, Enum, Integer, Float, Double, Boolean, String, Bytes
	from Type import Instance, List, Dict, FixedPoint, Python
	if isinstance(p, Enum):
		if p.pyType.__name__ not in types:
			values = []
			for i, e in sorted(p.pyType.__enum__.iteritems()):
				values.append("""
	'%s': %d,""" % (e.name, i))
			codes.append("""
%s = type('%s', (typy.Enum, ), {%s
})""" % (p.pyType.__name__, p.pyType.__name__, ''.join(values)))
			types.add(p.pyType.__name__)
		return _ProtoFormat(p, p.pyType.__name__)
	elif isinstance(p, (Integer, Float, Double, Boolean, String, Bytes)):
		return _ProtoFormat(p)
	elif isinstance(p, FixedPoint):
		return _ProtoFormat(p, '%d, %d' % (p.precision, p.floor))
	elif isinstance(p, Python):
		if p.pyType.__name__ not in types:
			codes.append("""
%s = type('%s', (), {})""" % (p.pyType.__name__, p.pyType.__name__))
			types.add(p.pyType.__name__)
		return _ProtoFormat(p, p.pyType.__name__)
	elif isinstance(p, Instance):
		pyType = []
		for t in p.pyType:
			if t is None:
				pyType.append('None')
			elif t.__name__ in MetaObject.Objects:
				pyType.append(t.__name__)
				if t.__name__ not in types:
					_GenerateObject(t.__name__, MetaObject.Objects[t.__name__], codes, types)
			elif not isinstance(t, type) or (isinstance(t, type) and issubclass(t, (Type.Type, Type._Enum))):
				pyType.append(_GetProtoFromTypy(toType(t), codes, types))
			else:
				pyType.append(_GetProtoFromTypy(Python(PyObject), codes, types))
		return _ProtoFormat(p, *sorted(pyType))
	elif isinstance(p, List):
		return _ProtoFormat(p, _GetProtoFromTypy(p.elementType, codes, types))
	elif isinstance(p, Dict):
		return _ProtoFormat(p, _GetProtoFromTypy(p.keyType, codes, types), _GetProtoFromTypy(p.valueType, codes, types))


def _GenerateObject(name, cls, codes, types):
	fields = []
	sortedProperties = SortedMessage(cls.____properties__, getattr(cls, '____propertySequence__', None))
	for a, p in sortedProperties:
		fields.append("""
	('%s', %s),""" % (a, _GetProtoFromTypy(p, codes, types)))

	codes.append("""
%s = typy.Proto.Object('%s', %s
)""" % (name, name, ''.join(fields)))
	types.add(name)


def Object(name, *fields):
	attrs = {}
	attrs['____propertySequence__'] = [a for a, _ in fields]
	attrs['____properties__'] = properties = {}
	for a, p in fields:
		properties[a] = p
	attrs.update(properties)
	return type(name, (), attrs)


def Increment(forceReload, path, proto_file, ignore):
	import os
	import sys
	import imp
	import codecs

	forceReload and ClearTypes()
	path = path.replace('\\', '/')
	if os.path.isfile(proto_file):
		with codecs.open(proto_file, 'r', 'utf-8') as f:
			proto = imp.new_module('typy.proto')
			proto.__package__ = 'typy'
			exec str(f.read()) in proto.__dict__
	else:
		class _Proto(object): pass
		proto = _Proto()
		proto.timestamps = {}

	def _scanScripts(sub):
		for i in os.listdir('%s/%s' % (path, sub)):
			sys.stdout.write('.')<commit_msg>fix bug<commit_after>#-*- coding: utf-8 -*-
# Copyright 2017 ibelie, <NAME>, Joungtao. All rights reserved.
# Use of this source code is governed by The MIT License
# that can be found in the LICENSE file.

# [joungtao] The field-number/tag in protobuf is auto generated by this sorting function.
#	The sorted field sequence of the encoding and decoding sides must match.
def SortedMessage(d, sequence):
	sequence = sequence or ()
	return [(k, d[k]) for k in sequence] + sorted([(k, v) for k, v in d.iteritems() if k not in sequence], key = lambda (k, v): k)


def CompareWrite(path, content):
	import os
	import codecs
	if os.path.isfile(path):
		with codecs.open(path, 'r', 'utf-8') as f:
			if f.read() == content:
				return
	with codecs.open(path, 'w', 'utf-8') as f:
		f.write(content)


def ShortName(prefix, name):
	import hashlib
	import base64
	if len(name) > 25:
		name = prefix + base64.b64encode(hashlib.md5(name).digest())[:-2].replace('+', '__').replace('/', '_')
	return name


def RecordNesting(prefix, types):
	from Type import toType, isEnum, List, Dict, Instance
	shortName = {
		'Integer': 'i',
		'Float': 'f',
		'Double': 'd',
		'FixedPoint': 'fp',
		'Boolean': 'b',
		'String': 's',
		'Bytes': 'by',
		'Enum': 'e',
	}
	properties = {'Enum' if isEnum(p) else p.__name__: toType(p) for p in types if p is not None}
	names = sorted([shortName[k] for k in properties if k in shortName])
	for name in sorted([k for k in properties if k not in shortName]):
		prop = properties[name]
		if isinstance(prop, Instance):
			if len(prop.pyType) == 1 and not isinstance(prop.pyType[0], List):
				names.append(prop.pyType[0].__name__)
			else:
				names.append(RecordNesting('V', prop.pyType)[0])
		elif isinstance(prop, List):
			names.append(RecordNesting('L', [prop.elementType])[0])
		elif isinstance(prop, Dict):
			names.append(RecordNesting('D', [prop.keyType, prop.valueType])[0])
		else:
			names.append(name)
	name = ShortName(prefix, '%s%s' % (prefix, ''.join(names)))
	return name, properties


def ClearTypes():
	from Object import MetaObject
	MetaObject.Objects = {}
	from Enum import MetaEnum
	MetaEnum.Enums = {}
	import Type
	Type.PythonTypes = {}


class PyObject(object): pass


def _ProtoFormat(t, *args):
	args = args + tuple(('%s = %s' % (k, repr(v)) for k, v in sorted(t.metadata.iteritems())))
	return '.'.join(['typy'] + sorted([k.__name__ for k in t.____keywords__]) + [t.__class__.__name__]) + ('(%s)' % ', '.join(args))


def _GetProtoFromTypy(p, codes, types):
	from Object import MetaObject
	import Type
	from Type import pb, toType, Enum, Integer, Float, Double, Boolean, String, Bytes
	from Type import Instance, List, Dict, FixedPoint, Python
	if isinstance(p, Enum):
		if p.pyType.__name__ not in types:
			values = []
			for i, e in sorted(p.pyType.__enum__.iteritems()):
				values.append("""
	'%s': %d,""" % (e.name, i))
			codes.append("""
%s = type('%s', (typy.Enum, ), {%s
})""" % (p.pyType.__name__, p.pyType.__name__, ''.join(values)))
			types.add(p.pyType.__name__)
		return _ProtoFormat(p, p.pyType.__name__)
	elif isinstance(p, (Integer, Float, Double, Boolean, String, Bytes)):
		return _ProtoFormat(p)
	elif isinstance(p, FixedPoint):
		return _ProtoFormat(p, '%d, %d' % (p.precision, p.floor))
	elif isinstance(p, Python):
		if p.pyType.__name__ not in types:
			if p.pyType.__name__ == 'PyObject':
				codes.append("""
PyObject = typy.Proto.PyObject""")
			else:
				codes.append("""
%s = type('%s', (), {})""" % (p.pyType.__name__, p.pyType.__name__))
			types.add(p.pyType.__name__)
		return _ProtoFormat(p, p.pyType.__name__)
	elif isinstance(p, Instance):
		pyType = []
		for t in p.pyType:
			if t is None:
				pyType.append('None')
			elif t.__name__ in MetaObject.Objects:
				pyType.append(t.__name__)
				if t.__name__ not in types:
					_GenerateObject(t.__name__, MetaObject.Objects[t.__name__], codes, types)
			elif not isinstance(t, type) or (isinstance(t, type) and issubclass(t, (Type.Type, Type._Enum))):
				pyType.append(_GetProtoFromTypy(toType(t), codes, types))
			else:
				pyType.append(_GetProtoFromTypy(Python(PyObject), codes, types))
		return _ProtoFormat(p, *sorted(pyType))
	elif isinstance(p, List):
		return _ProtoFormat(p, _GetProtoFromTypy(p.elementType, codes, types))
	elif isinstance(p, Dict):
		return _ProtoFormat(p, _GetProtoFromTypy(p.keyType, codes, types), _GetProtoFromTypy(p.valueType, codes, types))


def _GenerateObject(name, cls, codes, types):
	fields = []
	sortedProperties = SortedMessage(cls.____properties__, getattr(cls, '____propertySequence__', None))
	for a, p in sortedProperties:
		fields.append("""
	('%s', %s),""" % (a, _GetProtoFromTypy(p, codes, types)))

	codes.append("""
%s = typy.Proto.Object('%s', %s
)""" % (name, name, ''.join(fields)))
	types.add(name)


def Object(name, *fields):
	attrs = {}
	attrs['____propertySequence__'] = [a for a, _ in fields]
	attrs['____properties__'] = properties = {}
	for a, p in fields:
		properties[a] = p
	attrs.update(properties)
	return type(name, (), attrs)


def Increment(forceReload, path, proto_file, ignore):
	import os
	import sys
	import imp
	import codecs

	forceReload and ClearTypes()
	import Type
	Type.PythonTypes = {}
	path = path.replace('\\', '/')
	if os.path.isfile(proto_file):
		with codecs.open(proto_file, 'r', 'utf-8') as f:
			proto = imp.new_module('typy.proto')
			proto.__package__ = 'typy'
			exec str(f.read()) in proto.__dict__
	else:
		class _Proto(object): pass
		proto = _Proto()
		proto.timestamps = {}

	def _scanScripts(sub):
		for i in os.listdir('%s/%s' % (path, sub)):
			sys.stdout.write('.')
{'fraction_non_alphanumeric': 0.16472290028026398, 'fraction_numerical': 0.0043395714673176026, 'mean_word_length': 4.435872235872236, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7078573', 'n_tokens_mistral': 4227, 'n_tokens_neox': 3941, 'n_words': 1090}