<commit_before>from math import floor, sqrt
import numpy as np

def flatten(list_of_lists):
	"""[[a, b], [c, d]] -> [a, b, c, d]"""
	return reduce(list.__add__, list_of_lists, [])

def combine(head, *tail):
	"""
	Generate all possible combinations of a set of list
	[[a, b], [c, d]] -> [(a,c), (a,d), (b,c), (b,d)]
	(works on higher dimensions)
	"""
	if not tail:
		return [(h,) for h in head]
	return [(h,) + t for h in head for t in combine(tail[0], *tail[1:])]

class Point:
	def __init__(self, x=0, y=0, z=0, index=None):
		self.x = x
		self.y = y
		self.z = z
		self.index = index

	def __str__(self):
		return "Point<"+str(self.x)+", "+str(self.y)+", "+str(self.z)+", #"+str(self.index)+">"

	def __eq__(self, other):
		res = False
		if isinstance(other, tuple) or isinstance(other, list) and len(other) == 3:
			if self.x == other[0] and self.y == other[1] and self.z == other[2]:
				res = True
		elif isinstance(other, Point):
			if self.x == other.x and self.y == other.y and self.z == other.z:
				res = True
		return res

	def __ne__(self, other):
		return not (self == other)

	def __hash__(self):
		return hash((self.x, self.y, self.z, self.index))

	def distance(self, other):
		return sqrt((self.x-other.x)**2 + (self.y-other.y)**2 + (self.z-other.z)**2)

class VoxelSpace:
	""" VoxelSpace holds points within voxels. It makes it easier to find
		points that are close to each other for example"""
	def __init__(self, voxelSize=10):
		self.voxelSize = voxelSize
		self.voxels = {}
		self.highestPoint = None
		self.highestPointIndex = 1 # .obj files start counting vertices at 1, not 0

	def __str__(self):
		return 	"VoxelSpace<voxelSize="+str(self.voxelSize)+\
				", #voxels="+str(self.numberOfVoxels())+\
				", #points="+str(self.numberOfPoints())+\
				", avg(points/voxel)="+str(self.averagePointsPerVoxel())+">"
<commit_msg>[add] __sub__, __add__ and toNParray() to Point<commit_after>from math import floor, sqrt
import numpy as np

def flatten(list_of_lists):
	"""[[a, b], [c, d]] -> [a, b, c, d]"""
	return reduce(list.__add__, list_of_lists, [])

def combine(head, *tail):
	"""
	Generate all possible combinations of a set of list
	[[a, b], [c, d]] -> [(a,c), (a,d), (b,c), (b,d)]
	(works on higher dimensions)
	"""
	if not tail:
		return [(h,) for h in head]
	return [(h,) + t for h in head for t in combine(tail[0], *tail[1:])]

class Point:
	def __init__(self, x=0, y=0, z=0, index=None):
		self.x = x
		self.y = y
		self.z = z
		self.index = index

	def __str__(self):
		return "Point<"+str(self.x)+", "+str(self.y)+", "+str(self.z)+", #"+str(self.index)+">"

	def __eq__(self, other):
		res = False
		if isinstance(other, tuple) or isinstance(other, list) and len(other) == 3:
			if self.x == other[0] and self.y == other[1] and self.z == other[2]:
				res = True
		elif isinstance(other, Point):
			if self.x == other.x and self.y == other.y and self.z == other.z:
				res = True
		return res

	def __ne__(self, other):
		return not (self == other)

	def __hash__(self):
		return hash((self.x, self.y, self.z, self.index))

	def __add__(self, other):
		return Point(self.x+other.x, self.y+other.y, self.z+other.z)

	def __sub__(self, other):
		return Point(self.x-other.x, self.y-other.y, self.z-other.z)

	def toNPArray(self):
		return np.array([self.x, self.y, self.z])

	def distance(self, other):
		return sqrt((self.x-other.x)**2 + (self.y-other.y)**2 + (self.z-other.z)**2)

class VoxelSpace:
	""" VoxelSpace holds points within voxels. It makes it easier to find
		points that are close to each other for example"""
	def __init__(self, voxelSize=10):
		self.voxelSize = voxelSize
		self.voxels = {}
		self.highestPoint = None
		self.highestPointIndex = 1 # .obj files start counting vertices at 1, not 0

	def __str__(self):
		return 	"VoxelSpace<voxelSize="+str(self.voxelSize)+\
				", #voxels="+str(self.numberOfVoxels())+\
				", #points="+str(self.numberOfPoints())+\
				", avg(points/voxel)="+str(self.averagePointsPerVoxel())+">"

{'fraction_non_alphanumeric': 0.18446845941191253, 'fraction_numerical': 0.008544860517718019, 'mean_word_length': 3.9502487562189055, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3594721', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1520, 'n_words': 426}