<commit_before>from math import log

__author__ = 'demidovs'


class ApEn:
	def __init__(self, m):
		self.N = -1
		self.m = m
		self.r = -1
		self.u_list = []
		self.x_list = []
		self.c_list = []

	def read_series(self, fileName):
		self.u_list = []
		with open(fileName, "r") as f:
			for val in f.readlines():
				self.u_list.append(float(val.strip()))
		assert self.u_list, "File is either missed or corrupted"
		assert len(self.u_list) >= 300, "Sample length is too small. Need more than 300"
		self.N = len(self.u_list)

	def create_vectors(self, m):
		self.x_list = []
		for i in range(self.N - m + 1):
			self.x_list.append(self.u_list[i:i + m])

	def calculate_distance(self, x1, x2):
		assert len(x1) == len(x2), "Vectors should be of equal sizes: " + str(x1) + " : " + str(x2)
		res = []
		for i in range(len(x1)):
			res.append(abs(x1[i] - x2[i]))
		return max(res)

	def calculate_c(self, m):
		self.c_list = []
		assert self.r >= 0, "Filtering threshold should be positive"
		for i in range(0, self.N - m + 1):
			similar_vectors = 0
			for j in range(0, self.N - m + 1):
				res = self.calculate_distance(self.x_list[i], self.x_list[j])
				if (res < self.r):
					similar_vectors += 1
			self.c_list.append(similar_vectors / (self.N - m + 1))

	def _final(self, m):
		return sum([log(self.c_list[i]) for i in range(self.N - m + 1)]) \
		       * ((self.N - m + 1) ** (-1))

	def calculate_final(self, m):

		# 3. Form a sequence of vectors so that
		# x[i] = [u[i],u[i+1],...,u[i+m-1]]
		# i ~ 0:N-m because indexes start from 0
		self.create_vectors(m=m)

		# 4. Construct the C(i,m) - portion of vectors "similar" to i-th
		# similarity - d[x(j),x(i)], where d = max(a)|u(a)-u*(a)|
		# this is just the respective values subtraction
		self.calculate_c(m=m)

		res1 = self._final(m=m)
		return res1

	def calculate_apen(self, m):
		return self.calculate_final(m) - self.calculate_final(m + 1)

	def calculate_deviation(self):
		total_sum_norm = sum(self.u_list) / self.N
		self.r = (sum([(i - total_sum_norm) ** 2
		               for i in self.u_list])
		          / (self.N - 1)) ** (1 / 2)

	def prepare_calculate_apen(self,m,series):
		# tmpApEn = ApEn(m=2)
		self.read_series(series)
		self.calculate_deviation()
		return self.calculate_apen(m=m)

def makeReport(fileName="results/results.csv", filesList = None,apEnList=None,rList=None):
	if not filesList:
		print("Error in generating report")
	with open(fileName,"w") as f:
		for index,name in enumerate(filesList):
			f.write(",".join([name,str(apEnList[index]),str(rList[index])]) + '\n')<commit_msg>added abs to the result<commit_after>from math import log

__author__ = 'demidovs'


class ApEn:
	def __init__(self, m):
		self.N = -1
		self.m = m
		self.r = -1
		self.u_list = []
		self.x_list = []
		self.c_list = []

	def read_series(self, fileName):
		self.u_list = []
		with open(fileName, "r") as f:
			for val in f.readlines():
				self.u_list.append(float(val.strip()))
		assert self.u_list, "File is either missed or corrupted"
		assert len(self.u_list) >= 300, "Sample length is too small. Need more than 300"
		self.N = len(self.u_list)

	def create_vectors(self, m):
		self.x_list = []
		for i in range(self.N - m + 1):
			self.x_list.append(self.u_list[i:i + m])

	def calculate_distance(self, x1, x2):
		assert len(x1) == len(x2), "Vectors should be of equal sizes: " + str(x1) + " : " + str(x2)
		res = []
		for i in range(len(x1)):
			res.append(abs(x1[i] - x2[i]))
		return max(res)

	def calculate_c(self, m):
		self.c_list = []
		assert self.r >= 0, "Filtering threshold should be positive"
		for i in range(0, self.N - m + 1):
			similar_vectors = 0
			for j in range(0, self.N - m + 1):
				res = self.calculate_distance(self.x_list[i], self.x_list[j])
				if (res < self.r):
					similar_vectors += 1
			self.c_list.append(similar_vectors / (self.N - m + 1))

	def _final(self, m):
		return sum([log(self.c_list[i]) for i in range(self.N - m + 1)]) \
		       * ((self.N - m + 1) ** (-1))

	def calculate_final(self, m):

		# 3. Form a sequence of vectors so that
		# x[i] = [u[i],u[i+1],...,u[i+m-1]]
		# i ~ 0:N-m because indexes start from 0
		self.create_vectors(m=m)

		# 4. Construct the C(i,m) - portion of vectors "similar" to i-th
		# similarity - d[x(j),x(i)], where d = max(a)|u(a)-u*(a)|
		# this is just the respective values subtraction
		self.calculate_c(m=m)

		res1 = self._final(m=m)
		return res1

	def calculate_apen(self, m):
		return abs(self.calculate_final(m) - self.calculate_final(m + 1))

	def calculate_deviation(self):
		total_sum_norm = sum(self.u_list) / self.N
		self.r = (sum([(i - total_sum_norm) ** 2
		               for i in self.u_list])
		          / (self.N - 1)) ** (1 / 2)

	def prepare_calculate_apen(self,m,series):
		# tmpApEn = ApEn(m=2)
		self.read_series(series)
		self.calculate_deviation()
		return self.calculate_apen(m=m)

def makeReport(fileName="results/results.csv", filesList = None,apEnList=None,rList=None):
	if not filesList:
		print("Error in generating report")
	with open(fileName,"w") as f:
		for index,name in enumerate(filesList):
			f.write(",".join([name,str(apEnList[index]),str(rList[index])]) + '\n')
{'fraction_non_alphanumeric': 0.16560263106983944, 'fraction_numerical': 0.016637647514025924, 'mean_word_length': 3.348191757779647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023166', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2048, 'n_words': 499}