<commit_before>from pandas import DataFrame,Series
import pandas as pd
import numpy as np
from numbers import Number

def __validate_and_grab_columns(data,x=None,y=None):
	if not isinstance(data,DataFrame):
		raise Exception("Data argument is not a pandas DataFrame")

	if data.shape[1] < 2:
		raise Exception("DataFrame must have at least two columns")

	xColumn = None
	yColumn = None

	if x is None:
		xColumn = data.columns[0]
	else:
		if x not in data.columns:
			raise Exception("Column %s not found in the given DataFrame" % x)
		xColumn = x

	if y is None:
		yColumn = data.columns[1]
	else:
		if y not in data.columns:
			raise Exception("Column %s not found in the given DataFrame" % y)
		yColumn = y

	for i in list(range(data.shape[0])):
		if not __is_actual_number(data[xColumn][i]) or not __is_actual_number(data[yColumn][i]): 
			raise Exception("Non-numeric values found at row %d (%s,%s)" % (i+1),data[xColumn][i],data[yColumn][i])

	return (xColumn,yColumn)

def __is_actual_number(x):
	return isinstance(x,Number) and not np.isnan(x)

def __trim(data,x,y):
	return data[[x,y]]

def __ucc_x(data,x,y):
	return np.ediff1d(data.sort([x]).rank()[y].values).mean()

def __ucc_y(data,x,y):
	return __ucc_x(data,y,x)

def ucc(data,x=None,y=None):
	columns = __validate_and_grab_columns(data,x=x,y=y)
	
	x = columns[0]
	y = columns[1]

	dataFrame = __trim(data,x,y).drop_duplicates

	ucc_x = __ucc_x(data,x,y)
	ucc_y = __ucc_y(data,x,y)
	ucc = max([ucc_x,ucc_y])

	return Series([ucc_x,ucc_y,ucc],index=['ucc_x','ucc_y','ucc'])<commit_msg>Fixed calculation error, slight refactor.<commit_after>from pandas import DataFrame,Series
import pandas as pd
import numpy as np
from numbers import Number

def __validate_and_grab_columns(data,x=None,y=None):
	if not isinstance(data,DataFrame):
		raise Exception("Data argument is not a pandas DataFrame")

	if data.shape[1] < 2:
		raise Exception("DataFrame must have at least two columns")

	xColumn = None
	yColumn = None

	if x is None:
		xColumn = data.columns[0]
	else:
		if x not in data.columns:
			raise Exception("Column %s not found in the given DataFrame" % x)
		xColumn = x

	if y is None:
		yColumn = data.columns[1]
	else:
		if y not in data.columns:
			raise Exception("Column %s not found in the given DataFrame" % y)
		yColumn = y

	for i in list(range(data.shape[0])):
		if not __is_actual_number(data[xColumn][i]) or not __is_actual_number(data[yColumn][i]): 
			raise Exception("Non-numeric values found at row %d (%s,%s)" % (i+1),data[xColumn][i],data[yColumn][i])

	return (xColumn,yColumn)

def __is_actual_number(x):
	return isinstance(x,Number) and not np.isnan(x)

def __trim(data,x,y):
	return data[[x,y]]

def __avg_of_deltas(data,dependent,independent):
	return np.ediff1d(data.sort([independent]).rank()[dependent].values).mean()

def ucc(data,x=None,y=None,copy=False):
	columns = __validate_and_grab_columns(data,x=x,y=y)
	
	xColumn = columns[0]
	yColumn = columns[1]
	
	dataFrame = None
	
	if(copy):
		dataFrame = __trim(data,xColumn,yColumn).drop_duplicates
	else:
		dataFrame = data.drop_duplicates([xColumn,yColumn])
	
	n = dataFrame.shape[0]
	
	ucc_x = 1 - __avg_of_deltas(dataFrame,xColumn,yColumn) * 3 / n
	ucc_y = 1 - __avg_of_deltas(dataFrame,yColumn,xColumn) * 3 / n
	ucc = max([ucc_x,ucc_y])

	return Series([ucc_x,ucc_y,ucc],index=['ucc_x','ucc_y','ucc'])
{'fraction_non_alphanumeric': 0.1357398568019093, 'fraction_numerical': 0.006861575178997614, 'mean_word_length': 5.255597014925373, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2660960', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1258, 'n_words': 313}