# Path: pyBN/utils/independence_tests.py
# def are_independent(data, alpha=0.05, method='mi_test'):
# 	pval = mi_test(data)
# 	if pval < alpha:
# 		return True
# 	else:
# 		return False
# 
# def entropy(data):
# 	"""
# 	In the context of structure learning, and more specifically
# 	in constraint-based algorithms which rely on the mutual information
# 	test for conditional independence, it has been proven that the variable
# 	X in a set which MAXIMIZES mutual information is also the variable which
# 	MINIMIZES entropy. This fact can be used to reduce the computational
# 	requirements of tests based on the following relationship:
# 
# 		Entropy is related to marginal mutual information as follows:
# 			MI(X;Y) = H(X) - H(X|Y)
# 
# 		Entropy is related to conditional mutual information as follows:
# 			MI(X;Y|Z) = H(X|Z) - H(X|Y,Z)
# 
# 		For one varibale, H(X) is equal to the following:
# 			-1 * sum of p(x) * log(p(x))
# 
# 		For two variables H(X|Y) is equal to the following:
# 			sum over x,y of p(x,y)*log(p(y)/p(x,y))
# 		
# 		For three variables, H(X|Y,Z) is equal to the following:
# 			-1 * sum of p(x,y,z) * log(p(x|y,z)),
# 				where p(x|y,z) = p(x,y,z)/p(y)*p(z)
# 	Arguments
# 	----------
# 	*data* : a nested numpy array
# 		The data from which to learn - must have at least three
# 		variables. All conditioned variables (i.e. Z) are compressed
# 		into one variable.
# 
# 	Returns
# 	-------
# 	*H* : entropy value
# 
# 	"""
# 	try:
# 		cols = data.shape[1]
# 	except IndexError:
# 		cols = 1
# 
# 	#bins = np.amax(data,axis=0)
# 	bins = unique_bins(data)
# 
# 	if cols == 1:
# 		hist,_ = np.histogramdd(data, bins=(bins)) # frequency counts
# 		Px = hist/hist.sum()
# 		H = -1 * np.sum( Px * np.log( Px ) )
# 
# 	elif cols == 2: # two variables -> assume X then Y
# 		hist,_ = np.histogramdd(data, bins=bins[0:2]) # frequency counts
# 
# 		Pxy = hist / hist.sum()# joint probability distribution over X,Y,Z
# 		Py = np.sum(Pxy, axis = 0) # P(Y)	
# 		Py += 1e-7
# 		Pxy += 1e-7
# 		H = np.sum( Pxy * np.log( Py / Pxy ) )
# 
# 	else:
# 		# CHECK FOR > 3 COLUMNS -> concatenate Z into one column
# 		if cols  > 3:
# 			data = data.astype('str')
# 			ncols = len(bins)
# 			for i in range(len(data)):
# 				data[i,2] = ''.join(data[i,2:ncols])
# 			data = data.astype('int')[:,0:3]
# 
# 		bins = np.amax(data,axis=0)
# 		hist,_ = np.histogramdd(data, bins=bins) # frequency counts
# 
# 		Pxyz = hist / hist.sum()# joint probability distribution over X,Y,Z
# 		Pyz = np.sum(Pxyz, axis=0)
# 
# 		Pxyz += 1e-7 # for log -inf
# 		Pyz += 1e-7
# 		H = -1 * np.sum( Pxyz * np.log( Pxyz ) ) + np.sum( Pyz * np.log( Pyz ) ) 
# 
# 	return round(H,4)


import numpy as np
from pyBN.utils.independence_tests import are_independent, entropy
		_T = [feature_selection]

	# LEARN MARKOV BLANKET
	for T in _T:

		V = set(range(n_rv)) - {T}
		Mb_change=True

		# GROWING PHASE
		while Mb_change:
			Mb_change = False
			cols = tuple({T}) + tuple(Mb[T])
			H_tmb = entropy(data[:,cols])
			# find X1_min in V-Mb[T]-{T} that minimizes
			# entropy of T|X1_inMb[T]
			# i.e. min of entropy(data[:,(T,X,Mb[T])])
			min_val1, min_val2 = 1e7,1e7
			min_x1, min_x2 = None, None
			for X in V - Mb[T] - {T}:
				cols = (T,X)+tuple(Mb[T])
				ent_val = entropy(data[:,cols])
				if ent_val < min_val:
					min_val2, min_val1 = min_val1, ent_val
					min_x2,min_x1 = min_x1, X
					
			# if min_x1 is dependent on T given Mb[T]...
			cols = (min_x1,T) + tuple(Mb[T])
			if are_independent(data[:,cols]):
				if (min_val2 - L*min_val1) < ((1-L)*H_tmb):
					cols = (min_x2,T)+tuple(Mb[T])

# Path: pyBN/utils/independence_tests.py
# def are_independent(data, alpha=0.05, method='mi_test'):
# 	pval = mi_test(data)
# 	if pval < alpha:
# 		return True
# 	else:
# 		return False
# 
# def entropy(data):
# 	"""
# 	In the context of structure learning, and more specifically
# 	in constraint-based algorithms which rely on the mutual information
# 	test for conditional independence, it has been proven that the variable
# 	X in a set which MAXIMIZES mutual information is also the variable which
# 	MINIMIZES entropy. This fact can be used to reduce the computational
# 	requirements of tests based on the following relationship:
# 
# 		Entropy is related to marginal mutual information as follows:
# 			MI(X;Y) = H(X) - H(X|Y)
# 
# 		Entropy is related to conditional mutual information as follows:
# 			MI(X;Y|Z) = H(X|Z) - H(X|Y,Z)
# 
# 		For one varibale, H(X) is equal to the following:
# 			-1 * sum of p(x) * log(p(x))
# 
# 		For two variables H(X|Y) is equal to the following:
# 			sum over x,y of p(x,y)*log(p(y)/p(x,y))
# 		
# 		For three variables, H(X|Y,Z) is equal to the following:
# 			-1 * sum of p(x,y,z) * log(p(x|y,z)),
# 				where p(x|y,z) = p(x,y,z)/p(y)*p(z)
# 	Arguments
# 	----------
# 	*data* : a nested numpy array
# 		The data from which to learn - must have at least three
# 		variables. All conditioned variables (i.e. Z) are compressed
# 		into one variable.
# 
# 	Returns
# 	-------
# 	*H* : entropy value
# 
# 	"""
# 	try:
# 		cols = data.shape[1]
# 	except IndexError:
# 		cols = 1
# 
# 	#bins = np.amax(data,axis=0)
# 	bins = unique_bins(data)
# 
# 	if cols == 1:
# 		hist,_ = np.histogramdd(data, bins=(bins)) # frequency counts
# 		Px = hist/hist.sum()
# 		H = -1 * np.sum( Px * np.log( Px ) )
# 
# 	elif cols == 2: # two variables -> assume X then Y
# 		hist,_ = np.histogramdd(data, bins=bins[0:2]) # frequency counts
# 
# 		Pxy = hist / hist.sum()# joint probability distribution over X,Y,Z
# 		Py = np.sum(Pxy, axis = 0) # P(Y)	
# 		Py += 1e-7
# 		Pxy += 1e-7
# 		H = np.sum( Pxy * np.log( Py / Pxy ) )
# 
# 	else:
# 		# CHECK FOR > 3 COLUMNS -> concatenate Z into one column
# 		if cols  > 3:
# 			data = data.astype('str')
# 			ncols = len(bins)
# 			for i in range(len(data)):
# 				data[i,2] = ''.join(data[i,2:ncols])
# 			data = data.astype('int')[:,0:3]
# 
# 		bins = np.amax(data,axis=0)
# 		hist,_ = np.histogramdd(data, bins=bins) # frequency counts
# 
# 		Pxyz = hist / hist.sum()# joint probability distribution over X,Y,Z
# 		Pyz = np.sum(Pxyz, axis=0)
# 
# 		Pxyz += 1e-7 # for log -inf
# 		Pyz += 1e-7
# 		H = -1 * np.sum( Pxyz * np.log( Pxyz ) ) + np.sum( Pyz * np.log( Pyz ) ) 
# 
# 	return round(H,4)


# Path: pyBN/learning/structure/constraint/lambda_iamb.py
import numpy as np
from pyBN.utils.independence_tests import are_independent, entropy

		_T = [feature_selection]

	# LEARN MARKOV BLANKET
	for T in _T:

		V = set(range(n_rv)) - {T}
		Mb_change=True

		# GROWING PHASE
		while Mb_change:
			Mb_change = False
			cols = tuple({T}) + tuple(Mb[T])
			H_tmb = entropy(data[:,cols])
			# find X1_min in V-Mb[T]-{T} that minimizes
			# entropy of T|X1_inMb[T]
			# i.e. min of entropy(data[:,(T,X,Mb[T])])
			min_val1, min_val2 = 1e7,1e7
			min_x1, min_x2 = None, None
			for X in V - Mb[T] - {T}:
				cols = (T,X)+tuple(Mb[T])
				ent_val = entropy(data[:,cols])
				if ent_val < min_val:
					min_val2, min_val1 = min_val1, ent_val
					min_x2,min_x1 = min_x1, X
					
			# if min_x1 is dependent on T given Mb[T]...
			cols = (min_x1,T) + tuple(Mb[T])
			if are_independent(data[:,cols]):
				if (min_val2 - L*min_val1) < ((1-L)*H_tmb):
					cols = (min_x2,T)+tuple(Mb[T])

					if are_independent(data[:,cols]):