<commit_before>"""
Utilities for testing
"""

from shogun.Features import *
from shogun.PreProc import *
from shogun.Distance import *
from shogun.Kernel import *
from numpy import *


def check_accuracy (accuracy, **kwargs):
	acc=double(accuracy)
	output=[]

	for key, val in kwargs.iteritems():
		if val is not None:
			output.append('%s: %e' % (key, val))
	print ', '.join(output)+' <--- accuracy: %e' % accuracy

	for val in kwargs.itervalues():
		if val>acc:
			return False

	return True


def get_args (indata, prefix=''):
	"""
	Slightly esoteric function to build a tuple to be used as argument to
	constructor calls.

	Python dicts are not ordered, so we have to look at the number in
	the parameter's name and insert items appropriately into an
	ordered list
	"""

	ident=prefix+'arg'
	# need to pregenerate list for using indices in loop
	args=len(indata)*[None]

	for i in indata:
		if i.find(ident)==-1:
			continue

		try:
			idx=int(i[len(ident)])
		except ValueError:
			raise ValueError, 'Wrong indata data %s: "%s"!' % (ident, i)

		if i.find('_distance')!=-1: # DistanceKernel
			args[idx]=eval(indata[i]+'()')
		else:
			try:
				args[idx]=eval(indata[i])
			except TypeError: # no bool
				args[idx]=indata[i]

	# weed out superfluous Nones
	return filter(lambda arg: arg is not None, args)


def get_features(indata, prefix=''):
	fclass=prefix+'feature_class'
	if indata[fclass]=='simple':
		return get_feats_simple(indata, prefix)
	elif indata[fclass]=='string':
		return get_feats_string(indata, prefix)
	elif indata[fclass]=='string_complex':
		return get_feats_string_complex(indata, prefix)
	elif indata[fclass]=='wd':
		return get_feats_wd(indata, prefix)
	else:
		raise ValueError, \
			'Unknown feature class %s!'%indata[prefix+'feature_class']


def get_feats_simple (indata, prefix=''):
	ftype=indata[prefix+'feature_type']

	# have to explicitely set data type for numpy if not real
	as_types={
		'Byte': ubyte,
		'Real': double,
		'Word': ushort
	}
	data_train=indata[prefix+'data_train'].astype(as_types[ftype])
	data_test=indata[prefix+'data_test'].astype(as_types[ftype])

	if ftype=='Byte' or ftype=='Char':
		alphabet=eval(indata[prefix+'alphabet'])
		ftrain=eval(ftype+'Features(alphabet)')
		ftest=eval(ftype+'Features(alphabet)')
		ftrain.copy_feature_matrix(data_train)
		ftest.copy_feature_matrix(data_test)
	else:
		ftrain=eval(ftype+'Features(data_train)')
		ftest=eval(ftype+'Features(data_test)')

	if (indata[prefix+'name'].find('Sparse')!=-1 or (
		indata.has_key('classifier_type') and \
			indata['classifier_type']=='linear')):
		sparse_train=eval('Sparse'+ftype+'Features()')
		sparse_train.obtain_from_simple(ftrain)

		sparse_test=eval('Sparse'+ftype+'Features()')
		sparse_test.obtain_from_simple(ftest)

		return {'train':sparse_train, 'test':sparse_test}
	else:
		return {'train':ftrain, 'test':ftest}


def get_feats_string (indata, prefix=''):
	ftype=indata[prefix+'feature_type']
	alphabet=eval(indata[prefix+'alphabet'])
	feats={
		'train': eval('String'+ftype+'Features(alphabet)'),
		'test': eval('String'+ftype+'Features(alphabet)')
	}
	feats['train'].set_string_features(list(indata[prefix+'data_train'][0]))
	feats['test'].set_string_features(list(indata[prefix+'data_test'][0]))

	return feats


def get_feats_string_complex (indata, prefix=''):
	alphabet=eval(indata[prefix+'alphabet'])
	feats={
		'train': StringCharFeatures(DNA),
		'test': StringCharFeatures(DNA)
	}

	if alphabet==CUBE: # data_{train,test} ints due to test.py:_read_matrix
		data_train=[str(x) for x in list(indata[prefix+'data_train'][0])]
		data_test=[str(x) for x in list(indata[prefix+'data_test'][0])]
	else:
		data_train=list(indata[prefix+'data_train'][0])
		data_test=list(indata[prefix+'data_test'][0])

	feats['train'].set_string_features(data_train)
	feats['test'].set_string_features(data_test)

	feat=eval('String'+indata[prefix+'feature_type']+ \
		"Features(alphabet)")
	feat.obtain_from_char(feats['train'], indata[prefix+'order']-1,
		indata[prefix+'order'], indata[prefix+'gap'],
		eval(indata[prefix+'reverse']))
	feats['train']=feat

	feat=eval('String'+indata[prefix+'feature_type']+ \
		"Features(alphabet)")
	feat.obtain_from_char(feats['test'], indata[prefix+'order']-1,
		indata[prefix+'order'], indata[prefix+'gap'],
		eval(indata[prefix+'reverse']))<commit_msg>fix alphabet<commit_after>"""
Utilities for testing
"""

from shogun.Features import *
from shogun.PreProc import *
from shogun.Distance import *
from shogun.Kernel import *
from numpy import *


def check_accuracy (accuracy, **kwargs):
	acc=double(accuracy)
	output=[]

	for key, val in kwargs.iteritems():
		if val is not None:
			output.append('%s: %e' % (key, val))
	print ', '.join(output)+' <--- accuracy: %e' % accuracy

	for val in kwargs.itervalues():
		if val>acc:
			return False

	return True


def get_args (indata, prefix=''):
	"""
	Slightly esoteric function to build a tuple to be used as argument to
	constructor calls.

	Python dicts are not ordered, so we have to look at the number in
	the parameter's name and insert items appropriately into an
	ordered list
	"""

	ident=prefix+'arg'
	# need to pregenerate list for using indices in loop
	args=len(indata)*[None]

	for i in indata:
		if i.find(ident)==-1:
			continue

		try:
			idx=int(i[len(ident)])
		except ValueError:
			raise ValueError, 'Wrong indata data %s: "%s"!' % (ident, i)

		if i.find('_distance')!=-1: # DistanceKernel
			args[idx]=eval(indata[i]+'()')
		else:
			try:
				args[idx]=eval(indata[i])
			except TypeError: # no bool
				args[idx]=indata[i]

	# weed out superfluous Nones
	return filter(lambda arg: arg is not None, args)


def get_features(indata, prefix=''):
	fclass=prefix+'feature_class'
	if indata[fclass]=='simple':
		return get_feats_simple(indata, prefix)
	elif indata[fclass]=='string':
		return get_feats_string(indata, prefix)
	elif indata[fclass]=='string_complex':
		return get_feats_string_complex(indata, prefix)
	elif indata[fclass]=='wd':
		return get_feats_wd(indata, prefix)
	else:
		raise ValueError, \
			'Unknown feature class %s!'%indata[prefix+'feature_class']


def get_feats_simple (indata, prefix=''):
	ftype=indata[prefix+'feature_type']

	# have to explicitely set data type for numpy if not real
	as_types={
		'Byte': ubyte,
		'Real': double,
		'Word': ushort
	}
	data_train=indata[prefix+'data_train'].astype(as_types[ftype])
	data_test=indata[prefix+'data_test'].astype(as_types[ftype])

	if ftype=='Byte' or ftype=='Char':
		alphabet=eval(indata[prefix+'alphabet'])
		ftrain=eval(ftype+'Features(alphabet)')
		ftest=eval(ftype+'Features(alphabet)')
		ftrain.copy_feature_matrix(data_train)
		ftest.copy_feature_matrix(data_test)
	else:
		ftrain=eval(ftype+'Features(data_train)')
		ftest=eval(ftype+'Features(data_test)')

	if (indata[prefix+'name'].find('Sparse')!=-1 or (
		indata.has_key('classifier_type') and \
			indata['classifier_type']=='linear')):
		sparse_train=eval('Sparse'+ftype+'Features()')
		sparse_train.obtain_from_simple(ftrain)

		sparse_test=eval('Sparse'+ftype+'Features()')
		sparse_test.obtain_from_simple(ftest)

		return {'train':sparse_train, 'test':sparse_test}
	else:
		return {'train':ftrain, 'test':ftest}


def get_feats_string (indata, prefix=''):
	ftype=indata[prefix+'feature_type']
	alphabet=eval(indata[prefix+'alphabet'])
	feats={
		'train': eval('String'+ftype+'Features(alphabet)'),
		'test': eval('String'+ftype+'Features(alphabet)')
	}
	feats['train'].set_string_features(list(indata[prefix+'data_train'][0]))
	feats['test'].set_string_features(list(indata[prefix+'data_test'][0]))

	return feats


def get_feats_string_complex (indata, prefix=''):
	alphabet=eval(indata[prefix+'alphabet'])
	feats={
		'train': StringCharFeatures(alphabet),
		'test': StringCharFeatures(alphabet)
	}

	if alphabet==CUBE: # data_{train,test} ints due to test.py:_read_matrix
		data_train=[str(x) for x in list(indata[prefix+'data_train'][0])]
		data_test=[str(x) for x in list(indata[prefix+'data_test'][0])]
	else:
		data_train=list(indata[prefix+'data_train'][0])
		data_test=list(indata[prefix+'data_test'][0])

	feats['train'].set_string_features(data_train)
	feats['test'].set_string_features(data_test)

	feat=eval('String'+indata[prefix+'feature_type']+ \
		"Features(alphabet)")
	feat.obtain_from_char(feats['train'], indata[prefix+'order']-1,
		indata[prefix+'order'], indata[prefix+'gap'],
		eval(indata[prefix+'reverse']))
	feats['train']=feat

	feat=eval('String'+indata[prefix+'feature_type']+ \
		"Features(alphabet)")
	feat.obtain_from_char(feats['test'], indata[prefix+'order']-1,
		indata[prefix+'order'], indata[prefix+'gap'],
		eval(indata[prefix+'reverse']))
{'fraction_non_alphanumeric': 0.16387766878245816, 'fraction_numerical': 0.0025389497980380844, 'mean_word_length': 6.535652173913044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4101555', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3227, 'n_words': 600}