<commit_before>from .core import *
import numpy as np

__version__ = core.__version__

def __reorder_array(array):
	if array.flags['C_CONTIGUOUS']: return array

	array = array.transpose(np.argsort(array.strides)[::-1])
	return np.ascontiguousarray(array)

def __p_fix_array(func):
	def func_wrapper(array, *args, **kwargs):
		return func(__reorder_array(array), *args, **kwargs)
	return func_wrapper

def __get_fn(array, fn_2d, fn_3d):
	if hasattr(array, 'axistags'):
		if len(array.shape) == 2: return fn_2d
		elif len(array.shape) == 3:
			if array.axistags[2].isSpatial(): return fn_3d
			elif array.axistags[2].isChannel(): return fn_2d
			else: raise NotImplementedError("Invalid array shape.")
		elif len(array.shape) == 4 and array.axistags[3].isChannel(): return fn_3d
		else: raise NotImplementedError("Invalid array shape.")
	else:
		if len(array.shape) == 2: return fn_2d
		elif len(array.shape) == 3:
			if array.shape[2] > 3: return fn_3d
			else: return fn_2d
		elif len(array.shape) == 4: return fn_3d
		else: return NotImplementedError("Invalid array shape.")<commit_msg>only convert array if required<commit_after>from .core import *
import numpy as np

__version__ = core.__version__

def __p_fix_array(func):
	def func_wrapper(array, *args, **kwargs):
		permutation = []
		if array.dtype == np.float32 and not array.flags['C_CONTIGUOUS']:
			permutation = np.argsort(array.strides)[::-1]
			array = array.transpose(permutation)

			if array.strides[-1] != array.dtype.itemsize:
				array = np.ascontiguousarray(array)

		res = func(array, *args, **kwargs)

		if len(permutation) > 0:
			res = res.transpose(permutation[::-1])

		return res

	return func_wrapper

def __get_fn(array, fn_2d, fn_3d):
	if hasattr(array, 'axistags'):
		if len(array.shape) == 2: return fn_2d
		elif len(array.shape) == 3:
			if array.axistags[2].isSpatial(): return fn_3d
			elif array.axistags[2].isChannel(): return fn_2d
			else: raise NotImplementedError("Invalid array shape.")
		elif len(array.shape) == 4 and array.axistags[3].isChannel(): return fn_3d
		else: raise NotImplementedError("Invalid array shape.")
	else:
		if len(array.shape) == 2: return fn_2d
		elif len(array.shape) == 3:
			if array.shape[2] > 3: return fn_3d
			else: return fn_2d
		elif len(array.shape) == 4: return fn_3d
		else: return NotImplementedError("Invalid array shape.")
{'fraction_non_alphanumeric': 0.13364446330080612, 'fraction_numerical': 0.02078913873568095, 'mean_word_length': 4.737226277372263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1271372', 'n_tokens_mistral': 926, 'n_tokens_neox': 850, 'n_words': 212}