<commit_before>from __future__ import division, print_function, absolute_import
from abc import ABCMeta, abstractmethod
import warnings
import time
import numbers
import numpy as np
import pandas as pd

import h2o
from h2o.frame import H2OFrame
from h2o import H2OEstimator
from .base import _check_is_frame
from ..base import overrides

from sklearn.externals import six
from sklearn.base import _pprint
from sklearn.utils.fixes import signature, bincount
from sklearn.utils import check_random_state


__all__ = [
	'check_cv',
	'H2OKFold',
	'H2OStratifiedKFold'
]




def _build_repr(self):
	# XXX This is copied from sklearn.BaseEstimator's get_params
	cls = self.__class__
	init = getattr(cls.__init__, 'deprecated_original', cls.__init__)

	init_signature = signature(init)

	if init is object.__init__:
		args = []
	else:
		args = sorted([p.name for p in init_signature.parameters.values()
						if p.name != 'self' and p.kind != p.VAR_KEYWORD])

	class_name = self.__class__.__name__
	params = dict()
	for key in args:
		warnings.simplefilter("always", DeprecationWarning)
		try:
			with warnings.catch_warnings(record=True) as w:
				value = getattr(self, key, None)
			if len(w) and w[0].category == DeprecationWarning:
				continue
		finally:
			warnings.filters.pop(0)
		params[key] = value

	return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))


def check_cv(cv=3):
	if cv is None:
		cv = 3

	if isinstance(cv, numbers.Integral):
		return H2OKFold(cv)

	if not isinstance(cv, H2OBaseCrossValidator):
		raise ValueError('expected int or instance of '
						 'H2OBaseCrossValidator but got %s'
						 % type(cv))

	return cv

def _val_y(y):
	if isinstance(y, (str, unicode)):
		return str(y)
	elif y is None:
		return y
	raise TypeError('y must be a string. Got %s' % y)


class H2OBaseCrossValidator(six.with_metaclass(ABCMeta)):

	def __init__(self):
		pass

	def split(self, frame, y=None):
		"""Generate indices to split data into training and test.

		Parameters
		----------
		frame : H2OFrame
			The h2o frame to split

		Returns
		-------
		train : ndarray
			The training set indices for the split

		test : ndarray
			The testing set indices for that split
		"""

		_check_is_frame(frame)
		indices = np.arange(frame.shape[0])
		for test_index in self._iter_test_masks(frame):
			train_index = indices[np.logical_not(test_index)]
			test_index = indices[test_index]

			# h2o can't handle anything but lists...
			yield list(train_index), list(test_index)
<commit_msg>Fixed bug in h2o CV<commit_after>from __future__ import division, print_function, absolute_import
from abc import ABCMeta, abstractmethod
import warnings
import time
import numbers
import numpy as np
import pandas as pd

import h2o
from h2o.frame import H2OFrame
from h2o import H2OEstimator
from .base import _check_is_frame
from ..base import overrides

from sklearn.externals import six
from sklearn.base import _pprint
from sklearn.utils.fixes import signature, bincount
from sklearn.utils import check_random_state


__all__ = [
	'check_cv',
	'H2OKFold',
	'H2OStratifiedKFold'
]




def _build_repr(self):
	# XXX This is copied from sklearn.BaseEstimator's get_params
	cls = self.__class__
	init = getattr(cls.__init__, 'deprecated_original', cls.__init__)

	init_signature = signature(init)

	if init is object.__init__:
		args = []
	else:
		args = sorted([p.name for p in init_signature.parameters.values()
						if p.name != 'self' and p.kind != p.VAR_KEYWORD])

	class_name = self.__class__.__name__
	params = dict()
	for key in args:
		warnings.simplefilter("always", DeprecationWarning)
		try:
			with warnings.catch_warnings(record=True) as w:
				value = getattr(self, key, None)
			if len(w) and w[0].category == DeprecationWarning:
				continue
		finally:
			warnings.filters.pop(0)
		params[key] = value

	return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))


def check_cv(cv=3):
	if cv is None:
		cv = 3

	if isinstance(cv, numbers.Integral):
		return H2OKFold(cv)

	if not isinstance(cv, H2OBaseCrossValidator):
		raise ValueError('expected int or instance of '
						 'H2OBaseCrossValidator but got %s'
						 % type(cv))

	return cv

def _val_y(y):
	if isinstance(y, (str, unicode)):
		return str(y)
	elif y is None:
		return y
	raise TypeError('y must be a string. Got %s' % y)


class H2OBaseCrossValidator(six.with_metaclass(ABCMeta)):

	def __init__(self):
		pass

	def split(self, frame, y=None):
		"""Generate indices to split data into training and test.

		Parameters
		----------
		frame : H2OFrame
			The h2o frame to split

		Returns
		-------
		train : ndarray
			The training set indices for the split

		test : ndarray
			The testing set indices for that split
		"""

		_check_is_frame(frame)
		indices = np.arange(frame.shape[0])
		for test_index in self._iter_test_masks(frame, y):
			train_index = indices[np.logical_not(test_index)]
			test_index = indices[test_index]

			# h2o can't handle anything but lists...
			yield list(train_index), list(test_index)

{'fraction_non_alphanumeric': 0.09930348258706467, 'fraction_numerical': 0.007761194029850746, 'mean_word_length': 4.066532258064516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '191348', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1696, 'n_words': 507}