<commit_before>'''
Created on Feb 11, 2016

@author: jason
'''
from sklearn.base import BaseEstimator, clone, MetaEstimatorMixin
import numpy as np
from six import with_metaclass
from functools import update_wrapper
from inspect import getargspec
from sym import syms, sym_update, sym_predict, sym_transform_parts, sym_predict_parts
import pandas
from sympy.functions.elementary.miscellaneous import Max, Min
from sympy.core.numbers import RealNumber
from itertools import chain
from sym import sym_transform
from sympy.core.symbol import Symbol
from . import __version__
# 
# def if_delegate_has_method(*args, **kwargs):
#     return decorator(sklearn_if_delegate_has_method(*args, **kwargs))

# @decorator
def if_delegate_has_method(*args, **kwargs):
    return lambda x: x
#     return fn(*args, **kwargs)

def safe_call(fn, args):
    if hasattr(fn, '_spec'):
        spec = fn._spec
    else:   
        spec = getargspec(fn)
    if spec.keywords is not None:
        return fn(**args)
    else:
        safe_args = {arg: args[arg] for arg in spec.args[1:] if arg in args}
        try:
            return fn(**safe_args)
        except:
            1+1
            return fn(**safe_args)
        
def safer_call(fn, *args, **kwargs):
    kwargs = kwargs.copy()
    if hasattr(fn, '_spec'):
        spec = fn._spec
    else:
        try:
            spec = getargspec(fn)
        except TypeError:
            spec = getargspec(fn.__call__)
    spec_args = list(spec.args)
    if spec_args and spec_args[0] == 'self':
        spec_args = spec_args[1:]
    if spec.varargs is None:
        for i, arg in enumerate(args):
            name = spec_args[i]
            kwargs[name] = arg
            args = []
    if spec.keywords is None:
        for name in list(kwargs.keys()):
            if name not in spec_args:
                del kwargs[name]
    return fn(*args, **kwargs)

def safe_concat(arrays):
    return pandas.concat([pandas.DataFrame(arr) for arr in arrays], axis=0).as_matrix()

def _subset(data, idx):
    if len(data.shape) == 1:
        return data[idx]
    else:
        if hasattr(data, 'iloc'):
            return data.iloc[idx, :].reset_index(drop=True)
        else:
            return data[idx, :]

def _subset_data(data, idx):
    result = {}
    for k in data.keys():
        result[k] = _subset(data[k], idx)
    return result

def safe_assign_subset(arr, idx, value):
    try:
        arr.loc[idx, :] = value
    except AttributeError:
        try:
            arr[idx, :] = value.reshape(arr[idx, :].shape)
        except:
            arr.flat[idx] = value
    
def safe_column_names(arr):
    if hasattr(arr, 'columns'):
        return list(arr.columns)
    elif len(arr.shape) == 2:
        return map(lambda i: 'x%d'%i, range(arr.shape[1]))
    elif len(arr.shape) == 1:
        return ['x']
    else:
        raise ValueError()

def _fit_and_score(estimator, data, scorer, train, test):
    train_data = _subset_data(data, train)
    estimator_ = clone(estimator).fit(**train_data)
    test_data = _subset_data(data, test)
    score = safer_call(scorer, estimator_, **test_data)
    return (score, np.sum(test))

def _fit_and_predict(estimator, data, train, test):
    train_data = _subset_data(data, train)
    estimator_ = clone(estimator).fit(**train_data)
    test_data = _subset_data(data, test)
    prediction = safer_call(estimator_.predict, **test_data)
    return estimator_, prediction, test

class SklearnTool(object):
    _skversion = __version__
# 
# def name_estimator(estimator):
#     if hasattr(estimator, 'name'):
#         return estimator.name
#     else:
#         return estimator.__class__.__name__
# 
# def combine_named_estimators(names):
#     used_set = set()
#     result = []
#     for name in names:
#         new_name = name
#         i = 2
#         while new_name in used_set:
#             new_name = name + '_' + str(i)
#             i += 1
#             if i > 1e5:
#                 raise ValueError('Unable to name estimator %s in pipeline' % str(name))
#         used_set.add(new_name)
#         result.append(new_name)
#     return result
<commit_msg>Change _skversion to _version for standardization purposes<commit_after>'''
Created on Feb 11, 2016

@author: jason
'''
from sklearn.base import BaseEstimator, clone, MetaEstimatorMixin
import numpy as np
from six import with_metaclass
from functools import update_wrapper
from inspect import getargspec
from sym import syms, sym_update, sym_predict, sym_transform_parts, sym_predict_parts
import pandas
from sympy.functions.elementary.miscellaneous import Max, Min
from sympy.core.numbers import RealNumber
from itertools import chain
from sym import sym_transform
from sympy.core.symbol import Symbol
from . import __version__
# 
# def if_delegate_has_method(*args, **kwargs):
#     return decorator(sklearn_if_delegate_has_method(*args, **kwargs))

# @decorator
def if_delegate_has_method(*args, **kwargs):
    return lambda x: x
#     return fn(*args, **kwargs)

def safe_call(fn, args):
    if hasattr(fn, '_spec'):
        spec = fn._spec
    else:   
        spec = getargspec(fn)
    if spec.keywords is not None:
        return fn(**args)
    else:
        safe_args = {arg: args[arg] for arg in spec.args[1:] if arg in args}
        try:
            return fn(**safe_args)
        except:
            1+1
            return fn(**safe_args)
        
def safer_call(fn, *args, **kwargs):
    kwargs = kwargs.copy()
    if hasattr(fn, '_spec'):
        spec = fn._spec
    else:
        try:
            spec = getargspec(fn)
        except TypeError:
            spec = getargspec(fn.__call__)
    spec_args = list(spec.args)
    if spec_args and spec_args[0] == 'self':
        spec_args = spec_args[1:]
    if spec.varargs is None:
        for i, arg in enumerate(args):
            name = spec_args[i]
            kwargs[name] = arg
            args = []
    if spec.keywords is None:
        for name in list(kwargs.keys()):
            if name not in spec_args:
                del kwargs[name]
    return fn(*args, **kwargs)

def safe_concat(arrays):
    return pandas.concat([pandas.DataFrame(arr) for arr in arrays], axis=0).as_matrix()

def _subset(data, idx):
    if len(data.shape) == 1:
        return data[idx]
    else:
        if hasattr(data, 'iloc'):
            return data.iloc[idx, :].reset_index(drop=True)
        else:
            return data[idx, :]

def _subset_data(data, idx):
    result = {}
    for k in data.keys():
        result[k] = _subset(data[k], idx)
    return result

def safe_assign_subset(arr, idx, value):
    try:
        arr.loc[idx, :] = value
    except AttributeError:
        try:
            arr[idx, :] = value.reshape(arr[idx, :].shape)
        except:
            arr.flat[idx] = value
    
def safe_column_names(arr):
    if hasattr(arr, 'columns'):
        return list(arr.columns)
    elif len(arr.shape) == 2:
        return map(lambda i: 'x%d'%i, range(arr.shape[1]))
    elif len(arr.shape) == 1:
        return ['x']
    else:
        raise ValueError()

def _fit_and_score(estimator, data, scorer, train, test):
    train_data = _subset_data(data, train)
    estimator_ = clone(estimator).fit(**train_data)
    test_data = _subset_data(data, test)
    score = safer_call(scorer, estimator_, **test_data)
    return (score, np.sum(test))

def _fit_and_predict(estimator, data, train, test):
    train_data = _subset_data(data, train)
    estimator_ = clone(estimator).fit(**train_data)
    test_data = _subset_data(data, test)
    prediction = safer_call(estimator_.predict, **test_data)
    return estimator_, prediction, test

class SklearnTool(object):
    _version = __version__
# 
# def name_estimator(estimator):
#     if hasattr(estimator, 'name'):
#         return estimator.name
#     else:
#         return estimator.__class__.__name__
# 
# def combine_named_estimators(names):
#     used_set = set()
#     result = []
#     for name in names:
#         new_name = name
#         i = 2
#         while new_name in used_set:
#             new_name = name + '_' + str(i)
#             i += 1
#             if i > 1e5:
#                 raise ValueError('Unable to name estimator %s in pipeline' % str(name))
#         used_set.add(new_name)
#         result.append(new_name)
#     return result

{'fraction_non_alphanumeric': 0.11353501265517657, 'fraction_numerical': 0.004821019645655056, 'mean_word_length': 2.6346911957950065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '95673', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2669, 'n_words': 706}