# Path: sklearn/utils/testing.py
#     def assert_in(x, container):
#     def assert_not_in(x, container):
#     def assert_raises_regex(expected_exception, expected_regexp,
#                             callable_obj=None, *args, **kwargs):
# def _assert_less(a, b, msg=None):
# def _assert_greater(a, b, msg=None):
# def assert_less_equal(a, b, msg=None):
# def assert_greater_equal(a, b, msg=None):
# def assert_warns(warning_class, func, *args, **kw):
# def assert_warns_message(warning_class, message, func, *args, **kw):
# def assert_no_warnings(func, *args, **kw):
# def ignore_warnings(obj=None):
# def _ignore_warnings(fn):
#     def wrapper(*args, **kwargs):
#     def __init__(self):
#     def __repr__(self):
#     def __enter__(self):
#             def showwarning(*args, **kwargs):
#     def __exit__(self, *exc_info):
# def _assert_allclose(actual, desired, rtol=1e-7, atol=0,
#                      err_msg='', verbose=True):
# def assert_raise_message(exception, message, function, *args, **kwargs):
# def fake_mldata(columns_dict, dataname, matfile, ordering=None):
#     def __init__(self, mock_datasets):
#     def __call__(self, urlname):
# def install_mldata_mock(mock_datasets):
# def uninstall_mldata_mock():
# def all_estimators(include_meta_estimators=False, include_other=False,
#                    type_filter=None, include_dont_test=False):
#     def is_abstract(c):
# def set_random_state(estimator, random_state=0):
# def if_matplotlib(func):
#     def run_test(*args, **kwargs):
# def if_not_mac_os(versions=('10.7', '10.8', '10.9'),
#                   message='Multi-process bug in Mac OS X >= 10.7 '
#                           '(see issue #636)'):
#     def decorator(func):
#             def func(*args, **kwargs):
# def clean_warning_registry():
# def check_skip_network():
# def check_skip_travis():
# class _IgnoreWarnings(object):
# class mock_mldata_urlopen(object):
# META_ESTIMATORS = ["OneVsOneClassifier",
#                    "OutputCodeClassifier", "OneVsRestClassifier", "RFE",
#                    "RFECV", "BaseEnsemble"]
# OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV"]
# DONT_TEST = ['SparseCoder', 'EllipticEnvelope', 'DictVectorizer',
#              'LabelBinarizer', 'LabelEncoder', 'MultiLabelBinarizer',
#              'TfidfTransformer', 'IsotonicRegression', 'OneHotEncoder',
#              'RandomTreesEmbedding', 'FeatureHasher', 'DummyClassifier',
#              'DummyRegressor', 'TruncatedSVD', 'PolynomialFeatures',
#              'GaussianRandomProjectionHash']


import warnings
import numpy as np
import pickle
from sklearn.feature_extraction.text import strip_tags
from sklearn.feature_extraction.text import strip_accents_unicode
from sklearn.feature_extraction.text import strip_accents_ascii
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
from sklearn.cross_validation import train_test_split
from sklearn.cross_validation import cross_val_score
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.base import clone
from nose import SkipTest
from nose.tools import assert_equal
from nose.tools import assert_false
from nose.tools import assert_not_equal
from nose.tools import assert_true
from nose.tools import assert_almost_equal
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal
from numpy.testing import assert_raises
from sklearn.utils.testing import (assert_in, assert_less, assert_greater,
                                   assert_warns_message, assert_raise_message,
                                   clean_warning_registry)
from collections import defaultdict, Mapping
from functools import partial
from io import StringIO
    assert_true((tfidf >= 0).all())

    # check normalization
    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])

    # the lack of smoothing make IDF fragile in the presence of feature with
    # only zeros
    X = [[1, 1, 0],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')

    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        1. / np.array([0.])
        numpy_provides_div0_warning = len(w) == 1

    in_warning_message = 'divide by zero'
    tfidf = assert_warns_message(RuntimeWarning, in_warning_message,
                                 tr.fit_transform, X).toarray()
    if not numpy_provides_div0_warning:
        raise SkipTest("Numpy does not provide div 0 warnings.")


def test_sublinear_tf():
    X = [[1], [2], [3]]
    tr = TfidfTransformer(sublinear_tf=True, use_idf=False, norm=None)
    tfidf = tr.fit_transform(X).toarray()
    assert_equal(tfidf[0], 1)
    assert_greater(tfidf[1], tfidf[0])

# Path: sklearn/utils/testing.py
#     def assert_in(x, container):
#     def assert_not_in(x, container):
#     def assert_raises_regex(expected_exception, expected_regexp,
#                             callable_obj=None, *args, **kwargs):
# def _assert_less(a, b, msg=None):
# def _assert_greater(a, b, msg=None):
# def assert_less_equal(a, b, msg=None):
# def assert_greater_equal(a, b, msg=None):
# def assert_warns(warning_class, func, *args, **kw):
# def assert_warns_message(warning_class, message, func, *args, **kw):
# def assert_no_warnings(func, *args, **kw):
# def ignore_warnings(obj=None):
# def _ignore_warnings(fn):
#     def wrapper(*args, **kwargs):
#     def __init__(self):
#     def __repr__(self):
#     def __enter__(self):
#             def showwarning(*args, **kwargs):
#     def __exit__(self, *exc_info):
# def _assert_allclose(actual, desired, rtol=1e-7, atol=0,
#                      err_msg='', verbose=True):
# def assert_raise_message(exception, message, function, *args, **kwargs):
# def fake_mldata(columns_dict, dataname, matfile, ordering=None):
#     def __init__(self, mock_datasets):
#     def __call__(self, urlname):
# def install_mldata_mock(mock_datasets):
# def uninstall_mldata_mock():
# def all_estimators(include_meta_estimators=False, include_other=False,
#                    type_filter=None, include_dont_test=False):
#     def is_abstract(c):
# def set_random_state(estimator, random_state=0):
# def if_matplotlib(func):
#     def run_test(*args, **kwargs):
# def if_not_mac_os(versions=('10.7', '10.8', '10.9'),
#                   message='Multi-process bug in Mac OS X >= 10.7 '
#                           '(see issue #636)'):
#     def decorator(func):
#             def func(*args, **kwargs):
# def clean_warning_registry():
# def check_skip_network():
# def check_skip_travis():
# class _IgnoreWarnings(object):
# class mock_mldata_urlopen(object):
# META_ESTIMATORS = ["OneVsOneClassifier",
#                    "OutputCodeClassifier", "OneVsRestClassifier", "RFE",
#                    "RFECV", "BaseEnsemble"]
# OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV"]
# DONT_TEST = ['SparseCoder', 'EllipticEnvelope', 'DictVectorizer',
#              'LabelBinarizer', 'LabelEncoder', 'MultiLabelBinarizer',
#              'TfidfTransformer', 'IsotonicRegression', 'OneHotEncoder',
#              'RandomTreesEmbedding', 'FeatureHasher', 'DummyClassifier',
#              'DummyRegressor', 'TruncatedSVD', 'PolynomialFeatures',
#              'GaussianRandomProjectionHash']


# Path: sklearn/feature_extraction/tests/test_text.py
import warnings
import numpy as np
import pickle
from sklearn.feature_extraction.text import strip_tags
from sklearn.feature_extraction.text import strip_accents_unicode
from sklearn.feature_extraction.text import strip_accents_ascii
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
from sklearn.cross_validation import train_test_split
from sklearn.cross_validation import cross_val_score
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC
from sklearn.base import clone
from nose import SkipTest
from nose.tools import assert_equal
from nose.tools import assert_false
from nose.tools import assert_not_equal
from nose.tools import assert_true
from nose.tools import assert_almost_equal
from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal
from numpy.testing import assert_raises
from sklearn.utils.testing import (assert_in, assert_less, assert_greater,
                                   assert_warns_message, assert_raise_message,
                                   clean_warning_registry)
from collections import defaultdict, Mapping
from functools import partial
from io import StringIO

    assert_true((tfidf >= 0).all())

    # check normalization
    assert_array_almost_equal((tfidf ** 2).sum(axis=1), [1., 1., 1.])

    # the lack of smoothing make IDF fragile in the presence of feature with
    # only zeros
    X = [[1, 1, 0],
         [1, 1, 0],
         [1, 0, 0]]
    tr = TfidfTransformer(smooth_idf=False, norm='l2')

    clean_warning_registry()
    with warnings.catch_warnings(record=True) as w:
        1. / np.array([0.])
        numpy_provides_div0_warning = len(w) == 1

    in_warning_message = 'divide by zero'
    tfidf = assert_warns_message(RuntimeWarning, in_warning_message,
                                 tr.fit_transform, X).toarray()
    if not numpy_provides_div0_warning:
        raise SkipTest("Numpy does not provide div 0 warnings.")


def test_sublinear_tf():
    X = [[1], [2], [3]]
    tr = TfidfTransformer(sublinear_tf=True, use_idf=False, norm=None)
    tfidf = tr.fit_transform(X).toarray()
    assert_equal(tfidf[0], 1)
    assert_greater(tfidf[1], tfidf[0])

    assert_greater(tfidf[2], tfidf[1])