<commit_before># -*- coding: utf-8 -*-


import numpy as np
import pandas as pd
import pandas.util.testing as tm
import pytest
from pandas import DataFrame, MultiIndex, date_range
from pandas.compat import PY3, range
from pandas.util.testing import assert_almost_equal


def test_tolist(idx):
    result = idx.tolist()
    exp = list(idx.values)
    assert result == exp


def test_to_frame():
    tuples = [(1, 'one'), (1, 'two'), (2, 'one'), (2, 'two')]

    index = MultiIndex.from_tuples(tuples)
    result = index.to_frame(index=False)
    expected = DataFrame(tuples)
    tm.assert_frame_equal(result, expected)

    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)

    tuples = [(1, 'one'), (1, 'two'), (2, 'one'), (2, 'two')]
    index = MultiIndex.from_tuples(tuples, names=['first', 'second'])
    result = index.to_frame(index=False)
    expected = DataFrame(tuples)
    expected.columns = ['first', 'second']
    tm.assert_frame_equal(result, expected)

    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)

    index = MultiIndex.from_product([range(5),
                                     pd.date_range('20130101', periods=3)])
    result = index.to_frame(index=False)
    expected = DataFrame(
        {0: np.repeat(np.arange(5, dtype='int64'), 3),
            1: np.tile(pd.date_range('20130101', periods=3), 5)})
    tm.assert_frame_equal(result, expected)

    index = MultiIndex.from_product([range(5),
                                     pd.date_range('20130101', periods=3)])
    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)


def test_to_hierarchical():
    index = MultiIndex.from_tuples([(1, 'one'), (1, 'two'), (2, 'one'), (
        2, 'two')])
    result = index.to_hierarchical(3)
    expected = MultiIndex(levels=[[1, 2], ['one', 'two']],
                          labels=[[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1],
                                  [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1]])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names

    # K > 1
    result = index.to_hierarchical(3, 2)
    expected = MultiIndex(levels=[[1, 2], ['one', 'two']],
                          labels=[[0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
                                  [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names

    # non-sorted
    index = MultiIndex.from_tuples([(2, 'c'), (1, 'b'),
                                    (2, 'a'), (2, 'b')],
                                   names=['N1', 'N2'])

    result = index.to_hierarchical(2)
    expected = MultiIndex.from_tuples([(2, 'c'), (2, 'c'), (1, 'b'),
                                       (1, 'b'),
                                       (2, 'a'), (2, 'a'),
                                       (2, 'b'), (2, 'b')],
                                      names=['N1', 'N2'])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names


@pytest.mark.skipif(PY3, reason="testing legacy pickles not support on py3")
def test_legacy_pickle(datapath):

    path = datapath('indexes', 'multi', 'data', 'multiindex_v1.pickle')
    obj = pd.read_pickle(path)

    obj2 = MultiIndex.from_tuples(obj.values)
    assert obj.equals(obj2)

    res = obj.get_indexer(obj)
    exp = np.arange(len(obj), dtype=np.intp)
    assert_almost_equal(res, exp)

    res = obj.get_indexer(obj2[::-1])
    exp = obj.get_indexer(obj[::-1])
    exp2 = obj2.get_indexer(obj2[::-1])
    assert_almost_equal(res, exp)
    assert_almost_equal(exp, exp2)


def test_legacy_v2_unpickle(datapath):<commit_msg>TST: catch warnings for to_hierarchical deprecation (#21744)<commit_after># -*- coding: utf-8 -*-


import numpy as np
import pandas as pd
import pandas.util.testing as tm
import pytest
from pandas import DataFrame, MultiIndex, date_range
from pandas.compat import PY3, range
from pandas.util.testing import assert_almost_equal


def test_tolist(idx):
    result = idx.tolist()
    exp = list(idx.values)
    assert result == exp


def test_to_frame():
    tuples = [(1, 'one'), (1, 'two'), (2, 'one'), (2, 'two')]

    index = MultiIndex.from_tuples(tuples)
    result = index.to_frame(index=False)
    expected = DataFrame(tuples)
    tm.assert_frame_equal(result, expected)

    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)

    tuples = [(1, 'one'), (1, 'two'), (2, 'one'), (2, 'two')]
    index = MultiIndex.from_tuples(tuples, names=['first', 'second'])
    result = index.to_frame(index=False)
    expected = DataFrame(tuples)
    expected.columns = ['first', 'second']
    tm.assert_frame_equal(result, expected)

    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)

    index = MultiIndex.from_product([range(5),
                                     pd.date_range('20130101', periods=3)])
    result = index.to_frame(index=False)
    expected = DataFrame(
        {0: np.repeat(np.arange(5, dtype='int64'), 3),
            1: np.tile(pd.date_range('20130101', periods=3), 5)})
    tm.assert_frame_equal(result, expected)

    index = MultiIndex.from_product([range(5),
                                     pd.date_range('20130101', periods=3)])
    result = index.to_frame()
    expected.index = index
    tm.assert_frame_equal(result, expected)


def test_to_hierarchical():
    index = MultiIndex.from_tuples([(1, 'one'), (1, 'two'), (2, 'one'), (
        2, 'two')])
    with tm.assert_produces_warning(FutureWarning,
                                    check_stacklevel=False):
        result = index.to_hierarchical(3)
    expected = MultiIndex(levels=[[1, 2], ['one', 'two']],
                          labels=[[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1],
                                  [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1]])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names

    # K > 1
    with tm.assert_produces_warning(FutureWarning,
                                    check_stacklevel=False):
        result = index.to_hierarchical(3, 2)
    expected = MultiIndex(levels=[[1, 2], ['one', 'two']],
                          labels=[[0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
                                  [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names

    # non-sorted
    index = MultiIndex.from_tuples([(2, 'c'), (1, 'b'),
                                    (2, 'a'), (2, 'b')],
                                   names=['N1', 'N2'])

    with tm.assert_produces_warning(FutureWarning,
                                    check_stacklevel=False):
        result = index.to_hierarchical(2)
    expected = MultiIndex.from_tuples([(2, 'c'), (2, 'c'), (1, 'b'),
                                       (1, 'b'),
                                       (2, 'a'), (2, 'a'),
                                       (2, 'b'), (2, 'b')],
                                      names=['N1', 'N2'])
    tm.assert_index_equal(result, expected)
    assert result.names == index.names


@pytest.mark.skipif(PY3, reason="testing legacy pickles not support on py3")
def test_legacy_pickle(datapath):

    path = datapath('indexes', 'multi', 'data', 'multiindex_v1.pickle')
    obj = pd.read_pickle(path)

    obj2 = MultiIndex.from_tuples(obj.values)
    assert obj.equals(obj2)

    res = obj.get_indexer(obj)
    exp = np.arange(len(obj), dtype=np.intp)
    assert_almost_equal(res, exp)

    res = obj.get_indexer(obj2[::-1])
    exp = obj.get_indexer(obj[::-1])
    exp2 = obj2.get_indexer(obj2[::-1])
    assert_almost_equal(res, exp)
    assert_almost_equal(exp, exp2)


def test_legacy_v2_unpickle(datapath):
{'fraction_non_alphanumeric': 0.15531508593252705, 'fraction_numerical': 0.035518777848504136, 'mean_word_length': 2.3444018731375054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6951328', 'n_tokens_mistral': 2871, 'n_tokens_neox': 2650, 'n_words': 586}