<commit_before>import pytest
from mock import MagicMock
import numpy as np
from numpy.testing import assert_allclose

from .. import data_factories as df
from ..data import CategoricalComponent
from .util import make_file


def test_load_data():
    factory = MagicMock()
    d = df.load_data('test.fits', factory)
    factory.assert_called_once_with('test.fits')
    assert d.label == 'test'


def test_extension():
    assert df._extension('test.fits') == 'fits'
    assert df._extension('test.fits.gz') == 'fits.gz'
    assert df._extension('test.fits.gzip') == 'fits.gzip'
    assert df._extension('test.fits.bz') == 'fits.bz'
    assert df._extension('test.fits.bz2') == 'fits.bz2'
    assert df._extension('test.other.names.fits') == 'fits'


def test_data_label():
    assert df.data_label('test.fits') == 'test'
    assert df.data_label('/Leading/Path/test.fits') == 'test'
    assert df.data_label('') == ''
    assert df.data_label('/Leading/Path/no_extension') == 'no_extension'
    assert df.data_label('no_extension') == 'no_extension'


def test_png_loader():
    data = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x02\x00\x00\x00\x02\x08\x00\x00\x00\x00W\xddR\xf8\x00\x00\x00\x0eIDATx\x9ccdddab\x04\x00\x00&\x00\x0b\x8e`\xe7A\x00\x00\x00\x00IEND\xaeB`\x82'
    with make_file(data, '.png') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.img_data
    np.testing.assert_array_equal(d['PRIMARY'], [[3, 4], [1, 2]])


def test_fits_image_loader():
    with make_file(TEST_FITS_DATA, '.fits') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data
    np.testing.assert_array_equal(d['PRIMARY'], [1, 2, 3])


@pytest.mark.parametrize('suffix', ['.h5', '.hdf5', '.hd5', '.h5custom'])
def test_hdf5_loader(suffix):
    data = 'x\xda\xeb\xf4pq\xe3\xe5\x92\xe2b\x00\x01\x0e\x0e\x06\x16\x06\x01\x06d\xf0\x1f\n*8P\xf90\xf9\x04(\xcd\x08\xa5;\xa0\xf4\n&\x988#XN\x02*.\x085\x1f]]H\x90\xab+H\xf5\x7f4\x00\xb3\xc7\x80\x05Bs0\x8c\x82\x91\x08<\\\x1d\x03@t\x04\x94\x0fK\xa5\'\x98P\xd5U\xa0\xa5G\x0f\n\xeded`\x83\x98\xc5\x08\xe3CR2##D\x80\x19\xaa\x0eA\x0b\x80\x95\np\xc0\xd2\xaa\x03\x98d\x05\xf2@\xe2LLL\x8c\x90t,\x01\xe633&@\x93\xb4\x04\x8a\xbdBP\xdd 5\xc9\xd5]A\x0c\x0c\r\x83"\x1e\x82\xfd\xfc]@9\x1a\x96\x0f\x15\x98G\xd3\xe6(\x18\x05\xa3\x00W\xf9\t\x01Lh\xe5$\x00\xc2A.\xaf'
    with make_file(data, suffix, decompress=True) as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data
    np.testing.assert_array_equal(d['/x'], [1, 2, 3])


def test_fits_catalog_factory():
    data = '\x1f\x8b\x08\x08\x19\r\x9cQ\x02\x03test.fits\x00\xed\xd7AO\x830\x18\xc6\xf1\xe9\'yo\x1c\'\x1c\x8c\x97\x1d\x86c\xa6\x911"5\xc1c\x91n\x92\x8cBJ\x97\xb8o\xef\x06\xd3\x98H\xdd\x16\x97]|~\x17\x12H\xfeyI{h\x136\x8b\xc3\x80hD=8\r\xe9\xb5R\x8bJ\x97\r\x99\x8a\xa6\x8c\'\xd4\x18\xa1r\xa1s\xea\xe53\x1e\xb3\xd4\xd2\xbb\xdb\xf6\x84\xd6bC\xb90\x82\xcc\xa6\x96t@4NYB\x96\xde\xcd\xb6\xa7\xd6e&5U\x8b\xcfrQJ\xd5\x14\x95jz{A\xca\x83hb\xfd\xdf\x93\xb51\x00\x00\x00\x00\xf87v\xc7\xc9\x84\xcd\xa3\x119>\x8b\xf8\xd8\x0f\x03\xe7\xdb\xe7!e\x85\x12zCFd+I\xf2\xddt\x87Sk\xef\xa2\xe7g\xef\xf4\xf3s\xdbs\xfb{\xee\xed\xb6\xb7\x92ji\xdev\xbd\xaf\x12\xb9\x07\xe6\xf3,\xf3\xb9\x96\x9eg\xef\xc5\xf7\xf3\xe7\x88\x1fu_X\xeaj]S-\xb4(\xa5\x91\xba\xff\x7f\x1f~\xeb\xb9?{\xcd\x81\xf5\xe0S\x16\x84\x93\xe4\x98\xf5\xe8\xb6\xcc\xa2\x90\xab\xdc^\xe5\xfc%\x0e\xda\xf5p\xc4\xfe\x95\xf3\x97\xfd\xcc\xa7\xf3\xa7Y\xd7{<Ko7_\xbb\xbeNv\xb6\xf9\xbc\xf3\xcd\x87\xfb\x1b\x00\x00\xc0\xe5\r:W\xfb\xe7\xf5\x00\x00\x00\x00\x00\x00\xac>\x00\x04\x01*\xc7\xc0!\x00\x00'
    with make_file(data, '.fits') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


@pytest.mark.parametrize(('delim', 'suffix'),
                         ((',', '.csv'),
                          ('\t', '.tsv'),
                          ('|', '.txt'),
                          (' ', '.dat'),
                          ('\t', '.tbl')))
def test_ascii_catalog_factory(delim, suffix):
    data = "#a%sb\n1%s2" % (delim, delim)
    with make_file(data, suffix) as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


@pytest.mark.parametrize(('delim', 'suffix'),
                         ((',', '.csv'),
                          ('\t', '.tsv'),
                          ('|', '.txt'),
                          (' ', '.dat'),
                          ('\t', '.tbl')))
def test_pandas_parse_delimiters(delim, suffix):
    data = "a%sb\n1%s2" % (delim, delim)
    with make_file(data, suffix) as fname:
        d = df.load_data(fname, factory=df.pandas_read_table)

    print d
    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


def test_fits_gz_factory():
    data = '\x1f\x8b\x08\x08\xdd\x1a}R\x00\x03test.fits\x00\xed\xd1\xb1\n\xc20\x10\xc6q\x1f\xe5\xde@ZA]\x1cZ\x8d\x10\xd0ZL\x87\xe2\x16m\x0b\x1d\x9aHR\x87n>\xba\xa5".\tRq\x11\xbe_\xe6\xfb\x93\xe3\x04\xdf\xa7;F\xb4"\x87\x8c\xa6t\xd1\xaa\xd2\xa6\xb1\xd4j\xda\xf2L\x90m\xa5*\xa4)\\\x03D1\xcfR\x9e\xbb{\xc1\xbc\xefIcdG\x85l%\xb5\xdd\xb5tW\xde\x92(\xe7\x82<\xff\x0b\xfb\x9e\xba5\xe7\xd2\x90\xae^\xe5\xba)\x95\xad\xb5\xb2\xfe^\xe0\xed\x8d6\xf4\xc2\xdf\xf5X\x9e\xb1d\xe3\xbd\xc7h\xb1XG\xde\xfb\x06_\xf4N\xecx Go\x16.\xe6\xcb\xf1\xbdaY\x00\x00\x00\x80?r\x9f<\x1f\x00\x00\x00\x00\x00|\xf6\x00\x03v\xd8\xf6\x80\x16\x00\x00'

    with make_file(data, '.fits.gz') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data

    np.testing.assert_array_equal(d['PRIMARY'], [[0, 0], [0, 0]])


def test_csv_gz_factory():
    data = '\x1f\x8b\x08\x08z\x1e}R\x00\x03test.csv\x00\xab\xe02\xe42\xe22\xe6\x02\x00y\xffzx\x08\x00\x00\x00'
    with make_file(data, '.csv.gz') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['x'], [1, 2, 3])


def test_sextractor_factory():
    data = """#   1 NUMBER                 Running object number
#   2 X_IMAGE                Object position along x                                    [pixel]
#   3 Y_IMAGE                Object position along y                                    [pixel]
   1 2988.249    2.297
   2 2373.747    3.776
   3 3747.026    4.388"""
    with make_file(data, '.cat') as fname:
        d = df.load_data(fname, factory=df.sextractor_factory)
    assert_allclose(d['NUMBER'], [1, 2, 3])
    assert_allclose(d['X_IMAGE'], [2988.249, 2373.747, 3747.026])
    assert_allclose(d['Y_IMAGE'], [2.297, 3.776, 4.388])


def test_excel_factory():
    data = 'x\xda\xedX=h\x14A\x14\xfef\xf7\xfeI.\xbb\xe7EL\x84\xb0\x04\x8c\x1a\xd3\x04\x1b\x9bdOAS\x19\xa2\x16\x8a\x08z1\x0bJ\xc2E\x8e\x14\xc6\xc6h\xbcR\x10\xac\x14\x9b@\x1a\x9b\xa8\x8d?\x18A;\x0b!\xa2\x85 \x08w\n6V\x82B\x8a\xe4\xd67og\xcd%^q\x07\x1aT\xe6[\xe6\xcd\xdb7\xdf\xdby\xc7\xbcy{\xb3\xaf\x97\xec\xf2\xdc\x83\xce\n6`\x10&\xaa~\x12\xb1\x1a\x9b\xa0\x96\x0co,\xd0\xb8\xefK5\xec\x13\xd4|\x8d\x7f\n\xc9\x04-d,\x8a\xa7\xad\xaf\xe2r\r\xe5zW`\xe0~\xe4\x05I\xe0#\xb5S\xb8\x80\xe1\xc9\x82\xe7l"\x0ep\x0cy!c\x18 )p\x87,itpT\x19\x96gYnay\x8f\x99\x8b,sl\xb9\xcer\x80\xb8eq\x12K\xeep\xef>\x95\xc5\'\x8cn\x1eKC>\xf7\x11\xfb\xbcgK?\xda\xf1Rf\xf1\xe5\x1b"\xe0F\xb1\xbfx>?\xf1\x17\x0c\x08\x1aX4\x1a\xf3\xe8\x8a\xb4`\x1e\xb4\xa0C^\xc1+\xe6\'\xca\xc8\xd2\xca\xce\xe3\xbb\xef\x00\xdf\xc2-\xfc\xdc\xd1\xf6\xcd\xb5\x0b\x90}y\xbd=\xcevZ\xd8e\xdb\xce99\'\xb4\'\xea\xf0o\x1a\x11`\x06\xfe\x19\xde\x11%\xca\xe0\xcff\xb0k\x8f\x9d\xf3\xbc\xa9\xfe\x15*\xcb2E\x8c\xa0Z_\xa46\x9d\x92U\x9bw\xb9\xb5n\x97\xb7r\xf6\xb7\x90\x1cC\x1b\xeb6\xfbY4\xf3\xca\xdd\xafo\x0e\x8f\x8e\xb8\xa7\xd92\xc3\x95=\xa8\xff;d\x04\xf0qEz\x90s\x9aG",e(\xbd\xec\xb1\x87\xe5U~\xeav\xd6;Yf)A\xa9\xef\x19iW\xca\xa1Y\xe6\\\xe3\xd1\x1e\x9ag/\xe3\xad\xbb\xb3F\xdfEz\xe9\xcb\x91\xc7]\xa5O\xeen\xd2\x17\x86*\x97\xb2\x0b\xef\xdc9t\xd3\xfbh\x8c\xfc\xe55\x8b>\xd1\'n\xdf\x92x\xe2\x86\xbdP\xb5\xe2\x03\xcb\x8e_\xeaF\xc2\xb0T\xec\xbez\xc9\xb5a\x15)\xfe\xb56\x1b\x82;\x83\xee\x84\xe2\x8b\r\xfcg\xd8\xc66[\xd5\xa6\x18\xcb\xc0K\xf2\x8d:|\x83\xf9k\xcc\x0c\xd2?\xf9f\x1d\xbe\xc9\xfc5f\x86.\xc9\x1f42x\xc8U!W\xf3\xa6NACCCCCCCCC\xa3\x1e\x84:-\x98\xea_|T\x9d\x0e\xe2\xea\xbb\xce*\xb5\xaa\xfeL\xf2\xdf\xe2(&\xe9\x9a\xa2s\xe6A\x14\xa8/b\xba\xa9\xfc\xd9\x8a\xa8\x08\x9f%\x1a\xf4\t\xbf\x17J\x1c\xa7\xd9\x8b\x18\xc7(\xc71\xdet\xfe\xd2\xe9N\xd4\xfe\x9e\x86\x1d\xad\xdf\xb7\x85\x9a\x9d\xbf\xdaL\x9c\x7fx\xfe\x1f\xf5\x81\xcaV'
    with make_file(data, '.xlsx', decompress=True) as fname:
        d = df.load_data(fname)

    np.testing.assert_array_equal(d['x'], [1, 2, 3])
    np.testing.assert_array_equal(d['y'], [2, 3, 4])


def test_csv_pandas_factory():
    data = """a,b,c,d
1,2.1,some,True<commit_msg>Skip astropy0.3 test on old astropy<commit_after>from distutils.version import LooseVersion

import pytest
from mock import MagicMock
import numpy as np
from numpy.testing import assert_allclose
from astropy import __version__ as _astro_ver_

from .. import data_factories as df
from ..data import CategoricalComponent
from .util import make_file

needs_astropy_03 = pytest.mark.skipif(LooseVersion(_astro_ver_) < LooseVersion('0.3'),
                                      reason='Astropy >=0.3 feature')


def test_load_data():
    factory = MagicMock()
    d = df.load_data('test.fits', factory)
    factory.assert_called_once_with('test.fits')
    assert d.label == 'test'


def test_extension():
    assert df._extension('test.fits') == 'fits'
    assert df._extension('test.fits.gz') == 'fits.gz'
    assert df._extension('test.fits.gzip') == 'fits.gzip'
    assert df._extension('test.fits.bz') == 'fits.bz'
    assert df._extension('test.fits.bz2') == 'fits.bz2'
    assert df._extension('test.other.names.fits') == 'fits'


def test_data_label():
    assert df.data_label('test.fits') == 'test'
    assert df.data_label('/Leading/Path/test.fits') == 'test'
    assert df.data_label('') == ''
    assert df.data_label('/Leading/Path/no_extension') == 'no_extension'
    assert df.data_label('no_extension') == 'no_extension'


def test_png_loader():
    data = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x02\x00\x00\x00\x02\x08\x00\x00\x00\x00W\xddR\xf8\x00\x00\x00\x0eIDATx\x9ccdddab\x04\x00\x00&\x00\x0b\x8e`\xe7A\x00\x00\x00\x00IEND\xaeB`\x82'
    with make_file(data, '.png') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.img_data
    np.testing.assert_array_equal(d['PRIMARY'], [[3, 4], [1, 2]])


def test_fits_image_loader():
    with make_file(TEST_FITS_DATA, '.fits') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data
    np.testing.assert_array_equal(d['PRIMARY'], [1, 2, 3])


@pytest.mark.parametrize('suffix', ['.h5', '.hdf5', '.hd5', '.h5custom'])
def test_hdf5_loader(suffix):
    data = 'x\xda\xeb\xf4pq\xe3\xe5\x92\xe2b\x00\x01\x0e\x0e\x06\x16\x06\x01\x06d\xf0\x1f\n*8P\xf90\xf9\x04(\xcd\x08\xa5;\xa0\xf4\n&\x988#XN\x02*.\x085\x1f]]H\x90\xab+H\xf5\x7f4\x00\xb3\xc7\x80\x05Bs0\x8c\x82\x91\x08<\\\x1d\x03@t\x04\x94\x0fK\xa5\'\x98P\xd5U\xa0\xa5G\x0f\n\xeded`\x83\x98\xc5\x08\xe3CR2##D\x80\x19\xaa\x0eA\x0b\x80\x95\np\xc0\xd2\xaa\x03\x98d\x05\xf2@\xe2LLL\x8c\x90t,\x01\xe633&@\x93\xb4\x04\x8a\xbdBP\xdd 5\xc9\xd5]A\x0c\x0c\r\x83"\x1e\x82\xfd\xfc]@9\x1a\x96\x0f\x15\x98G\xd3\xe6(\x18\x05\xa3\x00W\xf9\t\x01Lh\xe5$\x00\xc2A.\xaf'
    with make_file(data, suffix, decompress=True) as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data
    np.testing.assert_array_equal(d['/x'], [1, 2, 3])


def test_fits_catalog_factory():
    data = '\x1f\x8b\x08\x08\x19\r\x9cQ\x02\x03test.fits\x00\xed\xd7AO\x830\x18\xc6\xf1\xe9\'yo\x1c\'\x1c\x8c\x97\x1d\x86c\xa6\x911"5\xc1c\x91n\x92\x8cBJ\x97\xb8o\xef\x06\xd3\x98H\xdd\x16\x97]|~\x17\x12H\xfeyI{h\x136\x8b\xc3\x80hD=8\r\xe9\xb5R\x8bJ\x97\r\x99\x8a\xa6\x8c\'\xd4\x18\xa1r\xa1s\xea\xe53\x1e\xb3\xd4\xd2\xbb\xdb\xf6\x84\xd6bC\xb90\x82\xcc\xa6\x96t@4NYB\x96\xde\xcd\xb6\xa7\xd6e&5U\x8b\xcfrQJ\xd5\x14\x95jz{A\xca\x83hb\xfd\xdf\x93\xb51\x00\x00\x00\x00\xf87v\xc7\xc9\x84\xcd\xa3\x119>\x8b\xf8\xd8\x0f\x03\xe7\xdb\xe7!e\x85\x12zCFd+I\xf2\xddt\x87Sk\xef\xa2\xe7g\xef\xf4\xf3s\xdbs\xfb{\xee\xed\xb6\xb7\x92ji\xdev\xbd\xaf\x12\xb9\x07\xe6\xf3,\xf3\xb9\x96\x9eg\xef\xc5\xf7\xf3\xe7\x88\x1fu_X\xeaj]S-\xb4(\xa5\x91\xba\xff\x7f\x1f~\xeb\xb9?{\xcd\x81\xf5\xe0S\x16\x84\x93\xe4\x98\xf5\xe8\xb6\xcc\xa2\x90\xab\xdc^\xe5\xfc%\x0e\xda\xf5p\xc4\xfe\x95\xf3\x97\xfd\xcc\xa7\xf3\xa7Y\xd7{<Ko7_\xbb\xbeNv\xb6\xf9\xbc\xf3\xcd\x87\xfb\x1b\x00\x00\xc0\xe5\r:W\xfb\xe7\xf5\x00\x00\x00\x00\x00\x00\xac>\x00\x04\x01*\xc7\xc0!\x00\x00'
    with make_file(data, '.fits') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


@pytest.mark.parametrize(('delim', 'suffix'),
                         ((',', '.csv'),
                          ('\t', '.tsv'),
                          ('|', '.txt'),
                          (' ', '.dat'),
                          ('\t', '.tbl')))
def test_ascii_catalog_factory(delim, suffix):
    data = "#a%sb\n1%s2" % (delim, delim)
    with make_file(data, suffix) as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


@pytest.mark.parametrize(('delim', 'suffix'),
                         ((',', '.csv'),
                          ('\t', '.tsv'),
                          ('|', '.txt'),
                          (' ', '.dat'),
                          ('\t', '.tbl')))
def test_pandas_parse_delimiters(delim, suffix):
    data = "a%sb\n1%s2" % (delim, delim)
    with make_file(data, suffix) as fname:
        d = df.load_data(fname, factory=df.pandas_read_table)

    print d
    np.testing.assert_array_equal(d['a'], [1])
    np.testing.assert_array_equal(d['b'], [2])


def test_fits_gz_factory():
    data = '\x1f\x8b\x08\x08\xdd\x1a}R\x00\x03test.fits\x00\xed\xd1\xb1\n\xc20\x10\xc6q\x1f\xe5\xde@ZA]\x1cZ\x8d\x10\xd0ZL\x87\xe2\x16m\x0b\x1d\x9aHR\x87n>\xba\xa5".\tRq\x11\xbe_\xe6\xfb\x93\xe3\x04\xdf\xa7;F\xb4"\x87\x8c\xa6t\xd1\xaa\xd2\xa6\xb1\xd4j\xda\xf2L\x90m\xa5*\xa4)\\\x03D1\xcfR\x9e\xbb{\xc1\xbc\xefIcdG\x85l%\xb5\xdd\xb5tW\xde\x92(\xe7\x82<\xff\x0b\xfb\x9e\xba5\xe7\xd2\x90\xae^\xe5\xba)\x95\xad\xb5\xb2\xfe^\xe0\xed\x8d6\xf4\xc2\xdf\xf5X\x9e\xb1d\xe3\xbd\xc7h\xb1XG\xde\xfb\x06_\xf4N\xecx Go\x16.\xe6\xcb\xf1\xbdaY\x00\x00\x00\x80?r\x9f<\x1f\x00\x00\x00\x00\x00|\xf6\x00\x03v\xd8\xf6\x80\x16\x00\x00'

    with make_file(data, '.fits.gz') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.gridded_data

    np.testing.assert_array_equal(d['PRIMARY'], [[0, 0], [0, 0]])


def test_csv_gz_factory():
    data = '\x1f\x8b\x08\x08z\x1e}R\x00\x03test.csv\x00\xab\xe02\xe42\xe22\xe6\x02\x00y\xffzx\x08\x00\x00\x00'
    with make_file(data, '.csv.gz') as fname:
        d = df.load_data(fname)
        assert df.find_factory(fname) is df.tabular_data

    np.testing.assert_array_equal(d['x'], [1, 2, 3])


@needs_astropy_03
def test_sextractor_factory():
    data = """#   1 NUMBER                 Running object number
#   2 X_IMAGE                Object position along x                                    [pixel]
#   3 Y_IMAGE                Object position along y                                    [pixel]
   1 2988.249    2.297
   2 2373.747    3.776
   3 3747.026    4.388"""
    with make_file(data, '.cat') as fname:
        d = df.load_data(fname, factory=df.sextractor_factory)
    assert_allclose(d['NUMBER'], [1, 2, 3])
    assert_allclose(d['X_IMAGE'], [2988.249, 2373.747, 3747.026])
    assert_allclose(d['Y_IMAGE'], [2.297, 3.776, 4.388])


def test_excel_factory():
    data = 'x\xda\xedX=h\x14A\x14\xfef\xf7\xfeI.\xbb\xe7EL\x84\xb0\x04\x8c\x1a\xd3\x04\x1b\x9bdOAS\x19\xa2\x16\x8a\x08z1\x0bJ\xc2E\x8e\x14\xc6\xc6h\xbcR\x10\xac\x14\x9b@\x1a\x9b\xa8\x8d?\x18A;\x0b!\xa2\x85 \x08w\n6V\x82B\x8a\xe4\xd67og\xcd%^q\x07\x1aT\xe6[\xe6\xcd\xdb7\xdf\xdby\xc7\xbcy{\xb3\xaf\x97\xec\xf2\xdc\x83\xce\n6`\x10&\xaa~\x12\xb1\x1a\x9b\xa0\x96\x0co,\xd0\xb8\xefK5\xec\x13\xd4|\x8d\x7f\n\xc9\x04-d,\x8a\xa7\xad\xaf\xe2r\r\xe5zW`\xe0~\xe4\x05I\xe0#\xb5S\xb8\x80\xe1\xc9\x82\xe7l"\x0ep\x0cy!c\x18 )p\x87,itpT\x19\x96gYnay\x8f\x99\x8b,sl\xb9\xcer\x80\xb8eq\x12K\xeep\xef>\x95\xc5\'\x8cn\x1eKC>\xf7\x11\xfb\xbcgK?\xda\xf1Rf\xf1\xe5\x1b"\xe0F\xb1\xbfx>?\xf1\x17\x0c\x08\x1aX4\x1a\xf3\xe8\x8a\xb4`\x1e\xb4\xa0C^\xc1+\xe6\'\xca\xc8\xd2\xca\xce\xe3\xbb\xef\x00\xdf\xc2-\xfc\xdc\xd1\xf6\xcd\xb5\x0b\x90}y\xbd=\xcevZ\xd8e\xdb\xce99\'\xb4\'\xea\xf0o\x1a\x11`\x06\xfe\x19\xde\x11%\xca\xe0\xcff\xb0k\x8f\x9d\xf3\xbc\xa9\xfe\x15*\xcb2E\x8c\xa0Z_\xa46\x9d\x92U\x9bw\xb9\xb5n\x97\xb7r\xf6\xb7\x90\x1cC\x1b\xeb6\xfbY4\xf3\xca\xdd\xafo\x0e\x8f\x8e\xb8\xa7\xd92\xc3\x95=\xa8\xff;d\x04\xf0qEz\x90s\x9aG",e(\xbd\xec\xb1\x87\xe5U~\xeav\xd6;Yf)A\xa9\xef\x19iW\xca\xa1Y\xe6\\\xe3\xd1\x1e\x9ag/\xe3\xad\xbb\xb3F\xdfEz\xe9\xcb\x91\xc7]\xa5O\xeen\xd2\x17\x86*\x97\xb2\x0b\xef\xdc9t\xd3\xfbh\x8c\xfc\xe55\x8b>\xd1\'n\xdf\x92x\xe2\x86\xbdP\xb5\xe2\x03\xcb\x8e_\xeaF\xc2\xb0T\xec\xbez\xc9\xb5a\x15)\xfe\xb56\x1b\x82;\x83\xee\x84\xe2\x8b\r\xfcg\xd8\xc66[\xd5\xa6\x18\xcb\xc0K\xf2\x8d:|\x83\xf9k\xcc\x0c\xd2?\xf9f\x1d\xbe\xc9\xfc5f\x86.\xc9\x1f42x\xc8U!W\xf3\xa6NACCCCCCCCC\xa3\x1e\x84:-\x98\xea_|T\x9d\x0e\xe2\xea\xbb\xce*\xb5\xaa\xfeL\xf2\xdf\xe2(&\xe9\x9a\xa2s\xe6A\x14\xa8/b\xba\xa9\xfc\xd9\x8a\xa8\x08\x9f%\x1a\xf4\t\xbf\x17J\x1c\xa7\xd9\x8b\x18\xc7(\xc71\xdet\xfe\xd2\xe9N\xd4\xfe\x9e\x86\x1d\xad\xdf\xb7\x85\x9a\x9d\xbf\xdaL\x9c\x7fx\xfe\x1f\xf5\x81\xcaV'
    with make_file(data, '.xlsx', decompress=True) as fname:
        d = df.load_data(fname)

    np.testing.assert_array_equal(d['x'], [1, 2, 3])
    np.testing.assert_array_equal(d['y'], [2, 3, 4])


def test_csv_pandas_factory():
    data = """a,b,c,d
1,2.1,some,True
{'fraction_non_alphanumeric': 0.20732713570603445, 'fraction_numerical': 0.12783981517135157, 'mean_word_length': 2.944456498155782, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6598238', 'n_tokens_mistral': 10781, 'n_tokens_neox': 9933, 'n_words': 691}