<commit_before>from os.path import join
from neuroimaging import ENTHOUGHT_TRAITS_DEF

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('utils', parent_package, top_path)

    config.add_data_dir('tests')

    config.add_subpackage('config')

    if not ENTHOUGHT_TRAITS_DEF:
        config.add_subpackage('enthought')
        config.add_extension('enthought.traits.ctraits',
          [join(*('enthought/traits/ctraits.c'.split('/')))])


    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())
<commit_msg>unzip data files before installing them. This addresses #79<commit_after>from os.path import join
from neuroimaging import ENTHOUGHT_TRAITS_DEF

def configuration(parent_package='',top_path=None):
    from numpy.distutils.misc_util import Configuration
    config = Configuration('utils', parent_package, top_path)

    # We want to unzip our data here where we have permissions
    # before moving it into the install location.
    from data_io.datasource import unzip
    import os
    path = "lib/neuroimaging/utils/tests/data"
    for filename in os.listdir(path):
        if filename.endswith('bz2'):
            filename = os.path.join(path, filename)
            unzip(filename)
            os.remove(filename)


    config.add_data_dir('tests')


    config.add_subpackage('config')

    if not ENTHOUGHT_TRAITS_DEF:
        config.add_subpackage('enthought')
        config.add_extension('enthought.traits.ctraits',
          [join(*('enthought/traits/ctraits.c'.split('/')))])


    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())

{'fraction_non_alphanumeric': 0.10981697171381032, 'fraction_numerical': 0.0016638935108153079, 'mean_word_length': 3.836461126005362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301401', 'n_tokens_mistral': 564, 'n_tokens_neox': 530, 'n_words': 117}