<commit_before>import os
import os.path
import six
from progressivis import ProgressiveError
from .random import generate_random_csv
from .wget import wget_file
import bz2
import zlib

from functools import partial

DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data'))
Z_CHUNK_SIZE = 1024*32

def get_dataset(name, **kwds):
    if not os.path.isdir(DATA_DIR):
        os.mkdir(DATA_DIR)
    if name == 'bigfile':
        return generate_random_csv('%s/bigfile.csv'%DATA_DIR, 1000000, 30)
    if name == 'smallfile':
        return generate_random_csv('%s/smallfile.csv'%DATA_DIR, 30000, 10)
    if name == 'warlogs':
        return wget_file(filename='%s/warlogs.vec.bz2'%DATA_DIR,
                         url='http://www.cs.ubc.ca/labs/imager/video/2014/QSNE/warlogs.vec.bz2',
                         **kwds)
    if name.startswith('cluster:'):
        fname = name[len('cluster:'):] + ".txt"
        return wget_file(filename='%s/%s'%(DATA_DIR, fname),
                         url='http://cs.joensuu.fi/sipu/datasets/%s'%fname)
    raise ProgressiveError('Unknown dataset %s'%name)

if six.PY3:
    import lzma
    compressors = dict(bz2=dict(ext='.bz2', factory=bz2.BZ2Compressor),
                        zlib=dict(ext='.zlib', factory=zlib.compressobj),
                        gzip=dict(ext='.gz', factory=partial(zlib.compressobj, wbits=zlib.MAX_WBITS|16)),<commit_msg>extend chunk size for compression<commit_after>import os
import os.path
import six
from progressivis import ProgressiveError
from .random import generate_random_csv
from .wget import wget_file
import bz2
import zlib

from functools import partial

DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data'))
Z_CHUNK_SIZE = 8*1024*32

def get_dataset(name, **kwds):
    if not os.path.isdir(DATA_DIR):
        os.mkdir(DATA_DIR)
    if name == 'bigfile':
        return generate_random_csv('%s/bigfile.csv'%DATA_DIR, 1000000, 30)
    if name == 'smallfile':
        return generate_random_csv('%s/smallfile.csv'%DATA_DIR, 30000, 10)
    if name == 'warlogs':
        return wget_file(filename='%s/warlogs.vec.bz2'%DATA_DIR,
                         url='http://www.cs.ubc.ca/labs/imager/video/2014/QSNE/warlogs.vec.bz2',
                         **kwds)
    if name.startswith('cluster:'):
        fname = name[len('cluster:'):] + ".txt"
        return wget_file(filename='%s/%s'%(DATA_DIR, fname),
                         url='http://cs.joensuu.fi/sipu/datasets/%s'%fname)
    raise ProgressiveError('Unknown dataset %s'%name)

if six.PY3:
    import lzma
    compressors = dict(bz2=dict(ext='.bz2', factory=bz2.BZ2Compressor),
                        zlib=dict(ext='.zlib', factory=zlib.compressobj),
                        gzip=dict(ext='.gz', factory=partial(zlib.compressobj, wbits=zlib.MAX_WBITS|16)),
{'fraction_non_alphanumeric': 0.14482029598308668, 'fraction_numerical': 0.025722339675828047, 'mean_word_length': 2.981767180925666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2211033', 'n_tokens_mistral': 1055, 'n_tokens_neox': 978, 'n_words': 171}