<commit_before>import backports.lzma as lzma
import json
import os
import random
import string
import StringIO
import tarfile

import base
import layers
import storage.local

# from mock import patch
# from mockredis import mock_strict_redis_client


class SeekableStringIO(StringIO.StringIO):
    def seekable(self):
        return True

StringIO.StringIO = SeekableStringIO


def comp(n, f, *args, **kwargs):
    return (f(*args, **kwargs) for i in xrange(n))


def rndstr(length=5):
    palette = string.ascii_uppercase + string.digits
    return ''.join(comp(length, random.choice, palette))


def _get_tarfile(filenames):
    tfobj = StringIO.StringIO()
    tar = tarfile.TarFile(fileobj=tfobj, mode='w')
    for filename in filenames:
        tarinfo = tarfile.TarInfo(filename)
        data = StringIO.StringIO()
        data.write(rndstr(512))
        data.seek(0)
        tar.addfile(tarinfo, data)
    tfobj.seek(0)
    return tfobj


def _get_xzfile(filenames):
    tar_data = _get_tarfile(filenames)
    lzma_fobj = StringIO.StringIO()
    xz_file = lzma.open(lzma_fobj, 'w')
    xz_file.write(tar_data.read())
    lzma_fobj.seek(0)
    return lzma_fobj


class TestLayers(base.TestCase):

    def setUp(self):
        self.store = storage.load(kind='local')
        self.filenames = list(comp(5, rndstr))

    def test_tar_archive(self):
        tfobj = _get_tarfile(self.filenames)

        archive = layers.Archive(tfobj)
        tar = tarfile.open(fileobj=archive)
        members = tar.getmembers()
        for tarinfo in members:
            self.assertIn(tarinfo.name, self.filenames)<commit_msg>Fixed test_xz_archive test<commit_after>import backports.lzma as lzma
import json
import os
import random
import string
import StringIO
import tarfile

import base
import layers
import storage.local

# from mock import patch
# from mockredis import mock_strict_redis_client


class SeekableStringIO(StringIO.StringIO):
    def seekable(self):
        return True

StringIO.StringIO = SeekableStringIO


def comp(n, f, *args, **kwargs):
    return (f(*args, **kwargs) for i in xrange(n))


def rndstr(length=5):
    palette = string.ascii_uppercase + string.digits
    return ''.join(comp(length, random.choice, palette))


def _get_tarfile(filenames):
    tfobj = StringIO.StringIO()
    tar = tarfile.TarFile(fileobj=tfobj, mode='w')
    for filename in filenames:
        tarinfo = tarfile.TarInfo(filename)
        data = StringIO.StringIO()
        data.write(rndstr(512))
        data.seek(0)
        tar.addfile(tarinfo, data)
    tfobj.seek(0)
    return tfobj


def _get_xzfile(filenames):
    tar_data = _get_tarfile(filenames)
    lzma_fobj = StringIO.StringIO()
    xz_file = lzma.open(lzma_fobj, 'w')
    xz_file.write(tar_data.read())
    xz_file.close()
    lzma_fobj.seek(0)
    return lzma_fobj


class TestLayers(base.TestCase):

    def setUp(self):
        self.store = storage.load(kind='local')
        self.filenames = list(comp(5, rndstr))

    def test_tar_archive(self):
        tfobj = _get_tarfile(self.filenames)

        archive = layers.Archive(tfobj)
        tar = tarfile.open(fileobj=archive)
        members = tar.getmembers()
        for tarinfo in members:
            self.assertIn(tarinfo.name, self.filenames)
{'fraction_non_alphanumeric': 0.10389212381244253, 'fraction_numerical': 0.004903463070793748, 'mean_word_length': 3.35781041388518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182842', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1125, 'n_words': 240}