<commit_before># -*- coding: utf-8 -*-

from io import StringIO, BufferedReader
import os

import py
import pytest

import hashfs


@pytest.fixture
def testpath(tmpdir):
    return tmpdir.mkdir('hashfs')


@pytest.fixture
def testfile(testpath):
    return testpath.join('hashfs.txt')


@pytest.fixture
def stringio():
    return StringIO(u'foo')


@pytest.yield_fixture
def fileio(testfile):
    with open(str(testfile), 'wb') as io:
        io.write(b'foo')

    io = open(str(testfile), 'rb')

    yield io

    io.close()


@pytest.fixture
def filepath(testfile):
    testfile.write(b'foo')
    return testfile


@pytest.fixture
def fs(testpath):
    return hashfs.HashFS(str(testpath))


def put_range(fs, count):
    return dict((addr.path, addr)
                for addr in (fs.put(StringIO(u'{0}'.format(i)))
                             for i in range(count)))


def assert_file_put(fs, address):
    path = address.path[len(fs.root):]
    directory = os.path.dirname(path)
    dir_parts = [part for part in directory.split(os.path.sep) if part]

    assert address.path in tuple(py.path.local(fs.root).visit())
    assert fs.exists(address.digest)
    assert os.path.splitext(path.replace(os.path.sep, ''))[0] == address.digest
    assert len(dir_parts) == fs.depth
    assert all(len(part) == fs.length for part in dir_parts)


def test_hashfs_put_stringio(fs, stringio):
    address = fs.put(stringio)

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == stringio.getvalue().encode('utf8')


def test_hashfs_put_fileobj(fs, fileio):
    address = fs.put(fileio)

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == fileio.read()


def test_hashfs_put_file(fs, filepath):
    address = fs.put(str(filepath))

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == filepath.read().encode('utf8')


@pytest.mark.parametrize('extension', [
    'txt',
    '.txt',
    'md',
    '.md'
])
def test_hashfs_put_extension(fs, stringio, extension):
    address = fs.put(stringio, extension)

    assert_file_put(fs, address)
    assert os.path.sep in address.path
    assert os.path.splitext(address.path)[1].endswith(extension)


def test_hashfs_put_error(fs):
    with pytest.raises(ValueError):
        fs.put('foo')


@pytest.mark.parametrize('extension,address_attr', [
    ('', 'digest'),
    ('.txt', 'digest'),
    ('txt', 'digest'),
    ('', 'path'),
    ('.txt', 'path'),
    ('txt', 'path'),
])
def test_hashfs_get(fs, stringio, extension, address_attr):
    address = fs.put(stringio, extension)

    fileobj = fs.get(getattr(address, address_attr))

    assert isinstance(fileobj, BufferedReader)
    assert fileobj.read() == stringio.getvalue()

    fileobj.close()


def test_hashfs_get_error(fs):
    with pytest.raises(IOError):
        fs.get('invalid')


@pytest.mark.parametrize('address_attr', [
   'digest',
   'path',
])
def test_hashfs_delete(fs, stringio, address_attr):
    address = fs.put(stringio)

    fs.delete(getattr(address, address_attr))
    assert len(os.listdir(fs.root)) == 0
<commit_msg>Make tests Py3 compatible.<commit_after># -*- coding: utf-8 -*-

from io import StringIO, BufferedReader
import os

import py
import pytest

import hashfs
from hashfs._compat import to_bytes


@pytest.fixture
def testpath(tmpdir):
    return tmpdir.mkdir('hashfs')


@pytest.fixture
def testfile(testpath):
    return testpath.join('hashfs.txt')


@pytest.fixture
def stringio():
    return StringIO(u'foo')


@pytest.yield_fixture
def fileio(testfile):
    with open(str(testfile), 'wb') as io:
        io.write(b'foo')

    io = open(str(testfile), 'rb')

    yield io

    io.close()


@pytest.fixture
def filepath(testfile):
    testfile.write(b'foo')
    return testfile


@pytest.fixture
def fs(testpath):
    return hashfs.HashFS(str(testpath))


def put_range(fs, count):
    return dict((addr.path, addr)
                for addr in (fs.put(StringIO(u'{0}'.format(i)))
                             for i in range(count)))


def assert_file_put(fs, address):
    path = address.path[len(fs.root):]
    directory = os.path.dirname(path)
    dir_parts = [part for part in directory.split(os.path.sep) if part]

    assert address.path in tuple(py.path.local(fs.root).visit())
    assert fs.exists(address.digest)
    assert os.path.splitext(path.replace(os.path.sep, ''))[0] == address.digest
    assert len(dir_parts) == fs.depth
    assert all(len(part) == fs.length for part in dir_parts)


def test_hashfs_put_stringio(fs, stringio):
    address = fs.put(stringio)

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == to_bytes(stringio.getvalue())


def test_hashfs_put_fileobj(fs, fileio):
    address = fs.put(fileio)

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == fileio.read()


def test_hashfs_put_file(fs, filepath):
    address = fs.put(str(filepath))

    assert_file_put(fs, address)

    with open(address.path, 'rb') as fileobj:
        assert fileobj.read() == to_bytes(filepath.read())


@pytest.mark.parametrize('extension', [
    'txt',
    '.txt',
    'md',
    '.md'
])
def test_hashfs_put_extension(fs, stringio, extension):
    address = fs.put(stringio, extension)

    assert_file_put(fs, address)
    assert os.path.sep in address.path
    assert os.path.splitext(address.path)[1].endswith(extension)


def test_hashfs_put_error(fs):
    with pytest.raises(ValueError):
        fs.put('foo')


@pytest.mark.parametrize('extension,address_attr', [
    ('', 'digest'),
    ('.txt', 'digest'),
    ('txt', 'digest'),
    ('', 'path'),
    ('.txt', 'path'),
    ('txt', 'path'),
])
def test_hashfs_get(fs, stringio, extension, address_attr):
    address = fs.put(stringio, extension)

    fileobj = fs.get(getattr(address, address_attr))

    assert isinstance(fileobj, BufferedReader)
    assert fileobj.read() == to_bytes(stringio.getvalue())

    fileobj.close()


def test_hashfs_get_error(fs):
    with pytest.raises(IOError):
        fs.get('invalid')


@pytest.mark.parametrize('address_attr', [
   'digest',
   'path',
])
def test_hashfs_delete(fs, stringio, address_attr):
    address = fs.put(stringio)

    fs.delete(getattr(address, address_attr))
    assert len(os.listdir(fs.root)) == 0

{'fraction_non_alphanumeric': 0.14880123743232793, 'fraction_numerical': 0.002010827532869296, 'mean_word_length': 3.6054131054131053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4634298', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2272, 'n_words': 455}