<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import os
import shutil
import hashlib

from path_and_hash import PathAndHash


# Functions & classes =========================================================
class BalancedDiscStorage(object):
    """
    Store files, make sure, that there are never more files in one directory
    than :attr:`_dir_limit`.
    """
    def __init__(self, path, dir_limit=32000):
        self.path = path  #: Path on which the storage operates.
        self._assert_path_is_rw()

        self.dir_limit = dir_limit  #: Maximum number of files in directory.
        self.read_bs = 2**16  #: File read blocksize.
        self.hash_builder = hashlib.sha256  #: Hashing function used for FN.

    def _assert_path_is_rw(self):
        """
        Make sure, that `self.path` exists, is directory a readable/writeable.

        Raises:
            IOError: In case that any of the assumptions failed.
            ValueError: In case that `self.path` is not set.
        """
        if not self.path:
            raise ValueError("`path` argument must be set!")

        if not os.path.exists(self.path):
            raise IOError("`%s` not found." % self.path)

        if not os.path.isdir(self.path):
            raise IOError("`%s` is not a directory!" % self.path)

        if not os.access(self.path, (os.R_OK or os.W_OK)):
            raise IOError(
                "Can't access `%s`, please check permissions." % self.path
            )

    def _get_file_iterator(self, file_obj):
        """
        For given `file_obj` return iterator, which will read the file in<commit_msg>#3: Bugfix.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import os
import shutil
import hashlib

from path_and_hash import PathAndHash


# Functions & classes =========================================================
class BalancedDiscStorage(object):
    """
    Store files, make sure, that there are never more files in one directory
    than :attr:`_dir_limit`.
    """
    def __init__(self, path, dir_limit=32000):
        self.path = path  #: Path on which the storage operates.
        self._assert_path_is_rw()

        self.dir_limit = dir_limit  #: Maximal number of files in directory.
        self.read_bs = 2**16  #: File read blocksize.
        self.hash_builder = hashlib.sha256  #: Hashing function used for FN.

    def _assert_path_is_rw(self):
        """
        Make sure, that `self.path` exists, is directory a readable/writeable.

        Raises:
            IOError: In case that any of the assumptions failed.
            ValueError: In case that `self.path` is not set.
        """
        if not self.path:
            raise ValueError("`path` argument must be set!")

        if not os.path.exists(self.path):
            raise IOError("`%s` not found." % self.path)

        if not os.path.isdir(self.path):
            raise IOError("`%s` is not a directory!" % self.path)

        if not os.access(self.path, (os.R_OK or os.W_OK)):
            raise IOError(
                "Can't access `%s`, please check permissions." % self.path
            )

    def _get_file_iterator(self, file_obj):
        """
        For given `file_obj` return iterator, which will read the file in
{'fraction_non_alphanumeric': 0.17048710601719197, 'fraction_numerical': 0.00830945558739255, 'mean_word_length': 2.5658835546475998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5824073', 'n_tokens_mistral': 1031, 'n_tokens_neox': 971, 'n_words': 314}