<commit_before>import gzip
import os
from json import JSONEncoder
import datetime

def __attributes_to_dict(object, fields):
    return {field:getattr(object, field) for field in fields}

def datetime_to_dict(dt):
    return __attributes_to_dict(dt, ('year', 'month', 'day', 'hour', 'minute', 'second'))

def deltatime_to_dict(dt):
    return __attributes_to_dict(dt, ('days', 'seconds'))

class JSONConfigEncoder(JSONEncoder):

    def default(self, o):
        if hasattr(o, '__iter__'):
            return [x for x in o]

        elif isinstance(o, datetime.datetime):
            return datetime_to_dict(o)

        elif isinstance(o, datetime.timedelta):
            return deltatime_to_dict(o)

        elif hasattr(o, 'to_json'):
            return o.to_json()

        return super().default(o)

class AutoSplittingFile:
    """
    This class can be used to store lines. Every matches_per_file lines it opens a new file to write to.
    """
    extension = ".json.gz"

    def __init__(self, dir_path, matches_per_file=1000, prefix="", file_name_postfix =""):
        self._dir = dir_path
        self._prefix = prefix
        self._matches_per_file = matches_per_file
        self._file = None
        self._postfix = file_name_postfix
        self._stored_matches = 0
        self._index = 0

    def open(self, path):
        if self._file:
            self.close()
        self._file = gzip.open(path, 'wt')

    def generate_file_path(self):
        date = datetime.datetime.now().isoformat()
        name =  '{0}_{1}_{2}_{3}'.format(self._prefix, date, self._postfix, self.extension)
        return os.path.join(self._dir, name)

    def close(self):
        if self._file:
            self._file.close()
            self._file = None
            self._stored_matches = 0

    def write(self, text):
        if self._matches_per_file and self._stored_matches >= self._matches_per_file:
            self.close()
        if not self._file:
            self.open(self.generate_file_path())
        elif self._stored_matches != 0:
            # the file is not new, so a line has been written before. Add a  new line
            self._file.write('\n')

        self._file.write(text)
        self._stored_matches += 1

class TierStore:
<commit_msg>Changed (":" becomes "-") the store file name so that it is a valid one in windows<commit_after>import gzip
import os
from json import JSONEncoder
import datetime

def __attributes_to_dict(object, fields):
    return {field:getattr(object, field) for field in fields}

def datetime_to_dict(dt):
    return __attributes_to_dict(dt, ('year', 'month', 'day', 'hour', 'minute', 'second'))

def deltatime_to_dict(dt):
    return __attributes_to_dict(dt, ('days', 'seconds'))

class JSONConfigEncoder(JSONEncoder):

    def default(self, o):
        if hasattr(o, '__iter__'):
            return [x for x in o]

        elif isinstance(o, datetime.datetime):
            return datetime_to_dict(o)

        elif isinstance(o, datetime.timedelta):
            return deltatime_to_dict(o)

        elif hasattr(o, 'to_json'):
            return o.to_json()

        return super().default(o)

class AutoSplittingFile:
    """
    This class can be used to store lines. Every matches_per_file lines it opens a new file to write to.
    """
    extension = ".json.gz"

    def __init__(self, dir_path, matches_per_file=1000, prefix="", file_name_postfix =""):
        self._dir = dir_path
        self._prefix = prefix
        self._matches_per_file = matches_per_file
        self._file = None
        self._postfix = file_name_postfix
        self._stored_matches = 0
        self._index = 0

    def open(self, path):
        if self._file:
            self.close()
        self._file = gzip.open(path, 'wt')

    def generate_file_path(self):
        date = datetime.datetime.now().isoformat().replace(":","-")
        name =  '{0}_{1}_{2}_{3}'.format(self._prefix, date, self._postfix, self.extension)
        return os.path.realpath(os.path.join(self._dir, name))

    def close(self):
        if self._file:
            self._file.close()
            self._file = None
            self._stored_matches = 0

    def write(self, text):
        if self._matches_per_file and self._stored_matches >= self._matches_per_file:
            self.close()
        if not self._file:
            self.open(self.generate_file_path())
        elif self._stored_matches != 0:
            # the file is not new, so a line has been written before. Add a  new line
            self._file.write('\n')

        self._file.write(text)
        self._stored_matches += 1

class TierStore:

{'fraction_non_alphanumeric': 0.11120726958026828, 'fraction_numerical': 0.005625270445694505, 'mean_word_length': 2.5182648401826486, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1291265', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1395, 'n_words': 370}