<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from . import Dataset
from deepy.utils import FakeGenerator, StreamPickler, global_rand

import logging as loggers
logging = loggers.getLogger(__name__)

class OnDiskDataset(Dataset):
    """
    On-disk dataset.
    The data should be dumped with deepy.utils.StreamPickler.
    You must convert the data to mini-batches before dump it to a file.
    """

    def __init__(self, train_path, valid_path=None, test_path=None, train_size=None, cache_on_memory=False, post_processing=None):
        self._train_path = train_path
        self._valid_path = valid_path
        self._test_path = test_path
        self._train_size = train_size
        self._cache_on_memory = cache_on_memory
        self._cached_train_data = None
        self._post_processing = post_processing if post_processing else lambda x: x
        self._skip_amount = 0
        if self._cache_on_memory:
            logging.info("Cache on memory")
            self._cached_train_data = list(map(self._post_processing, StreamPickler.load(open(self._train_path))))
            global_rand.shuffle(self._cached_train_data)

    def skip(self, amount):
        """
        Skip N training batches.
        Do not support on-memory data set.
        """
        self._skip_amount = amount

    def generate_train_data(self):
        for data in StreamPickler.load(open(self._train_path)):
            if self._skip_amount > 0:
                self._skip_amount -= 1
                continue
            yield self._post_processing(data)

    def generate_valid_data(self):
        for data in StreamPickler.load(open(self._valid_path)):
            yield self._post_processing(data)

    def generate_test_data(self):
        for data in StreamPickler.load(open(self._test_path)):
            yield self._post_processing(data)

    def train_set(self):
        if self._cache_on_memory:
            return self._cached_train_data
        if not self._train_path:
            return None
        return FakeGenerator(self, "generate_train_data")

    def valid_set(self):
        if not self._valid_path:
            return None
        return FakeGenerator(self, "generate_valid_data")

    def test_set(self):
        if not self._test_path:
            return None
        return FakeGenerator(self, "generate_test_data")

    def train_size(self):
        return self._train_size
<commit_msg>ondisk<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from . import Dataset
from deepy.utils import FakeGenerator, StreamPickler, global_rand

import logging as loggers
logging = loggers.getLogger(__name__)

class OnDiskDataset(Dataset):
    """
    On-disk dataset.
    The data should be dumped with deepy.utils.StreamPickler.
    You must convert the data to mini-batches before dump it to a file.
    """

    def __init__(self, train_path, valid_path=None, test_path=None, train_size=None,
                 cache_on_memory=False, post_processing=None, shuffle_memory=False):
        self._train_path = train_path
        self._valid_path = valid_path
        self._test_path = test_path
        self._train_size = train_size
        self._cache_on_memory = cache_on_memory
        self._cached_train_data = None
        self._post_processing = post_processing if post_processing else lambda x: x
        self._skip_amount = 0
        self._shuffle_memory = shuffle_memory
        if self._cache_on_memory:
            logging.info("Cache on memory")
            self._cached_train_data = list(map(self._post_processing, StreamPickler.load(open(self._train_path))))
            if self._shuffle_memory:
                logging.info("Shuffle on-memory data")
                global_rand.shuffle(self._cached_train_data)

    def skip(self, amount):
        """
        Skip N training batches.
        Do not support on-memory data set.
        """
        self._skip_amount = amount

    def generate_train_data(self):
        for data in StreamPickler.load(open(self._train_path)):
            if self._skip_amount > 0:
                self._skip_amount -= 1
                continue
            yield self._post_processing(data)

    def generate_valid_data(self):
        for data in StreamPickler.load(open(self._valid_path)):
            yield self._post_processing(data)

    def generate_test_data(self):
        for data in StreamPickler.load(open(self._test_path)):
            yield self._post_processing(data)

    def train_set(self):
        if self._cache_on_memory:
            return self._cached_train_data
        if not self._train_path:
            return None
        return FakeGenerator(self, "generate_train_data")

    def valid_set(self):
        if not self._valid_path:
            return None
        return FakeGenerator(self, "generate_valid_data")

    def test_set(self):
        if not self._test_path:
            return None
        return FakeGenerator(self, "generate_test_data")

    def train_size(self):
        return self._train_size

{'fraction_non_alphanumeric': 0.08379013312451057, 'fraction_numerical': 0.001957713390759593, 'mean_word_length': 2.5210199862164027, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645564', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1456, 'n_words': 346}