<commit_before>import os
import unittest

from chainer import dataset
from chainer import testing


class TestGetSetDatasetRoot(unittest.TestCase):

    def test_set_dataset_root(self):
        orig_root = dataset.get_dataset_root()
        new_root = '/tmp/dataset_root'
        try:
            dataset.set_dataset_root(new_root)
            self.assertEqual(dataset.get_dataset_root(), new_root)
        finally:
            dataset.set_dataset_root(orig_root)


class TestGetDatasetDirectory(unittest.TestCase):

    def test_get_dataset_directory(self):
        root = dataset.get_dataset_root()
        path = dataset.get_dataset_directory('test', False)
        self.assertEqual(path, os.path.join(root, 'test'))


testing.run_module(__name__, __file__)<commit_msg>Write a test of download module<commit_after>import os
import shutil
import tempfile
import unittest

import mock

from chainer import dataset
from chainer import testing


class TestGetSetDatasetRoot(unittest.TestCase):

    def test_set_dataset_root(self):
        orig_root = dataset.get_dataset_root()
        new_root = '/tmp/dataset_root'
        try:
            dataset.set_dataset_root(new_root)
            self.assertEqual(dataset.get_dataset_root(), new_root)
        finally:
            dataset.set_dataset_root(orig_root)


class TestGetDatasetDirectory(unittest.TestCase):

    def test_get_dataset_directory(self):
        root = dataset.get_dataset_root()
        path = dataset.get_dataset_directory('test', False)
        self.assertEqual(path, os.path.join(root, 'test'))


class TestCacheOrLoadFile(unittest.TestCase):

    def test_cache_exists(self):
        creator = mock.Mock()
        loader = mock.Mock()

        f = tempfile.NamedTemporaryFile(delete=False)
        f.close()

        try:
            dataset.cache_or_load_file(f.name, creator, loader)
        finally:
            os.remove(f.name)

        self.assertFalse(creator.called)
        loader.assert_called_once_with(f.name)

    def test_new_file(self):
        def create(path):
            with open(path, 'w') as f:
                f.write('test')

        creator = mock.Mock()
        creator.side_effect = create
        loader = mock.Mock()

        dir_path = tempfile.mkdtemp()
        # This file always does not exists as the directory is new.
        path = os.path.join(dir_path, 'cahche')

        try:
            dataset.cache_or_load_file(path, creator, loader)

            self.assertEqual(creator.call_count, 1)
            self.assertFalse(loader.called)

            self.assertTrue(os.path.exists(path))
            with open(path) as f:
                self.assertEqual(f.read(), 'test')

        finally:
            shutil.rmtree(dir_path)


class TestCachedDownload(unittest.TestCase):

    def setUp(self):
        self.default_dataset_root = dataset.get_dataset_root()
        self.temp_dir = tempfile.mkdtemp()
        dataset.set_dataset_root(self.temp_dir)

    def tearDown(self):
        dataset.set_dataset_root(self.default_dataset_root)
        shutil.rmtree(self.temp_dir)

    def test_fail_to_make_dir(self):
        with mock.patch('os.makedirs') as f:
            f.side_effect = OSError()
            with self.assertRaises(RuntimeError):
                dataset.cached_download('http://example.com')

    def test_cached_download(self):
        with mock.patch('urllib.urlretrieve') as f:
            def download(url, path):
                with open(path, 'w') as f:
                    f.write('test')
            f.side_effect = download

            cache_path = dataset.cached_download('http://example.com')

        self.assertEqual(f.call_count, 1)
        args, kwargs = f.call_args
        self.assertEqual(kwargs, {})
        self.assertEqual(len(args), 2)
        # The second argument is a temporary path, and it is removed
        self.assertEqual(args[0], 'http://example.com')

        self.assertTrue(os.path.exists(cache_path))
        with open(cache_path) as f:
            stored_data = f.read()
        self.assertEqual(stored_data, 'test')


testing.run_module(__name__, __file__)
{'fraction_non_alphanumeric': 0.09863945578231292, 'fraction_numerical': 0.0009718172983479105, 'mean_word_length': 2.5737847222222223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825370', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1211, 'n_words': 230}