<commit_before>import os
from tempfile import mkstemp
from unittest import TestCase

LINE = '1;2;1.23456;-2.3;1;2;3;4;0;0;2013-04-20 02:59:43;2013-04-20 02:59:43;1'


class TestLoadFile(TestCase):

    def _make_one(self):
        from ichnaea.importer import load_file
        tmpfile = mkstemp()
        settings = {'celldb': 'sqlite://'}
        return load_file, settings, tmpfile

    def test_no_lines(self):
        func, settings, tmpfile = self._make_one()
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 0)

    def test_one_line(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE)
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 1)

    def test_batch(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE)
        os.write(tmpfile[0], '\n1' + LINE)
        os.write(tmpfile[0], '\n2' + LINE)
        counter = func(settings, tmpfile[1], batch_size=2)
        self.assertEqual(counter, 3)

    def test_corrupt_lines(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE + '\n')
        os.write(tmpfile[0], '2;\\N\n')
        os.write(tmpfile[0], '3;\\N;abc\n')
        os.write(tmpfile[0], '4;\\N;1.0;2.1;abc\n')
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 1)


class TestMain(TestCase):

    def _make_one(self):
        from ichnaea.importer import main
        config = mkstemp()
        data = mkstemp()
        return config, data, main

    def test_main(self):
        config, data, func = self._make_one()<commit_msg>fix tests after last commit<commit_after>import os
from tempfile import mkstemp
from unittest import TestCase

LINE = '2;2;1.23456;-2.3;1;2;3;4;0;0;2013-04-20 02:59:43;2013-04-20 02:59:43;1'


class TestLoadFile(TestCase):

    def _make_one(self):
        from ichnaea.importer import load_file
        tmpfile = mkstemp()
        settings = {'celldb': 'sqlite://'}
        return load_file, settings, tmpfile

    def test_no_lines(self):
        func, settings, tmpfile = self._make_one()
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 0)

    def test_one_line(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE)
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 1)

    def test_batch(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE)
        os.write(tmpfile[0], '\n1' + LINE)
        os.write(tmpfile[0], '\n2' + LINE)
        counter = func(settings, tmpfile[1], batch_size=2)
        self.assertEqual(counter, 3)

    def test_corrupt_lines(self):
        func, settings, tmpfile = self._make_one()
        os.write(tmpfile[0], LINE + '\n')
        os.write(tmpfile[0], '3;\\N\n')
        os.write(tmpfile[0], '4;\\N;abc\n')
        os.write(tmpfile[0], '5;\\N;1.0;2.1;abc\n')
        counter = func(settings, tmpfile[1])
        self.assertEqual(counter, 1)


class TestMain(TestCase):

    def _make_one(self):
        from ichnaea.importer import main
        config = mkstemp()
        data = mkstemp()
        return config, data, main

    def test_main(self):
        config, data, func = self._make_one()
{'fraction_non_alphanumeric': 0.14345991561181434, 'fraction_numerical': 0.04279686558167571, 'mean_word_length': 2.6918798665183536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5692651', 'n_tokens_mistral': 1223, 'n_tokens_neox': 1144, 'n_words': 237}