<commit_before>import unittest
from core import train
import os
import core.settings

class TestTrain(unittest.TestCase):

    def test_train(self):
        path = os.path.join(core.settings.BASE_DIR,'sources.json')
        texts = [{u"Period": period,
                  u"Author": author,
                  u"Title": title,
                  u"URL": url}
                for author, title, period, url
                in train.parse_json(path)]
        test_text = {
            u"Period": u"Romantic",
            u"Author": u"<NAME>",
            u"Title": u"A Defence of Poetry and Other Essays",
            u"URL": u"http://www.gutenberg.org/cache/epub/5428/pg5428.txt"
        }
        for t in texts:
            print t
        self.assertIn(test_text, texts)<commit_msg>Changed indentation<commit_after>import unittest
from core import train
import os
import core.settings

class TestTrain(unittest.TestCase):

    def test_train(self):
        path = os.path.join(core.settings.BASE_DIR, 'sources.json')
        texts = [{u"Period": period,
                  u"Author": author,
                  u"Title": title,
                  u"URL": url}
                 for author, title, period, url
                 in train.parse_json(path)
                ]
        test_text = {
            u"Period": u"Romantic",
            u"Author": u"<NAME>",
            u"Title": u"A Defence of Poetry and Other Essays",
            u"URL": u"http://www.gutenberg.org/cache/epub/5428/pg5428.txt"
        }
        for t in texts:
            print t
        self.assertIn(test_text, texts)
{'fraction_non_alphanumeric': 0.11460957178841309, 'fraction_numerical': 0.010075566750629723, 'mean_word_length': 1.7586805555555556, 'pattern_counts': {'":': 16, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2599602', 'n_tokens_mistral': 486, 'n_tokens_neox': 459, 'n_words': 111}