<commit_before><commit_msg>add tests for coretools module<commit_after>from unittest import TestCase
from libextract.coretools import pipeline, histogram,\
        above_threshold, argmax


class PipelineTest(TestCase):
    def setUp(self):
        self.pipe = [lambda x: x+1,
                     lambda x: x+2,
                     lambda x: x/2.0]

    def runTest(self):
        self.assertEqual(pipeline(1, self.pipe), 2)
        self.assertEqual(pipeline(2, self.pipe), 2.5)


class HistogramTest(TestCase):
    def setUp(self):
        self.hist = histogram((('i', 10),
                               ('h', 20),
                               ('i', -5),
                               ('j', 50)))

    def runTest(self):
        self.assertEqual(self.hist['i'], 5)
        self.assertEqual(self.hist['h'], 20)
        self.assertEqual(self.hist['j'], 50)


class ArgmaxTest(HistogramTest):
    def runTest(self):
        self.assertEqual(argmax(self.hist), ('j', 50))
        self.hist.pop('j')
        self.assertEqual(argmax(self.hist), ('h', 20))


class AboveThresholdTest(TestCase):
    dataset = (('i', 8),
               ('h', 50),
               ('i', 10),
               ('k', 9))

    def setUp(self):
        self.func = above_threshold(9)

    def runTest(self):
        self.assertEqual(list(self.func(self.dataset)),
                         [('h', 50),
                          ('i', 10),
                          ('k', 9)])
{'fraction_non_alphanumeric': 0.15193370165745856, 'fraction_numerical': 0.025552486187845305, 'mean_word_length': 1.7339622641509433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3591036', 'n_tokens_mistral': 460, 'n_tokens_neox': 420, 'n_words': 93}