<commit_before>
# -*- coding: iso-8859-15 -*-

import unittest
from mycroft.util.parse import normalize
from mycroft.util.parse import extractnumber


class TestNormalize(unittest.TestCase):
    def test_articles(self):
        self.assertEqual(normalize("this is a test", remove_articles=True),
                         "this is test")
        self.assertEqual(normalize("this is the test", remove_articles=True),
                         "this is test")
        self.assertEqual(normalize("and another test", remove_articles=True),
                         "and another test")
        self.assertEqual(normalize("this is an extra test",
                                   remove_articles=False),
                         "this is an extra test")

    def test_extractnumber(self):
        self.assertEqual(extractnumber("this is the first test"), 1)
        self.assertEqual(extractnumber("this is 2 test"), 2)
        self.assertEqual(extractnumber("this is second test"), 2)
        self.assertEqual(extractnumber("this is the third test"), 1.0/3.0)
        self.assertEqual(extractnumber("this is test number 4"), 4)
        self.assertEqual(extractnumber("one third of a cup"), 1.0/3.0)
        self.assertEqual(extractnumber("three cups"), 3)
        self.assertEqual(extractnumber("1/3 cups"), 1.0/3.0)
        self.assertEqual(extractnumber("quarter cup"), 0.25)
        self.assertEqual(extractnumber("1/4 cup"), 0.25)
        self.assertEqual(extractnumber("one fourth cup"), 0.25)
        self.assertEqual(extractnumber("2/3 cups"), 2.0/3.0)
        self.assertEqual(extractnumber("3/4 cups"), 3.0/4.0)
        self.assertEqual(extractnumber("1 and 3/4 cups"), 1.75)
        self.assertEqual(extractnumber("1 cup and a half"), 1.5)
        self.assertEqual(extractnumber("one cup and a half"), 1.5)
        self.assertEqual(extractnumber("one and a half cups"), 1.5)
        self.assertEqual(extractnumber("one and one half cups"), 1.5)
        self.assertEqual(extractnumber("three quarter cups"), 3.0/4.0)
        self.assertEqual(extractnumber("three quarters cups"), 3.0/4.0)


    def test_spaces(self):
        self.assertEqual(normalize("  this   is  a    test"),
                         "this is test")
        self.assertEqual(normalize("  this   is  a    test  "),
                         "this is test")
        self.assertEqual(normalize("  this   is  one    test"),
                         "this is 1 test")

    def test_numbers(self):
        self.assertEqual(normalize("this is a one two three  test"),
                         "this is 1 2 3 test")
        self.assertEqual(normalize("  it's  a four five six  test"),
                         "it is 4 5 6 test")
        self.assertEqual(normalize("it's  a seven eight nine test"),
                         "it is 7 8 9 test")
        self.assertEqual(normalize("it's a seven eight nine  test"),
                         "it is 7 8 9 test")
        self.assertEqual(normalize("that's a ten eleven twelve test"),
                         "that is 10 11 12 test")
        self.assertEqual(normalize("that's a thirteen fourteen test"),
                         "that is 13 14 test")
        self.assertEqual(normalize("that's fifteen sixteen seventeen"),
                         "that is 15 16 17")
        self.assertEqual(normalize("that's eighteen nineteen twenty"),
                         "that is 18 19 20")

    def test_contractions(self):
        self.assertEqual(normalize("ain't"), "is not")
        self.assertEqual(normalize("aren't"), "are not")<commit_msg>PEP8 Fix To Test File<commit_after>
# -*- coding: iso-8859-15 -*-

import unittest
from mycroft.util.parse import normalize
from mycroft.util.parse import extractnumber


class TestNormalize(unittest.TestCase):
    def test_articles(self):
        self.assertEqual(normalize("this is a test", remove_articles=True),
                         "this is test")
        self.assertEqual(normalize("this is the test", remove_articles=True),
                         "this is test")
        self.assertEqual(normalize("and another test", remove_articles=True),
                         "and another test")
        self.assertEqual(normalize("this is an extra test",
                                   remove_articles=False),
                         "this is an extra test")

    def test_extractnumber(self):
        self.assertEqual(extractnumber("this is the first test"), 1)
        self.assertEqual(extractnumber("this is 2 test"), 2)
        self.assertEqual(extractnumber("this is second test"), 2)
        self.assertEqual(extractnumber("this is the third test"), 1.0/3.0)
        self.assertEqual(extractnumber("this is test number 4"), 4)
        self.assertEqual(extractnumber("one third of a cup"), 1.0/3.0)
        self.assertEqual(extractnumber("three cups"), 3)
        self.assertEqual(extractnumber("1/3 cups"), 1.0/3.0)
        self.assertEqual(extractnumber("quarter cup"), 0.25)
        self.assertEqual(extractnumber("1/4 cup"), 0.25)
        self.assertEqual(extractnumber("one fourth cup"), 0.25)
        self.assertEqual(extractnumber("2/3 cups"), 2.0/3.0)
        self.assertEqual(extractnumber("3/4 cups"), 3.0/4.0)
        self.assertEqual(extractnumber("1 and 3/4 cups"), 1.75)
        self.assertEqual(extractnumber("1 cup and a half"), 1.5)
        self.assertEqual(extractnumber("one cup and a half"), 1.5)
        self.assertEqual(extractnumber("one and a half cups"), 1.5)
        self.assertEqual(extractnumber("one and one half cups"), 1.5)
        self.assertEqual(extractnumber("three quarter cups"), 3.0/4.0)
        self.assertEqual(extractnumber("three quarters cups"), 3.0/4.0)

    def test_spaces(self):
        self.assertEqual(normalize("  this   is  a    test"),
                         "this is test")
        self.assertEqual(normalize("  this   is  a    test  "),
                         "this is test")
        self.assertEqual(normalize("  this   is  one    test"),
                         "this is 1 test")

    def test_numbers(self):
        self.assertEqual(normalize("this is a one two three  test"),
                         "this is 1 2 3 test")
        self.assertEqual(normalize("  it's  a four five six  test"),
                         "it is 4 5 6 test")
        self.assertEqual(normalize("it's  a seven eight nine test"),
                         "it is 7 8 9 test")
        self.assertEqual(normalize("it's a seven eight nine  test"),
                         "it is 7 8 9 test")
        self.assertEqual(normalize("that's a ten eleven twelve test"),
                         "that is 10 11 12 test")
        self.assertEqual(normalize("that's a thirteen fourteen test"),
                         "that is 13 14 test")
        self.assertEqual(normalize("that's fifteen sixteen seventeen"),
                         "that is 15 16 17")
        self.assertEqual(normalize("that's eighteen nineteen twenty"),
                         "that is 18 19 20")

    def test_contractions(self):
        self.assertEqual(normalize("ain't"), "is not")
        self.assertEqual(normalize("aren't"), "are not")
{'fraction_non_alphanumeric': 0.11728308254816482, 'fraction_numerical': 0.030516101814090844, 'mean_word_length': 2.1693404634581106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4085834', 'n_tokens_mistral': 2074, 'n_tokens_neox': 1848, 'n_words': 549}