<commit_before># coding: utf-8
"""
This module contains unit testing functions.
"""

import random
import logging
import unittest
import copy
import os
import json
from flatisfy import tools
from flatisfy.filters import duplicates
from flatisfy.filters.cache import ImageCache
from flatisfy.constants import BACKENDS_BY_PRECEDENCE

LOGGER = logging.getLogger(__name__)
TESTS_DATA_DIR = os.path.dirname(os.path.realpath(__file__)) + "/test_files/"

class TestTexts(unittest.TestCase):
    def test_roman_numbers(self):
        """
        Checks roman numbers replacement.
        """
        tester = tools.RomanNumbers()
        self.assertTrue(tester.check_valid("XIV"))
        self.assertTrue(not tester.check_valid("ABC"))

        self.assertEqual(
            "14",
            tester.convert_to_arabic("XIV")
        )

        self.assertEqual(
            "1987",
            tester.convert_to_arabic("MCMLXXXVII")
        )

        self.assertEqual(
            "Dans le 15e arrondissement",
            tester.convert_to_arabic_in_text("Dans le XVe arrondissement")
        )

        self.assertEqual(
            "20eme arr.",
            tester.convert_to_arabic_in_text("XXeme arr.")
        )

        self.assertEqual(
            "A AIX EN PROVENCE",
            tester.convert_to_arabic_in_text("A AIX EN PROVENCE")
        )

    def test_roman_numbers_in_text(self):
        self.assertEqual(
            "dans le 15e arrondissement",
            tools.normalize_string("Dans le XVe arrondissement")
        )

    def test_multiple_whitespaces(self):
        """
        Checks whitespaces are collapsed.
        """
        self.assertEqual(
            "avec   ascenseur",
            tools.normalize_string("avec ascenseur")
        )

    def test_accents(self):
        """
        Checks accents are replaced.
        """
        self.assertEqual(
            "éèêàüï",
            tools.normalize_string("eeeaui")
        )

class TestPhoneNumbers(unittest.TestCase):
    def test_prefix(self):<commit_msg>Fix tests<commit_after># coding: utf-8
"""
This module contains unit testing functions.
"""

import random
import logging
import unittest
import copy
import os
import json
from flatisfy import tools
from flatisfy.filters import duplicates
from flatisfy.filters.cache import ImageCache
from flatisfy.constants import BACKENDS_BY_PRECEDENCE

LOGGER = logging.getLogger(__name__)
TESTS_DATA_DIR = os.path.dirname(os.path.realpath(__file__)) + "/test_files/"

class TestTexts(unittest.TestCase):
    def test_roman_numbers(self):
        """
        Checks roman numbers replacement.
        """
        tester = tools.RomanNumbers()
        self.assertTrue(tester.check_valid("XIV"))
        self.assertTrue(not tester.check_valid("ABC"))

        self.assertEqual(
            "14",
            tester.convert_to_arabic("XIV")
        )

        self.assertEqual(
            "1987",
            tester.convert_to_arabic("MCMLXXXVII")
        )

        self.assertEqual(
            "Dans le 15e arrondissement",
            tester.convert_to_arabic_in_text("Dans le XVe arrondissement")
        )

        self.assertEqual(
            "20eme arr.",
            tester.convert_to_arabic_in_text("XXeme arr.")
        )

        self.assertEqual(
            "A AIX EN PROVENCE",
            tester.convert_to_arabic_in_text("A AIX EN PROVENCE")
        )

    def test_roman_numbers_in_text(self):
        self.assertEqual(
            "dans le 15e arrondissement",
            tools.normalize_string("Dans le XVe arrondissement")
        )

    def test_multiple_whitespaces(self):
        """
        Checks whitespaces are collapsed.
        """
        self.assertEqual(
            "avec ascenseur",
            tools.normalize_string("avec   ascenseur")
        )

    def test_accents(self):
        """
        Checks accents are replaced.
        """
        self.assertEqual(
            "eeeaui",
            tools.normalize_string(u"éèêàüï")
        )

class TestPhoneNumbers(unittest.TestCase):
    def test_prefix(self):
{'fraction_non_alphanumeric': 0.09285187914517318, 'fraction_numerical': 0.006386637189879637, 'mean_word_length': 2.231746031746032, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3346002', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1221, 'n_words': 238}