<commit_before>import os
import shutil
import unittest

from fff.fuzzyindex import FuzzyIndex
from fff import MATCH_LEVELS, IGNORE_DIRS, IGNORE_FILES

ROOT = 'tmp'

FOCUS_FILES = ['dir0_file1.ext']

IGNORE = [
'tmp/dir2/py.pyc',
'tmp/dir2/__init__.py',
'tmp/.git/nomatch.ext'
]

FILES = [
'tmp/dir0_file1.ext',
'tmp/dir1/dir1_file1.ext',
'tmp/dir1/dir1_file2.ext',
'tmp/dir2/dir2_file1.ext',
'tmp/dir2/dir2_file2.ext',
]

FILE_SYSTEM = FILES + IGNORE

class TestFuzzyIndex(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        """ setup the test directory. """
        def makefile(path):
            basedir = os.path.dirname(path)
            if not os.path.exists(basedir):
                os.makedirs(basedir)
            open(path, 'a').close()

        for f in FILE_SYSTEM:
            makefile(f)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(ROOT)

    def setUp(self):
        self.fi = FuzzyIndex(ROOT, ignore_dirs=IGNORE_DIRS, ignore_files=IGNORE_FILES)

    def test_init(self):
        self.assertEqual(len(self.fi.files), 5)

    def test_append(self):
        FILE = '/path/to/file.ext'
        self.fi.append(FILE)
        self.assertEqual(self.fi.files[-1].path, FILE)

    def test_generate_paths(self):
        paths = [x for x in self.fi.generate_paths(ROOT, ignore_dirs=IGNORE_DIRS, ignore_files=IGNORE_FILES)]
        paths.sort()
        FILES.sort()
        self.assertEqual(paths, FILES)

    def test_generate_paths_focus(self):
        paths = [os.path.basename(x) for x in self.fi.generate_paths(ROOT, focus_files=FOCUS_FILES)]
        paths.sort()
        FOCUS_FILES.sort()
        self.assertEqual(len(paths), 1)
        self.assertEqual(paths, FOCUS_FILES)

    def test_generate_patterns(self):
        patts = [x.pattern for x in self.fi.generate_patterns('ab')]
        self.assertEqual(len(patts), MATCH_LEVELS)
        patterns = ['^(?P<head>.*?)a(.{{,{0}}}?)b(?P<tail>.*?)$'.format(i) for i in range(MATCH_LEVELS)]
        self.assertEqual(patts, patterns)        

    def test_match(self):
        NO_MATCH = ' '
        self.assertEqual(self.fi.match(NO_MATCH), None)<commit_msg>100% coverage for fuzzyindex now<commit_after>import os
import shutil
import unittest

from fff.fuzzyindex import FuzzyIndex
from fff import MATCH_LEVELS, IGNORE_DIRS, IGNORE_FILES

ROOT = 'tmp'

FOCUS_FILES = ['dir0_file1.ext']

IGNORE = [
'tmp/dir2/py.pyc',
'tmp/dir2/__init__.py',
'tmp/.git/nomatch.ext'
]

FILES = [
'tmp/dir0_file1.ext',
'tmp/dir1/dir1_file1.ext',
'tmp/dir1/dir1_file2.ext',
'tmp/dir2/dir2_file1.ext',
'tmp/dir2/dir2_file2.ext',
]

FILE_SYSTEM = FILES + IGNORE

class TestFuzzyIndex(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        """ setup the test directory. """
        def makefile(path):
            basedir = os.path.dirname(path)
            if not os.path.exists(basedir):
                os.makedirs(basedir)
            open(path, 'a').close()

        for f in FILE_SYSTEM:
            makefile(f)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(ROOT)

    def setUp(self):
        self.fi = FuzzyIndex(ROOT, ignore_dirs=IGNORE_DIRS, ignore_files=IGNORE_FILES)

    def test_init(self):
        self.assertEqual(len(self.fi.files), 5)

    def test_append(self):
        FILE = '/path/to/file.ext'
        self.fi.append(FILE)
        self.assertEqual(self.fi.files[-1].path, FILE)

    def test_generate_paths(self):
        paths = list(self.fi.generate_paths(ROOT))
        paths.sort()
        FILE_SYSTEM.sort()
        self.assertEqual(paths, FILE_SYSTEM)

    def test_generate_paths_filter_dirs(self):
        p = list(self.fi.generate_paths(ROOT, ignore_dirs=['dir1', 'dir2']))
        self.assertEqual(p, ['tmp/dir0_file1.ext', 'tmp/.git/nomatch.ext'])

    def test_generate_paths_filter_files(self):
        p = list(self.fi.generate_paths(ROOT, ignore_files=[r'\.pyc$', r'\.py$', r'\.ext$']))
        self.assertEqual(p, [])

    def test_generate_paths_focus(self):
        paths = [os.path.basename(x) for x in self.fi.generate_paths(ROOT, focus_files=FOCUS_FILES)]
        paths.sort()
        FOCUS_FILES.sort()
        self.assertEqual(len(paths), 1)
        self.assertEqual(paths, FOCUS_FILES)

    def test_generate_patterns(self):
        patts = [x.pattern for x in self.fi.generate_patterns('ab')]
        self.assertEqual(len(patts), MATCH_LEVELS)
        patterns = ['^(?P<head>.*?)a(.{{,{0}}}?)b(?P<tail>.*?)$'.format(i) for i in range(MATCH_LEVELS)]
        self.assertEqual(patts, patterns)        

    def test_match_none(self):
        NO_MATCH = ' '
        self.assertEqual(self.fi.match(NO_MATCH), None)

    def test_match(self):
        m = self.fi.match('d1_f1')
        self.assertEqual(m.path, 'tmp/dir1/dir1_file1.ext')

    def test_match_include_dir(self):
        m = self.fi.match('/file1')
        self.assertEqual(m.path, 'tmp/dir0_file1.ext')

    def test_match_list(self):
        m = self.fi.match('dir1_', list_files=True)
        self.assertTrue(type(m) == list)
        self.assertEqual(len(m), 2)
        L1 = [x.path for x in m]
        L1.sort()
        L2 = ['tmp/dir1/dir1_file1.ext',
              'tmp/dir1/dir1_file2.ext']
        self.assertEqual(L1, L2)
{'fraction_non_alphanumeric': 0.14356435643564355, 'fraction_numerical': 0.013709063214013708, 'mean_word_length': 2.851173020527859, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2097294', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1884, 'n_words': 307}