<commit_before># # -*- coding: utf-8 -*-
import os
import shutil
from sync_settings.libs import helper
from tests import test_file_path
from unittest import TestCase

class TestHelper(TestCase):
  def test_difference(self):
    l = helper.get_difference([1, 2, 3, 4], [1, 2, 3])
    self.assertEqual(len(l), 1)
    self.assertEqual(l[0], 4)

  def test_home_path(self):
    home_path = os.path.expanduser('~')
    self.assertEqual(helper.join_path((home_path, 'foo')), helper.get_home_path('foo'))
    self.assertEqual(home_path, helper.get_home_path())
    #Wrong Cases
    self.assertEqual(home_path, helper.get_home_path(None))
    self.assertEqual(home_path, helper.get_home_path(1234))

  def test_exists_path(self):
    self.assertFalse(helper.exists_path(test_file_path, True))
    with self.assertRaises(TypeError): helper.exists_path()
    with self.assertRaises(TypeError): helper.exists_path(isFolder=True)
    self.assertFalse(helper.exists_path(helper.join_path((os.getcwd(), 'tests'))))
    self.assertTrue(helper.exists_path(test_file_path))
    self.assertTrue(helper.exists_path(helper.join_path((os.getcwd(), 'tests')), True))

  def test_join_path(self):
    self.assertIsNone(helper.join_path(''))
    self.assertIsNone(helper.join_path([]))
    self.assertIsNone(helper.join_path(1234))
    self.assertIsNone(helper.join_path(('')))
    self.assertIsNotNone(helper.join_path(('123', '1234')))

  def test_get_files(self):
    with self.assertRaises(TypeError): helper.get_files()
    self.assertListEqual(helper.get_files('t'), [])
    self.assertListEqual(helper.get_files(1234), [])
    self.assertListEqual(helper.get_files(1234), [])
    self.assertGreater(len(helper.get_files(helper.join_path((os.getcwd(), 'tests')))), 0)

    #Create a test folder structure
    os.makedirs(helper.join_path((os.getcwd(), 'tests', 'hello', 'world')))
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'bar.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'world', 'foo.txt')), 'a').close()
    allFiles = sorted([
      helper.join_path((os.getcwd(), 'tests', 'hello', 'bar.txt')),
      helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt')),
      helper.join_path((os.getcwd(), 'tests', 'hello', 'world', 'foo.txt'))
    ])
    files = sorted(helper.get_files(helper.join_path((os.getcwd(), 'tests', 'hello'))))
    self.assertEqual(len(files), 3)
    self.assertListEqual(files, allFiles)
    shutil.rmtree(helper.join_path((os.getcwd(), 'tests', 'hello')))

  def test_is_folder_pattern(self):
    file_path = helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt'))
    folder_path = helper.join_path((os.getcwd(), 'tests', 'hello'))

    os.makedirs(folder_path)
    open(file_path, 'a').close()

    self.assertFalse(helper.is_folder_pattern(file_path, file_path))
    self.assertFalse(helper.is_folder_pattern(folder_path, folder_path))
    self.assertTrue(helper.is_folder_pattern(file_path, folder_path))

    shutil.rmtree(helper.join_path((os.getcwd(), 'tests', 'hello')))

  def test_exclude_files_by_patterns(self):
    #Assuming <../tests/foo> is <../User/>
    os.makedirs(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar')))
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.py')), 'a').close()

    files = helper.get_files(helper.join_path((os.getcwd(), 'tests', 'foo')))

    #Unfiltered
    self.assertEqual(len(files), 4)
    self.assertListEqual(helper.exclude_files_by_patterns(files, []), files)
    self.assertListEqual(helper.exclude_files_by_patterns(files, ['.boo']), files)

    # By extension
    filteredFiles = helper.exclude_files_by_patterns(files, [
      helper.join_path((os.getcwd(), 'tests', 'foo', '.txt'))
    ])

    self.assertEqual(len(filteredFiles), 1)
    self.assertListEqual(filteredFiles, [
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.py'))
    ])

    filteredFiles = sorted(helper.exclude_files_by_patterns(files, [
      helper.join_path((os.getcwd(), 'tests', 'foo', '.py'))
    ]))
    self.assertEqual(len(filteredFiles), 3)
    self.assertListEqual(filteredFiles, sorted([
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar.txt')),
      helper.join_path((os.getcwd(), 'tests', 'foo', 'foo.txt')),
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.txt'))
    ]))<commit_msg>fix filter by extension<commit_after># # -*- coding: utf-8 -*-
import os
import shutil
from sync_settings.libs import helper
from tests import test_file_path
from unittest import TestCase

class TestHelper(TestCase):
  def test_difference(self):
    l = helper.get_difference([1, 2, 3, 4], [1, 2, 3])
    self.assertEqual(len(l), 1)
    self.assertEqual(l[0], 4)

  def test_home_path(self):
    home_path = os.path.expanduser('~')
    self.assertEqual(helper.join_path((home_path, 'foo')), helper.get_home_path('foo'))
    self.assertEqual(home_path, helper.get_home_path())
    #Wrong Cases
    self.assertEqual(home_path, helper.get_home_path(None))
    self.assertEqual(home_path, helper.get_home_path(1234))

  def test_exists_path(self):
    self.assertFalse(helper.exists_path(test_file_path, True))
    with self.assertRaises(TypeError): helper.exists_path()
    with self.assertRaises(TypeError): helper.exists_path(isFolder=True)
    self.assertFalse(helper.exists_path(helper.join_path((os.getcwd(), 'tests'))))
    self.assertTrue(helper.exists_path(test_file_path))
    self.assertTrue(helper.exists_path(helper.join_path((os.getcwd(), 'tests')), True))

  def test_join_path(self):
    self.assertIsNone(helper.join_path(''))
    self.assertIsNone(helper.join_path([]))
    self.assertIsNone(helper.join_path(1234))
    self.assertIsNone(helper.join_path(('')))
    self.assertIsNotNone(helper.join_path(('123', '1234')))

  def test_get_files(self):
    with self.assertRaises(TypeError): helper.get_files()
    self.assertListEqual(helper.get_files('t'), [])
    self.assertListEqual(helper.get_files(1234), [])
    self.assertListEqual(helper.get_files(1234), [])
    self.assertGreater(len(helper.get_files(helper.join_path((os.getcwd(), 'tests')))), 0)

    #Create a test folder structure
    os.makedirs(helper.join_path((os.getcwd(), 'tests', 'hello', 'world')))
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'bar.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'hello', 'world', 'foo.txt')), 'a').close()
    allFiles = sorted([
      helper.join_path((os.getcwd(), 'tests', 'hello', 'bar.txt')),
      helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt')),
      helper.join_path((os.getcwd(), 'tests', 'hello', 'world', 'foo.txt'))
    ])
    files = sorted(helper.get_files(helper.join_path((os.getcwd(), 'tests', 'hello'))))
    self.assertEqual(len(files), 3)
    self.assertListEqual(files, allFiles)
    shutil.rmtree(helper.join_path((os.getcwd(), 'tests', 'hello')))

  def test_is_folder_pattern(self):
    file_path = helper.join_path((os.getcwd(), 'tests', 'hello', 'foo.txt'))
    folder_path = helper.join_path((os.getcwd(), 'tests', 'hello'))

    os.makedirs(folder_path)
    open(file_path, 'a').close()

    self.assertFalse(helper.is_folder_pattern(file_path, file_path))
    self.assertFalse(helper.is_folder_pattern(folder_path, folder_path))
    self.assertTrue(helper.is_folder_pattern(file_path, folder_path))

    shutil.rmtree(helper.join_path((os.getcwd(), 'tests', 'hello')))

  def test_exclude_files_by_patterns(self):
    #Assuming <../tests/foo> is <../User/>
    os.makedirs(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar')))
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.txt')), 'a').close()
    open(helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.py')), 'a').close()

    files = helper.get_files(helper.join_path((os.getcwd(), 'tests', 'foo')))

    #Unfiltered
    self.assertEqual(len(files), 4)
    self.assertListEqual(helper.exclude_files_by_patterns(files, []), files)
    self.assertListEqual(helper.exclude_files_by_patterns(files, ['.boo']), files)

    # By extension
    filteredFiles = helper.exclude_files_by_patterns(files, ['.txt'])

    self.assertEqual(len(filteredFiles), 1)
    self.assertListEqual(filteredFiles, [
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.py'))
    ])

    filteredFiles = sorted(helper.exclude_files_by_patterns(files, ['.py']))
    self.assertEqual(len(filteredFiles), 3)
    self.assertListEqual(filteredFiles, sorted([
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar.txt')),
      helper.join_path((os.getcwd(), 'tests', 'foo', 'foo.txt')),
      helper.join_path((os.getcwd(), 'tests', 'foo', 'bar', 'foo.txt'))
    ]))
{'fraction_non_alphanumeric': 0.19534282018111254, 'fraction_numerical': 0.008408796895213453, 'mean_word_length': 5.063398692810457, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1043622', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3279, 'n_words': 481}