<commit_before>import os
import sys
reload(sys)
sys.setdefaultencoding('utf8')

def __is_visible_directory__(directory):
    return not '.' in directory and not "__" in directory

def __all_subdirectories__(directory):
    return [x[0] for x in os.walk(wikiquotes_directory)]

def _look_for(subdirectory_name, directories):
    # We add list to support filter in python 3.x (because it returns an iterable)
    # In python 2.x, list(list) behaves as identity (list([1,2]) is [1,2])
    return list(filter(lambda subdirectory: subdirectory_name in subdirectory, directories))[0]

wikiquotes_directory = os.path.abspath(os.path.join(os.path.dirname(__file__)))
visible_subdirectories = list(filter(__is_visible_directory__, __all_subdirectories__(wikiquotes_directory)))

for subdirectory in visible_subdirectories:
    sys.path.append(subdirectory)

languages_directory = _look_for("languages", visible_subdirectories)
authors_directory = _look_for("authors", visible_subdirectories)
logs_directory = _look_for("logs", visible_subdirectories)
log_path = os.path.join(logs_directory, "debug.log")<commit_msg>Fixing python 3.x by just changing encoding to 2.x<commit_after>import sys
def _get_python_version():
    version = sys.version_info
    return (version.major, version.minor)

python_version = _get_python_version()

import os

if python_version[0] == 2:
    # In python 3.x, the default encoding is already utf8.
    reload(sys)
    sys.setdefaultencoding('utf-8')

def __is_visible_directory__(directory):
    return not '.' in directory and not "__" in directory

def __all_subdirectories__(directory):
    return [x[0] for x in os.walk(wikiquotes_directory)]

def _look_for(subdirectory_name, directories):
    # We add list to support filter in python 3.x (because it returns an iterable)
    # In python 2.x, list(list) behaves as identity (list([1,2]) is [1,2])
    return list(filter(lambda subdirectory: subdirectory_name in subdirectory, directories))[0]

wikiquotes_directory = os.path.abspath(os.path.join(os.path.dirname(__file__)))
visible_subdirectories = list(filter(__is_visible_directory__, __all_subdirectories__(wikiquotes_directory)))

for subdirectory in visible_subdirectories:
    sys.path.append(subdirectory)

languages_directory = _look_for("languages", visible_subdirectories)
authors_directory = _look_for("authors", visible_subdirectories)
logs_directory = _look_for("logs", visible_subdirectories)
log_path = os.path.join(logs_directory, "debug.log")
{'fraction_non_alphanumeric': 0.1001609010458568, 'fraction_numerical': 0.009654062751407884, 'mean_word_length': 6.423880597014925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426404', 'n_tokens_mistral': 786, 'n_tokens_neox': 767, 'n_words': 188}