<commit_before>import os
import glob

from nalaf.features import FeatureGenerator
from nalaf.utils.hdfs import maybe_get_hdfs_client, walk_hdfs_directory


class DictionaryFeatureGenerator(FeatureGenerator):

    def __init__(self, name, words_set, case_sensitive=False):
        self.name = name
        self.words_set = words_set
        self.key = "dics." + name
        self.case_sensitive = case_sensitive

    def generate(self, dataset):
        for token in dataset.tokens():
            normalized_token = token if self.case_sensitive else token.word.lower()
            token.features[self.key] = normalized_token in self.words_set

    @staticmethod
    def construct_words_set(file_reader, string_tokenizer, case_sensitive, stop_words):
        """
        Note, the file_reader descriptor is not closed. The client is responsible for this.
        """
        ret = set()
        for name in file_reader:
            tokens = string_tokenizer(name)
            normalized_tokens = tokens if case_sensitive else (x.lower() for x in tokens)
            filtered_normalized_tokens = (x for x in normalized_tokens if ((x not in stop_words) and DictionaryFeatureGenerator.default_stop_rules(x)))

            ret.update(filtered_normalized_tokens)

        return ret

    @staticmethod
    def default_stop_rules(token):
        return len(token) > 1

    @staticmethod
    def __normalize_stop_words(stop_words):
        if stop_words is None:
            return set()
        elif type(stop_words) is str:
            return set(stop_words.split())
        else:
            return stop_words

    @staticmethod
    def __get_filename(path):
        return os.path.splitext(os.path.basename(path))[0]

    @staticmethod
    def __read_dictionaries(dic_paths, read_function, string_tokenizer, case_sensitive, stop_words):
        stop_words = DictionaryFeatureGenerator.__normalize_stop_words(stop_words)

        ret = []

        for dic_path in dic_paths:
            reader = read_function(dic_path)
            try:
                name = DictionaryFeatureGenerator.__get_filename(dic_path)
                words_set = DictionaryFeatureGenerator.construct_words_set(reader, string_tokenizer, case_sensitive, stop_words)
                generator = DictionaryFeatureGenerator(name, words_set, case_sensitive)
                ret.append(generator)
            finally:
                reader.close()

        return ret

    @staticmethod
    def __localfs_read_function(dic_path):
        return open(dic_path, "r")

    @staticmethod
    def __hdfs_read_funciton(hdfs_client):
        return lambda dic_path: hdfs_client.read(dic_path)

    @staticmethod
    def construct_all_from_paths(dictionaries_paths, string_tokenizer=(lambda x: x.split()), case_sensitive=False, hdfs_url=None, hdfs_user=None, stop_words=None, accepted_extensions=[".dic", "dict", ".txt", ".tsv", ".csv"]):

        hdfs_client = maybe_get_hdfs_client(hdfs_url, hdfs_user)

        if hdfs_client:
            read_function = DictionaryFeatureGenerator.__hdfs_read_funciton(hdfs_client)

        else:
            read_function = DictionaryFeatureGenerator.__localfs_read_function

        #

        return DictionaryFeatureGenerator.__read_dictionaries(dictionaries_paths, read_function, string_tokenizer, case_sensitive, stop_words)

    @staticmethod
    def construct_all_from_folder(dictionaries_folder, string_tokenizer=(lambda x: x.split()), case_sensitive=False, hdfs_url=None, hdfs_user=None, stop_words=None, accepted_extensions=[".dic", "dict", ".txt", ".tsv", ".csv"]):

        def accept_filename_fun(filename):
            return any(filename.endswith(accepted_extension) for accepted_extension in accepted_extensions)

        hdfs_client = maybe_get_hdfs_client(hdfs_url, hdfs_user)

        if hdfs_client:
            # hdfs<commit_msg>minor utility<commit_after>import os
import glob

from nalaf.features import FeatureGenerator
from nalaf.utils.hdfs import maybe_get_hdfs_client, walk_hdfs_directory


class DictionaryFeatureGenerator(FeatureGenerator):

    def __init__(self, name, words_set, case_sensitive=False):
        self.name = name
        self.words_set = words_set
        self.key = "dics." + name
        self.case_sensitive = case_sensitive

    def generate(self, dataset):
        for token in dataset.tokens():
            normalized_token = token if self.case_sensitive else token.word.lower()
            token.features[self.key] = normalized_token in self.words_set

    @staticmethod
    def construct_words_set(file_reader, string_tokenizer, case_sensitive, stop_words):
        """
        Note, the file_reader descriptor is not closed. The client is responsible for this.
        """
        ret = set()
        for name in file_reader:
            tokens = string_tokenizer(name)
            normalized_tokens = tokens if case_sensitive else (x.lower() for x in tokens)
            filtered_normalized_tokens = (x for x in normalized_tokens if ((x not in stop_words) and DictionaryFeatureGenerator.default_stop_rules(x)))

            ret.update(filtered_normalized_tokens)

        return ret

    @staticmethod
    def default_stop_rules(token):
        return len(token) > 1

    @staticmethod
    def __normalize_stop_words(stop_words):
        if stop_words is None:
            return set()
        elif type(stop_words) is str:
            return set(stop_words.split())
        else:
            return stop_words

    @staticmethod
    def __get_filename(path):
        return os.path.splitext(os.path.basename(path))[0]

    @staticmethod
    def __read_dictionaries(dic_paths, read_function, string_tokenizer, case_sensitive, stop_words):
        stop_words = DictionaryFeatureGenerator.__normalize_stop_words(stop_words)

        ret = []

        for dic_path in dic_paths:
            reader = read_function(dic_path)
            try:
                name = DictionaryFeatureGenerator.__get_filename(dic_path)
                words_set = DictionaryFeatureGenerator.construct_words_set(reader, string_tokenizer, case_sensitive, stop_words)
                generator = DictionaryFeatureGenerator(name, words_set, case_sensitive)
                ret.append(generator)
            finally:
                reader.close()

        return ret

    @staticmethod
    def __localfs_read_function(dic_path):
        return open(dic_path, "r")

    @staticmethod
    def __hdfs_read_funciton(hdfs_client):
        return lambda dic_path: hdfs_client.read(dic_path)

    @staticmethod
    def construct_all_from_paths(dictionaries_paths, string_tokenizer=(lambda x: x.split()), case_sensitive=False, hdfs_url=None, hdfs_user=None, stop_words=None, accepted_extensions=[".dic", "dict", ".txt", ".tsv", ".csv"]):
        if type(dictionaries_paths) is str:
            dictionaries_paths = dictionaries_paths.split()

        hdfs_client = maybe_get_hdfs_client(hdfs_url, hdfs_user)

        if hdfs_client:
            read_function = DictionaryFeatureGenerator.__hdfs_read_funciton(hdfs_client)

        else:
            read_function = DictionaryFeatureGenerator.__localfs_read_function

        #

        return DictionaryFeatureGenerator.__read_dictionaries(dictionaries_paths, read_function, string_tokenizer, case_sensitive, stop_words)

    @staticmethod
    def construct_all_from_folder(dictionaries_folder, string_tokenizer=(lambda x: x.split()), case_sensitive=False, hdfs_url=None, hdfs_user=None, stop_words=None, accepted_extensions=[".dic", "dict", ".txt", ".tsv", ".csv"]):

        def accept_filename_fun(filename):
            return any(filename.endswith(accepted_extension) for accepted_extension in accepted_extensions)

        hdfs_client = maybe_get_hdfs_client(hdfs_url, hdfs_user)

        if hdfs_client:
            # hdfs
{'fraction_non_alphanumeric': 0.0794200667179882, 'fraction_numerical': 0.0005132152938157557, 'mean_word_length': 3.150692225772098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530814', 'n_tokens_mistral': 2306, 'n_tokens_neox': 2206, 'n_words': 474}