<commit_before># -*- coding: utf-8 -*-

class AnDiP(object):
    def __init__(self, provider, backoff = None):
        assert isinstance(provider, DataProvider)

        self.__provider = provider
        self.__backoff = backoff;
        self.__unsaved_items = list()

    def get_word(self, conf):
        """
        Get a specified word from a dictionary.

        @param conf a tuple build from three parts: part of a speech, base word and dict of properties

        @return word in a specified configuration
        """

        try:
            retval = self.__backoff.get_word(conf)
        except Exception:
            retval = self.__provider.get_word(conf)

        if isinstance(retval, tuple):
            self.__unsaved_items.append(retval[1])
            return retval[0]
        else:
            return retval

    def get_conf(self, word):
        """
        Get the configuration for a given word.

        Each configuration is a tuple build from three parts:
        - part of a speech
        - base word
        - dict of properties

        @return a list with configurations
        """

        try:
            return self.__backoff.get_conf(word)
        except Exception:
            return self.__provider.get_conf(word)

    def save(self, items = None):
        """
        Saves all unsaved items using all providers including backoff.
        """
        if items == None:
            self.__backoff.save(self.__unsaved_items)
        else:
            for item in items:
                self.__provider.save(item)

class DataProvider(object):
    def get_word(self, conf):
        raise Exception("abstract method")

    def get_conf(self, word):
        raise Exception("abstract method")

    def save(self):
        raise Exception("abstract method")
<commit_msg>use NotImplementedError instead of Exception<commit_after># -*- coding: utf-8 -*-

class AnDiP(object):
    def __init__(self, provider, backoff = None):
        assert isinstance(provider, DataProvider)

        self.__provider = provider
        self.__backoff = backoff;
        self.__unsaved_items = list()

    def get_word(self, conf):
        """
        Get a specified word from a dictionary.

        @param conf a tuple build from three parts: part of a speech, base word and dict of properties

        @return word in a specified configuration
        """

        try:
            retval = self.__backoff.get_word(conf)
        except Exception:
            retval = self.__provider.get_word(conf)

        if isinstance(retval, tuple):
            self.__unsaved_items.append(retval[1])
            return retval[0]
        else:
            return retval

    def get_conf(self, word):
        """
        Get the configuration for a given word.

        Each configuration is a tuple build from three parts:
        - part of a speech
        - base word
        - dict of properties

        @return a list with configurations
        """

        try:
            return self.__backoff.get_conf(word)
        except Exception:
            return self.__provider.get_conf(word)

    def save(self, items = None):
        """
        Saves all unsaved items using all providers including backoff.
        """
        if items == None:
            self.__backoff.save(self.__unsaved_items)
        else:
            for item in items:
                self.__provider.save(item)

class DataProvider(object):
    def get_word(self, conf):
        raise NotImplementedError("abstract method")

    def get_conf(self, word):
        raise NotImplementedError("abstract method")

    def save(self):
        raise NotImplementedError("abstract method")
{'fraction_non_alphanumeric': 0.08233369683751363, 'fraction_numerical': 0.0016357688113413304, 'mean_word_length': 1.9780844155844155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2092379', 'n_tokens_mistral': 950, 'n_tokens_neox': 925, 'n_words': 306}