<commit_before>#!/usr/bin/env python
"""
:Author <NAME>
:Created 22.07.2017 13:19
:Licence GNUv3
Part of grammpy

"""
from collections import Iterable


class HashContainer:
    def __init__(self, items=None):
        if items is None:
            items = []
        self.__items = {}
        self.add(items)

    # Helpers
    @staticmethod
    def is_iterable(param: Iterable) -> bool:
        return isinstance(param, Iterable)

    @staticmethod
    def to_iterable(param: Iterable):
        # standardize it to iterable object
        if not HashContainer.is_iterable(param):
            return [param]
        return param

    def add(self, item):
        items = HashContainer.to_iterable(item)
        # iterace throught items
        for t in items:
            self.__items[hash(t)] = t
        return items

    def remove(self, item=None):
        if item is None:
            all = list(self.__items.values())
            self.__items.clear()
            return all
        item = HashContainer.to_iterable(item)
        deleted = []
        # iterate through items
        for t in item:
            h = hash(t)
            if h in self.__items:
                deleted.append(self.__items[h])
                del self.__items[h]
        return deleted

    def have(self, item):
        item = HashContainer.to_iterable(item)
        for t in item:
            if hash(t) not in self.__items:
                return False
        return True

    def get(self, item=None):
        # if no parameter is passed than return all terminals
        if item is None:
            # Maybe lazy evaluation, but it cannot be combined with return
            return self.all()
        # else return relevant to parameter
        transformed = HashContainer.to_iterable(item)
        ret = []
        for t in transformed:
            ret.append(t if hash(t) in self.__items else None)
        if not HashContainer.is_iterable(item):
            return ret[0]
        return ret

    def all(self):
        return [item for _, item in self.__items.items()]

    def count(self):
        return len(self.__items)
<commit_msg>Reiplement HashCotnainer.add method so only elements that are newly add are return<commit_after>#!/usr/bin/env python
"""
:Author <NAME>
:Created 22.07.2017 13:19
:Licence GNUv3
Part of grammpy

"""
from collections import Iterable


class HashContainer:
    def __init__(self, items=None):
        if items is None:
            items = []
        self.__items = {}
        self.add(items)

    # Helpers
    @staticmethod
    def is_iterable(param: Iterable) -> bool:
        return isinstance(param, Iterable)

    @staticmethod
    def to_iterable(param: Iterable):
        # standardize it to iterable object
        if not HashContainer.is_iterable(param):
            return [param]
        return param

    def add(self, item):
        items = HashContainer.to_iterable(item)
        # iterace throught items
        add = []
        for t in items:
            h = hash(t)
            if h not in self.__items:
                self.__items[h] = t
                add.append(t)
        return add

    def remove(self, item=None):
        if item is None:
            all = list(self.__items.values())
            self.__items.clear()
            return all
        item = HashContainer.to_iterable(item)
        deleted = []
        # iterate through items
        for t in item:
            h = hash(t)
            if h in self.__items:
                deleted.append(self.__items[h])
                del self.__items[h]
        return deleted

    def have(self, item):
        item = HashContainer.to_iterable(item)
        for t in item:
            if hash(t) not in self.__items:
                return False
        return True

    def get(self, item=None):
        # if no parameter is passed than return all terminals
        if item is None:
            # Maybe lazy evaluation, but it cannot be combined with return
            return self.all()
        # else return relevant to parameter
        transformed = HashContainer.to_iterable(item)
        ret = []
        for t in transformed:
            ret.append(t if hash(t) in self.__items else None)
        if not HashContainer.is_iterable(item):
            return ret[0]
        return ret

    def all(self):
        return [item for _, item in self.__items.items()]

    def count(self):
        return len(self.__items)

{'fraction_non_alphanumeric': 0.08355916892502259, 'fraction_numerical': 0.006323396567299007, 'mean_word_length': 1.822817080943276, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4406047', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1212, 'n_words': 394}