<commit_before>"""
Created on 12 Apr 2019

@author: <NAME> (<EMAIL>)

Warning: only use this on data sets where there are no missing timeline intervals!
"""

from scs_core.data.path_dict import PathDict


# --------------------------------------------------------------------------------------------------------------------

class NodeShifter(object):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, offset, fill, source_sub_path, target_sub_path=None):
        """
        Constructor
        """
        self.__offset = offset                                                                      # int
        self.__fill = fill                                                                          # bool

        self.__source_sub_path = source_sub_path                                                    # string
        self.__target_sub_path = source_sub_path if target_sub_path is None else target_sub_path    # string

        self.__documents = []                                                                       # array of PathDict
        self.__enqueued = 0                                                                         # int


    def __len__(self):
        return len(self.__documents)


    # ----------------------------------------------------------------------------------------------------------------

    def shift(self, document: PathDict):
        self.append(document)

        return self.pop()


    def append(self, document: PathDict):
        self.__documents.append(document)
        self.__enqueued += 1


    def pop(self):
        length = len(self)

        if length == 0:
            return None

        # indices...
        if self.offset > 0:
            outer_index = length - self.enqueued
            inner_index = outer_index - self.offset

        else:
            inner_index = length - self.enqueued
            outer_index = inner_index + self.offset

        # sources...
        outer = self.__documents[outer_index] if outer_index in range(length) else None
        inner = self.__documents[inner_index] if inner_index in range(length) else None

        self.__enqueued -= 1

        # trim the list...
        if outer is not None and inner is not None:
            del self.__documents[0]

        # check...
        if outer is None:
            return None

        if inner is None and not self.fill:
            return None

        # target...
        document = PathDict()

        for source_path in outer.paths():
            if PathDict.sub_path_includes_path(self.source_sub_path, source_path):
                source = inner
                path = source_path.replace(self.source_sub_path, self.target_sub_path)

            else:
                source = outer
                path = source_path

            node = None if source is None else source.node(source_path)

            document.append(path, node)

        return document


    # ----------------------------------------------------------------------------------------------------------------

    @property
    def offset(self):
        return self.__offset


    @property
    def fill(self):
        return self.__fill


    @property
    def source_sub_path(self):
        return self.__source_sub_path


    @property
    def target_sub_path(self):
        return self.__target_sub_path


    @property
    def enqueued(self):
        return self.__enqueued


    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "NodeShifter:{offset:%s, fill:%s, source_sub_path:%s, target_sub_path:%s, enqueued:%s, len:%s}" % \
               (self.offset, self.fill, self.source_sub_path, self.target_sub_path, self.enqueued, len(self))
<commit_msg>Fixed a bug in NodeShifter.<commit_after>"""
Created on 12 Apr 2019

@author: <NAME> (<EMAIL>)

Warning: only use this on data sets where there are no missing timeline intervals!
"""

from scs_core.data.path_dict import PathDict


# --------------------------------------------------------------------------------------------------------------------

class NodeShifter(object):
    """
    classdocs
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self, offset, fill, source_sub_path, target_sub_path=None):
        """
        Constructor
        """
        self.__offset = offset                                                                      # int
        self.__fill = fill                                                                          # bool

        self.__source_sub_path = source_sub_path                                                    # string
        self.__target_sub_path = source_sub_path if target_sub_path is None else target_sub_path    # string

        self.__documents = []                                                                       # array of PathDict
        self.__enqueued = 0                                                                         # int


    def __len__(self):
        return len(self.__documents)


    # ----------------------------------------------------------------------------------------------------------------

    def shift(self, document: PathDict):
        self.append(document)

        return self.pop()


    def append(self, document: PathDict):
        self.__documents.append(document)
        self.__enqueued += 1


    def pop(self):
        length = len(self)

        if length == 0:
            return None

        # indices...
        if self.offset > 0:
            outer_index = length - self.enqueued
            inner_index = outer_index - self.offset

        else:
            inner_index = length - self.enqueued
            outer_index = inner_index + self.offset

        # sources...
        outer = self.__documents[outer_index] if outer_index in range(length) else None
        inner = self.__documents[inner_index] if inner_index in range(length) else None

        self.__enqueued -= 1

        # trim the list...
        if outer is not None and inner is not None:
            del self.__documents[0]

        # check...
        if outer is None:
            return None

        if inner is None and not self.fill:
            return None

        # target...
        document = PathDict()

        for source_path in outer.paths():
            if PathDict.sub_path_includes_path(self.source_sub_path, source_path):
                source = inner
                path = source_path.replace(self.source_sub_path, self.target_sub_path, 1)

            else:
                source = outer
                path = source_path

            node = None if source is None else source.node(source_path)

            document.append(path, node)

        return document


    # ----------------------------------------------------------------------------------------------------------------

    @property
    def offset(self):
        return self.__offset


    @property
    def fill(self):
        return self.__fill


    @property
    def source_sub_path(self):
        return self.__source_sub_path


    @property
    def target_sub_path(self):
        return self.__target_sub_path


    @property
    def enqueued(self):
        return self.__enqueued


    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "NodeShifter:{offset:%s, fill:%s, source_sub_path:%s, target_sub_path:%s, enqueued:%s, len:%s}" % \
               (self.offset, self.fill, self.source_sub_path, self.target_sub_path, self.enqueued, len(self))

{'fraction_non_alphanumeric': 0.2111801242236025, 'fraction_numerical': 0.003168969451134491, 'mean_word_length': 1.0541525644363448, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5266262', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1704, 'n_words': 465}