<commit_before>class MatrixPointer:


    def __init__(self, data, size=None, offset=(0,0)):
        """Create a matrix pointer.

        Keyword arguments:
        data -- array, matrix of matrix pointer;
        size -- tuple (width, height) with size of current image fragment;
            default is None, but you can use it only when data is a matrix;
        offset -- tuple (x, y) with offset of current image fragment
            from the left top element of the matrix; default is (0, 0).
        """

        if size is None:
            if type(data) is not list or type(data[0]) is not list:
                raise ValueError('If size is not set, data should be a matrix')
            size = (len(data[0]), len(data))

        if isinstance(data, MatrixPointer):
            self.__data = data.get_data()
            self.__original_size = data.get_size()
            self.__size = size
            self.__offset = data.get_offset(offset)
        else:
            self.__data = self.__flatten(data)
            self.__original_size = size
            self.__size = size
            self.__offset = offset


    def __flatten(self, lst):
        """Convert lst to one-dimensional list.
        
        Works with non-lists, lists and matrices.
        """
        if type(lst) is not list:
            return [lst]
        elif type(lst[0]) is not list:
            return lst
        return sum(lst, [])


    def get_data(self, source=False):
        """Get matrix or a simple list with data, current pointer points to.
        
        Keyword arguments:
        source -- whether you need the source data (True),
            or cropped fragment (False); default is False.
        """
        if source:
            return self.__data
        result = []
        for offset in range(self.__offset[1], self.__offset[1]+self.__size[1]):
            result.extend(self.__data[
                          self.__original_size[0]*offset+self.__offset[0]:
                          self.__original_size[0]*offset+self.__offset[0]
                                                        +self.__size[0]])
        return result


    def get_offset(self, initial_offset=(0,0), y=0):
        """Get current pointer offset and add new offset to it.
        
        Useful for the case, when you create new pointer
        on the basis of existing one.

        Keyword arguments:
        initial_offset -- tuple (x, y) with offset from current pointer's
            left upper corner either horizontal offset;
        y -- vertical offset, if `initial_offset` is a horizontal offset.
        """

        if type(initial_offset) is tuple:
            return (self.__offset[0]+initial_offset[0],
                    self.__offset[1]+initial_offset[1])<commit_msg>Added generator to matrix pointer<commit_after>class MatrixPointer:


    def __init__(self, data, size=None, offset=(0,0)):
        """Create a matrix pointer.

        Keyword arguments:
        data -- array, matrix of matrix pointer;
        size -- tuple (width, height) with size of current image fragment;
            default is None, but you can use it only when data is a matrix;
        offset -- tuple (x, y) with offset of current image fragment
            from the left top element of the matrix; default is (0, 0).
        """

        if size is None:
            if type(data) is not list or type(data[0]) is not list:
                raise ValueError('If size is not set, data should be a matrix')
            size = (len(data[0]), len(data))

        if isinstance(data, MatrixPointer):
            self.__data = data.get_data()
            self.__original_size = data.get_size()
            self.__size = size
            self.__offset = data.get_offset(offset)
        else:
            self.__data = self.__flatten(data)
            self.__original_size = size
            self.__size = size
            self.__offset = offset


    def __flatten(self, lst):
        """Convert lst to one-dimensional list.
        
        Works with non-lists, lists and matrices.
        """
        if type(lst) is not list:
            return [lst]
        elif type(lst[0]) is not list:
            return lst
        return sum(lst, [])


    def get_data(self, source=False):
        """Get list with data, current pointer points to.
        
        Keyword arguments:
        source -- whether you need the source data (True),
            or cropped fragment (False); default is False.
        """
        if source:
            return self.__data
        return [e for e in self.get_iterator()]


    def get_iterator(self):
        """Get generator of list with data, current pointer points to."""
        result = []
        for y in range(self.__offset[1], self.__offset[1]+self.__size[1]):
            for e in self.__data[self.__original_size[0]*y+self.__offset[0]:
                                 self.__original_size[0]*y+self.__offset[0]
                                 +self.__size[0]]:
                yield e


    def get_offset(self, initial_offset=(0,0), y=0):
        """Get current pointer offset and add new offset to it.
        
        Useful for the case, when you create new pointer
        on the basis of existing one.

        Keyword arguments:
        initial_offset -- tuple (x, y) with offset from current pointer's
            left upper corner either horizontal offset;
        y -- vertical offset, if `initial_offset` is a horizontal offset.
        """

        if type(initial_offset) is tuple:
            return (self.__offset[0]+initial_offset[0],
                    self.__offset[1]+initial_offset[1])
{'fraction_non_alphanumeric': 0.0917170280838962, 'fraction_numerical': 0.007820831852115179, 'mean_word_length': 1.805084745762712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '99432', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1429, 'n_words': 525}