<commit_before># coding: utf8

from io import StringIO
from collections import deque


class StreamEntity:
    def __init__(self, args, kwargs, stream_class=None):
        self.streamClass = stream_class
        self.args = args
        self.kwargs = kwargs


class StreamReader(StreamEntity):
    def __init__(self, *args, stream_class=StringIO, **kwargs):
        super(StreamReader, self).__init__(args, kwargs, stream_class=stream_class)

    def read(self, parsing_pipeline):
        parsing_pipeline.reset()
        min_position = parsing_pipeline.get_min_position()
        max_position = parsing_pipeline.get_max_position()
        length = max_position - min_position + 1
        stream = self.streamClass(*self.args, **self.kwargs)
        current_position = -min_position
        ar_index = list()
        element = deque(stream.read(length))
        if len(element) == length:
            while True:
                result = parsing_pipeline.check(element, ref_position=-min_position)
                if result is not None and result[0]:
                    ar_index.append((current_position, element[-min_position]))
                next_character = stream.read(1)
                if next_character and result is not None:
                    element.popleft()
                    element.append(next_character)
                else:
                    break
                current_position += 1

            stream.close()
            return ParsingResult(self.streamClass, self.args, self.kwargs, -min_position, current_position, ar_index)
        else:
            stream.close()
            raise ValueError("Not enough characters to parse : " + str(len(element)))


class StreamWriter(StreamEntity):
    def __init__(self, *args, stream_class=None, **kwargs):
        super(StreamReader, self).__init__(args, kwargs, stream_class=stream_class)

    def write(self, parsing_result):
        pass


class ParsingResult:
    def __init__(self, parsing_class, args, kwargs, initial_character_index, final_character_index, ar_index):
        self.parsingClass = parsing_class
        self.arInput = {'args': args, 'kwargs': kwargs}
        self.initialCharacterIndex = initial_character_index
        self.finalCharacterIndex = final_character_index
        self.arIndex = ar_index

    def __str__(self):
        return str({'Stream class': str(self.parsingClass.__name__), 'Inputs': str(self.arInput),
                    'from': str(self.initialCharacterIndex), 'to': str(self.finalCharacterIndex),
                    'Index result': str(self.arIndex)})

    def __repr__(self):
        return 'Parsing result :' + '\n' + '   Stream class : ' + str(self.parsingClass.__name__) + '\n' \
               + '   Inputs : ' + str(self.arInput) + '\n' + '   Parsed from character ' \
               + str(self.initialCharacterIndex) + ' to character ' + str(self.finalCharacterIndex) + '\n' \
               + '   Index result : ' + str(self.arIndex)

    def check_indexes(self):
        previous_index_value = -1
        for index in self.arIndex:
            if index[0] > previous_index_value:
                previous_index_value = index[0]
            else:
                raise ValueError('Indexes have to be sorted in ascending order')
            if len(index[1]) != 1 or not isinstance(index[1], str):
                raise TypeError("Indexes's characters have to be 'str' objects with a length of 1")

        return True
<commit_msg>Implement StreamWriter class<commit_after># coding: utf8

from io import StringIO
from collections import deque
import abc
import copy


class StreamEntity(metaclass=abc.ABCMeta):
    @abc.abstractmethod
    def __init__(self, args, kwargs, stream_class=None):
        self.streamClass = stream_class
        self.args = args
        self.kwargs = kwargs


class StreamReader(StreamEntity):
    def __init__(self, *args, stream_class=StringIO, **kwargs):
        super(StreamReader, self).__init__(args, kwargs, stream_class=stream_class)

    def read(self, parsing_pipeline):
        parsing_pipeline.reset()
        min_position = parsing_pipeline.get_min_position()
        max_position = parsing_pipeline.get_max_position()
        length = max_position - min_position + 1
        stream = self.streamClass(*self.args, **self.kwargs)
        current_position = -min_position
        ar_index = list()
        element = deque(stream.read(length))
        if len(element) == length:
            while True:
                result = parsing_pipeline.check(element, ref_position=-min_position)
                if result is not None and result[0]:
                    ar_index.append((current_position, element[-min_position]))
                next_character = stream.read(1)
                if next_character and result is not None:
                    element.popleft()
                    element.append(next_character)
                else:
                    break
                current_position += 1

            stream.close()
            return ParsingResult(self.streamClass, self.args, self.kwargs, -min_position, current_position, ar_index)
        else:
            stream.close()
            raise ValueError("Not enough characters to parse : " + str(len(element)))


class StreamWriter(StreamEntity):
    def __init__(self, *args, stream_class=None, **kwargs):
        super(StreamReader, self).__init__(args, kwargs, stream_class=stream_class)

    def write(self, parsing_result, parsing_class=None, args=None, kwargs=None):
        input_parsing_result = copy.deepcopy(parsing_result)
        if parsing_class is not None:
            input_parsing_result.parsing_class = parsing_class
        if args is not None:
            input_parsing_result.arInput['args'] = args
        if kwargs is not None:
            input_parsing_result.arInput['kwargs'] = kwargs

        input_stream = input_parsing_result.streamClass(*input_parsing_result.args, **input_parsing_result.kwargs)

        #TO DO

        return ParsingResult(self.streamClass, self.args, self.kwargs, 0, 0, {})


class ParsingResult:
    def __init__(self, parsing_class, args, kwargs, initial_character_index, final_character_index, ar_index):
        self.parsingClass = parsing_class
        self.arInput = {'args': args, 'kwargs': kwargs}
        self.initialCharacterIndex = initial_character_index
        self.finalCharacterIndex = final_character_index
        self.arIndex = ar_index

    def __str__(self):
        return str({'Stream class': str(self.parsingClass.__name__), 'Inputs': str(self.arInput),
                    'from': str(self.initialCharacterIndex), 'to': str(self.finalCharacterIndex),
                    'Index result': str(self.arIndex)})

    def __repr__(self):
        return 'Parsing result :' + '\n' + '   Stream class : ' + str(self.parsingClass.__name__) + '\n' \
               + '   Inputs : ' + str(self.arInput) + '\n' + '   Parsed from character ' \
               + str(self.initialCharacterIndex) + ' to character ' + str(self.finalCharacterIndex) + '\n' \
               + '   Index result : ' + str(self.arIndex)

    def __copy__(self):
        return ParsingResult(self.parsingClass, self.arInput['args'], self.arInput['kwargs'],
                             self.initialCharacterIndex, self.finalCharacterIndex, self.arIndex)

    def __deepcopy__(self):
        copy_ar_input = copy.deepcopy(self.arInput)
        copy_ar_index = copy.deepcopy(self.arIndex)
        return ParsingResult(self.parsingClass, copy_ar_input['args'], copy_ar_input['kwargs'],
                             self.initialCharacterIndex, self.finalCharacterIndex, copy_ar_index)

    def check_indexes(self):
        previous_index_value = -1
        for index in self.arIndex:
            if index[0] > previous_index_value:
                previous_index_value = index[0]
            else:
                raise ValueError('Indexes have to be sorted in ascending order')
            if len(index[1]) != 1 or not isinstance(index[1], str):
                raise TypeError("Indexes's characters have to be 'str' objects with a length of 1")

        return True
{'fraction_non_alphanumeric': 0.09879429133858268, 'fraction_numerical': 0.0031988188976377952, 'mean_word_length': 2.514483354950281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6559594', 'n_tokens_mistral': 2280, 'n_tokens_neox': 2223, 'n_words': 519}