<commit_before>"""
parse data by line (naive and reckless)
"""

__author__ = "siznax"
__verion__ = "2 Oct 2015"


class WPLineParser:

    MAX_ELEM_BYTES = 1024**2
    _found_end = False
    _found_start = False
    byte_count = 0
    elem = ""
    elems_found = 0
    elems_processed = 0

    def __init__(self, start="<page>", end="</page>"):
        self.start = start
        self.end = end

    def process(self, elem):
        """override this method to process elements"""
        pass

    def parse(self, chunk):
        """scan uncompressed chunk for start/end"""
        if len(self.elem) > self.MAX_ELEM_BYTES:
            print self.elem[:1024]
            raise RuntimeError("elem grew too big!")
        if type(chunk) is list:
            for line in chunk:
                self._scan(line)
                self.byte_count += len(line)
        else:
            self._scan(chunk)
            self.byte_count += len(chunk)

    def _scan(self, line):
        if self._found_start:
            self.elem += line
            self._find_end(line)
        else:
            self._find_start(line)<commit_msg>increase MAX_ELEM_BYTES to 10 MiB in wp_parser<commit_after>"""
parse data by line (naive and reckless)
"""

__author__ = "siznax"
__verion__ = "2 Oct 2015"


class WPLineParser:

    MAX_ELEM_BYTES = 1024**2 * 10  # 10 MiB
    _found_end = False
    _found_start = False
    byte_count = 0
    elem = ""
    elems_found = 0
    elems_processed = 0

    def __init__(self, start="<page>", end="</page>"):
        self.start = start
        self.end = end

    def process(self, elem):
        """override this method to process elements"""
        pass

    def parse(self, chunk):
        """scan uncompressed chunk for start/end"""
        if len(self.elem) > self.MAX_ELEM_BYTES:
            msg = "WPLineParser elem grew too big!"
            print(msg)
            raise RuntimeError(msg)
        if type(chunk) is list:
            for line in chunk:
                self._scan(line)
                self.byte_count += len(line)
        else:
            self._scan(chunk)
            self.byte_count += len(chunk)

    def _scan(self, line):
        if self._found_start:
            self.elem += line
            self._find_end(line)
        else:
            self._find_start(line)
{'fraction_non_alphanumeric': 0.10610653962754439, 'fraction_numerical': 0.015591165006496318, 'mean_word_length': 1.9277566539923954, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6045347', 'n_tokens_mistral': 766, 'n_tokens_neox': 705, 'n_words': 190}