<commit_before>import base
from blocks import Blocks
import re

class Sentences(base.Base):
    _type = 'sentences'
    child_class = Blocks

    @staticmethod
    def parse(text):
        #re.split('\!|\?|\. | \.',text)
        result = []
        prev = 0

        # we allow .09 as not end of sentences
        #for m in re.finditer('[\!\?]+|\.+(?:\s+|$|\?|\!)', text):
        for m in re.finditer('\.+(?:\s+|$)|(\.*)[\!\?]+(\.+(?:\s+|$))*', text):
            curr, _next = m.start(), m.end()
            # if prev position of delimiter < current - between exists text
            # at least 1 symbol.
            if prev < curr:
                node = text[prev:curr].strip()
                if node != '':
                    result.append(node)
            result.append(list( text[curr:_next].strip() ))
            prev = _next
        if len(text) > prev:
            result.append(text[prev:].strip())
        return result<commit_msg>Optimize from 5-6s to 2.9-3.0<commit_after>import base
from blocks import Blocks
import re

class Sentences(base.Base):
    _type = 'sentences'
    child_class = Blocks

    @staticmethod
    def parse(text):
        _len = len(text)
        result = []
        prev = 0

        # we allow .09 as not end of sentences
        for m in re.finditer('[\!\?\.]+', text):
            curr, _next = m.start(), m.end()
            items = list( text[curr: _next].strip() )

            if (_len > _next) and not (text[_next] == ' '):
                # delete ending '.' if they not before space or end of string
                while (len(items) > 0) and (items[-1] == '.'):
                    items.pop()
                    _next = _next - 1

            if len(items) > 0:
                # if prev position of delimiter < current - between exists text
                # at least 1 symbol.
                if prev < curr:
                    node = text[prev:curr].strip()
                    if node != '':
                        result.append(node)
                result.append( items )
                prev = _next
        if _len > prev:
            result.append(text[prev:].strip())
        return result
{'fraction_non_alphanumeric': 0.14510894761242465, 'fraction_numerical': 0.008344923504867872, 'mean_word_length': 1.4833141542002302, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '777220', 'n_tokens_mistral': 657, 'n_tokens_neox': 619, 'n_words': 181}