<commit_before>"""
SDoc

Copyright 2016 Set Based IT Consultancy

Licence MIT
"""
# ----------------------------------------------------------------------------------------------------------------------
import html
import re
import sdoc
from sdoc.sdoc2 import node_store
from sdoc.sdoc2.node.Node import Node
from sdoc.sdoc2.node.EndParagraphNode import EndParagraphNode


class TextNode(Node):
    """
    SDoc2 node for items.
    """

    # ------------------------------------------------------------------------------------------------------------------
    def __init__(self, options, argument):
        """
        Object constructor.

        :param dict[str,str] options: Not used.
        :param str argument: The actual text.
        """
        super().__init__('TEXT', options, argument)

    # ------------------------------------------------------------------------------------------------------------------
    def print_info(self, level):
        """
        Temp function for development.

        :param int level: the level of block commands.
        """
        print("%s%4d %s %s" % (' ' * 4 * level, self.id, self.name, ''))

    # ------------------------------------------------------------------------------------------------------------------
    def generate_html(self, file):
        """
        Generates the HTML code for this node.

        :param file file: The output stream to with the generated HTML will be written.
        """
        file.write(html.escape(self._argument))

        super().generate_html(file)

    # ------------------------------------------------------------------------------------------------------------------
    def is_block_command(self):
        """
        Returns False.

        :rtype: bool
        """
        return False

    # ------------------------------------------------------------------------------------------------------------------
    def is_inline_command(self):
        """
        Returns False.

        :rtype: bool
        """
        return False

    # ------------------------------------------------------------------------------------------------------------------
    def is_phrasing(self):
        """
        Returns True.

        :rtype: bool
        """
        return True

    # ------------------------------------------------------------------------------------------------------------------
    def split_by_paragraph(self):
        """
        Splits this text node into text nodes without a paragraph separator (i.e. a double new line) in to a list of
        text nodes without paragraph separator each paragraph separator is replace with a end paragraph node. Each
        paragraph separator is replaced wth a end paragraph node.

        Returns a list of node IDs.

        :rtype: list[int]
        """
        text_ids = []
        list_of_texts = re.split("\n\n", self._argument)

        #print(list_of_texts)

        # Cleaning the text parts.
        if "\n" in list_of_texts:
            list_of_texts[list_of_texts.index("\n")] = ' '
        if list_of_texts[0] == '':
            list_of_texts.remove('')

        # Checking the range.
        if list_of_texts:
            if not list_of_texts[-1]:
                to = len(list_of_texts) - 1
            else:<commit_msg>Removed bogus statement.<commit_after>"""
SDoc

Copyright 2016 Set Based IT Consultancy

Licence MIT
"""
# ----------------------------------------------------------------------------------------------------------------------
import html
import re
import sdoc
from sdoc.sdoc2 import node_store
from sdoc.sdoc2.node.Node import Node
from sdoc.sdoc2.node.EndParagraphNode import EndParagraphNode


class TextNode(Node):
    """
    SDoc2 node for items.
    """

    # ------------------------------------------------------------------------------------------------------------------
    def __init__(self, options, argument):
        """
        Object constructor.

        :param dict[str,str] options: Not used.
        :param str argument: The actual text.
        """
        super().__init__('TEXT', options, argument)

    # ------------------------------------------------------------------------------------------------------------------
    def print_info(self, level):
        """
        Temp function for development.

        :param int level: the level of block commands.
        """
        print("%s%4d %s %s" % (' ' * 4 * level, self.id, self.name, ''))

    # ------------------------------------------------------------------------------------------------------------------
    def generate_html(self, file):
        """
        Generates the HTML code for this node.

        :param file file: The output stream to with the generated HTML will be written.
        """
        file.write(html.escape(self._argument))

        super().generate_html(file)

    # ------------------------------------------------------------------------------------------------------------------
    def is_block_command(self):
        """
        Returns False.

        :rtype: bool
        """
        return False

    # ------------------------------------------------------------------------------------------------------------------
    def is_inline_command(self):
        """
        Returns False.

        :rtype: bool
        """
        return False

    # ------------------------------------------------------------------------------------------------------------------
    def is_phrasing(self):
        """
        Returns True.

        :rtype: bool
        """
        return True

    # ------------------------------------------------------------------------------------------------------------------
    def split_by_paragraph(self):
        """
        Splits this text node into text nodes without a paragraph separator (i.e. a double new line) in to a list of
        text nodes without paragraph separator each paragraph separator is replace with a end paragraph node. Each
        paragraph separator is replaced wth a end paragraph node.

        Returns a list of node IDs.

        :rtype: list[int]
        """
        text_ids = []
        list_of_texts = re.split("\n\n", self._argument)

        # Cleaning the text parts.
        if "\n" in list_of_texts:
            list_of_texts[list_of_texts.index("\n")] = ' '
        if list_of_texts[0] == '':
            list_of_texts.remove('')

        # Checking the range.
        if list_of_texts:
            if not list_of_texts[-1]:
                to = len(list_of_texts) - 1
            else:
{'fraction_non_alphanumeric': 0.3510800121691512, 'fraction_numerical': 0.003954974140553696, 'mean_word_length': 0.8817973669147109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5295781', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1390, 'n_words': 422}