<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
Elements used for rendering (parts) of the canvas.

The :class:`Line` determines how a single line is rendered.
"""
from html import unescape

from inscriptis.model.css import HtmlElement


class Canvas:
    """
    The Canvas on which we write our HTML page.
    """

    def __init__(self):
        self.blocks = []
        self.indent = ''
        self.head = ''
        self.tail = ''

    def write_block(self, tag: HtmlElement, text: str):
        self.flush_inline()
        self.head = text

    def write_inline(self, tag: HtmlElement, text: str):
        self.tail += text

    def flush_inline(self):
        if self.head or self.tail:
            self.blocks.append(self.head + ' '.join(self.tail.split()))
            self.indent = ''
            self.head = ''
            self.tail = ''
<commit_msg>fix: improved whitespace handling; passes test_whitespace_handling<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
Elements used for rendering (parts) of the canvas.

The :class:`Line` determines how a single line is rendered.
"""
from collections import namedtuple
from enum import Enum
from html import unescape

from inscriptis.html_properties import WhiteSpace
from inscriptis.model.css import HtmlElement

TextSnippet = namedtuple("TextSnippet", "text whitespace")


class Canvas:
    """
    The Canvas on which we write our HTML page.
    """

    def __init__(self):
        """
        Contains the completed blocks. Each block spawns at least a line
        """
        self.blocks = []
        self.current_block = []

    def write_block(self, tag: HtmlElement, text: str):
        self.flush_inline()
        self.current_block.append(TextSnippet(text, whitespace=0))

    def write_inline(self, tag: HtmlElement, text: str):
        if tag.whitespace == WhiteSpace.pre:
            self.current_block.append(TextSnippet(text, whitespace=0))
        else:
            space_handling = 1 + 2 * text[0].isspace() + 4 * (text[-1].isspace() and len(text) > 1)
            self.current_block.append(TextSnippet(text, whitespace=space_handling))

    @staticmethod
    def normalize_text(text_snippet):
        return ''.join((' ' if (text_snippet.whitespace & 2) else '',
                        ' '.join(text_snippet.text.split()),
                        ' ' if (text_snippet.whitespace & 4) else ''))

    def flush_inline(self):
        if self.current_block:
            print(self.current_block)
            block = ''.join((self.normalize_text(b) if b.whitespace else b.text
                             for b in self.current_block))
            self.blocks.append(block)
            self.current_block = []

{'fraction_non_alphanumeric': 0.11414850387883266, 'fraction_numerical': 0.004432951606944958, 'mean_word_length': 2.471794871794872, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7125474', 'n_tokens_mistral': 817, 'n_tokens_neox': 790, 'n_words': 213}