<commit_before># coding: utf8
"""
    weasyprint.formatting_structure.build
    -------------------------------------

    Turn an element tree with associated CSS style (computed values)
    into a "before layout" formatting structure / box tree.

    This includes creating anonymous boxes and processing whitespace
    as necessary.

    :copyright: Copyright 2011-2012 <NAME> and contributors, see AUTHORS.
    :license: BSD, see LICENSE for details.

"""

from __future__ import division, unicode_literals

import re

from tinycss.color3 import COLOR_KEYWORDS

from . import boxes, counters
from .. import html
from ..css import properties
from ..css.computed_values import ZERO_PIXELS
from ..compat import basestring, xrange


# Maps values of the ``display`` CSS property to box types.
BOX_TYPE_FROM_DISPLAY = {
    'block': boxes.BlockBox,
    'list-item': boxes.BlockBox,
    'inline': boxes.InlineBox,
    'inline-block': boxes.InlineBlockBox,
    'table': boxes.TableBox,
    'inline-table': boxes.InlineTableBox,
    'table-row': boxes.TableRowBox,
    'table-row-group': boxes.TableRowGroupBox,
    'table-header-group': boxes.TableRowGroupBox,
    'table-footer-group': boxes.TableRowGroupBox,
    'table-column': boxes.TableColumnBox,
    'table-column-group': boxes.TableColumnGroupBox,
    'table-cell': boxes.TableCellBox,
    'table-caption': boxes.TableCaptionBox,
}


def build_formatting_structure(element_tree, style_for, get_image_from_uri):
    """Build a formatting structure (box tree) from an element tree."""
    box, = element_to_box(element_tree, style_for, get_image_from_uri)
    box.is_for_root_element = True
    # If this is changed, maybe update weasy.layout.pages.make_margin_boxes()
    process_whitespace(box)
    box = anonymous_table_boxes(box)
    box = inline_in_block(box)
    box = block_in_inline(box)
    box = resolve_bookmark_labels(box)
    box = set_viewport_overflow(box)
    return box


def make_box(element_tag, sourceline, style, content, get_image_from_uri):
    if (style.display in ('table', 'inline-table')
            and style.border_collapse == 'collapse'):
        # Padding do not apply
        for side in ['top', 'bottom', 'left', 'right']:
            style['padding_' + side] = ZERO_PIXELS
    if style.display.startswith('table-') and style.display != 'table-caption':
        # Margins do not apply
        for side in ['top', 'bottom', 'left', 'right']:
            style['margin_' + side] = ZERO_PIXELS

    return BOX_TYPE_FROM_DISPLAY[style.display](element_tag, sourceline,
                                                style, content)

def element_to_box(element, style_for, get_image_from_uri, state=None):
    """Convert an element and its children into a box with children.

    Return a list of boxes. Most of the time the list will have one item but
    may have zero or more than one.

    Eg.::

        <p>Some <em>emphasised</em> text.</p>

    gives (not actual syntax)::

        BlockBox[
            TextBox['Some '],
            InlineBox[
                TextBox['emphasised'],
            ],
            TextBox[' text.'],
        ]

    ``TextBox``es are anonymous inline boxes:
    See http://www.w3.org/TR/CSS21/visuren.html#anonymous

    """
    if not isinstance(element.tag, basestring):
        # lxml.html already converts HTML entities to text.
        # Here we ignore comments and XML processing instructions.
        return []

    style = style_for(element)

    # TODO: should be the used value. When does the used value for `display`
    # differ from the computer value?
    display = style.display
    if display == 'none':
        return []

    box = make_box(element.tag, element.sourceline, style, [],
                   get_image_from_uri)

    if state is None:
        # use a list to have a shared mutable object
        state = (
            # Shared mutable objects:
            [0],  # quote_depth: single integer
            {},  # counter_values: name -> stacked/scoped values
            [set()]  # counter_scopes: element tree depths -> counter names
        )
    _quote_depth, counter_values, counter_scopes = state

    update_counters(state, style)

    children = []
    if display == 'list-item':
        children.extend(add_box_marker(
            box, counter_values, get_image_from_uri))

    # If this element’s direct children create new scopes, the counter
    # names will be in this new list
    counter_scopes.append(set())

    children.extend(pseudo_to_box(
        element, 'before', state, style_for, get_image_from_uri))
    text = element.text
    if text:
        children.append(boxes.TextBox.anonymous_from(box, text))
    for child_element in element:
        children.extend(element_to_box(
            child_element, style_for, get_image_from_uri, state))
        text = child_element.tail
        if text:
            children.append(boxes.TextBox.anonymous_from(box, text))
    children.extend(pseudo_to_box(
        element, 'after', state, style_for, get_image_from_uri))

    # Scopes created by this element’s children stop here.
    for name in counter_scopes.pop():
        counter_values[name].pop()<commit_msg>Collapse consecutive text boxes<commit_after># coding: utf8
"""
    weasyprint.formatting_structure.build
    -------------------------------------

    Turn an element tree with associated CSS style (computed values)
    into a "before layout" formatting structure / box tree.

    This includes creating anonymous boxes and processing whitespace
    as necessary.

    :copyright: Copyright 2011-2012 <NAME> and contributors, see AUTHORS.
    :license: BSD, see LICENSE for details.

"""

from __future__ import division, unicode_literals

import re

from tinycss.color3 import COLOR_KEYWORDS

from . import boxes, counters
from .. import html
from ..css import properties
from ..css.computed_values import ZERO_PIXELS
from ..compat import basestring, xrange


# Maps values of the ``display`` CSS property to box types.
BOX_TYPE_FROM_DISPLAY = {
    'block': boxes.BlockBox,
    'list-item': boxes.BlockBox,
    'inline': boxes.InlineBox,
    'inline-block': boxes.InlineBlockBox,
    'table': boxes.TableBox,
    'inline-table': boxes.InlineTableBox,
    'table-row': boxes.TableRowBox,
    'table-row-group': boxes.TableRowGroupBox,
    'table-header-group': boxes.TableRowGroupBox,
    'table-footer-group': boxes.TableRowGroupBox,
    'table-column': boxes.TableColumnBox,
    'table-column-group': boxes.TableColumnGroupBox,
    'table-cell': boxes.TableCellBox,
    'table-caption': boxes.TableCaptionBox,
}


def build_formatting_structure(element_tree, style_for, get_image_from_uri):
    """Build a formatting structure (box tree) from an element tree."""
    box, = element_to_box(element_tree, style_for, get_image_from_uri)
    box.is_for_root_element = True
    # If this is changed, maybe update weasy.layout.pages.make_margin_boxes()
    process_whitespace(box)
    box = anonymous_table_boxes(box)
    box = inline_in_block(box)
    box = block_in_inline(box)
    box = resolve_bookmark_labels(box)
    box = set_viewport_overflow(box)
    return box


def make_box(element_tag, sourceline, style, content, get_image_from_uri):
    if (style.display in ('table', 'inline-table')
            and style.border_collapse == 'collapse'):
        # Padding do not apply
        for side in ['top', 'bottom', 'left', 'right']:
            style['padding_' + side] = ZERO_PIXELS
    if style.display.startswith('table-') and style.display != 'table-caption':
        # Margins do not apply
        for side in ['top', 'bottom', 'left', 'right']:
            style['margin_' + side] = ZERO_PIXELS

    return BOX_TYPE_FROM_DISPLAY[style.display](element_tag, sourceline,
                                                style, content)

def element_to_box(element, style_for, get_image_from_uri, state=None):
    """Convert an element and its children into a box with children.

    Return a list of boxes. Most of the time the list will have one item but
    may have zero or more than one.

    Eg.::

        <p>Some <em>emphasised</em> text.</p>

    gives (not actual syntax)::

        BlockBox[
            TextBox['Some '],
            InlineBox[
                TextBox['emphasised'],
            ],
            TextBox[' text.'],
        ]

    ``TextBox``es are anonymous inline boxes:
    See http://www.w3.org/TR/CSS21/visuren.html#anonymous

    """
    if not isinstance(element.tag, basestring):
        # lxml.html already converts HTML entities to text.
        # Here we ignore comments and XML processing instructions.
        return []

    style = style_for(element)

    # TODO: should be the used value. When does the used value for `display`
    # differ from the computer value?
    display = style.display
    if display == 'none':
        return []

    box = make_box(element.tag, element.sourceline, style, [],
                   get_image_from_uri)

    if state is None:
        # use a list to have a shared mutable object
        state = (
            # Shared mutable objects:
            [0],  # quote_depth: single integer
            {},  # counter_values: name -> stacked/scoped values
            [set()]  # counter_scopes: element tree depths -> counter names
        )
    _quote_depth, counter_values, counter_scopes = state

    update_counters(state, style)

    children = []
    if display == 'list-item':
        children.extend(add_box_marker(
            box, counter_values, get_image_from_uri))

    # If this element’s direct children create new scopes, the counter
    # names will be in this new list
    counter_scopes.append(set())

    children.extend(pseudo_to_box(
        element, 'before', state, style_for, get_image_from_uri))
    text = element.text
    if text:
        children.append(boxes.TextBox.anonymous_from(box, text))
    for child_element in element:
        children.extend(element_to_box(
            child_element, style_for, get_image_from_uri, state))
        text = child_element.tail
        if text:
            text_box = boxes.TextBox.anonymous_from(box, text)
            if children and isinstance(children[-1], boxes.TextBox):
                children[-1].text += text_box.text
            else:
                children.append(text_box)
    children.extend(pseudo_to_box(
        element, 'after', state, style_for, get_image_from_uri))

    # Scopes created by this element’s children stop here.
    for name in counter_scopes.pop():
        counter_values[name].pop()
{'fraction_non_alphanumeric': 0.1051783004552352, 'fraction_numerical': 0.0028452200303490135, 'mean_word_length': 2.7728085867620753, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2388283', 'n_tokens_mistral': 3133, 'n_tokens_neox': 3052, 'n_words': 870}