<commit_before>"""texlib.wrap

Implements TeX's algorithm for breaking paragraphs into lines.

This module provides a straightforward implementation of the algorithm
used by TeX to format paragraphs.  The algorithm uses dynamic
programming to find a globally optimal division into lines, enabling
it to produce more attractive results than a first-fit or best-fit
algorithm can.  For a full description, see the reference.

The module provides the ObjectList class, which is a list of Box,
Glue, and Penalty instances.  The elements making up a paragraph of
text should be assembled into a single ObjectList.  Boxes represent
characters of type, and their only attribute is width.  Glue
represents a space of variable size; in addition to a preferred width,
glue can also stretch and shrink, to an amount that's specified by the
user.  Penalties are used to encourage or discourage breaking a line
at a given point.  Positive values discourage line breaks at a given
point, and a value of INFINITY forbids breaking the line at the
penalty.  Negative penalty values encourage line breaks at a given
point, and a value of -INFINITY forces a line break at a particular
point.

The compute_breakpoints() method of ObjectList returns a list of
integers containing the indexes at which the paragraph should be
broken.  If you're setting the text to be ragged-right (or
ragged-left, I suppose), then simply loop over the text and insert
breaks at the appropriate points.  For full justification, you'll have
to loop over each line's contents, calculate its adjustment ratio by
calling compute_adjustment_ratio(), and for each bit of glue, call its
compute_width() method to figure out how long this dab of glue should
be.

Reference:
    "Breaking Paragraphs into Lines", <NAME> and <NAME>,
    chapter 3 of _Digital Typography_, CSLI Lecture Notes #78.
"""

from __future__ import print_function

import sys, string
import UserList

__version__ = "1.01"

INFINITY = 1000

# Three classes defining the three different types of object that
# can go into an ObjectList.

class Box:
    """Class representing a glyph or character.  Boxes have a fixed
    width that doesn't change.
    """

    def __init__(self, width, character=None):
        self.character = character
        self.width = width
        self.stretch = self.shrink = 0
        self.penalty = 0
        self.flagged = 0

    def is_glue(self):         return 0
    def is_box(self):          return 1
    def is_penalty(self):      return 0
    def is_forced_break(self): return 0

class Glue:
    """Class representing a bit of glue.  Glue has a preferred width,
    but it can stretch up to an additional distance, and can shrink
    by a certain amount.  Line breaks can be placed at any point where
    glue immediately follows a box.
    """

    def __init__(self, width, stretch, shrink):
        self.width, self.stretch, self.shrink = width, stretch, shrink

    def compute_width(self, r):
        """Return how long this glue should be, for the given adjustment
        ratio r."""

        if r < 0: return self.width + r*self.shrink
        else:     return self.width + r*self.stretch

    def is_glue(self):         return 1
    def is_box(self):          return 0
    def is_penalty(self):      return 0
    def is_forced_break(self): return 0

class Penalty:
    """Class representing a penalty.  Negative penalty values
    encourage line breaks at a given point, and positive values
    discourage breaks.  A value of INFINITY either absolutely requires
    or forbids a break.  Penalties have a width of zero unless a break
    is taken at the penalty point, at which point the value of the
    penalty's 'width' attribute is used.

    """

    def __init__(self, width, penalty, flagged = 0):
        self.width = width
        self.penalty = penalty
        self.flagged = flagged
        self.stretch = self.shrink = 0

    def is_glue(self):         return 0
    def is_box(self):          return 0
    def is_penalty(self):      return 1
    def is_forced_break(self):
        return (self.penalty == -INFINITY)


class _BreakNode:
    "Internal class representing an active breakpoint."

    def __init__(self, position, line, fitness_class,
                 totalwidth, totalstretch, totalshrink,
                 demerits, previous = None):
        self.position, self.line = position, line
        self.fitness_class = fitness_class
        self.totalwidth, self.totalstretch = totalwidth, totalstretch
        self.totalshrink, self.demerits = totalshrink, demerits
        self.previous = previous

    def __repr__(self):
        return '<_BreakNode at %i>' % self.position

class ObjectList(UserList.UserList):

    """Class representing a list of Box, Glue, and Penalty objects.
    Supports the same methods as regular Python lists.
    """

    # Set this to 1 to trace the execution of the algorithm.
    debug = 0

    def add_closing_penalty (self):
        "Add the standard glue and penalty for the end of a paragraph"
        self.append( Penalty( 0, INFINITY, 0 ) )
        self.append( Glue(0, INFINITY, 0) )
        self.append( Penalty(0, -INFINITY, 1) )<commit_msg>Python 3 compatibility: just subclass list instead of UserList<commit_after>"""texlib.wrap

Implements TeX's algorithm for breaking paragraphs into lines.

This module provides a straightforward implementation of the algorithm
used by TeX to format paragraphs.  The algorithm uses dynamic
programming to find a globally optimal division into lines, enabling
it to produce more attractive results than a first-fit or best-fit
algorithm can.  For a full description, see the reference.

The module provides the ObjectList class, which is a list of Box,
Glue, and Penalty instances.  The elements making up a paragraph of
text should be assembled into a single ObjectList.  Boxes represent
characters of type, and their only attribute is width.  Glue
represents a space of variable size; in addition to a preferred width,
glue can also stretch and shrink, to an amount that's specified by the
user.  Penalties are used to encourage or discourage breaking a line
at a given point.  Positive values discourage line breaks at a given
point, and a value of INFINITY forbids breaking the line at the
penalty.  Negative penalty values encourage line breaks at a given
point, and a value of -INFINITY forces a line break at a particular
point.

The compute_breakpoints() method of ObjectList returns a list of
integers containing the indexes at which the paragraph should be
broken.  If you're setting the text to be ragged-right (or
ragged-left, I suppose), then simply loop over the text and insert
breaks at the appropriate points.  For full justification, you'll have
to loop over each line's contents, calculate its adjustment ratio by
calling compute_adjustment_ratio(), and for each bit of glue, call its
compute_width() method to figure out how long this dab of glue should
be.

Reference:
    "Breaking Paragraphs into Lines", <NAME> and <NAME>,
    chapter 3 of _Digital Typography_, CSLI Lecture Notes #78.
"""

from __future__ import print_function

import sys, string

__version__ = "1.01"

INFINITY = 1000

# Three classes defining the three different types of object that
# can go into an ObjectList.

class Box:
    """Class representing a glyph or character.  Boxes have a fixed
    width that doesn't change.
    """

    def __init__(self, width, character=None):
        self.character = character
        self.width = width
        self.stretch = self.shrink = 0
        self.penalty = 0
        self.flagged = 0

    def is_glue(self):         return 0
    def is_box(self):          return 1
    def is_penalty(self):      return 0
    def is_forced_break(self): return 0

class Glue:
    """Class representing a bit of glue.  Glue has a preferred width,
    but it can stretch up to an additional distance, and can shrink
    by a certain amount.  Line breaks can be placed at any point where
    glue immediately follows a box.
    """

    def __init__(self, width, stretch, shrink):
        self.width, self.stretch, self.shrink = width, stretch, shrink

    def compute_width(self, r):
        """Return how long this glue should be, for the given adjustment
        ratio r."""

        if r < 0: return self.width + r*self.shrink
        else:     return self.width + r*self.stretch

    def is_glue(self):         return 1
    def is_box(self):          return 0
    def is_penalty(self):      return 0
    def is_forced_break(self): return 0

class Penalty:
    """Class representing a penalty.  Negative penalty values
    encourage line breaks at a given point, and positive values
    discourage breaks.  A value of INFINITY either absolutely requires
    or forbids a break.  Penalties have a width of zero unless a break
    is taken at the penalty point, at which point the value of the
    penalty's 'width' attribute is used.

    """

    def __init__(self, width, penalty, flagged = 0):
        self.width = width
        self.penalty = penalty
        self.flagged = flagged
        self.stretch = self.shrink = 0

    def is_glue(self):         return 0
    def is_box(self):          return 0
    def is_penalty(self):      return 1
    def is_forced_break(self):
        return (self.penalty == -INFINITY)


class _BreakNode:
    "Internal class representing an active breakpoint."

    def __init__(self, position, line, fitness_class,
                 totalwidth, totalstretch, totalshrink,
                 demerits, previous = None):
        self.position, self.line = position, line
        self.fitness_class = fitness_class
        self.totalwidth, self.totalstretch = totalwidth, totalstretch
        self.totalshrink, self.demerits = totalshrink, demerits
        self.previous = previous

    def __repr__(self):
        return '<_BreakNode at %i>' % self.position

class ObjectList(list):

    """Class representing a list of Box, Glue, and Penalty objects.
    Supports the same methods as regular Python lists.
    """

    # Set this to 1 to trace the execution of the algorithm.
    debug = 0

    def add_closing_penalty (self):
        "Add the standard glue and penalty for the end of a paragraph"
        self.append( Penalty( 0, INFINITY, 0 ) )
        self.append( Glue(0, INFINITY, 0) )
        self.append( Penalty(0, -INFINITY, 1) )
{'fraction_non_alphanumeric': 0.06194519490544191, 'fraction_numerical': 0.00685063681976071, 'mean_word_length': 3.0456674473067915, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5674338', 'n_tokens_mistral': 2972, 'n_tokens_neox': 2770, 'n_words': 1308}