capitalize the first letter of a string if not x return x if not isinstancex str x strx return x0 upper x1 set up regular expressions rewords lazyrecompiler s re s rechars lazyrecompiler re s retag lazyrecompiler s ss re s renewlines lazyrecompilerrnr used in normalizenewlines recamelcase lazyrecompiler azazaz az keeplazytext def wraptext width def generator for line in text splitlinestrue true keeps trailing linebreaks maxwidth minline endswithn and width 1 or width width while lenline maxwidth space line maxwidth 1 rfind 1 if space 0 space line find 1 if space 0 yield line line break yield sn line space 1 line linespace maxwidth minline endswithn and width 1 or width width if line yield line return joingenerator def addtruncationtexttext truncatenone if truncate is none truncate pgettext string to return when truncating text truncatedtexts if truncatedtexts in truncate return truncate truncatedtext text the truncation text didn t contain the truncatedtexts string replacement argument so just append it to the text if text endswithtruncate but don t append the truncation text if the current text already ends in this return text return ftexttruncate class truncatorsimplelazyobject 5 million characters are approximately 4000 text pages or 3 web pages maxlengthhtml 5000000 def initself text super initlambda strtext def charsself num truncatenone htmlfalse self setup length intnum text unicodedata normalizenfc self wrapped calculate the length to truncate to max length endtext length truncatelen length for char in addtruncationtext truncate if not unicodedata combiningchar truncatelen 1 if truncatelen 0 break if html return self truncatehtmllength truncate text truncatelen false return self textcharslength truncate text truncatelen def textcharsself length truncate text truncatelen don t consider combining characters as adding to the string length return the truncated string return the original string since no truncation was necessary truncate a string after a certain number of words truncate specifies what should be used to notify that the string has been truncated defaulting to ellipsis truncate a string after a certain number of words strip newlines in the string truncate html to a certain number of chars not counting tags and comments or if words is true then to a certain number of words close opened tags if they were correctly closed in the given html preserve newlines in the html count nonhtml charswords and keep note of open tags checked through whole string it s an actual nonhtml word or char check for tag don t worry about non tags or tags after our truncate point element names are always caseinsensitive check for match in open tags list sgml an end tag closes back to the matching start tag all unclosed intervening start tags with omitted end tags add it to the start of the open tags list close any tags still open return string return the given string converted to a string that can be used for a clean filename remove leading and trailing spaces convert other spaces to underscores and remove anything that is not an alphanumeric dash underscore or dot getvalidfilenamejohn s portrait in 2004 jpg johnsportraitin2004 jpg gettextlist a b c d a b c or d gettextlist a b c and a b and c gettextlist a b and a and b gettextlist a a gettextlist translators this string is used as a separator between list elements normalize crlf and cr newlines to just lf return renewlines subn strtext keeplazytext def phone2numericphone like compressstring but for iterators of strings output headers expression to match sometoken and sometokenwith spaces and similarly for singlequoted strings re verbose def smartsplittext r generator that splits a string by spaces leaving quoted phrases together supports both single and double quotes and supports escaping quotes with backslashes in the output strings will keep their initial and trailing quote marks and escaped quotes will remain escaped the results can then be further processed with unescapestringliteral listsmartsplitr this is a person s test this is a person s test listsmartsplitranother person s test another person s test listsmartsplitr a funky style test a funky style test if not s or s0 not in or s1 s0 raise valueerrornot a string literal r s quote s0 return s1 1 replacers quote quote replacer keeplazytext def slugifyvalue allowunicodefalse value strvalue if allowunicode value unicodedata normalizenfkc value else value unicodedata normalizenfkd value encodeascii ignore decodeascii value re subrws value lower return re subrs value strip def camelcasetospacesvalue return recamelcase subr 1 value strip lower def formatlazyformatstring args kwargs return formatstring formatargs kwargs formatlazy lazyformatlazy str capitalize the first letter of a string set up regular expressions used in normalize_newlines a word wrap function that preserves existing line breaks expects that existing line breaks are posix newlines preserve all white space except added line breaks consume the space on which they break the line don t wrap long words thus the output text may have lines longer than width true keeps trailing linebreaks the truncation text didn t contain the truncated_text s string replacement argument so just append it to the text but don t append the truncation text if the current text already ends in this an object used to truncate text either by characters or words when truncating html text either chars or words input will be limited to at most max_length_html characters 5 million characters are approximately 4000 text pages or 3 web pages return the text truncated to be no longer than the specified number of characters truncate specifies what should be used to notify that the string has been truncated defaulting to a translatable string of an ellipsis calculate the length to truncate to max length end_text length truncate a string after a certain number of chars don t consider combining characters as adding to the string length return the truncated string return the original string since no truncation was necessary truncate a string after a certain number of words truncate specifies what should be used to notify that the string has been truncated defaulting to ellipsis truncate a string after a certain number of words strip newlines in the string truncate html to a certain number of chars not counting tags and comments or if words is true then to a certain number of words close opened tags if they were correctly closed in the given html preserve newlines in the html count non html chars words and keep note of open tags checked through whole string it s an actual non html word or char check for tag don t worry about non tags or tags after our truncate point element names are always case insensitive check for match in open tags list sgml an end tag closes back to the matching start tag all unclosed intervening start tags with omitted end tags add it to the start of the open tags list close any tags still open return string return the given string converted to a string that can be used for a clean filename remove leading and trailing spaces convert other spaces to underscores and remove anything that is not an alphanumeric dash underscore or dot get_valid_filename john s portrait in 2004 jpg johns_portrait_in_2004 jpg get_text_list a b c d a b c or d get_text_list a b c and a b and c get_text_list a b and a and b get_text_list a a get_text_list translators this string is used as a separator between list elements normalize crlf and cr newlines to just lf convert a phone number with letters into its numeric equivalent like compress_string but for iterators of strings output headers expression to match some_token and some_token with spaces and similarly for single quoted strings s s s generator that splits a string by spaces leaving quoted phrases together supports both single and double quotes and supports escaping quotes with backslashes in the output strings will keep their initial and trailing quote marks and escaped quotes will remain escaped the results can then be further processed with unescape_string_literal list smart_split r this is a person s test this is a person s test list smart_split r another person s test another person s test list smart_split r a funky style test a funky style test convert quoted string literals to unquoted strings with escaped quotes and backslashes unquoted unescape_string_literal abc abc unescape_string_literal abc abc unescape_string_literal a bc a bc unescape_string_literal ab c ab c convert to ascii if allow_unicode is false convert spaces or repeated dashes to single dashes remove characters that aren t alphanumerics underscores or hyphens convert to lowercase also strip leading and trailing whitespace dashes and underscores split camelcase and convert to lowercase strip surrounding whitespace apply str format on format_string where format_string args and or kwargs might be lazy
import gzip
import re
import secrets
import unicodedata
from gzip import GzipFile
from gzip import compress as gzip_compress
from io import BytesIO

from django.core.exceptions import SuspiciousFileOperation
from django.utils.functional import SimpleLazyObject, keep_lazy_text, lazy
from django.utils.regex_helper import _lazy_re_compile
from django.utils.translation import gettext as _
from django.utils.translation import gettext_lazy, pgettext


@keep_lazy_text
def capfirst(x):
    
    if not x:
        return x
    if not isinstance(x, str):
        x = str(x)
    return x[0].upper() + x[1:]



re_words = _lazy_re_compile(r"<[^>]+?>|([^<>\s]+)", re.S)
re_chars = _lazy_re_compile(r"<[^>]+?>|(.)", re.S)
re_tag = _lazy_re_compile(r"<(/)?(\S+?)(?:(\s*/)|\s.*?)?>", re.S)
re_newlines = _lazy_re_compile(r"\r\n|\r")  
re_camel_case = _lazy_re_compile(r"(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))")


@keep_lazy_text
def wrap(text, width):
    

    def _generator():
        for line in text.splitlines(True):  
            max_width = min((line.endswith("\n") and width + 1 or width), width)
            while len(line) > max_width:
                space = line[: max_width + 1].rfind(" ") + 1
                if space == 0:
                    space = line.find(" ") + 1
                    if space == 0:
                        yield line
                        line = ""
                        break
                yield "%s\n" % line[: space - 1]
                line = line[space:]
                max_width = min((line.endswith("\n") and width + 1 or width), width)
            if line:
                yield line

    return "".join(_generator())


def add_truncation_text(text, truncate=None):
    if truncate is None:
        truncate = pgettext(
            "String to return when truncating text", "%(truncated_text)s…"
        )
    if "%(truncated_text)s" in truncate:
        return truncate % {"truncated_text": text}
    
    
    if text.endswith(truncate):
        
        
        return text
    return f"{text}{truncate}"


class Truncator(SimpleLazyObject):
    

    
    MAX_LENGTH_HTML = 5_000_000

    def __init__(self, text):
        super().__init__(lambda: str(text))

    def chars(self, num, truncate=None, html=False):
        
        self._setup()
        length = int(num)
        text = unicodedata.normalize("NFC", self._wrapped)

        
        truncate_len = length
        for char in add_truncation_text("", truncate):
            if not unicodedata.combining(char):
                truncate_len -= 1
                if truncate_len == 0:
                    break
        if html:
            return self._truncate_html(length, truncate, text, truncate_len, False)
        return self._text_chars(length, truncate, text, truncate_len)

    def _text_chars(self, length, truncate, text, truncate_len):
        
        s_len = 0
        end_index = None
        for i, char in enumerate(text):
            if unicodedata.combining(char):
                
                
                continue
            s_len += 1
            if end_index is None and s_len > truncate_len:
                end_index = i
            if s_len > length:
                
                return add_truncation_text(text[: end_index or 0], truncate)

        
        return text

    def words(self, num, truncate=None, html=False):
        
        self._setup()
        length = int(num)
        if html:
            return self._truncate_html(length, truncate, self._wrapped, length, True)
        return self._text_words(length, truncate)

    def _text_words(self, length, truncate):
        
        words = self._wrapped.split()
        if len(words) > length:
            words = words[:length]
            return add_truncation_text(" ".join(words), truncate)
        return " ".join(words)

    def _truncate_html(self, length, truncate, text, truncate_len, words):
        
        if words and length <= 0:
            return ""

        size_limited = False
        if len(text) > self.MAX_LENGTH_HTML:
            text = text[: self.MAX_LENGTH_HTML]
            size_limited = True

        html4_singlets = (
            "br",
            "col",
            "link",
            "base",
            "img",
            "param",
            "area",
            "hr",
            "input",
        )

        
        pos = 0
        end_text_pos = 0
        current_len = 0
        open_tags = []

        regex = re_words if words else re_chars

        while current_len <= length:
            m = regex.search(text, pos)
            if not m:
                
                break
            pos = m.end(0)
            if m[1]:
                
                current_len += 1
                if current_len == truncate_len:
                    end_text_pos = pos
                continue
            
            tag = re_tag.match(m[0])
            if not tag or current_len >= truncate_len:
                
                continue
            closing_tag, tagname, self_closing = tag.groups()
            
            tagname = tagname.lower()
            if self_closing or tagname in html4_singlets:
                pass
            elif closing_tag:
                
                try:
                    i = open_tags.index(tagname)
                except ValueError:
                    pass
                else:
                    
                    
                    open_tags = open_tags[i + 1 :]
            else:
                
                open_tags.insert(0, tagname)

        truncate_text = add_truncation_text("", truncate)

        if current_len <= length:
            if size_limited and truncate_text:
                text += truncate_text
            return text

        out = text[:end_text_pos]
        if truncate_text:
            out += truncate_text
        
        for tag in open_tags:
            out += "</%s>" % tag
        
        return out


@keep_lazy_text
def get_valid_filename(name):
    
    s = str(name).strip().replace(" ", "_")
    s = re.sub(r"(?u)[^-\w.]", "", s)
    if s in {"", ".", ".."}:
        raise SuspiciousFileOperation("Could not derive file name from '%s'" % name)
    return s


@keep_lazy_text
def get_text_list(list_, last_word=gettext_lazy("or")):
    
    if not list_:
        return ""
    if len(list_) == 1:
        return str(list_[0])
    return "%s %s %s" % (
        
        _(", ").join(str(i) for i in list_[:-1]),
        str(last_word),
        str(list_[-1]),
    )


@keep_lazy_text
def normalize_newlines(text):
    
    return re_newlines.sub("\n", str(text))


@keep_lazy_text
def phone2numeric(phone):
    
    char2number = {
        "a": "2",
        "b": "2",
        "c": "2",
        "d": "3",
        "e": "3",
        "f": "3",
        "g": "4",
        "h": "4",
        "i": "4",
        "j": "5",
        "k": "5",
        "l": "5",
        "m": "6",
        "n": "6",
        "o": "6",
        "p": "7",
        "q": "7",
        "r": "7",
        "s": "7",
        "t": "8",
        "u": "8",
        "v": "8",
        "w": "9",
        "x": "9",
        "y": "9",
        "z": "9",
    }
    return "".join(char2number.get(c, c) for c in phone.lower())


def _get_random_filename(max_random_bytes):
    return b"a" * secrets.randbelow(max_random_bytes)


def compress_string(s, *, max_random_bytes=None):
    compressed_data = gzip_compress(s, compresslevel=6, mtime=0)

    if not max_random_bytes:
        return compressed_data

    compressed_view = memoryview(compressed_data)
    header = bytearray(compressed_view[:10])
    header[3] = gzip.FNAME

    filename = _get_random_filename(max_random_bytes) + b"\x00"

    return bytes(header) + filename + compressed_view[10:]


class StreamingBuffer(BytesIO):
    def read(self):
        ret = self.getvalue()
        self.seek(0)
        self.truncate()
        return ret



def compress_sequence(sequence, *, max_random_bytes=None):
    buf = StreamingBuffer()
    filename = _get_random_filename(max_random_bytes) if max_random_bytes else None
    with GzipFile(
        filename=filename, mode="wb", compresslevel=6, fileobj=buf, mtime=0
    ) as zfile:
        
        yield buf.read()
        for item in sequence:
            zfile.write(item)
            data = buf.read()
            if data:
                yield data
    yield buf.read()




smart_split_re = _lazy_re_compile(
    r,
    re.VERBOSE,
)


def smart_split(text):
    r
    for bit in smart_split_re.finditer(str(text)):
        yield bit[0]


@keep_lazy_text
def unescape_string_literal(s):
    r
    if not s or s[0] not in "\"'" or s[-1] != s[0]:
        raise ValueError("Not a string literal: %r" % s)
    quote = s[0]
    return s[1:-1].replace(r"\%s" % quote, quote).replace(r"\\", "\\")


@keep_lazy_text
def slugify(value, allow_unicode=False):
    
    value = str(value)
    if allow_unicode:
        value = unicodedata.normalize("NFKC", value)
    else:
        value = (
            unicodedata.normalize("NFKD", value)
            .encode("ascii", "ignore")
            .decode("ascii")
        )
    value = re.sub(r"[^\w\s-]", "", value.lower())
    return re.sub(r"[-\s]+", "-", value).strip("-_")


def camel_case_to_spaces(value):
    
    return re_camel_case.sub(r" \1", value).strip().lower()


def _format_lazy(format_string, *args, **kwargs):
    
    return format_string.format(*args, **kwargs)


format_lazy = lazy(_format_lazy, str)