<commit_before>import unicode, algorithm, strutils, sequtils
import nimx.font, nimx.types, nimx.unistring, nimx.utils.lower_bound


type
    FormattedText* = ref object
        mText: string
        mAttributes: seq[Attributes]
        lines: seq[LineInfo]
        mTotalHeight: float32
        horizontalAlignment*: HorizontalTextAlignment
        verticalAlignment*: VerticalAlignment
        mBoundingSize: Size
        cacheValid: bool
        canBreakOnAnyChar: bool
        overrideColor*: Color
        shadowAttrs: seq[int]
        strokeAttrs: seq[int]
        shadowMultiplier*: Size
        mLineSpacing: float32

    LineInfo* = object
        startByte: int
        startRune: int
        width: float32
        height: float32
        top: float32
        baseline: float32 # distance from top of the line to baseline
        firstAttr: int

    Attributes = object
        startByte: int # Index of char where attributes start.
        startRune: int # Index of rune where attributes start.
        font: Font
        textColor: Color

        # Richer attributes. TODO: Move to separate ref object
        shadowColor: Color
        strokeColor1: Color
        strokeColor2: Color
        textColor2: Color
        backgroundColor: Color
        shadowOffset: Size
        shadowRadius: float32
        shadowSpread: float32
        strokeSize: float32
        tracking: float32
        isTextGradient: bool
        isStrokeGradient: bool

    VerticalAlignment* = enum
        vaTop
        vaCenter
        vaBottom

    HorizontalTextAlignment* = enum
        haLeft
        haRight
        haCenter
        haJustify

proc defaultAttributes(): Attributes =
    result.font = systemFont()
    result.textColor = blackColor()

proc `text=`*(t: FormattedText, s: string) =
    if s.isNil:
        t.mText = ""
    else:
        t.mText = s
    t.cacheValid = false

template text*(t: FormattedText): string = t.mText

proc `lineSpacing=`*(t: FormattedText, s: float32) =
    t.mLineSpacing = s
    t.cacheValid = false

template lineSpacing*(t: FormattedText): float32 = t.mLineSpacing

proc newFormattedText*(s: string = ""): FormattedText =
    result.new()
    result.mText = s
    result.mAttributes = @[defaultAttributes()]
    result.lines = @[]
    result.shadowAttrs = @[]
    result.strokeAttrs = @[]
    result.shadowMultiplier = newSize(1, 1)
    result.mLineSpacing = 2

proc updateCache(t: FormattedText) =
    t.cacheValid = true
    t.lines.setLen(0)
    t.shadowAttrs.setLen(0)
    t.strokeAttrs.setLen(0)
    t.mTotalHeight = 0

    var curLineInfo: LineInfo
    curLineInfo.height = t.mAttributes[0].font.height

    # In this context "word" means minimal sequence of runes that can not have
    # line break
    var curWordWidth = 0'f32
    var curWordHeight = 0'f32
    var curWordBaseline = 0'f32

    var curWordStartByte = 0
    var curWordStartRune = 0
    var curWordFirstAttr = 0

    var i = 0
    var curAttrIndex = 0
    var nextAttrStartIndex = -1
    if t.mAttributes.len > 1:
        nextAttrStartIndex = t.mAttributes[1].startByte

    var boundingWidth = t.mBoundingSize.width
    if boundingWidth == 0: boundingWidth = Inf

    var c: Rune
    let textLen = t.mText.len
    var curRune = 0

    if t.mAttributes[0].strokeSize > 0: t.strokeAttrs.add(0)
    if t.mAttributes[0].shadowColor.a != 0: t.shadowAttrs.add(0)

    template mustBreakLine(): bool =
        c == Rune('\l')

    template canBreakLine(): bool =
        t.canBreakOnAnyChar or c == Rune(' ') or c == Rune('-') or i == textLen or mustBreakLine()

    while i < textLen:
        let font = t.mAttributes[curAttrIndex].font
        let charStart = i

        fastRuneAt(t.mText, i, c, true)

        let runeWidth = font.getAdvanceForRune(c)

        curWordWidth += runeWidth
        curWordHeight = max(curWordHeight, font.height)
        curWordBaseline = max(curWordBaseline, font.ascent)

        if canBreakLine():
            # commit current word
            if (curLineInfo.width + curWordWidth < boundingWidth or curLineInfo.width == 0):
                # Word fits in the line
                curLineInfo.width += curWordWidth
                curLineInfo.height = max(curLineInfo.height, curWordHeight)
                curLineInfo.baseline = max(curLineInfo.baseline, curWordBaseline)
                if mustBreakLine():
                    let tmp = curLineInfo # JS bug workaround. Copy to temp object.
                    t.lines.add(tmp)
                    t.mTotalHeight += curLineInfo.height + t.mLineSpacing
                    curLineInfo.top = t.mTotalHeight
                    curLineInfo.startByte = i
                    curLineInfo.startRune = curRune + 1
                    curLineInfo.width = 0
                    curLineInfo.height = 0
                    curLineInfo.baseline = 0
                    curLineInfo.firstAttr = curAttrIndex
            else:
                # Complete current line
                let tmp = curLineInfo # JS bug workaround. Copy to temp object.
                t.lines.add(tmp)
                t.mTotalHeight += curLineInfo.height + t.mLineSpacing
                curLineInfo.top = t.mTotalHeight
                curLineInfo.startByte = curWordStartByte
                curLineInfo.startRune = curWordStartRune
                curLineInfo.width = curWordWidth
                curLineInfo.height = curWordHeight
                curLineInfo.baseline = curWordBaseline
                curLineInfo.firstAttr = curWordFirstAttr

            curWordWidth = 0
            curWordHeight = 0
            curWordBaseline = 0
            curWordStartByte = i
            curWordStartRune = curRune + 1
            curWordFirstAttr = curAttrIndex

        # Switch to next attribute if its time
        if charStart + 1 == nextAttrStartIndex:<commit_msg>Fixed foramtted text layout for unicode chars<commit_after>import unicode, algorithm, strutils, sequtils
import nimx.font, nimx.types, nimx.unistring, nimx.utils.lower_bound


type
    FormattedText* = ref object
        mText: string
        mAttributes: seq[Attributes]
        lines: seq[LineInfo]
        mTotalHeight: float32
        horizontalAlignment*: HorizontalTextAlignment
        verticalAlignment*: VerticalAlignment
        mBoundingSize: Size
        cacheValid: bool
        canBreakOnAnyChar: bool
        overrideColor*: Color
        shadowAttrs: seq[int]
        strokeAttrs: seq[int]
        shadowMultiplier*: Size
        mLineSpacing: float32

    LineInfo* = object
        startByte: int
        startRune: int
        width: float32
        height: float32
        top: float32
        baseline: float32 # distance from top of the line to baseline
        firstAttr: int

    Attributes = object
        startByte: int # Index of char where attributes start.
        startRune: int # Index of rune where attributes start.
        font: Font
        textColor: Color

        # Richer attributes. TODO: Move to separate ref object
        shadowColor: Color
        strokeColor1: Color
        strokeColor2: Color
        textColor2: Color
        backgroundColor: Color
        shadowOffset: Size
        shadowRadius: float32
        shadowSpread: float32
        strokeSize: float32
        tracking: float32
        isTextGradient: bool
        isStrokeGradient: bool

    VerticalAlignment* = enum
        vaTop
        vaCenter
        vaBottom

    HorizontalTextAlignment* = enum
        haLeft
        haRight
        haCenter
        haJustify

proc defaultAttributes(): Attributes =
    result.font = systemFont()
    result.textColor = blackColor()

proc `text=`*(t: FormattedText, s: string) =
    if s.isNil:
        t.mText = ""
    else:
        t.mText = s
    t.cacheValid = false

template text*(t: FormattedText): string = t.mText

proc `lineSpacing=`*(t: FormattedText, s: float32) =
    t.mLineSpacing = s
    t.cacheValid = false

template lineSpacing*(t: FormattedText): float32 = t.mLineSpacing

proc newFormattedText*(s: string = ""): FormattedText =
    result.new()
    result.mText = s
    result.mAttributes = @[defaultAttributes()]
    result.lines = @[]
    result.shadowAttrs = @[]
    result.strokeAttrs = @[]
    result.shadowMultiplier = newSize(1, 1)
    result.mLineSpacing = 2

proc updateCache(t: FormattedText) =
    t.cacheValid = true
    t.lines.setLen(0)
    t.shadowAttrs.setLen(0)
    t.strokeAttrs.setLen(0)
    t.mTotalHeight = 0

    var curLineInfo: LineInfo
    curLineInfo.height = t.mAttributes[0].font.height

    # In this context "word" means minimal sequence of runes that can not have
    # line break
    var curWordWidth = 0'f32
    var curWordHeight = 0'f32
    var curWordBaseline = 0'f32

    var curWordStartByte = 0
    var curWordStartRune = 0
    var curWordFirstAttr = 0

    var i = 0
    var curAttrIndex = 0
    var nextAttrStartIndex = -1
    if t.mAttributes.len > 1:
        nextAttrStartIndex = t.mAttributes[1].startByte

    var boundingWidth = t.mBoundingSize.width
    if boundingWidth == 0: boundingWidth = Inf

    var c: Rune
    let textLen = t.mText.len
    var curRune = 0

    if t.mAttributes[0].strokeSize > 0: t.strokeAttrs.add(0)
    if t.mAttributes[0].shadowColor.a != 0: t.shadowAttrs.add(0)

    template mustBreakLine(): bool =
        c == Rune('\l')

    template canBreakLine(): bool =
        t.canBreakOnAnyChar or c == Rune(' ') or c == Rune('-') or i == textLen or mustBreakLine()

    while i < textLen:
        let font = t.mAttributes[curAttrIndex].font

        fastRuneAt(t.mText, i, c, true)

        let runeWidth = font.getAdvanceForRune(c)

        curWordWidth += runeWidth
        curWordHeight = max(curWordHeight, font.height)
        curWordBaseline = max(curWordBaseline, font.ascent)

        if canBreakLine():
            # commit current word
            if (curLineInfo.width + curWordWidth < boundingWidth or curLineInfo.width == 0):
                # Word fits in the line
                curLineInfo.width += curWordWidth
                curLineInfo.height = max(curLineInfo.height, curWordHeight)
                curLineInfo.baseline = max(curLineInfo.baseline, curWordBaseline)
                if mustBreakLine():
                    let tmp = curLineInfo # JS bug workaround. Copy to temp object.
                    t.lines.add(tmp)
                    t.mTotalHeight += curLineInfo.height + t.mLineSpacing
                    curLineInfo.top = t.mTotalHeight
                    curLineInfo.startByte = i
                    curLineInfo.startRune = curRune + 1
                    curLineInfo.width = 0
                    curLineInfo.height = 0
                    curLineInfo.baseline = 0
                    curLineInfo.firstAttr = curAttrIndex
            else:
                # Complete current line
                let tmp = curLineInfo # JS bug workaround. Copy to temp object.
                t.lines.add(tmp)
                t.mTotalHeight += curLineInfo.height + t.mLineSpacing
                curLineInfo.top = t.mTotalHeight
                curLineInfo.startByte = curWordStartByte
                curLineInfo.startRune = curWordStartRune
                curLineInfo.width = curWordWidth
                curLineInfo.height = curWordHeight
                curLineInfo.baseline = curWordBaseline
                curLineInfo.firstAttr = curWordFirstAttr

            curWordWidth = 0
            curWordHeight = 0
            curWordBaseline = 0
            curWordStartByte = i
            curWordStartRune = curRune + 1
            curWordFirstAttr = curAttrIndex

        # Switch to next attribute if its time
        if i == nextAttrStartIndex:
{'fraction_non_alphanumeric': 0.0754295532646048, 'fraction_numerical': 0.011941580756013745, 'mean_word_length': 2.0433986928104577, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7355900', 'n_tokens_mistral': 3488, 'n_tokens_neox': 3257, 'n_words': 861}