<commit_before>module Rouge
module Formatters
# Transforms a token stream into an array of 
# formatted text fragments for use with Prawn.
class Prawn < Formatter
  tag 'prawn'

  LF = %(\n)
  NoBreakSpace = %(\u00a0)
  InnerIndent = %(\n )
  GuardedIndent = %(\u00a0)
  GuardedInnerIndent = %(\n\u00a0)
  BoldStyle = [:bold].to_set
  ItalicStyle = [:italic].to_set
  BoldItalicStyle = [:bold, :italic].to_set

  def initialize opts = {}
    unless ::Rouge::Theme === (theme = opts[:theme])
      unless theme && (theme = ::Rouge::Theme.find theme)
        theme = ::Rouge::Themes::Pastie
      end
      theme = theme.new
    end
    @theme = theme
    @normalized_colors = {}
    @linenum_fragment_base = (create_fragment Token['Generic.Lineno']).merge linenum: true
  end

  # Override format method so fragments don't get flatted to a string
  # and to add an options Hash.
  def format tokens, opts = {}
    stream tokens, opts
  end

  def stream tokens, opts = {}
    if opts[:line_numbers]
      # TODO implement line number start (offset)
      linenum = 0
      fragments = []
      fragments << (create_linenum_fragment linenum += 1)
      tokens.each do |tok, val|
        if val == LF
          fragments << { text: LF }
          fragments << (create_linenum_fragment linenum += 1)
        elsif val.include? LF
          base_fragment = create_fragment tok, val
          val.each_line do |line|
            fragments << (base_fragment.merge text: line)
            # NOTE append linenum fragment if there's a next line; only works if source doesn't have trailing endline
            if line.end_with? LF
              fragments << (create_linenum_fragment linenum += 1)
            end
          end
        else
          fragments << (create_fragment tok, val)
        end
      end
      # NOTE drop orphaned linenum fragment (due to trailing endline in source)
      fragments.pop if (last_fragment = fragments[-1]) && last_fragment[:linenum]
      # NOTE pad numbers that have less digits than the largest line number
      if (linenum_w = linenum.to_s.size) > 1
        # NOTE extra column is the trailing space after the line number
        linenum_w += 1
        fragments.each do |fragment|
          fragment[:text] = %(#{fragment[:text].rjust linenum_w, NoBreakSpace}) if fragment[:linenum]
        end
      end
      fragments
    else
      start_of_line = true
      tokens.map do |tok, val|
        # match one or more consecutive endlines
        if val == LF || (val == (LF * val.length))
          start_of_line = true
          { text: val }
        else
          val[0] = GuardedIndent if start_of_line && (val.start_with? ' ')
          val.gsub! InnerIndent, GuardedInnerIndent if val.include? InnerIndent
          start_of_line = val.end_with? LF
          # NOTE this optimization assumes we don't support/use background colors
          val.rstrip.empty? ? { text: val } : (create_fragment tok, val)
        end
      end
      # QUESTION should we strip trailing newline?
    end
  end

  # TODO method could still be optimized (for instance, check if val is LF or empty)
  def create_fragment tok, val = nil
    fragment = val ? { text: val } : {}
    if (style_rules = @theme.style_for tok)
      # TODO support background color
      if (fg = normalize_color style_rules.fg)
        fragment[:color] = fg
      end
      if style_rules[:bold]
        fragment[:styles] = style_rules[:italic] ? BoldItalicStyle : BoldStyle
      elsif style_rules[:italic]
        fragment[:styles] = ItalicStyle
      end
    end
    fragment
  end

  def create_linenum_fragment linenum
    @linenum_fragment_base.merge text: %(#{linenum} )
  end

  def normalize_color raw
    return unless raw<commit_msg>resolves #665 support underline style for token in Rouge theme (PR #666)<commit_after>module Rouge
module Formatters
# Transforms a token stream into an array of 
# formatted text fragments for use with Prawn.
class Prawn < Formatter
  tag 'prawn'

  LF = %(\n)
  NoBreakSpace = %(\u00a0)
  InnerIndent = %(\n )
  GuardedIndent = %(\u00a0)
  GuardedInnerIndent = %(\n\u00a0)
  BoldStyle = [:bold].to_set
  ItalicStyle = [:italic].to_set
  BoldItalicStyle = [:bold, :italic].to_set
  UnderlineStyle = [:underline].to_set

  def initialize opts = {}
    unless ::Rouge::Theme === (theme = opts[:theme])
      unless theme && (theme = ::Rouge::Theme.find theme)
        theme = ::Rouge::Themes::Pastie
      end
      theme = theme.new
    end
    @theme = theme
    @normalized_colors = {}
    @linenum_fragment_base = (create_fragment Token['Generic.Lineno']).merge linenum: true
  end

  # Override format method so fragments don't get flatted to a string
  # and to add an options Hash.
  def format tokens, opts = {}
    stream tokens, opts
  end

  def stream tokens, opts = {}
    if opts[:line_numbers]
      # TODO implement line number start (offset)
      linenum = 0
      fragments = []
      fragments << (create_linenum_fragment linenum += 1)
      tokens.each do |tok, val|
        if val == LF
          fragments << { text: LF }
          fragments << (create_linenum_fragment linenum += 1)
        elsif val.include? LF
          base_fragment = create_fragment tok, val
          val.each_line do |line|
            fragments << (base_fragment.merge text: line)
            # NOTE append linenum fragment if there's a next line; only works if source doesn't have trailing endline
            if line.end_with? LF
              fragments << (create_linenum_fragment linenum += 1)
            end
          end
        else
          fragments << (create_fragment tok, val)
        end
      end
      # NOTE drop orphaned linenum fragment (due to trailing endline in source)
      fragments.pop if (last_fragment = fragments[-1]) && last_fragment[:linenum]
      # NOTE pad numbers that have less digits than the largest line number
      if (linenum_w = linenum.to_s.size) > 1
        # NOTE extra column is the trailing space after the line number
        linenum_w += 1
        fragments.each do |fragment|
          fragment[:text] = %(#{fragment[:text].rjust linenum_w, NoBreakSpace}) if fragment[:linenum]
        end
      end
      fragments
    else
      start_of_line = true
      tokens.map do |tok, val|
        # match one or more consecutive endlines
        if val == LF || (val == (LF * val.length))
          start_of_line = true
          { text: val }
        else
          val[0] = GuardedIndent if start_of_line && (val.start_with? ' ')
          val.gsub! InnerIndent, GuardedInnerIndent if val.include? InnerIndent
          start_of_line = val.end_with? LF
          # NOTE this optimization assumes we don't support/use background colors
          val.rstrip.empty? ? { text: val } : (create_fragment tok, val)
        end
      end
      # QUESTION should we strip trailing newline?
    end
  end

  # TODO method could still be optimized (for instance, check if val is LF or empty)
  def create_fragment tok, val = nil
    fragment = val ? { text: val } : {}
    if (style_rules = @theme.style_for tok)
      # TODO support background color
      if (fg = normalize_color style_rules.fg)
        fragment[:color] = fg
      end
      if style_rules[:bold]
        fragment[:styles] = style_rules[:italic] ? BoldItalicStyle.dup : BoldStyle.dup
      elsif style_rules[:italic]
        fragment[:styles] = ItalicStyle.dup
      end
      if style_rules[:underline]
        if fragment.key? :styles
          fragment[:styles] << UnderlineStyle.first
        else
          fragment[:styles] = UnderlineStyle.dup
        end
      end
    end
    fragment
  end

  def create_linenum_fragment linenum
    @linenum_fragment_base.merge text: %(#{linenum} )
  end

  def normalize_color raw
    return unless raw
{'fraction_non_alphanumeric': 0.0907573812580231, 'fraction_numerical': 0.005134788189987163, 'mean_word_length': 2.444297082228117, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2361356', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2300, 'n_words': 742}