<commit_before>module Asciidoctor
module Pdf
module FormattedText
class Formatter
  FormattingSnifferPattern = /[<&]/
  EOL = "\n"

  def initialize options = {}
    @parser = MarkupParser.new
    @transform = Transform.new merge_adjacent_text_nodes: true, theme: options[:theme]
  end

  def format string, *args
    options = args[0] || {}
    string = string.tr_s(EOL, ' ') if options[:normalize]
    return [text: string] unless string.match(FormattingSnifferPattern)
    if (parsed = @parser.parse(string))
      @transform.apply(parsed.content)
    else
      warn %(Failed to parse formatted text: #{string})
      [text: string]
    end
  end
end
end
end
end<commit_msg>resolves #355 collapse all whitespace when normalizing (PR #438)<commit_after>module Asciidoctor
module Pdf
module FormattedText
class Formatter
  FormattingSnifferPattern = /[<&]/
  WHITESPACE = " \t\n"

  def initialize options = {}
    @parser = MarkupParser.new
    @transform = Transform.new merge_adjacent_text_nodes: true, theme: options[:theme]
  end

  def format string, *args
    options = args[0] || {}
    string = string.tr_s(WHITESPACE, ' ') if options[:normalize]
    return [text: string] unless string.match(FormattingSnifferPattern)
    if (parsed = @parser.parse(string))
      @transform.apply(parsed.content)
    else
      warn %(Failed to parse formatted text: #{string})
      [text: string]
    end
  end
end
end
end
end
{'fraction_non_alphanumeric': 0.11025280898876405, 'fraction_numerical': 0.0056179775280898875, 'mean_word_length': 3.73421926910299, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2972090', 'n_tokens_mistral': 459, 'n_tokens_neox': 449, 'n_words': 123}