<commit_before>require 'nokogiri'

module MarkupHelper
  def plain?(filename)
    Gitlab::MarkupHelper.plain?(filename)
  end

  def markup?(filename)
    Gitlab::MarkupHelper.markup?(filename)
  end

  def gitlab_markdown?(filename)
    Gitlab::MarkupHelper.gitlab_markdown?(filename)
  end

  def asciidoc?(filename)
    Gitlab::MarkupHelper.asciidoc?(filename)
  end

  # Use this in places where you would normally use link_to(gfm(...), ...).
  #
  # It solves a problem occurring with nested links (i.e.
  # "<a>outer text <a>gfm ref</a> more outer text</a>"). This will not be
  # interpreted as intended. Browsers will parse something like
  # "<a>outer text </a><a>gfm ref</a> more outer text" (notice the last part is
  # not linked any more). link_to_gfm corrects that. It wraps all parts to
  # explicitly produce the correct linking behavior (i.e.
  # "<a>outer text </a><a>gfm ref</a><a> more outer text</a>").
  def link_to_gfm(body, url, html_options = {})
    return '' if body.blank?

    context = {
      project: @project,
      current_user: (current_user if defined?(current_user)),
      pipeline: :single_line,
    }
    gfm_body = Banzai.render(body, context)

    fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
    if fragment.children.size == 1 && fragment.children[0].name == 'a'
      # Fragment has only one node, and it's a link generated by `gfm`.
      # Replace it with our requested link.
      text = fragment.children[0].text
      fragment.children[0].replace(link_to(text, url, html_options))
    else
      # Traverse the fragment's first generation of children looking for pure
      # text, wrapping anything found in the requested link
      fragment.children.each do |node|
        next unless node.text?
        node.replace(link_to(node.text, url, html_options))
      end
    end

    # Add any custom CSS classes to the GFM-generated reference links
    if html_options[:class]
      fragment.css('a.gfm').add_class(html_options[:class])
    end

    fragment.to_html.html_safe
  end

  # Return the first line of +text+, up to +max_chars+, after parsing the line
  # as Markdown.  HTML tags in the parsed output are not counted toward the
  # +max_chars+ limit.  If the length limit falls within a tag's contents, then
  # the tag contents are truncated without removing the closing tag.
  def first_line_in_markdown(text, max_chars = nil, options = {})
    md = markdown(text, options).strip

    truncate_visible(md, max_chars || md.length) if md.present?
  end

  def markdown(text, context = {})
    return '' unless text.present?

    context[:project] ||= @project
    html = context.delete(:rendered) || markdown_unsafe(text, context)
    banzai_postprocess(html, context)
  end

  def markdown_field(object, field)
    object = object.for_display if object.respond_to?(:for_display)
    return '' unless object.present?

    html = Banzai.render_field(object, field)
    banzai_postprocess(html, object.banzai_render_context(field))
  end<commit_msg>`markdown` helper is not used with pre-rendered content<commit_after>require 'nokogiri'

module MarkupHelper
  def plain?(filename)
    Gitlab::MarkupHelper.plain?(filename)
  end

  def markup?(filename)
    Gitlab::MarkupHelper.markup?(filename)
  end

  def gitlab_markdown?(filename)
    Gitlab::MarkupHelper.gitlab_markdown?(filename)
  end

  def asciidoc?(filename)
    Gitlab::MarkupHelper.asciidoc?(filename)
  end

  # Use this in places where you would normally use link_to(gfm(...), ...).
  #
  # It solves a problem occurring with nested links (i.e.
  # "<a>outer text <a>gfm ref</a> more outer text</a>"). This will not be
  # interpreted as intended. Browsers will parse something like
  # "<a>outer text </a><a>gfm ref</a> more outer text" (notice the last part is
  # not linked any more). link_to_gfm corrects that. It wraps all parts to
  # explicitly produce the correct linking behavior (i.e.
  # "<a>outer text </a><a>gfm ref</a><a> more outer text</a>").
  def link_to_gfm(body, url, html_options = {})
    return '' if body.blank?

    context = {
      project: @project,
      current_user: (current_user if defined?(current_user)),
      pipeline: :single_line,
    }
    gfm_body = Banzai.render(body, context)

    fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
    if fragment.children.size == 1 && fragment.children[0].name == 'a'
      # Fragment has only one node, and it's a link generated by `gfm`.
      # Replace it with our requested link.
      text = fragment.children[0].text
      fragment.children[0].replace(link_to(text, url, html_options))
    else
      # Traverse the fragment's first generation of children looking for pure
      # text, wrapping anything found in the requested link
      fragment.children.each do |node|
        next unless node.text?
        node.replace(link_to(node.text, url, html_options))
      end
    end

    # Add any custom CSS classes to the GFM-generated reference links
    if html_options[:class]
      fragment.css('a.gfm').add_class(html_options[:class])
    end

    fragment.to_html.html_safe
  end

  # Return the first line of +text+, up to +max_chars+, after parsing the line
  # as Markdown.  HTML tags in the parsed output are not counted toward the
  # +max_chars+ limit.  If the length limit falls within a tag's contents, then
  # the tag contents are truncated without removing the closing tag.
  def first_line_in_markdown(text, max_chars = nil, options = {})
    md = markdown(text, options).strip

    truncate_visible(md, max_chars || md.length) if md.present?
  end

  def markdown(text, context = {})
    return '' unless text.present?

    context[:project] ||= @project
    html = markdown_unsafe(text, context)
    banzai_postprocess(html, context)
  end

  def markdown_field(object, field)
    object = object.for_display if object.respond_to?(:for_display)
    return '' unless object.present?

    html = Banzai.render_field(object, field)
    banzai_postprocess(html, object.banzai_render_context(field))
  end
{'fraction_non_alphanumeric': 0.10688914587807699, 'fraction_numerical': 0.001321658681645465, 'mean_word_length': 3.5725075528700905, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4249904', 'n_tokens_mistral': 1914, 'n_tokens_neox': 1871, 'n_words': 573}