<commit_before>module Gitlab
  # Custom parser for Gitlab-flavored Markdown
  #
  # It replaces references in the text with links to the appropriate items in Gitlab.
  #
  # Supported reference formats are:
  #   * @foo for team members
  #   * #123 for issues
  #   * !123 for merge requests
  #   * $123 for snippets
  #   * 123456 for commits
  #
  # Examples
  #
  #   >> m = Markdown.new(...)
  #
  #   >> m.parse("Hey @david, can you fix this?")
  #   => "Hey <a href="/gitlab/team_members/1">@david</a>, can you fix this?"
  #
  #   >> m.parse("Commit 35d5f7c closes #1234")
  #   => "Commit <a href="/gitlab/commits/35d5f7c">35d5f7c</a> closes <a href="/gitlab/issues/1234">#1234</a>"
  class Markdown
    include Rails.application.routes.url_helpers
    include ActionView::Helpers

    REFERENCE_PATTERN = %r{
      ([^\w&;])?      # Prefix (1)
      (               # Reference (2)
        @([\w\._]+)   # User name (3)
        |[#!$](\d+)   # Issue/MR/Snippet ID (4)
        |([\h]{6,40}) # Commit ID (5)
      )
      ([^\w&;])?      # Suffix (6)
    }x.freeze

    attr_reader :html_options

    def initialize(project, html_options = {})
      @project      = project
      @html_options = html_options
    end

    def parse(text)
      text.gsub(REFERENCE_PATTERN) do |match|
        prefix     = $1 || ''
        reference  = $2
        identifier = $3 || $4 || $5
        suffix     = $6 || ''

        if ref_link = reference_link(reference, identifier)
          prefix + ref_link + suffix
        else
          match
        end
      end
    end

    private

    # Private: Dispatches to a dedicated processing method based on reference
    #
    # reference  - Object reference ("@1234", "!567", etc.)
    # identifier - Object identifier (Issue ID, SHA hash, etc.)
    #
    # Returns string rendered by the processing method
    def reference_link(reference, identifier)
      case reference
      when /^@/  then reference_user(identifier)
      when /^#/  then reference_issue(identifier)
      when /^!/  then reference_merge_request(identifier)
      when /^\$/ then reference_snippet(identifier)
      when /^\h/ then reference_commit(identifier)
      end
    end
<commit_msg>Change Gitlab::Markdown to a module; add emoji parsing<commit_after>module Gitlab
  # Custom parser for Gitlab-flavored Markdown
  #
  # It replaces references in the text with links to the appropriate items in
  # Gitlab.
  #
  # Supported reference formats are:
  #   * @foo for team members
  #   * #123 for issues
  #   * !123 for merge requests
  #   * $123 for snippets
  #   * 123456 for commits
  #
  # It also parses Emoji codes to insert images. See
  # http://www.emoji-cheat-sheet.com/ for a list of the supported icons.
  #
  # Examples
  #
  #   >> gfm("Hey @david, can you fix this?")
  #   => "Hey <a href="/gitlab/team_members/1">@david</a>, can you fix this?"
  #
  #   >> gfm("Commit 35d5f7c closes #1234")
  #   => "Commit <a href="/gitlab/commits/35d5f7c">35d5f7c</a> closes <a href="/gitlab/issues/1234">#1234</a>"
  #
  #   >> gfm(":trollface:")
  #   => "<img alt=\":trollface:\" class=\"emoji\" src=\"/images/trollface.png" title=\":trollface:\" />
  module Markdown
    REFERENCE_PATTERN = %r{
      ([^\w&;])?      # Prefix (1)
      (               # Reference (2)
        @([\w\._]+)   # User name (3)
        |[#!$](\d+)   # Issue/MR/Snippet ID (4)
        |([\h]{6,40}) # Commit ID (5)
      )
      ([^\w&;])?      # Suffix (6)
    }x.freeze

    EMOJI_PATTERN = %r{(:([\w\-\+]+):)}.freeze

    attr_reader :html_options

    # Public: Parse the provided text with GitLab-Flavored Markdown
    #
    # text         - the source text
    # html_options - extra options for the reference links as given to link_to
    #
    # Note: reference links will only be generated if @project is set
    def gfm(text, html_options = {})
      return text if text.nil?
      return text if @project.nil?

      @html_options = html_options

      # Extract pre blocks so they are not altered
      # from http://github.github.com/github-flavored-markdown/
      extractions = {}
      text.gsub!(%r{<pre>.*?</pre>|<code>.*?</code>}m) do |match|
        md5 = Digest::MD5.hexdigest(match)
        extractions[md5] = match
        "{gfm-extraction-#{md5}}"
      end

      # TODO: add popups with additional information

      text = parse(text)

      # Insert pre block extractions
      text.gsub!(/\{gfm-extraction-(\h{32})\}/) do
        extractions[$1]
      end

      sanitize text.html_safe, attributes: ActionView::Base.sanitized_allowed_attributes + %w(id class)
    end

    private

    # Private: Parses text for references and emoji
    #
    # text - Text to parse
    #
    # Returns parsed text
    def parse(text)
      text = text.gsub(REFERENCE_PATTERN) do |match|
        prefix     = $1 || ''
        reference  = $2
        identifier = $3 || $4 || $5
        suffix     = $6 || ''

        if ref_link = reference_link(reference, identifier)
          prefix + ref_link + suffix
        else
          match
        end
      end

      text = text.gsub(EMOJI_PATTERN) do |match|
        if valid_emoji?($2)
          helper.image_tag("#{$2}.png", class: 'emoji', title: $1, alt: $1)
        else
          match
        end
      end

      text
    end

    # Private: Checks if an emoji icon exists in the image asset directory
    #
    # emoji - Identifier of the emoji as a string (e.g., "+1", "heart")
    #
    # Returns boolean
    def valid_emoji?(emoji)
      File.exists?(Rails.root.join('app', 'assets', 'images', 'emoji', "#{emoji}.png"))
    end

    # Private: Dispatches to a dedicated processing method based on reference
    #
    # reference  - Object reference ("@1234", "!567", etc.)
    # identifier - Object identifier (Issue ID, SHA hash, etc.)
    #
    # Returns string rendered by the processing method
    def reference_link(reference, identifier)
      case reference
      when /^@/  then reference_user(identifier)
      when /^#/  then reference_issue(identifier)
      when /^!/  then reference_merge_request(identifier)
      when /^\$/ then reference_snippet(identifier)
      when /^\h/ then reference_commit(identifier)
      end
    end

{'fraction_non_alphanumeric': 0.1340504093755017, 'fraction_numerical': 0.02183336008990207, 'mean_word_length': 2.270341207349081, 'pattern_counts': {'":': 3, '<': 20, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7183567', 'n_tokens_mistral': 2139, 'n_tokens_neox': 2035, 'n_words': 552}