<commit_before>require 'spec_helper'
require 'erb'

# This feature spec is intended to be a comprehensive exercising of all of
# GitLab's non-standard Markdown parsing and the integration thereof.
#
# These tests should be very high-level. Anything low-level belongs in the specs
# for the corresponding HTML::Pipeline filter or helper method.
#
# The idea is to pass a Markdown document through our entire processing stack.
#
# The process looks like this:
#
#   Raw Markdown
#   -> `markdown` helper
#     -> Redcarpet::Render::GitlabHTML converts Markdown to HTML
#       -> Post-process HTML
#         -> `gfm_with_options` helper
#           -> HTML::Pipeline
#             -> Sanitize
#             -> Emoji
#             -> Table of Contents
#             -> Autolinks
#               -> Rinku (http, https, ftp)
#               -> Other schemes
#             -> References
#           -> `html_safe`
#           -> Template
#
# See the MarkdownFeature class for setup details.

describe 'GitLab Markdown' do
  include ActionView::Helpers::TagHelper
  include ActionView::Helpers::UrlHelper
  include Capybara::Node::Matchers
  include GitlabMarkdownHelper

  # `markdown` calls these two methods
  def current_user
    @feat.user
  end

  def user_color_scheme_class
    :white
  end

  # Let's only parse this thing once
  before(:all) do
    @feat = MarkdownFeature.new

    # `markdown` expects a `@project` variable
    @project = @feat.project

    @md = markdown(@feat.raw_markdown)
    @doc = Nokogiri::HTML::DocumentFragment.parse(@md)
  end

  after(:all) do
    @feat.teardown
  end

  # Given a header ID, goes to that element's parent (the header), then to its
  # second sibling (the body).
  def get_section(id)
    @doc.at_css("##{id}").parent.next.next
  end

  # it 'writes to a file' do
  #   File.open(Rails.root.join('tmp/capybara/markdown_spec.html'), 'w') do |file|
  #     file.puts @md
  #   end
  # end

  describe 'Markdown' do
    describe 'No Intra Emphasis' do
      it 'does not parse emphasis inside of words' do
        body = get_section('no-intra-emphasis')
        expect(body.to_html).not_to match('foo<em>bar</em>baz')
      end
    end

    describe 'Tables' do
      it 'parses table Markdown' do
        body = get_section('tables')
        expect(body).to have_selector('th:contains("Header")')
        expect(body).to have_selector('th:contains("Row")')
        expect(body).to have_selector('th:contains("Example")')
      end

      it 'allows Markdown in tables' do
        expect(@doc.at_css('td:contains("Baz")').children.to_html).
          to eq '<strong>Baz</strong>'
      end
    end

    describe 'Fenced Code Blocks' do
      it 'parses fenced code blocks' do
        expect(@doc).to have_selector('pre.code.highlight.white.c')
        expect(@doc).to have_selector('pre.code.highlight.white.python')
      end
    end

    describe 'Strikethrough' do
      it 'parses strikethroughs' do
        expect(@doc).to have_selector(%{del:contains("and this text doesn't")})
      end
    end

    describe 'Superscript' do
      it 'parses superscript' do
        body = get_section('superscript')
        expect(body.to_html).to match('1<sup>st</sup>')
        expect(body.to_html).to match('2<sup>nd</sup>')
      end
    end
  end

  describe 'HTML::Pipeline' do
    describe 'SanitizationFilter' do
      it 'uses a permissive whitelist' do
        expect(@doc).to have_selector('b#manual-b')
        expect(@doc).to have_selector('em#manual-em')
        expect(@doc).to have_selector("code#manual-code")
        expect(@doc).to have_selector('kbd:contains("s")')
        expect(@doc).to have_selector('strike:contains(Emoji)')
        expect(@doc).to have_selector('img#manual-img')
        expect(@doc).to have_selector('br#manual-br')
        expect(@doc).to have_selector('hr#manual-hr')
      end

      it 'permits span elements' do
        expect(@doc).to have_selector('span#span-class-light.light')
      end

      it 'permits table alignment' do
        expect(@doc.at_css('th:contains("Header")')['style']).to eq 'text-align: center'
        expect(@doc.at_css('th:contains("Row")')['style']).to eq 'text-align: right'
        expect(@doc.at_css('th:contains("Example")')['style']).to eq 'text-align: left'

        expect(@doc.at_css('td:contains("Foo")')['style']).to eq 'text-align: center'
        expect(@doc.at_css('td:contains("Bar")')['style']).to eq 'text-align: right'
        expect(@doc.at_css('td:contains("Baz")')['style']).to eq 'text-align: left'
      end

      it 'removes `rel` attribute from links' do
        expect(@doc).to have_selector('a#a-rel-nofollow')
        expect(@doc).not_to have_selector('a#a-rel-nofollow[rel]')
      end

      it "removes `href` from `a` elements if it's fishy" do
        expect(@doc).to have_selector('a#a-href-javascript')
        expect(@doc).not_to have_selector('a#a-href-javascript[href]')
      end
    end

    describe 'Escaping' do
      let(:table) { @doc.css('table').last.at_css('tbody') }

      it 'escapes non-tag angle brackets' do
        expect(table.at_xpath('.//tr[1]/td[3]').inner_html).to eq '1 &lt; 3 &amp; 5'
      end
    end

    describe 'Edge Cases' do
      it 'allows markup inside link elements' do
        expect(@doc.at_css('a[href="#link-emphasis"]').to_html).
          to eq %{<a href="#link-emphasis"><em>text</em></a>}

        expect(@doc.at_css('a[href="#link-strong"]').to_html).
          to eq %{<a href="#link-strong"><strong>text</strong></a>}

        expect(@doc.at_css('a[href="#link-code"]').to_html).
          to eq %{<a href="#link-code"><code>text</code></a>}
      end
    end

    describe 'EmojiFilter' do
      it 'parses Emoji' do
        expect(@doc).to have_selector('img.emoji', count: 10)
      end
    end

    describe 'TableOfContentsFilter' do
      it 'creates anchors inside header elements' do
        expect(@doc).to have_selector('h1 a#gitlab-markdown')
        expect(@doc).to have_selector('h2 a#markdown')
        expect(@doc).to have_selector('h3 a#autolinkfilter')
      end
    end

    describe 'AutolinkFilter' do
      let(:list) { get_section('autolinkfilter').parent.search('ul') }

      def item(index)
        list.at_css("li:nth-child(#{index})")
      end

      it 'autolinks http://' do
        expect(item(1).children.first.name).to eq 'a'
        expect(item(1).children.first['href']).to eq 'http://about.gitlab.com/'
      end

      it 'autolinks https://' do
        expect(item(2).children.first.name).to eq 'a'
        expect(item(2).children.first['href']).to eq 'https://google.com/'
      end

      it 'autolinks ftp://' do
        expect(item(3).children.first.name).to eq 'a'
        expect(item(3).children.first['href']).to eq 'ftp://ftp.us.debian.org/debian/'
      end

      it 'autolinks smb://' do<commit_msg>Fix Markdown feature spec<commit_after>require 'spec_helper'
require 'erb'

# This feature spec is intended to be a comprehensive exercising of all of
# GitLab's non-standard Markdown parsing and the integration thereof.
#
# These tests should be very high-level. Anything low-level belongs in the specs
# for the corresponding HTML::Pipeline filter or helper method.
#
# The idea is to pass a Markdown document through our entire processing stack.
#
# The process looks like this:
#
#   Raw Markdown
#   -> `markdown` helper
#     -> Redcarpet::Render::GitlabHTML converts Markdown to HTML
#       -> Post-process HTML
#         -> `gfm_with_options` helper
#           -> HTML::Pipeline
#             -> Sanitize
#             -> Emoji
#             -> Table of Contents
#             -> Autolinks
#               -> Rinku (http, https, ftp)
#               -> Other schemes
#             -> References
#           -> `html_safe`
#           -> Template
#
# See the MarkdownFeature class for setup details.

describe 'GitLab Markdown' do
  include ActionView::Helpers::TagHelper
  include ActionView::Helpers::UrlHelper
  include Capybara::Node::Matchers
  include GitlabMarkdownHelper

  # `markdown` calls these two methods
  def current_user
    @feat.user
  end

  def user_color_scheme_class
    :white
  end

  # Let's only parse this thing once
  before(:all) do
    @feat = MarkdownFeature.new

    # `markdown` expects a `@project` variable
    @project = @feat.project

    @md = markdown(@feat.raw_markdown)
    @doc = Nokogiri::HTML::DocumentFragment.parse(@md)
  end

  after(:all) do
    @feat.teardown
  end

  # Given a header ID, goes to that element's parent (the header), then to its
  # second sibling (the body).
  def get_section(id)
    @doc.at_css("##{id}").parent.next_element
  end

  # it 'writes to a file' do
  #   File.open(Rails.root.join('tmp/capybara/markdown_spec.html'), 'w') do |file|
  #     file.puts @md
  #   end
  # end

  describe 'Markdown' do
    describe 'No Intra Emphasis' do
      it 'does not parse emphasis inside of words' do
        body = get_section('no-intra-emphasis')
        expect(body.to_html).not_to match('foo<em>bar</em>baz')
      end
    end

    describe 'Tables' do
      it 'parses table Markdown' do
        body = get_section('tables')
        expect(body).to have_selector('th:contains("Header")')
        expect(body).to have_selector('th:contains("Row")')
        expect(body).to have_selector('th:contains("Example")')
      end

      it 'allows Markdown in tables' do
        expect(@doc.at_css('td:contains("Baz")').children.to_html).
          to eq '<strong>Baz</strong>'
      end
    end

    describe 'Fenced Code Blocks' do
      it 'parses fenced code blocks' do
        expect(@doc).to have_selector('pre.code.highlight.white.c')
        expect(@doc).to have_selector('pre.code.highlight.white.python')
      end
    end

    describe 'Strikethrough' do
      it 'parses strikethroughs' do
        expect(@doc).to have_selector(%{del:contains("and this text doesn't")})
      end
    end

    describe 'Superscript' do
      it 'parses superscript' do
        body = get_section('superscript')
        expect(body.to_html).to match('1<sup>st</sup>')
        expect(body.to_html).to match('2<sup>nd</sup>')
      end
    end
  end

  describe 'HTML::Pipeline' do
    describe 'SanitizationFilter' do
      it 'uses a permissive whitelist' do
        expect(@doc).to have_selector('b#manual-b')
        expect(@doc).to have_selector('em#manual-em')
        expect(@doc).to have_selector("code#manual-code")
        expect(@doc).to have_selector('kbd:contains("s")')
        expect(@doc).to have_selector('strike:contains(Emoji)')
        expect(@doc).to have_selector('img#manual-img')
        expect(@doc).to have_selector('br#manual-br')
        expect(@doc).to have_selector('hr#manual-hr')
      end

      it 'permits span elements' do
        expect(@doc).to have_selector('span#span-class-light.light')
      end

      it 'permits table alignment' do
        expect(@doc.at_css('th:contains("Header")')['style']).to eq 'text-align: center'
        expect(@doc.at_css('th:contains("Row")')['style']).to eq 'text-align: right'
        expect(@doc.at_css('th:contains("Example")')['style']).to eq 'text-align: left'

        expect(@doc.at_css('td:contains("Foo")')['style']).to eq 'text-align: center'
        expect(@doc.at_css('td:contains("Bar")')['style']).to eq 'text-align: right'
        expect(@doc.at_css('td:contains("Baz")')['style']).to eq 'text-align: left'
      end

      it 'removes `rel` attribute from links' do
        expect(@doc).to have_selector('a#a-rel-nofollow')
        expect(@doc).not_to have_selector('a#a-rel-nofollow[rel]')
      end

      it "removes `href` from `a` elements if it's fishy" do
        expect(@doc).to have_selector('a#a-href-javascript')
        expect(@doc).not_to have_selector('a#a-href-javascript[href]')
      end
    end

    describe 'Escaping' do
      let(:table) { @doc.css('table').last.at_css('tbody') }

      it 'escapes non-tag angle brackets' do
        expect(table.at_xpath('.//tr[1]/td[3]').inner_html).to eq '1 &lt; 3 &amp; 5'
      end
    end

    describe 'Edge Cases' do
      it 'allows markup inside link elements' do
        expect(@doc.at_css('a[href="#link-emphasis"]').to_html).
          to eq %{<a href="#link-emphasis"><em>text</em></a>}

        expect(@doc.at_css('a[href="#link-strong"]').to_html).
          to eq %{<a href="#link-strong"><strong>text</strong></a>}

        expect(@doc.at_css('a[href="#link-code"]').to_html).
          to eq %{<a href="#link-code"><code>text</code></a>}
      end
    end

    describe 'EmojiFilter' do
      it 'parses Emoji' do
        expect(@doc).to have_selector('img.emoji', count: 10)
      end
    end

    describe 'TableOfContentsFilter' do
      it 'creates anchors inside header elements' do
        expect(@doc).to have_selector('h1 a#gitlab-markdown')
        expect(@doc).to have_selector('h2 a#markdown')
        expect(@doc).to have_selector('h3 a#autolinkfilter')
      end
    end

    describe 'AutolinkFilter' do
      let(:list) { get_section('autolinkfilter').next_element }

      def item(index)
        list.at_css("li:nth-child(#{index})")
      end

      it 'autolinks http://' do
        expect(item(1).children.first.name).to eq 'a'
        expect(item(1).children.first['href']).to eq 'http://about.gitlab.com/'
      end

      it 'autolinks https://' do
        expect(item(2).children.first.name).to eq 'a'
        expect(item(2).children.first['href']).to eq 'https://google.com/'
      end

      it 'autolinks ftp://' do
        expect(item(3).children.first.name).to eq 'a'
        expect(item(3).children.first['href']).to eq 'ftp://ftp.us.debian.org/debian/'
      end

      it 'autolinks smb://' do
{'fraction_non_alphanumeric': 0.14813193228254523, 'fraction_numerical': 0.002626970227670753, 'mean_word_length': 2.8218070273284996, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 71, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6790751', 'n_tokens_mistral': 4530, 'n_tokens_neox': 4353, 'n_words': 1018}