<commit_before># encoding: utf-8

require './spec/spec_helper'

require './lib/gimli'

describe Gimli::Markup do

  it 'should give correct code block' do

    output_with_pygments = "<p>a</p>\n<div class=\"highlight\"><pre>" +
             "<span class=\"n\">x</span> <span class=\"o\">=</span> " +
             "<span class=\"mi\">1</span>\n</pre>\n</div>\n\n<p>b</p>"

    output_without_pygments = "<p>a</p>\n<pre><code>x = 1</code></pre>\n<p>b</p>"

    file = Gimli::MarkupFile.new File.expand_path('../../fixtures/code_block.textile', __FILE__)
    markup = Gimli::Markup.new file

    # Check if the html seems to be highlighted
    if markup.render.include? "<div class=\"highlight\">"
      markup.render.should == output_with_pygments
    else
      markup.render.should == output_without_pygments
    end
  end

  it 'should give correct code block for utf8' do

    output_with_pygments = "<p>Abcåäö</p>\n<div class=\"highlight\"><pre>" +
                           "<span class=\"nt\">&lt;h1&gt;</span>Abcåäö<span class=\"nt\">&lt;/h1&gt;</span>\n" +
                           "<span class=\"nt\">&lt;img</span> <span class=\"na\">src=</span><span class=\"s\">\"åäö.png\"" +
                           "</span> <span class=\"na\">alt=</span><span class=\"s\">\"ÅÄÖ\"" +
                           "</span> <span class=\"nt\">/&gt;</span>\n</pre>\n</div>\n"

    output_without_pygments = "<p>Abcåäö</p>\n<pre><code>&lt;h1&gt;Abcåäö&lt;/h1&gt;\n&lt;img " +
                              "src=\"åäö.png\" alt=\"ÅÄÖ\" /&gt;</code></pre>"

    file = Gimli::MarkupFile.new File.expand_path('../../fixtures/code_block_with_utf8.textile', __FILE__)
    markup = Gimli::Markup.new file

    # Check if the html seems to be highlighted
    if markup.render.include? "<div class=\"highlight\">"
      markup.render.should == output_with_pygments
    else
      markup.render.should == output_without_pygments
    end
  end<commit_msg>Fix output html to match nokogiri 1.5 in test<commit_after># encoding: utf-8

require './spec/spec_helper'

require './lib/gimli'

describe Gimli::Markup do

  it 'should give correct code block' do

    output_with_pygments = "<p>a</p>\n<div class=\"highlight\"><pre>" +
             "<span class=\"n\">x</span> <span class=\"o\">=</span> " +
             "<span class=\"mi\">1</span>\n</pre>\n</div>\n\n<p>b</p>"

    output_without_pygments = "<p>a</p>\n<pre>\n  <code>x = 1</code>\n</pre>\n<p>b</p>"

    file = Gimli::MarkupFile.new File.expand_path('../../fixtures/code_block.textile', __FILE__)
    markup = Gimli::Markup.new file

    # Check if the html seems to be highlighted
    if markup.render.include? "<div class=\"highlight\">"
      markup.render.should == output_with_pygments
    else
      markup.render.should == output_without_pygments
    end
  end

  it 'should give correct code block for utf8' do

    output_with_pygments = "<p>Abcåäö</p>\n<div class=\"highlight\"><pre>" +
                           "<span class=\"nt\">&lt;h1&gt;</span>Abcåäö<span class=\"nt\">&lt;/h1&gt;</span>\n" +
                           "<span class=\"nt\">&lt;img</span> <span class=\"na\">src=</span><span class=\"s\">\"åäö.png\"" +
                           "</span> <span class=\"na\">alt=</span><span class=\"s\">\"ÅÄÖ\"" +
                           "</span> <span class=\"nt\">/&gt;</span>\n</pre>\n</div>\n"

    output_without_pygments = "<p>Abcåäö</p>\n<pre>\n  <code>&lt;h1&gt;Abcåäö&lt;/h1&gt;\n&lt;img " +
                              "src=\"åäö.png\" alt=\"ÅÄÖ\" /&gt;</code>\n</pre>"

    file = Gimli::MarkupFile.new File.expand_path('../../fixtures/code_block_with_utf8.textile', __FILE__)
    markup = Gimli::Markup.new file

    # Check if the html seems to be highlighted
    if markup.render.include? "<div class=\"highlight\">"
      markup.render.should == output_with_pygments
    else
      markup.render.should == output_without_pygments
    end
  end
{'fraction_non_alphanumeric': 0.20361875637104995, 'fraction_numerical': 0.0050968399592252805, 'mean_word_length': 2.620848708487085, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5363818', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1387, 'n_words': 235}