<commit_before>require 'nokogiri'
require 'html-proofer'

namespace :static_checks do
  def should_not_run_external_url_checks?
    if ENV['CI'] || ENV['SNAP_CI']
      false
    else
      ENV['RUN_EXTERNAL_CHECKS'].nil? || ENV['RUN_EXTERNAL_CHECKS'] == 'false'
    end
  end

  options = {
    :disable_external     => should_not_run_external_url_checks?,
    :url_ignore           => ['http://localhost:8153'],
    :allow_hash_href      => true,

    :check_html           => true,
    :validation           => {
      :report_invalid_tags  => false,
      :report_script_embeds => false,
      :report_missing_names => true,
    },

    :empty_alt_ignore     => true,
    :log_level            => :info,
    :file_ignore          => [/googled355c365863be8d8/, /talkpython/, /thechangelog/]
  }

  class ProperHTMLCheck < ::HTMLProofer::Check
    VALID_HTML5_ENTITIES = ['copy', 'ldquo', 'rdquo']
    def add_error error
      if error.respond_to?('line') and error.respond_to?('message')
        add_issue "Proper HTML Check: #{error.message}", line: error.line
      else
        add_issue "Proper HTML Check (unknown error): #{error.to_s}", line: 1
      end
    end

    def is_entity_error_and_is_valid? error
      error.is_a? Nokogiri::XML::SyntaxError and error.code == 26 and VALID_HTML5_ENTITIES.include? error.str1<commit_msg>Added talkpython and changelog to escape ampersand<commit_after>require 'nokogiri'
require 'html-proofer'

namespace :static_checks do
  def should_not_run_external_url_checks?
    if ENV['CI'] || ENV['SNAP_CI']
      false
    else
      ENV['RUN_EXTERNAL_CHECKS'].nil? || ENV['RUN_EXTERNAL_CHECKS'] == 'false'
    end
  end

  options = {
    :disable_external     => should_not_run_external_url_checks?,
    :url_ignore           => ['http://localhost:8153'],
    :allow_hash_href      => true,

    :check_html           => true,
    :validation           => {
      :report_invalid_tags  => false,
      :report_script_embeds => false,
      :report_missing_names => true,
    },

    :empty_alt_ignore     => true,
    :log_level            => :info,
    :file_ignore          => [/googled355c365863be8d8/, /talkpython/, /changelog/, /infoq/]
  }

  class ProperHTMLCheck < ::HTMLProofer::Check
    VALID_HTML5_ENTITIES = ['copy', 'ldquo', 'rdquo']
    def add_error error
      if error.respond_to?('line') and error.respond_to?('message')
        add_issue "Proper HTML Check: #{error.message}", line: error.line
      else
        add_issue "Proper HTML Check (unknown error): #{error.to_s}", line: 1
      end
    end

    def is_entity_error_and_is_valid? error
      error.is_a? Nokogiri::XML::SyntaxError and error.code == 26 and VALID_HTML5_ENTITIES.include? error.str1
{'fraction_non_alphanumeric': 0.12302607418288652, 'fraction_numerical': 0.015424164524421594, 'mean_word_length': 2.9364161849710984, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947131', 'n_tokens_mistral': 959, 'n_tokens_neox': 910, 'n_words': 180}