<commit_before>require 'rubygems'
require 'bundler/setup'
require 'nokogiri'

require 'net/http'
require 'uri'

desc "Verifies all of the links in the generated documentation"
namespace :vivisection do
  task :verify_links do

    doc_path = Rake.original_dir + "/docs"
    paths = []

    Dir[doc_path + "/**/*.html"].each do |doc|
      Nokogiri::HTML.parse( File.read(doc) ).css('a').each do |link|
        path = link.attribute('href').to_s

        unless path =~ /^#/
          path = File.expand_path( File.dirname(doc) + "/" + path ) unless path =~ /^http/
          paths << path unless paths.include?(path)
        end
      end
    end

    failures = []

    paths.each do |path|
      if path =~ /^http/
        is_live = live_url?(path)
        case is_live
        when true then ok
        when false then fail(path)
        else
          error(path, is_live)
        end
      else
        if File.exists?( path )
          ok
        else
          fail path
        end
      end
    end

    report @@failures, "Paths that did not exists"
    report @@errors, "Paths that generated an error"

    puts "\n"

  end
end

private

def ok
  print "."
  STDOUT.flush
end

@@failures = []
@@errors = []

def fail( path )
  @@failures << path
  print "F"
  STDOUT.flush
end

def error( path, message )
  @@errors << [ path, message ]
  print "E"
  STDOUT.flush
end

def report( paths, message )
  unless paths.empty?
    puts "\n\n"
    puts "#{message}:"
    y paths
  end
end

def live_url?(url)
  uri = URI.parse(url)
  response = nil
  begin
    Net::HTTP.start(uri.host, uri.port) do |http|
      response = http.head(uri.path.size > 0 ? uri.path : "/")
    end
  rescue
    $!.message
  else
    response.code =~ /(200)?(300)?/ ? true : false
  end
end<commit_msg>Added some better link verification logic and reporting.<commit_after>require 'rubygems'
require 'bundler/setup'
require 'nokogiri'

require 'net/http'
require 'uri'

@@doc_path = Rake.original_dir + "/docs"
@@failures = []
@@errors = []

desc "Verifies all of the links in the generated documentation"
namespace :vivisection do
  task :verify_links do

    paths = {}

    Dir[@@doc_path + "/**/*.html"].each do |doc|
      Nokogiri::HTML.parse( File.read(doc) ).css('a').each do |link|
        path = link.attribute('href').to_s

        unless path =~ /^#/
          path = File.expand_path( File.dirname(doc) + "/" + path ) unless path =~ /^http/
          paths[path] = doc unless paths.has_key?(path)
        end
      end
    end

    failures = []

    paths.each do |path, doc|
      if path =~ /^http/
        is_live = live_url?(path)
        case is_live
        when true then ok
        when false then fail(path, doc)
        else
          error(path, doc, is_live)
        end
      else
        if File.exists?( path )
          ok
        else
          fail(path, doc)
        end
      end
    end

    report @@failures, "Paths that did not exist"
    report @@errors, "Paths that generated an error"

    puts "\n"

  end
end

private

def ok
  print "."
  STDOUT.flush
end

def fail( path, doc )
  @@failures << [ path, doc ]
  print "F"
  STDOUT.flush
end

def error( path, doc, message )
  @@errors << [ path, doc, message ]
  print "E"
  STDOUT.flush
end

def report( paths, message )
  unless paths.empty?
    puts "\n\n"
    puts "#{message}:"
    paths.each_with_index do |(path, doc), index|
      path = trim_root( path, Rake.original_dir )
      doc  = trim_root( doc, Rake.original_dir )
      puts "#{index}: #{doc} is linking to #{path}"
    end
  end
end

def live_url?(url)
  uri = URI.parse(url)
  response = nil
  begin
    Net::HTTP.start(uri.host, uri.port) do |http|
      response = http.head(uri.path.size > 0 ? uri.path : "/")
    end
  rescue
    $!.message
  else
    response.code =~ /(200)?(300)?/ ? true : false
  end
end

def trim_root(abs_path, relative_to)
  abs_path.gsub(relative_to + "/", '')
end
{'fraction_non_alphanumeric': 0.1315722631445263, 'fraction_numerical': 0.003556007112014224, 'mean_word_length': 2.2545454545454544, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6694419', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1345, 'n_words': 389}