<commit_before>require 'rubygems'
require 'mechanize'

def usage
  puts <<USAGE
  #{__FILE__} parses a site for you

  Usage:
    #{__FILE__} <site> [<output pdf file>]
    
USAGE
end

def get_uris(site)
  agent = Mechanize.new
  visited = []
  to_visit = [site]
  while !to_visit.empty?
    visited << site
    to_visit.delete(site)
    agent.get(site).links.each do |link|
      href = link.href
      if href != '/' and !URI.parse(href).host and !visited.include?(href) and !to_visit.include?(href)
        to_visit << href
      end
    end
    site = to_visit.shift
  end
  visited
end

site = ARGV[0]

if not site
  usage
  exit
end

if not site =~ /^https?:\/\//
  puts "Neither http or https protocol detected, using http"
  site = "http://" + site
end

if not URI.parse(site)
  puts "#{site} is not a valid URI"
  exit(false)
end
<commit_msg>Credits to "the Tin man" at stack overflow<commit_after>require 'rubygems'
require 'mechanize'

def usage
  puts <<USAGE
  #{__FILE__} parses a site for you

  Usage:
    #{__FILE__} <site> [<output pdf file>]
    
USAGE
end

def get_uris(site)
  agent = Mechanize.new
  visited = []
  to_visit = [site]
  while !to_visit.empty?
    visited << site
    to_visit.delete(site)
    agent.get(site).links.each do |link|
      href = link.href
      # URI.parse tips at http://stackoverflow.com/q/2719009/49879
      if href != '/' and !URI.parse(href).host and !visited.include?(href) and !to_visit.include?(href)
        to_visit << href
      end
    end
    site = to_visit.shift
  end
  visited
end

site = ARGV[0]

if not site
  usage
  exit
end

if not site =~ /^https?:\/\//
  puts "Neither http or https protocol detected, using http"
  site = "http://" + site
end

if not URI.parse(site)
  puts "#{site} is not a valid URI"
  exit(false)
end

{'fraction_non_alphanumeric': 0.12513904338153503, 'fraction_numerical': 0.00778642936596218, 'mean_word_length': 2.844017094017094, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6361921', 'n_tokens_mistral': 670, 'n_tokens_neox': 669, 'n_words': 197}