<commit_before>require 'HTTParty'
require 'Nokogiri'
require 'JSON'
require 'Pry'
require 'csv'

agent = 'User-Agent'
page = nil

if File.file?('tmp.html')
  page = open('tmp.html', 'r')
else
  page = HTTParty.get('http://www.bjjheroes.com/a-z-bjj-fighters-list',
                      headers: {agent =>
                                    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'})
  target = open('tmp.html', 'w')
  target.write(page)
  target.close
end

parsed_page = Nokogiri::HTML(page)
page.close if page.is_a?(File)

tds = parsed_page.css('.column-2 , .column-1')
tds = tds.drop(2)

tds.map do |i|
  a = i.css('a')
  puts i.text if i.text != ''
  puts a if i.text != ''
  url = a.attribute('href').to_s
  if url.start_with?('http://') && a != nil
    puts url
  elsif url.start_with?('/')
    puts 'http://www.bjjheroes.com' + url
  end
end
#Pry.start(binding)<commit_msg>STDOUT printing fix, ignoring a few verbose prints.<commit_after>require 'HTTParty'
require 'Nokogiri'
require 'JSON'
require 'Pry'
require 'csv'

agent = 'User-Agent'
page = nil

if File.file?('tmp.html')
  page = open('tmp.html', 'r')
else
  page = HTTParty.get('http://www.bjjheroes.com/a-z-bjj-fighters-list',
                      headers: {agent =>
                                    'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'})
  target = open('tmp.html', 'w')
  target.write(page)
  target.close
end

parsed_page = Nokogiri::HTML(page)
page.close if page.is_a?(File)

tds = parsed_page.css('.column-2 , .column-1')
tds = tds.drop(2)

class Athlete
  attr_accessor :array, :name, :last, :url

  def initialize
    @array = []
    @name = ''
    @last = ''
    @url = ''
  end

end

num = 1
tds.map do |i|
  puts i.text if i.text != ''
  if num.even?
    a = i.css('a')
    url = a.attribute('href').to_s
    if url.start_with?('http://') && a != nil
      puts url
    elsif url.start_with?('/')
      puts 'http://www.bjjheroes.com' + url
      puts
      end
  end
  num += 1
end

#Pry.start(binding)
{'fraction_non_alphanumeric': 0.1737689393939394, 'fraction_numerical': 0.02462121212121212, 'mean_word_length': 2.6557093425605536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3573179', 'n_tokens_mistral': 862, 'n_tokens_neox': 837, 'n_words': 183}