<commit_before># Have fun. Use at your own risk.
# Copyright (c) 2013 <NAME>

require 'rubygems'
require 'fileutils'
require 'net/http'

begin
  require 'mechanize'
  require 'json'
  require 'highline/import'
rescue LoadError => e
  puts
  puts "You need to have the mechanize, json and highline gems installed."
  puts "Install them by running"
  puts
  puts "  bundle install"
  puts
  puts "and then run the script like this:"
  puts
  puts "  bundle exec ruby lib/wwdcdownloader.rb"
  puts
  exit
end

class WWDCDownloader
  BASE_URI = 'https://developer.apple.com/wwdc-services/cy4p09ns/a4363cb15472b00287b/sessions.json'

  WWDC_LIBRARIES = [{:base => 'https://developer.apple.com/library/prerelease/ios', :lib => '/navigation/library.json'}, 
                    {:base => 'https://developer.apple.com/library/prerelease/mac', :lib => '/navigation/library.json'}]
  
  attr_accessor :downloaded_files, :dl_dir, :mech, :min_date
  
  def initialize(dl_dir, min_date)
    self.dl_dir = dl_dir
    self.min_date = min_date
    self.mech = Mechanize.new
    self.downloaded_files = []
  end

  # Creates the given directory if it doesn't exist already.
  def mkdir(dir)
    if File.exists?(dir)
      false
    else
      Dir.mkdir dir 
      true
    end
  end

  # Login<commit_msg>Turn off SSL certificate validation in Mechanizer.<commit_after># Have fun. Use at your own risk.
# Copyright (c) 2013 <NAME>

require 'rubygems'
require 'fileutils'
require 'net/http'

begin
  require 'mechanize'
  require 'json'
  require 'highline/import'
rescue LoadError => e
  puts
  puts "You need to have the mechanize, json and highline gems installed."
  puts "Install them by running"
  puts
  puts "  bundle install"
  puts
  puts "and then run the script like this:"
  puts
  puts "  bundle exec ruby lib/wwdcdownloader.rb"
  puts
  exit
end

class WWDCDownloader
  BASE_URI = 'https://developer.apple.com/wwdc-services/cy4p09ns/a4363cb15472b00287b/sessions.json'

  WWDC_LIBRARIES = [{:base => 'https://developer.apple.com/library/prerelease/ios', :lib => '/navigation/library.json'}, 
                    {:base => 'https://developer.apple.com/library/prerelease/mac', :lib => '/navigation/library.json'}]
  
  attr_accessor :downloaded_files, :dl_dir, :mech, :min_date
  
  def initialize(dl_dir, min_date)
    self.dl_dir = dl_dir
    self.min_date = min_date
    self.mech = Mechanize.new
    self.mech.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    self.downloaded_files = []
  end

  # Creates the given directory if it doesn't exist already.
  def mkdir(dir)
    if File.exists?(dir)
      false
    else
      Dir.mkdir dir 
      true
    end
  end

  # Login
{'fraction_non_alphanumeric': 0.10828976848394324, 'fraction_numerical': 0.01568334578043316, 'mean_word_length': 3.2322274881516586, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5861779', 'n_tokens_mistral': 962, 'n_tokens_neox': 921, 'n_words': 243}