<commit_before>require "byebug"
require "capybara/poltergeist"
require "scraperwiki"

VERSION = "0.0.1"

@session = Capybara::Session.new(:poltergeist)
@session.driver.headers = {"User-Agent" => "Morph.io Scraper https://github.com/tatey/new-zealand-radio-spectrum-licenses (Scaper #{VERSION}) (Ruby #{RUBY_VERSION}/#{RUBY_PLATFORM})"}
@logger = Logger.new(STDOUT)

@page = 0
@licences = {}

def scrape_search_page
  @logger.info "Scraping search page..."

  @session.visit("https://www.rsm.govt.nz/smart-web/smart/page/-smart/domain/licence/SelectLicencePage.wdk?showExit=Yes")
  districts_select = @session.find("#multi-district-select", visible: false)
  districts_select.all("option", visible: false).each do |district_option|
    text = district_option.text(:all)
    if !["New Zealand", "Overseas"].include?(text)
      districts_select.select(text, visible: false)
    end
  end
  @session.find(".formButton[title=Search]").click # Search
end

def scrape_list_page
  @logger.info "Scraping page #{@page += 1}..."

  @session.find(".listTable").all("tbody tr")[1..-1].map do |tr|
    tr.all("td").map(&:text)
  end.each do |values|<commit_msg>Suppress noisy phantomjs logger.<commit_after>require "byebug"
require "capybara/poltergeist"
require "scraperwiki"

VERSION = "0.0.1"

Capybara.register_driver :poltergeist_with_suppressed_logger do |app|
  Capybara::Poltergeist::Driver.new(app, phantomjs_logger: StringIO.new)
end

@session = Capybara::Session.new(:poltergeist_with_suppressed_logger)
@session.driver.headers = {"User-Agent" => "Morph.io Scraper https://github.com/tatey/new-zealand-radio-spectrum-licenses (Scaper #{VERSION}) (Ruby #{RUBY_VERSION}/#{RUBY_PLATFORM})"}
@logger = Logger.new(STDOUT)

@page = 0
@licences = {}

def scrape_search_page
  @logger.info "Scraping search page..."

  @session.visit("https://www.rsm.govt.nz/smart-web/smart/page/-smart/domain/licence/SelectLicencePage.wdk?showExit=Yes")
  districts_select = @session.find("#multi-district-select", visible: false)
  districts_select.all("option", visible: false).each do |district_option|
    text = district_option.text(:all)
    if !["New Zealand", "Overseas"].include?(text)
      districts_select.select(text, visible: false)
    end
  end
  @session.find(".formButton[title=Search]").click # Search
end

def scrape_list_page
  @logger.info "Scraping page #{@page += 1}..."

  @session.find(".listTable").all("tbody tr")[1..-1].map do |tr|
    tr.all("td").map(&:text)
  end.each do |values|
{'fraction_non_alphanumeric': 0.17081993569131831, 'fraction_numerical': 0.005627009646302251, 'mean_word_length': 5.206982543640898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5160487', 'n_tokens_mistral': 911, 'n_tokens_neox': 883, 'n_words': 148}