<commit_before>require 'atdisplanningalertsfeed/version'
require 'atdis'
require 'scraperwiki-morph'

module ATDISPlanningAlertsFeed
  def self.save(url, options = {})
    feed = ATDIS::Feed.new(url)
    options[:lodgement_date_start] = (options[:lodgement_date_start] || Date.today - 30)
    options[:lodgement_date_end] = (options[:lodgement_date_end] || Date.today)
    page = feed.applications(lodgement_date_start: options[:lodgement_date_start], lodgement_date_end: options[:lodgement_date_end])

    # Save the first page
    save_page(page)

    while page = page.next_page
      save_page(page)
    end
  end

  def self.save_page(page)
    puts "Saving page #{page.pagination.current} of #{page.pagination.pages}"

    page.response.each do |item|
      application = item.application

      # TODO: Only using the first address because PA doesn't support multiple addresses right now
      address = application.locations.first.address.street + ', ' +
                application.locations.first.address.suburb + ', ' +
                application.locations.first.address.state  +
                application.locations.first.address.postcode

      record = {
        council_reference: application.info.dat_id,
        address:           address,
        description:       application.info.description,
        info_url:          application.reference.more_info_url.to_s,
        comment_url:       application.reference.comments_url.to_s,
        date_scraped:      Date.today,
        date_received:     application.info.lodgement_date,
        on_notice_from:    application.info.notification_start_date,
        on_notice_to:      application.info.notification_end_date
      }

      if (ScraperWikiMorph.select("* from data where `council_reference`='#{record[:council_reference]}'").empty? rescue true)
        ScraperWikiMorph.save_sqlite([:council_reference], record)
      else
        puts "Skipping already saved record " + record[:council_reference]
      end
    end
  end
end<commit_msg>Unescape council_reference to make it a bit more readable<commit_after>require 'atdisplanningalertsfeed/version'
require 'atdis'
require 'scraperwiki-morph'
require 'cgi'

module ATDISPlanningAlertsFeed
  def self.save(url, options = {})
    feed = ATDIS::Feed.new(url)
    options[:lodgement_date_start] = (options[:lodgement_date_start] || Date.today - 30)
    options[:lodgement_date_end] = (options[:lodgement_date_end] || Date.today)
    page = feed.applications(lodgement_date_start: options[:lodgement_date_start], lodgement_date_end: options[:lodgement_date_end])

    # Save the first page
    save_page(page)

    while page = page.next_page
      save_page(page)
    end
  end

  def self.save_page(page)
    puts "Saving page #{page.pagination.current} of #{page.pagination.pages}"

    page.response.each do |item|
      application = item.application

      # TODO: Only using the first address because PA doesn't support multiple addresses right now
      address = application.locations.first.address.street + ', ' +
                application.locations.first.address.suburb + ', ' +
                application.locations.first.address.state  +
                application.locations.first.address.postcode

      record = {
        council_reference: CGI.unescape(application.info.dat_id),
        address:           address,
        description:       application.info.description,
        info_url:          application.reference.more_info_url.to_s,
        comment_url:       application.reference.comments_url.to_s,
        date_scraped:      Date.today,
        date_received:     application.info.lodgement_date,
        on_notice_from:    application.info.notification_start_date,
        on_notice_to:      application.info.notification_end_date
      }

      if (ScraperWikiMorph.select("* from data where `council_reference`='#{record[:council_reference]}'").empty? rescue true)
        ScraperWikiMorph.save_sqlite([:council_reference], record)
      else
        puts "Skipping already saved record " + record[:council_reference]
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09483178937103852, 'fraction_numerical': 0.0009751340809361287, 'mean_word_length': 3.161257606490872, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3842576', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1193, 'n_words': 225}