<commit_before>require 'spec_helper'

describe Distiller::Distil do

  before(:each) do
    @address =  {
      "saon" => {
        "name" => nil
      },
      "paon" => {
        "name" => "123"
      },
      "street" => {
        "name" => "TEST ROAD",
        "geometry" => {
          "type" => "Point",
          "coordinates" => [406043, 286921]
        }
      },
      "locality" => {
        "name" => "KINGS HEATH"
      },
      "town" => {
        "name" => "BIRMINGHAM"
      },
      "postcode" => {
        "name" => "B1 2NN",
        "geometry" => {
          "type" => "Point",
          "coordinates" => [406137,286927]
        }
      }
    }

  end

  context "distiller" do

    before(:each) do
      json = JSON.parse(File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")))

      json['addresses'].each do |address|
        FactoryGirl.create(:street, name: address['street']['name'], easting_northing: [address['street']['geometry']['coordinates'][1], address['street']['geometry']['coordinates'][0]] )
        FactoryGirl.create(:locality, name: address['locality']['name']) unless address['locality']['name'].blank?
        FactoryGirl.create(:town, name: address['town']['name'])
        FactoryGirl.create(:postcode, name: address['postcode']['name'])
      end
    end

    it "imports one page of addresses" do
      Timecop.freeze("2014-01-01")
      allow(Distiller::Distil).to receive(:current_sha).and_return("sdasdasdasd")

      stub_request(:get, ENV['ERNEST_ADDRESS_ENDPOINT']).
        to_return(body: File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")),
                  headers: {"Content-Type" => "application/json"})

      stub_request(:get, "#{ENV['ERNEST_ADDRESS_ENDPOINT']}?page=1").
        to_return(body: File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")),
                  headers: {"Content-Type" => "application/json"})

      Distiller::Distil.perform

      expect(Address.count).to eq 25

      address = Address.where("street.name" => "<NAME>").first

      expect(address.pao).to eq("57")
      expect(address.street.name).to eq("<NAME>")
      expect(address.town.name).to eq("LEEDS")
      expect(address.postcode.name).to eq("LS10 2RU")
      expect(address.provenance).to eq({
        "activity" => {
          "executed_at" => Time.parse("2014-01-01").utc,
          "processing_scripts" => "https://github.com/OpenAddressesUK/distiller",
          "derived_from" => [
            {
              "type" => "Source",
              "urls" => [
                "http://ernest.openaddressesuk.org/addresses/1"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {
              "type" => "Source",
              "urls" => [
                "http://alpha.openaddressesuk.org/postcodes/#{address.postcode.token}"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {
              "type" => "Source",
              "urls" => [
                "http://alpha.openaddressesuk.org/streets/#{address.street.token}"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {<commit_msg>Check source is recorded correctly<commit_after>require 'spec_helper'

describe Distiller::Distil do

  before(:each) do
    @address =  {
      "saon" => {
        "name" => nil
      },
      "paon" => {
        "name" => "123"
      },
      "street" => {
        "name" => "TEST ROAD",
        "geometry" => {
          "type" => "Point",
          "coordinates" => [406043, 286921]
        }
      },
      "locality" => {
        "name" => "KINGS HEATH"
      },
      "town" => {
        "name" => "BIRMINGHAM"
      },
      "postcode" => {
        "name" => "B1 2NN",
        "geometry" => {
          "type" => "Point",
          "coordinates" => [406137,286927]
        }
      }
    }

  end

  context "distiller" do

    before(:each) do
      json = JSON.parse(File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")))

      json['addresses'].each do |address|
        FactoryGirl.create(:street, name: address['street']['name'], easting_northing: [address['street']['geometry']['coordinates'][1], address['street']['geometry']['coordinates'][0]] )
        FactoryGirl.create(:locality, name: address['locality']['name']) unless address['locality']['name'].blank?
        FactoryGirl.create(:town, name: address['town']['name'])
        FactoryGirl.create(:postcode, name: address['postcode']['name'])
      end
    end

    it "imports one page of addresses" do
      Timecop.freeze("2014-01-01")
      allow(Distiller::Distil).to receive(:current_sha).and_return("sdasdasdasd")

      stub_request(:get, ENV['ERNEST_ADDRESS_ENDPOINT']).
        to_return(body: File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")),
                  headers: {"Content-Type" => "application/json"})

      stub_request(:get, "#{ENV['ERNEST_ADDRESS_ENDPOINT']}?page=1").
        to_return(body: File.read(File.join(File.dirname(__FILE__), "fixtures", "one-page.json")),
                  headers: {"Content-Type" => "application/json"})

      Distiller::Distil.perform

      expect(Address.count).to eq 25

      address = Address.where("street.name" => "PEPPER ROAD").first

      expect(address.pao).to eq("57")
      expect(address.street.name).to eq("PE<NAME>AD")
      expect(address.town.name).to eq("LEEDS")
      expect(address.postcode.name).to eq("LS10 2RU")
      expect(address.source).to eq("Source")
      expect(address.provenance).to eq({
        "activity" => {
          "executed_at" => Time.parse("2014-01-01").utc,
          "processing_scripts" => "https://github.com/OpenAddressesUK/distiller",
          "derived_from" => [
            {
              "type" => "Source",
              "urls" => [
                "http://ernest.openaddressesuk.org/addresses/1"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {
              "type" => "Source",
              "urls" => [
                "http://alpha.openaddressesuk.org/postcodes/#{address.postcode.token}"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {
              "type" => "Source",
              "urls" => [
                "http://alpha.openaddressesuk.org/streets/#{address.street.token}"
              ],
              "downloaded_at" => Time.parse("2014-01-01").utc,
              "processing_script" => "https://github.com/OpenAddressesUK/distiller/tree/sdasdasdasd/lib/distil.rb"
            },
            {
{'fraction_non_alphanumeric': 0.17975065077407865, 'fraction_numerical': 0.02192081107000959, 'mean_word_length': 2.2017543859649122, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 80, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3350942', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2289, 'n_words': 303}