<commit_before>require 'spec_helper'

describe Confidence do

  context "testing the basics" do

    before :all do
      # Make a pair of tags
      pc_type = FactoryGirl.create(:tag_type, label: "postcode")
      town_type = FactoryGirl.create(:tag_type, label: "town")
      @pc = FactoryGirl.create(:tag, label: "XX1 1XX", tag_type: pc_type)
      @town = FactoryGirl.create(:tag, label: "Fooville", tag_type: town_type)
    end

    it "can have a manually-set value" do
      # Valid
      c = Confidence.new(value: 0.85, left: @pc, right: @town)
      expect(c.value).to eq 0.85
      expect(c.valid?).to eq true
    end

    it "must have a value" do
      c = Confidence.new(left: @pc, right: @town)
      c.value = nil
      expect(c.valid?).to eq false
      expect(c.errors[:value]).to eq ["can't be blank"]
    end

    it "can relate the two together" do
      # Relate them together
      c = Confidence.new(value: 0.85, left: @pc, right: @town)
      # Check the relation
      expect(c.left).to eq @pc
      expect(c.right).to eq @town
      expect(c.valid?).to eq true
    end

    it "must have both sides" do
      # Relate them together
      c = Confidence.new(value: 0.85, left: nil, right: nil)
      # Check the relation
      expect(c.valid?).to eq false
      expect(c.errors[:left]).to eq ["can't be blank"]
      expect(c.errors[:right]).to eq ["can't be blank"]
    end

  end

  context "creating confidence measures" do

    it "should be able to create a confidence measure given a pair of tags" do
      # Make a pair of tags
      pc_type = FactoryGirl.create(:tag_type, label: "postcode")
      town_type = FactoryGirl.create(:tag_type, label: "town")
      pc = FactoryGirl.create(:tag, label: "XX1 1XX", tag_type: pc_type)
      town = FactoryGirl.create(:tag, label: "Fooville", tag_type: town_type)
      # Create the confidence measure with just the tags
      c = Confidence.new(left: pc, right: town)
      expect(c.value).to be_present
      expect(c.valid?).to eq true
      # Confidence should have generated a value for us
      expect(c.value).to be_present
    end

    context "with a load of data" do
      before :all do
        35.times do |n|
          FactoryGirl.create(:address, tags: [
            FactoryGirl.create(:tag, label: "SW1A 1AA", tag_type: FactoryGirl.create(:tag_type, label: "postcode")),
            FactoryGirl.create(:tag, label: "The Shire", tag_type: FactoryGirl.create(:tag_type, label: "town"), point: "POINT (309250 411754)"),
            FactoryGirl.create(:tag, label: "Hobbitton", tag_type: FactoryGirl.create(:tag_type, label: "locality")),
            FactoryGirl.create(:tag, label: "Hobbit Drive", tag_type: FactoryGirl.create(:tag_type, label: "street")),
            FactoryGirl.create(:tag, label: n, tag_type: FactoryGirl.create(:tag_type, label: "paon"))
          ])
        end
      end

      it "should calculate confidence between postcodes and towns" do
        pc = Tag.where(label: "SW1A 1AA").last
        town = Tag.where(label: "The Shire").last
        c = Confidence.new(left: pc, right: town)
        expect(c.value).to be_within(0.0001).of(0.8284)
      end

      it "should calculate confidence between towns and postcodes" do
        pc = Tag.where(label: "SW1A 1AA").last
        town = Tag.where(label: "The Shire").last
        c = Confidence.new(left: town, right: pc)
        expect(c.value).to be_within(0.0001).of(0.8284)
      end

      it "should calculate confidence between postcodes and streets" do
        pc = Tag.where(label: "SW1A 1AA").last
        street = Tag.where(label: "Hobbit Drive").last
        c = Confidence.new(left: pc, right: street)
        expect(c.value).to be_within(0.0001).of(0.8284)
      end

      it "should calculate confidence between streets and postcodes" do
        pc = Tag.where(label: "SW1A 1AA").last
        street = Tag.where(label: "Hobbit Drive").last
        c = Confidence.new(left: street, right: pc)
        expect(c.value).to be_within(0.0001).of(0.8284)
      end

    end

  end

  context "with stubbed data" do
<commit_msg>Tweak expectations<commit_after>require 'spec_helper'

describe Confidence do

  context "testing the basics" do

    before :all do
      # Make a pair of tags
      pc_type = FactoryGirl.create(:tag_type, label: "postcode")
      town_type = FactoryGirl.create(:tag_type, label: "town")
      @pc = FactoryGirl.create(:tag, label: "XX1 1XX", tag_type: pc_type)
      @town = FactoryGirl.create(:tag, label: "Fooville", tag_type: town_type)
    end

    it "can have a manually-set value" do
      # Valid
      c = Confidence.new(value: 0.85, left: @pc, right: @town)
      expect(c.value).to eq 0.85
      expect(c.valid?).to eq true
    end

    it "must have a value" do
      c = Confidence.new(left: @pc, right: @town)
      c.value = nil
      expect(c.valid?).to eq false
      expect(c.errors[:value]).to eq ["can't be blank"]
    end

    it "can relate the two together" do
      # Relate them together
      c = Confidence.new(value: 0.85, left: @pc, right: @town)
      # Check the relation
      expect(c.left).to eq @pc
      expect(c.right).to eq @town
      expect(c.valid?).to eq true
    end

    it "must have both sides" do
      # Relate them together
      c = Confidence.new(value: 0.85, left: nil, right: nil)
      # Check the relation
      expect(c.valid?).to eq false
      expect(c.errors[:left]).to eq ["can't be blank"]
      expect(c.errors[:right]).to eq ["can't be blank"]
    end

  end

  context "creating confidence measures" do

    it "should be able to create a confidence measure given a pair of tags" do
      # Make a pair of tags
      pc_type = FactoryGirl.create(:tag_type, label: "postcode")
      town_type = FactoryGirl.create(:tag_type, label: "town")
      pc = FactoryGirl.create(:tag, label: "XX1 1XX", tag_type: pc_type)
      town = FactoryGirl.create(:tag, label: "Fooville", tag_type: town_type)
      # Create the confidence measure with just the tags
      c = Confidence.new(left: pc, right: town)
      expect(c.value).to be_present
      expect(c.valid?).to eq true
      # Confidence should have generated a value for us
      expect(c.value).to be_present
    end

    context "with a load of data" do
      before :all do
        35.times do |n|
          FactoryGirl.create(:address, tags: [
            FactoryGirl.create(:tag, label: "SW1A 1AA", tag_type: FactoryGirl.create(:tag_type, label: "postcode")),
            FactoryGirl.create(:tag, label: "The Shire", tag_type: FactoryGirl.create(:tag_type, label: "town"), point: "POINT (309250 411754)"),
            FactoryGirl.create(:tag, label: "Hobbitton", tag_type: FactoryGirl.create(:tag_type, label: "locality")),
            FactoryGirl.create(:tag, label: "Hobbit Drive", tag_type: FactoryGirl.create(:tag_type, label: "street")),
            FactoryGirl.create(:tag, label: n, tag_type: FactoryGirl.create(:tag_type, label: "paon"))
          ])
        end
      end

      it "should calculate confidence between postcodes and towns" do
        pc = Tag.where(label: "SW1A 1AA").last
        town = Tag.where(label: "The Shire").last
        c = Confidence.new(left: pc, right: town)
        expect(c.value).to be_within(0.0001).of(0.7599)
      end

      it "should calculate confidence between towns and postcodes" do
        pc = Tag.where(label: "SW1A 1AA").last
        town = Tag.where(label: "The Shire").last
        c = Confidence.new(left: town, right: pc)
        expect(c.value).to be_within(0.0001).of(0.7599)
      end

      it "should calculate confidence between postcodes and streets" do
        pc = Tag.where(label: "SW1A 1AA").last
        street = Tag.where(label: "Hobbit Drive").last
        c = Confidence.new(left: pc, right: street)
        expect(c.value).to be_within(0.0001).of(0.7599)
      end

      it "should calculate confidence between streets and postcodes" do
        pc = Tag.where(label: "SW1A 1AA").last
        street = Tag.where(label: "Hobbit Drive").last
        c = Confidence.new(left: street, right: pc)
        expect(c.value).to be_within(0.0001).of(0.7599)
      end

    end

  end

  context "with stubbed data" do

{'fraction_non_alphanumeric': 0.12339958541641263, 'fraction_numerical': 0.01950981587611267, 'mean_word_length': 2.7012635379061374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078633', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2594, 'n_words': 779}