<commit_before>require 'statsample'
require 'uk_postcode'

class Confidence < ActiveRecord::Base

  belongs_to :left, :class_name => 'Tag'
  belongs_to :right, :class_name => 'Tag'

  validates :value, presence: true
  validates :left,  presence: true
  validates :right, presence: true

  after_initialize :calculate_value

  private

  def calculate_value
    # If we already have a value, EJECT
    return if value
    # Work that shit out
    send(:"calculate_for_#{left.tag_type.label}_and_#{right.tag_type.label}", left, right)
  end

  def calculate_for_postcode_and_town(postcode, town)
    calculate_for_town_and_postcode(town, postcode)
  end

  def calculate_for_town_and_postcode(town, postcode)
    pc = UKPostcode.parse(postcode.label)
    sector = "#{pc.outcode} #{pc.sector}"
    postcodes = Tag.where("label LIKE ?", "#{sector}%")
    addresses = postcodes.map { |p| p.addresses }.flatten
    town_count = addresses.select { |a| a.town.label == town.label }.count
    pc_count = postcodes.count
    confidence(town_count, pc_count)
  end
  
  def calculate_for_postcode_and_street(postcode, street)
    calculate_for_street_and_postcode(street, postcode)
  end

  def calculate_for_street_and_postcode(street, postcode)
    postcodes = Tag.where(label: postcode.label)
    addresses = postcodes.map { |p| p.addresses }.flatten
    street_count = addresses.select { |a| a.street.label == street.label }.count
    pc_count = postcodes.count
    confidence(street_count, pc_count)
  end

  def confidence(number, total)
    pc = number.to_f / total.to_f
    standard_error = Math.sqrt(0.25/total.to_f)
    tval = Statsample::Test.t_critical(0.95,total.to_f)
    margin = standard_error * tval
    if pc > 0.0
      self.value = [1-(margin/pc),0.0].max
    else
      self.value = 0.0
    end
  end


end<commit_msg>refactor for stubbing<commit_after>require 'statsample'
require 'uk_postcode'

class Confidence < ActiveRecord::Base

  belongs_to :left, :class_name => 'Tag'
  belongs_to :right, :class_name => 'Tag'

  validates :value, presence: true
  validates :left,  presence: true
  validates :right, presence: true

  after_initialize :calculate_value

  private

  def calculate_value
    # If we already have a value, EJECT
    return if value
    # Work that shit out
    send(:"calculate_for_#{left.tag_type.label}_and_#{right.tag_type.label}", left, right)
  end

  def calculate_for_postcode_and_town(postcode, town)
    calculate_for_town_and_postcode(town, postcode)
  end

  def calculate_for_town_and_postcode(town, postcode)
    postcodes = get_postcodes_in_sector(postcode)
    addresses = addresses_from_postcodes(postcodes)
    town_count = towns_from_addresses(addresses, town)
    pc_count = postcodes.count
    confidence(town_count, pc_count)
  end
  
  def calculate_for_postcode_and_street(postcode, street)
    calculate_for_street_and_postcode(street, postcode)
  end

  def calculate_for_street_and_postcode(street, postcode)
    postcodes = get_postcodes(postcode)
    addresses = addresses_from_postcodes(postcodes)
    street_count = streets_from_addresses(addresses, street)
    pc_count = postcodes.count
    confidence(street_count, pc_count)
  end

  def confidence(number, total)
    pc = number.to_f / total.to_f
    standard_error = Math.sqrt(0.25/total.to_f)
    tval = Statsample::Test.t_critical(0.95,total.to_f)
    margin = standard_error * tval
    if pc > 0.0
      self.value = [1-(margin/pc),0.0].max
    else
      self.value = 0.0
    end
  end
  
  # Split out so we can stub and test easily

  def get_postcodes(postcode)
    postcodes = Tag.where(label: postcode.label)
  end

  def get_postcodes_in_sector(postcode)
    pc = UKPostcode.parse(postcode.label)
    sector = "#{pc.outcode} #{pc.sector}"
    postcodes = Tag.where("label LIKE ?", "#{sector}%")
  end

  def streets_from_addresses(addresses, street)
    addresses.select { |a| a.street.label == street.label }.count
  end

  def towns_from_addresses(addresses, town)
    addresses.select { |a| a.town.label == town.label }.count
  end

  def addresses_from_postcodes(postcodes)
    postcodes.map { |p| p.addresses }.flatten
  end

end
{'fraction_non_alphanumeric': 0.10086455331412104, 'fraction_numerical': 0.006243996157540826, 'mean_word_length': 3.9524375743162903, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1647806', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1434, 'n_words': 305}