<commit_before>class KittenData < ActiveRecord::Base
  belongs_to :response_set

  attr_accessible :data, :url, :response_set

  serialize :data

  after_initialize :request_data, :if => :new_record?

  DATA_LICENCES = {
    "http://opendatacommons.org/licenses/by/" => "odc_by",
    "http://opendatacommons.org/licenses/odbl/" => "odc_odbl",
    "http://opendatacommons.org/licenses/pddl/" => "odc_pddl",
    "http://creativecommons.org/publicdomain/zero/1.0/" => "cc_zero",
    "http://reference.data.gov.uk/id/open-government-licence" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" => "ogl_uk"
  }

  CONTENT_LICENCES = {
    "http://creativecommons.org/licenses/by/2.0/uk/" => "cc_by",
    "http://creativecommons.org/licenses/by-sa/2.0/uk/" => "cc_by_sa",
    "http://creativecommons.org/publicdomain/zero/1.0/" => "cc_zero",
    "http://reference.data.gov.uk/id/open-government-licence" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" => "ogl_uk"
  }

  def dataset
    @dataset
  end

  def distributions
    dataset_field(:distributions, []).map { |distribution|
      {
        :title       => distribution.title,
        :description => distribution.description,
        :access_url  => distribution.access_url,
        :extension   => distribution.format.try(:extension),
        :open        => distribution.format.try(:open?),
        :structured  => distribution.format.try(:structured?)
      }
    }
  end

  def fields
    @fields || compute_fields
  end

  def compute_fields
    return {} if data.blank?

    begin
      @fields = {}

      KittenData.instance_methods(false).each do |method|
        self.send(method) if method.match(/set_[a-z_]+/)
      end
    rescue => ex
      if defined? notify_airbrake
        notify_airbrake ex
      end
    end

    @fields
  end

  def contacts_with_email
    agents = [data.values_at(:publishers, :maintainers, :contributors)].flatten.compact
    agents.select { |contact| contact.mbox.present? }
  end

  def has_data?
    data.present?
  end

  def get(field)
    data[field].presence if has_data?
  end

  def get_list(field)
    get(field) || []
  end

  def uri
    @uri ||= URI(url)
  end

  def hostname
    uri.hostname
  end

  protected

  def package_name
    uri.path.split("/").last
  end

  def set_title
    @fields["dataTitle"] = data[:title]
  end

  def set_publisher
    return unless data[:publishers].any?

    @fields["publisher"] = data[:publishers][0].name
    @fields["publisherUrl"] = data[:publishers][0].homepage
    @fields["contactEmail"] = data[:publishers][0].mbox
  end

  def set_rights
    return unless data[:rights]

    @fields["publisherRights"] = "yes"
    @fields["copyrightURL"] = data[:rights].uri
    @fields["dataLicence"] = KittenData::DATA_LICENCES[data[:rights].dataLicense]
    @fields["contentLicence"] = KittenData::CONTENT_LICENCES[data[:rights].contentLicense]
  end

  def set_license
    return if @fields["dataLicence"]
    return unless data[:licenses].any?

    @fields["publisherRights"] = "yes"
    @fields["dataLicence"] = KittenData::DATA_LICENCES[data[:licenses][0].uri]

    @fields["contentLicence"] = "ogl_uk" if @fields["dataLicence"] == "ogl_uk"

    # Settings for ordnance survey licences
    if data[:licenses][0].uri == "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["dataLicence"] = "other"
      @fields["contentLicence"] = "other"
      @fields["otherDataLicenceName"] = "OS OpenData Licence"
      @fields["otherDataLicenceURL"] = "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["otherDataLicenceOpen"] = "true"
      @fields["otherContentLicenceName"] = "OS OpenData Licence"
      @fields["otherContentLicenceURL"] = "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["otherContentLicenceOpen"] = "true"
    end
  end

  def set_release_type
    if data[:title].include?("API") || data[:description].include?("API")
      @fields["releaseType"] = "service"
    else
      @fields["releaseType"] = check_frequency
    end
  end

  def set_basic_requirements
    # These should probably only be assumed on imports from government portals
    # see discussion on https://github.com/theodi/open-data-certificate/issues/1090
    @fields["publisherOrigin"] = "true"
    @fields["dataPersonal"] = "not_personal"
    @fields["contentRights"] = "samerights"
    @fields["codelists"] = "false"
    @fields["vocabulary"] = "false"
  end

  def set_ckan_assumptions
    # Assumptions for data.gov.uk
    # also for data.london.gov.uk
    # copied from dgu assumptions with confirmation from <NAME> that they seemed sensible
    # and data.gov
    return unless %w[data.gov.uk data.london.gov.uk catalog.data.gov].include?(hostname)

    @fields["copyrightURL"] = url
    @fields["frequentChanges"] = "false"
    @fields["listed"] = "true"
    @fields["listing"] = "#{uri.scheme}://#{hostname}"
    @fields["versionManagement"] = ["list"]
    @fields["versionsUrl"] = "http://#{hostname}/api/rest/package/#{package_name}"
  end

  def set_structured_open
    # Checks if any of the distributions are machine readable or open
    @fields["machineReadable"] = "true" if data[:distributions].detect{|d| d[:structured] }
    @fields["openStandard"] = "true" if data[:distributions].detect{|d| d[:open] }
  end

  def set_metadata
    @fields["documentationMetadata"] = []

    # Does your data documentation contain machine readable documentation for:
    {
      "title" => :title,
      "description" =>  :description,
      "accrualPeriodicity" => :update_frequency,
      "publisher" => :publishers,
      "keyword" => :keywords,
      "distribution" => :distributions,
      "issued" => :release_date,
      "modified" => :modified_date
    }.each do |k,v|
        @fields["documentationMetadata"].push(k) unless data[v].blank?
    end

    @fields["documentationMetadata"].push("temporal") unless data[:temporal_coverage].start.nil? && data[:temporal_coverage].end.nil?
  end

  def check_frequency
    num_distributions = data[:distributions].length
    if data[:update_frequency].empty?
      num_distributions == 1 ? "oneoff" : "collection"
    elsif num_distributions > 1
      "series"
    else
      "oneoff"
    end
  end

  def set_dataset_url
    if check_frequency == "oneoff"
      @fields["datasetUrl"] = data[:distributions][0][:access_url]
    end
  end

  private
  def dataset_field(method, default = nil)
    @dataset.try(method) || default
  end

  def request_data
    @dataset ||= DataKitten::Dataset.new(access_url: url) rescue nil

    if @dataset && @dataset.supported? && dataset_field(:data_title)
      self.data = {
        :title             => dataset_field(:data_title, ''),
        :description       => dataset_field(:description, ''),
        :publishers        => dataset_field(:publishers, []),
        :maintainers       => dataset_field(:maintainers, []),
        :contributors      => dataset_field(:contributors, []),
        :rights            => dataset_field(:rights),
        :licenses          => dataset_field(:licenses, []),
        :update_frequency  => dataset_field(:update_frequency, ''),
        :keywords          => dataset_field(:keywords, []),
        :release_date      => dataset_field(:issued),
        :modified_date     => dataset_field(:modified),
        :temporal_coverage => dataset_field(:temporal, DataKitten::Temporal.new({})),
        :distributions     => distributions
      }
    else
      self.data = {}
    end
  end

end
<commit_msg>Recognise UK OGL v3<commit_after>class KittenData < ActiveRecord::Base
  belongs_to :response_set

  attr_accessible :data, :url, :response_set

  serialize :data

  after_initialize :request_data, :if => :new_record?

  DATA_LICENCES = {
    "http://opendatacommons.org/licenses/by/" => "odc_by",
    "http://opendatacommons.org/licenses/odbl/" => "odc_odbl",
    "http://opendatacommons.org/licenses/pddl/" => "odc_pddl",
    "http://creativecommons.org/publicdomain/zero/1.0/" => "cc_zero",
    "http://reference.data.gov.uk/id/open-government-licence" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" => "ogl_uk"
  }

  CONTENT_LICENCES = {
    "http://creativecommons.org/licenses/by/2.0/uk/" => "cc_by",
    "http://creativecommons.org/licenses/by-sa/2.0/uk/" => "cc_by_sa",
    "http://creativecommons.org/publicdomain/zero/1.0/" => "cc_zero",
    "http://reference.data.gov.uk/id/open-government-licence" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/2/" => "ogl_uk",
    "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" => "ogl_uk"
  }

  def dataset
    @dataset
  end

  def distributions
    dataset_field(:distributions, []).map { |distribution|
      {
        :title       => distribution.title,
        :description => distribution.description,
        :access_url  => distribution.access_url,
        :extension   => distribution.format.try(:extension),
        :open        => distribution.format.try(:open?),
        :structured  => distribution.format.try(:structured?)
      }
    }
  end

  def fields
    @fields || compute_fields
  end

  def compute_fields
    return {} if data.blank?

    begin
      @fields = {}

      KittenData.instance_methods(false).each do |method|
        self.send(method) if method.match(/set_[a-z_]+/)
      end
    rescue => ex
      if defined? notify_airbrake
        notify_airbrake ex
      end
    end

    @fields
  end

  def contacts_with_email
    agents = [data.values_at(:publishers, :maintainers, :contributors)].flatten.compact
    agents.select { |contact| contact.mbox.present? }
  end

  def has_data?
    data.present?
  end

  def get(field)
    data[field].presence if has_data?
  end

  def get_list(field)
    get(field) || []
  end

  def uri
    @uri ||= URI(url)
  end

  def hostname
    uri.hostname
  end

  protected

  def package_name
    uri.path.split("/").last
  end

  def set_title
    @fields["dataTitle"] = data[:title]
  end

  def set_publisher
    return unless data[:publishers].any?

    @fields["publisher"] = data[:publishers][0].name
    @fields["publisherUrl"] = data[:publishers][0].homepage
    @fields["contactEmail"] = data[:publishers][0].mbox
  end

  def set_rights
    return unless data[:rights]

    @fields["publisherRights"] = "yes"
    @fields["copyrightURL"] = data[:rights].uri
    @fields["dataLicence"] = KittenData::DATA_LICENCES[data[:rights].dataLicense]
    @fields["contentLicence"] = KittenData::CONTENT_LICENCES[data[:rights].contentLicense]
  end

  def set_license
    return if @fields["dataLicence"]
    return unless data[:licenses].any?

    @fields["publisherRights"] = "yes"
    @fields["dataLicence"] = KittenData::DATA_LICENCES[data[:licenses][0].uri]

    @fields["contentLicence"] = "ogl_uk" if @fields["dataLicence"] == "ogl_uk"

    # Settings for ordnance survey licences
    if data[:licenses][0].uri == "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["dataLicence"] = "other"
      @fields["contentLicence"] = "other"
      @fields["otherDataLicenceName"] = "OS OpenData Licence"
      @fields["otherDataLicenceURL"] = "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["otherDataLicenceOpen"] = "true"
      @fields["otherContentLicenceName"] = "OS OpenData Licence"
      @fields["otherContentLicenceURL"] = "http://www.ordnancesurvey.co.uk/docs/licences/os-opendata-licence.pdf"
      @fields["otherContentLicenceOpen"] = "true"
    end
  end

  def set_release_type
    if data[:title].include?("API") || data[:description].include?("API")
      @fields["releaseType"] = "service"
    else
      @fields["releaseType"] = check_frequency
    end
  end

  def set_basic_requirements
    # These should probably only be assumed on imports from government portals
    # see discussion on https://github.com/theodi/open-data-certificate/issues/1090
    @fields["publisherOrigin"] = "true"
    @fields["dataPersonal"] = "not_personal"
    @fields["contentRights"] = "samerights"
    @fields["codelists"] = "false"
    @fields["vocabulary"] = "false"
  end

  def set_ckan_assumptions
    # Assumptions for data.gov.uk
    # also for data.london.gov.uk
    # copied from dgu assumptions with confirmation from <NAME> that they seemed sensible
    # and data.gov
    return unless %w[data.gov.uk data.london.gov.uk catalog.data.gov].include?(hostname)

    @fields["copyrightURL"] = url
    @fields["frequentChanges"] = "false"
    @fields["listed"] = "true"
    @fields["listing"] = "#{uri.scheme}://#{hostname}"
    @fields["versionManagement"] = ["list"]
    @fields["versionsUrl"] = "http://#{hostname}/api/rest/package/#{package_name}"
  end

  def set_structured_open
    # Checks if any of the distributions are machine readable or open
    @fields["machineReadable"] = "true" if data[:distributions].detect{|d| d[:structured] }
    @fields["openStandard"] = "true" if data[:distributions].detect{|d| d[:open] }
  end

  def set_metadata
    @fields["documentationMetadata"] = []

    # Does your data documentation contain machine readable documentation for:
    {
      "title" => :title,
      "description" =>  :description,
      "accrualPeriodicity" => :update_frequency,
      "publisher" => :publishers,
      "keyword" => :keywords,
      "distribution" => :distributions,
      "issued" => :release_date,
      "modified" => :modified_date
    }.each do |k,v|
        @fields["documentationMetadata"].push(k) unless data[v].blank?
    end

    @fields["documentationMetadata"].push("temporal") unless data[:temporal_coverage].start.nil? && data[:temporal_coverage].end.nil?
  end

  def check_frequency
    num_distributions = data[:distributions].length
    if data[:update_frequency].empty?
      num_distributions == 1 ? "oneoff" : "collection"
    elsif num_distributions > 1
      "series"
    else
      "oneoff"
    end
  end

  def set_dataset_url
    if check_frequency == "oneoff"
      @fields["datasetUrl"] = data[:distributions][0][:access_url]
    end
  end

  private
  def dataset_field(method, default = nil)
    @dataset.try(method) || default
  end

  def request_data
    @dataset ||= DataKitten::Dataset.new(access_url: url) rescue nil

    if @dataset && @dataset.supported? && dataset_field(:data_title)
      self.data = {
        :title             => dataset_field(:data_title, ''),
        :description       => dataset_field(:description, ''),
        :publishers        => dataset_field(:publishers, []),
        :maintainers       => dataset_field(:maintainers, []),
        :contributors      => dataset_field(:contributors, []),
        :rights            => dataset_field(:rights),
        :licenses          => dataset_field(:licenses, []),
        :update_frequency  => dataset_field(:update_frequency, ''),
        :keywords          => dataset_field(:keywords, []),
        :release_date      => dataset_field(:issued),
        :modified_date     => dataset_field(:modified),
        :temporal_coverage => dataset_field(:temporal, DataKitten::Temporal.new({})),
        :distributions     => distributions
      }
    else
      self.data = {}
    end
  end

end

{'fraction_non_alphanumeric': 0.14870801033591732, 'fraction_numerical': 0.003036175710594315, 'mean_word_length': 3.333986562150056, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 89, 'https://': 2, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '630179', 'n_tokens_mistral': 5124, 'n_tokens_neox': 4906, 'n_words': 867}