<commit_before>class ApplicationImporter

  attr_accessor :options

  def logger
    @logger = Rails.logger
  end

  def initialize(options={})
    self.options = HashWithIndifferentAccess.new(options)
  end

  def source_name
    raise NotImplementedError.new("Subclass must implement source_name")
  end

  def distributor_tag
    ENV['DISTRIBUTOR_TAG'] || 'PRX'
  end

  def import(options)
    self.options.merge!(options)
  end

  def pmp
    @pmp ||= PMP::Client.new(client_id: pmp_client_id, client_secret: pmp_client_secret, endpoint: pmp_endpoint)
  end

  def reset_pmp
    @pmp = nil
  end

  def pmp_client_id
    options[:pmp_client_id] || ENV['PMP_CLIENT_ID']
  end

  def pmp_client_secret
    options[:pmp_client_secret] || ENV['PMP_CLIENT_SECRET']
  end

  def pmp_endpoint
    options[:pmp_endpoint] || ENV['PMP_ENDPOINT'] || 'https://api.pmp.io/'
  end

  def pmp_url(*path)
    URI.join(pmp_endpoint, *path.collect(&:to_s).join('/')).to_s
  end

  def pmp_doc_by_guid(guid)
    response = pmp.query["urn:collectiondoc:hreftpl:docs"].where(guid: guid).get

    response = nil if (response.response.raw['status'] == '404')
    response = nil if response.response.raw['body'].blank?

    response
  end

  def pmp_doc_find_first(conditions)
    response = pmp.query["urn:collectiondoc:query:docs"].where(conditions).get
    response.items.first
  end

  def retrieve_doc(type, url)
    doc = nil

    guid = PMPGuidMapping.find_guid(source_name, type, url)

    if guid
      doc = pmp_doc_by_guid(guid)
    end

    # no guid yet? look to see if a doc has the right tag
    if !doc
      itag = tag_for_url(source_name, url)
      doc = pmp_doc_find_first(itag: itag)

      if doc && !guid
        PMPGuidMapping.create(source_name: source_name, source_type: type, source_id: url, guid: doc.guid)
      end
    end

    doc
  end

  def set_standard_tags(tag_doc, url)
    add_tag_to_doc(tag_doc, distributor_tag)

    add_itag_to_doc(tag_doc, "#{distributor_tag.downcase}_test") unless Rails.env.production?
    add_itag_to_doc(tag_doc, tag_for_url(source_name, url))
  end

  def tag_for_url(source, url)
    "#{source}:#{url}"
  end

  def pmp_doc_profile(doc)
    # puts "doc.links['profile']: #{doc.links['profile'].inspect}"
    if profile_link = doc.links['profile']
      profile_link = profile_link.is_a?(Array) ? profile_link.first : profile_link
      profile_link.href.split('/').last.downcase
    end
  end

  def add_tag_to_doc(doc, tag)
    tag = tag.strip
    doc.tags ||= []
    return if doc.tags.include?(tag)<commit_msg>get rid of default PRX tag<commit_after>class ApplicationImporter

  attr_accessor :options

  def logger
    @logger = Rails.logger
  end

  def initialize(options={})
    self.options = HashWithIndifferentAccess.new(options)
  end

  def source_name
    raise NotImplementedError.new("Subclass must implement source_name")
  end

  def distributor_tag
    ENV['DISTRIBUTOR_TAG'] || 'PRX'
  end

  def import(options)
    self.options.merge!(options)
  end

  def pmp
    @pmp ||= PMP::Client.new(client_id: pmp_client_id, client_secret: pmp_client_secret, endpoint: pmp_endpoint)
  end

  def reset_pmp
    @pmp = nil
  end

  def pmp_client_id
    options[:pmp_client_id] || ENV['PMP_CLIENT_ID']
  end

  def pmp_client_secret
    options[:pmp_client_secret] || ENV['PMP_CLIENT_SECRET']
  end

  def pmp_endpoint
    options[:pmp_endpoint] || ENV['PMP_ENDPOINT'] || 'https://api.pmp.io/'
  end

  def pmp_url(*path)
    URI.join(pmp_endpoint, *path.collect(&:to_s).join('/')).to_s
  end

  def pmp_doc_by_guid(guid)
    response = pmp.query["urn:collectiondoc:hreftpl:docs"].where(guid: guid).get

    response = nil if (response.response.raw['status'] == '404')
    response = nil if response.response.raw['body'].blank?

    response
  end

  def pmp_doc_find_first(conditions)
    response = pmp.query["urn:collectiondoc:query:docs"].where(conditions).get
    response.items.first
  end

  def retrieve_doc(type, url)
    doc = nil

    guid = PMPGuidMapping.find_guid(source_name, type, url)

    if guid
      doc = pmp_doc_by_guid(guid)
    end

    # no guid yet? look to see if a doc has the right tag
    if !doc
      itag = tag_for_url(source_name, url)
      doc = pmp_doc_find_first(itag: itag)

      if doc && !guid
        PMPGuidMapping.create(source_name: source_name, source_type: type, source_id: url, guid: doc.guid)
      end
    end

    doc
  end

  def set_standard_tags(tag_doc, url)
    add_itag_to_doc(tag_doc, "#{distributor_tag.downcase}_test") unless Rails.env.production?
    add_itag_to_doc(tag_doc, tag_for_url(source_name, url))
  end

  def tag_for_url(source, url)
    "#{source}:#{url}"
  end

  def pmp_doc_profile(doc)
    # puts "doc.links['profile']: #{doc.links['profile'].inspect}"
    if profile_link = doc.links['profile']
      profile_link = profile_link.is_a?(Array) ? profile_link.first : profile_link
      profile_link.href.split('/').last.downcase
    end
  end

  def add_tag_to_doc(doc, tag)
    tag = tag.strip
    doc.tags ||= []
    return if doc.tags.include?(tag)
{'fraction_non_alphanumeric': 0.11661750245821043, 'fraction_numerical': 0.0011799410029498525, 'mean_word_length': 3.636280765724704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '909198', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1823, 'n_words': 364}