<commit_before>require 'source_helper'
require 'cgi'
require 'ostruct'

class Source < ActiveRecord::Base
  include SourceHelper

  has_many :retrieval_statuses, :dependent => :destroy
  belongs_to :group

  serialize :config, OpenStruct

  validates_presence_of :display_name
  validates_numericality_of :timeout, :only_integer => true, :greater_than => 0
  validates_numericality_of :workers, :only_integer => true, :greater_than => 0, :less_than => 10
  validates_numericality_of :wait_time, :only_integer => true, :greater_than => 0

  # for job priority
  TOP_PRIORITY = 0

  # counter data can only be queried at very specific time frame only.
  scope :data_retrievable_sources, lambda { where("active = true and name != 'counter'") }

  scope :public_sources, lambda { where("private = false") }
  scope :private_sources, lambda { where("private = true") }

  def get_data(article, options={})
    raise NotImplementedError, 'Children classes should override get_data method'
  end

  def queue_all_articles
    # determine if the source is active
    if active && (disable_until.nil? || disable_until < Time.now.utc)

      # reset disable_until value
      unless self.disable_until.nil?
        self.disable_until = nil
        save
      end

      # grab all the articles
      retrieval_statuses = RetrievalStatus.joins(:article, :source).<commit_msg>PDEV-438 added biod and nature to the sources that should not be refreshed<commit_after>require 'source_helper'
require 'cgi'
require 'ostruct'

class Source < ActiveRecord::Base
  include SourceHelper

  has_many :retrieval_statuses, :dependent => :destroy
  belongs_to :group

  serialize :config, OpenStruct

  validates_presence_of :display_name
  validates_numericality_of :timeout, :only_integer => true, :greater_than => 0
  validates_numericality_of :workers, :only_integer => true, :greater_than => 0, :less_than => 10
  validates_numericality_of :wait_time, :only_integer => true, :greater_than => 0

  # for job priority
  TOP_PRIORITY = 0

  # TODO: it might be a good idea to set a flag for special type of sources
  # counter, biod, and nature data can only be queried at very specific time frame only.
  scope :data_retrievable_sources, lambda { where("active = true and name not in ('counter', 'biod', 'nature')") }

  scope :public_sources, lambda { where("private = false") }
  scope :private_sources, lambda { where("private = true") }

  def get_data(article, options={})
    raise NotImplementedError, 'Children classes should override get_data method'
  end

  def queue_all_articles
    # determine if the source is active
    if active && (disable_until.nil? || disable_until < Time.now.utc)

      # reset disable_until value
      unless self.disable_until.nil?
        self.disable_until = nil
        save
      end

      # grab all the articles
      retrieval_statuses = RetrievalStatus.joins(:article, :source).
{'fraction_non_alphanumeric': 0.0890952872377021, 'fraction_numerical': 0.005159958720330237, 'mean_word_length': 4.220825852782765, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6302587', 'n_tokens_mistral': 903, 'n_tokens_neox': 855, 'n_words': 270}