<commit_before>require 'new_relic/agent/method_tracer'

class Scraper < ActiveRecord::Base
  include Sync::Actions

  belongs_to :owner, inverse_of: :scrapers
  belongs_to :forked_by, class_name: "User"

  has_many :runs, inverse_of: :scraper
  has_many :metrics, through: :runs
  has_many :contributors, through: :contributions, source: :user
  has_many :contributions
  has_many :watches, class_name: "Alert", foreign_key: :watch_id
  has_many :watchers, through: :watches, source: :user

  has_one :last_run, -> { order "queued_at DESC" }, class_name: "Run"

  validates :name, format: { with: /\A[a-zA-Z0-9_-]+\z/, message: "can only have letters, numbers, '_' and '-'" }
  validates :owner, presence: true
  validates :name, uniqueness: { scope: :owner, message: 'is already taken on Morph' }
  validate :not_used_on_github, on: :create, unless: :github_id
  with_options if: :scraperwiki_shortname, on: :create do |s|
    s.validate :exists_on_scraperwiki
    s.validate :public_on_scraperwiki
    s.validate :not_scraperwiki_view
  end

  extend FriendlyId
  friendly_id :full_name

  delegate :queued?, :running?, to: :last_run, allow_nil: true

  def all_watchers
    (watchers + owner.watchers).uniq
  end

  # Given a scraper name on github populates the fields for a morph scraper but doesn't save it
  def self.new_from_github(full_name)
    repo = Octokit.repository(full_name)
    repo_owner = Owner.find_by_nickname(repo.owner.login)
    # Populate a new scraper with information from the repo
    Scraper.new(name: repo.name, full_name: repo.full_name,
      description: repo.description, github_id: repo.id, owner_id: repo_owner.id,
      github_url: repo.rels[:html].href, git_url: repo.rels[:git].href)
  end

  # Find a user related to this scraper that we can use them to make authenticated github requests
  def related_user
    if owner.user?
      owner
    else
      owner.users.first
    end
  end

  def update_contributors
    # We can't use unauthenticated requests because we will go over our rate limit
    begin
      c = related_user.octokit_client.contributors(full_name).map do |c|
        User.find_or_create_by_nickname(c["login"])
      end
    rescue Octokit::NotFound
      c = []
    end
    update_attributes(contributors: c)
  end

  def successful_runs
    runs.includes(:log_lines).order(finished_at: :desc).select{|r| r.finished_successfully?}
  end

  def latest_successful_run_time
    latest_successful_run = successful_runs.first
    latest_successful_run.finished_at if latest_successful_run
  end

  def finished_runs
    runs.where("finished_at IS NOT NULL").order(finished_at: :desc).includes(:log_lines, :metric)
  end

  # For successful runs calculates the average wall clock time that this scraper takes
  # Handy for the user to know how long it should expect to run for
  # Returns nil if not able to calculate this
  # TODO Refactor this using scopes
  def average_successful_wall_time
    successful_runs.sum(&:wall_time) / successful_runs.count if successful_runs.count > 0
  end

  def total_wall_time
    runs.to_a.sum(&:wall_time)
  end

  def utime
    metrics.sum(:utime)
  end

  def stime
    metrics.sum(:stime)
  end

  def cpu_time
    utime + stime
  end

  def update_sqlite_db_size
    update_attributes(sqlite_db_size: database.sqlite_db_size)
  end

  def total_disk_usage
    repo_size + sqlite_db_size
  end

  add_method_tracer :average_successful_wall_time, 'Custom/Scraper/average_successful_wall_time'
  add_method_tracer :total_wall_time, 'Custom/Scraper/total_wall_time'
  add_method_tracer :cpu_time, 'Custom/Scraper/cpu_time'
  add_method_tracer :total_disk_usage, 'Custom/Scraper/total_disk_usage'

  # Let's say a scraper requires attention if it's set to run automatically and the last run failed
  def requires_attention?
    auto_run && last_run && last_run.finished_with_errors?
  end

  def self.can_write?(user, owner)
    user && (owner == user || user.organizations.include?(owner) || user.admin?)
  end

  def can_write?(user)
    Scraper.can_write?(user, owner)
  end

  def destroy_repo_and_data
    FileUtils::rm_rf repo_path
    FileUtils::rm_rf data_path
  end<commit_msg>Revert "Allow admins to administer scrapers"<commit_after>require 'new_relic/agent/method_tracer'

class Scraper < ActiveRecord::Base
  include Sync::Actions

  belongs_to :owner, inverse_of: :scrapers
  belongs_to :forked_by, class_name: "User"

  has_many :runs, inverse_of: :scraper
  has_many :metrics, through: :runs
  has_many :contributors, through: :contributions, source: :user
  has_many :contributions
  has_many :watches, class_name: "Alert", foreign_key: :watch_id
  has_many :watchers, through: :watches, source: :user

  has_one :last_run, -> { order "queued_at DESC" }, class_name: "Run"

  validates :name, format: { with: /\A[a-zA-Z0-9_-]+\z/, message: "can only have letters, numbers, '_' and '-'" }
  validates :owner, presence: true
  validates :name, uniqueness: { scope: :owner, message: 'is already taken on Morph' }
  validate :not_used_on_github, on: :create, unless: :github_id
  with_options if: :scraperwiki_shortname, on: :create do |s|
    s.validate :exists_on_scraperwiki
    s.validate :public_on_scraperwiki
    s.validate :not_scraperwiki_view
  end

  extend FriendlyId
  friendly_id :full_name

  delegate :queued?, :running?, to: :last_run, allow_nil: true

  def all_watchers
    (watchers + owner.watchers).uniq
  end

  # Given a scraper name on github populates the fields for a morph scraper but doesn't save it
  def self.new_from_github(full_name)
    repo = Octokit.repository(full_name)
    repo_owner = Owner.find_by_nickname(repo.owner.login)
    # Populate a new scraper with information from the repo
    Scraper.new(name: repo.name, full_name: repo.full_name,
      description: repo.description, github_id: repo.id, owner_id: repo_owner.id,
      github_url: repo.rels[:html].href, git_url: repo.rels[:git].href)
  end

  # Find a user related to this scraper that we can use them to make authenticated github requests
  def related_user
    if owner.user?
      owner
    else
      owner.users.first
    end
  end

  def update_contributors
    # We can't use unauthenticated requests because we will go over our rate limit
    begin
      c = related_user.octokit_client.contributors(full_name).map do |c|
        User.find_or_create_by_nickname(c["login"])
      end
    rescue Octokit::NotFound
      c = []
    end
    update_attributes(contributors: c)
  end

  def successful_runs
    runs.includes(:log_lines).order(finished_at: :desc).select{|r| r.finished_successfully?}
  end

  def latest_successful_run_time
    latest_successful_run = successful_runs.first
    latest_successful_run.finished_at if latest_successful_run
  end

  def finished_runs
    runs.where("finished_at IS NOT NULL").order(finished_at: :desc).includes(:log_lines, :metric)
  end

  # For successful runs calculates the average wall clock time that this scraper takes
  # Handy for the user to know how long it should expect to run for
  # Returns nil if not able to calculate this
  # TODO Refactor this using scopes
  def average_successful_wall_time
    successful_runs.sum(&:wall_time) / successful_runs.count if successful_runs.count > 0
  end

  def total_wall_time
    runs.to_a.sum(&:wall_time)
  end

  def utime
    metrics.sum(:utime)
  end

  def stime
    metrics.sum(:stime)
  end

  def cpu_time
    utime + stime
  end

  def update_sqlite_db_size
    update_attributes(sqlite_db_size: database.sqlite_db_size)
  end

  def total_disk_usage
    repo_size + sqlite_db_size
  end

  add_method_tracer :average_successful_wall_time, 'Custom/Scraper/average_successful_wall_time'
  add_method_tracer :total_wall_time, 'Custom/Scraper/total_wall_time'
  add_method_tracer :cpu_time, 'Custom/Scraper/cpu_time'
  add_method_tracer :total_disk_usage, 'Custom/Scraper/total_disk_usage'

  # Let's say a scraper requires attention if it's set to run automatically and the last run failed
  def requires_attention?
    auto_run && last_run && last_run.finished_with_errors?
  end

  def self.can_write?(user, owner)
    user && (owner == user || user.organizations.include?(owner))
  end

  def can_write?(user)
    Scraper.can_write?(user, owner)
  end

  def destroy_repo_and_data
    FileUtils::rm_rf repo_path
    FileUtils::rm_rf data_path
  end
{'fraction_non_alphanumeric': 0.08454740864580845, 'fraction_numerical': 0.0007165034631000717, 'mean_word_length': 4.153846153846154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5356101', 'n_tokens_mistral': 2895, 'n_tokens_neox': 2814, 'n_words': 741}