<commit_before>class Crawl < ApplicationRecord
  has_many :evaluations, dependent: :destroy

  class CrawlDuplicateError < StandardError
  end

  class << self
    START_OFFSET = [21, 21, 3, 3, 9, 9].freeze

    # @param [Date, Time, DateTime, ActiveSupport::TimeWithZone] date
    # @return [ActiveSupport::TimeWithZone]
    def start_time(date)
      (date = utc_time(date))
        .change(hour: START_OFFSET[date.strftime('%j').to_i % 6])
        .in_time_zone(Time.zone)
    end

    # @param [Date, Time, DateTime, ActiveSupport::TimeWithZone] date
    # @return [Symbol] :asc or :desc
    def order(date)
      (utc_time(date).strftime('%j').to_i % 2).zero? ? :asc : :desc
    end

    # Start crawler job asynchronously
    #
    # @param [Array<Integer>] id an array of endpoint id
    def start!(*id)
      if Crawl.find_by(started_at: Date.current.all_day)
        raise CrawlDuplicateError, "Crawl for #{Date.current} is already exist, "\
          'Stop and remove it before start new crawl.'
      end

      crawl = create!(started_at: Time.current)
      endpoints = Endpoint.active.order(id: order(Date.current))
      endpoints = endpoints.where(id: id) if id.present?

      endpoints.each do |endpoint|
        CrawlerJob.perform_async(crawl.id, endpoint.id)
      end
    end

    # @return [Array<Endpoint>] enqueued endpoint
    def restart!(*id)
      crawl = Crawl.find_by(started_at: Date.current.all_day)

      endpoints = Endpoint.active
      endpoints = endpoints.where(id: id) if id.present?

      (endpoints - (crawl.finished_endpoints | crawl.queued_endpoints)).tap do |missing|
        missing.each { |endpoint| CrawlerJob.perform_async(crawl.id, endpoint.id) }
      end
    end
<commit_msg>change function name not to conflict with ActiveRecord<commit_after>class Crawl < ApplicationRecord
  has_many :evaluations, dependent: :destroy

  class CrawlDuplicateError < StandardError
  end

  class << self
    START_OFFSET = [21, 21, 3, 3, 9, 9].freeze

    # @param [Date, Time, DateTime, ActiveSupport::TimeWithZone] date
    # @return [ActiveSupport::TimeWithZone]
    def start_time(date)
      (date = utc_time(date))
        .change(hour: START_OFFSET[date.strftime('%j').to_i % 6])
        .in_time_zone(Time.zone)
    end

    # @param [Date, Time, DateTime, ActiveSupport::TimeWithZone] date
    # @return [Symbol] :asc or :desc
    def queue_order(date)
      (utc_time(date).strftime('%j').to_i % 2).zero? ? :asc : :desc
    end

    # Start crawler job asynchronously
    #
    # @param [Array<Integer>] id an array of endpoint id
    def start!(*id)
      if Crawl.find_by(started_at: Date.current.all_day)
        raise CrawlDuplicateError, "Crawl for #{Date.current} is already exist, "\
          'Stop and remove it before start new crawl.'
      end

      crawl = create!(started_at: Time.current)
      endpoints = Endpoint.active.order(id: self.class.queue_order(Date.current))
      endpoints = endpoints.where(id: id) if id.present?

      endpoints.each do |endpoint|
        CrawlerJob.perform_async(crawl.id, endpoint.id)
      end
    end

    # @return [Array<Endpoint>] enqueued endpoint
    def restart!(*id)
      crawl = Crawl.find_by(started_at: Date.current.all_day)

      endpoints = Endpoint.active
      endpoints = endpoints.where(id: id) if id.present?

      (endpoints - (crawl.finished_endpoints | crawl.queued_endpoints)).tap do |missing|
        missing.each { |endpoint| CrawlerJob.perform_async(crawl.id, endpoint.id) }
      end
    end

{'fraction_non_alphanumeric': 0.11647727272727272, 'fraction_numerical': 0.005681818181818182, 'mean_word_length': 3.1326291079812205, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5857734', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1126, 'n_words': 275}