<commit_before># frozen_string_literal: true

class HistoryCrawlWorker
  include Sidekiq::Worker

  sidekiq_options queue: "slack_parser"

  def perform(channel_id)
    Rails.logger.info("[HistoryCrawlWorker] Started")

    channel = Channel.find(channel_id)
    slack_access_token = channel.user_with_access&.slack_access_token

    unless slack_access_token
      Rails.logger.info("[HistoryCrawlWorker] Can't get access token for Channel with ID = #{channel.id}")
      return
    end

    if channel.oldest_crawled.nil?
      channel.update!(oldest_crawled: Time.current, latest_crawled: Time.current)
    end

    crawl_by_oldest(channel, slack_access_token)
    crawl_by_latest(channel, slack_access_token)

    channel.next_crawl_time += channel.next_crawl_cycle.hours
    channel.save!

    Rails.logger.info("[HistoryCrawlWorker] Finished for Channel ID = #{channel.id}")
  end

  private

  def crawl_by_oldest(channel, slack_access_token)
    epoch = Time.zone.at(0)
    slack_api_client = SlackApiClient.new(slack_access_token)

    while channel.oldest_crawled > epoch
      history = slack_api_client.channel_history(channel, oldest: true)
      messages = history["messages"]

      if messages&.any?
        latest = messages[-1]["ts"]
        has_more = history["has_more"]
        channel.oldest_crawled = has_more ? Time.zone.at(latest.to_d) : epoch
        MessageAdder.add(messages, channel)
      else
        channel.oldest_crawled = epoch
      end
    end
  end

  def crawl_by_latest(channel, slack_access_token)
    now = Time.current

    slack_api_client = SlackApiClient.new(slack_access_token)
    while channel.latest_crawled < now
      channel.latest_crawled += 12.hours
      channel.latest_crawled = now if channel.latest_crawled > now

      history = slack_api_client.channel_history(channel, latest: true)
      messages = history["messages"]

      MessageAdder.add(messages, channel) if messages&.any?
    end
  end
end<commit_msg>Save channel date in DB while parsing<commit_after># frozen_string_literal: true

class HistoryCrawlWorker
  include Sidekiq::Worker

  sidekiq_options queue: "slack_parser"

  def perform(channel_id)
    Rails.logger.info("[HistoryCrawlWorker] Started")

    channel = Channel.find(channel_id)
    slack_access_token = channel.user_with_access&.slack_access_token

    unless slack_access_token
      Rails.logger.info("[HistoryCrawlWorker] Can't get access token for Channel with ID = #{channel.id}")
      return
    end

    if channel.oldest_crawled.nil?
      channel.update!(oldest_crawled: Time.current, latest_crawled: Time.current)
    end

    crawl_by_oldest(channel, slack_access_token)
    crawl_by_latest(channel, slack_access_token)

    channel.next_crawl_time += channel.next_crawl_cycle.hours
    channel.save!

    Rails.logger.info("[HistoryCrawlWorker] Finished for Channel ID = #{channel.id}")
  end

  private

  def crawl_by_oldest(channel, slack_access_token)
    epoch = Time.zone.at(0)
    slack_api_client = SlackApiClient.new(slack_access_token)

    while channel.oldest_crawled > epoch
      history = slack_api_client.channel_history(channel, oldest: true)
      messages = history["messages"]

      if messages&.any?
        latest = messages[-1]["ts"]
        has_more = history["has_more"]
        channel.oldest_crawled = has_more ? Time.zone.at(latest.to_d) : epoch
        MessageAdder.add(messages, channel)
      else
        channel.oldest_crawled = epoch
      end

      channel.save! if channel.changed?
    end
  end

  def crawl_by_latest(channel, slack_access_token)
    now = Time.current

    slack_api_client = SlackApiClient.new(slack_access_token)
    while channel.latest_crawled < now
      channel.latest_crawled += 12.hours
      channel.latest_crawled = now if channel.latest_crawled > now
      channel.save! if channel.changed?

      history = slack_api_client.channel_history(channel, latest: true)
      messages = history["messages"]

      MessageAdder.add(messages, channel) if messages&.any?
    end
  end
end
{'fraction_non_alphanumeric': 0.08742899481353421, 'fraction_numerical': 0.001975796492961225, 'mean_word_length': 3.6875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5545459', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1290, 'n_words': 247}