<commit_before>class CacheBrowserChannelsJsonJobV3 < ApplicationJob
  queue_as :heavy

  MAX_RETRY = 10
  TOTALS_CACHE_KEY = 'browser_channels_json_v3_totals'
  LAST_WRITTEN_AT_KEY = "CacheBrowserChannelsJsonJobV3_last_written_at".freeze

  def perform
    last_written_at = Rails.cache.fetch(LAST_WRITTEN_AT_KEY)
    return if last_written_at.present? && last_written_at > 2.hours.ago

    if ENV["RAILS_ENV"].in?(["staging", "development"])
      @channels_json = gather_channels(staging_info, production_info).to_json
    else
      @channels_json = JsonBuilders::ChannelsJsonBuilderV3.new.build
    end
    retry_count = 0
    result = nil

    loop do
      result = Rails.cache.write(Api::V3::Public::ChannelsController::REDIS_KEY, @channels_json)
      break if result || retry_count > MAX_RETRY

      retry_count += 1
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not write to Redis result: #{result}. Retrying: #{retry_count}/#{MAX_RETRY}")
    end

    if result
      Rails.cache.write(LAST_WRITTEN_AT_KEY, Time.now)
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 updated the cached browser channels json.")
    else
      SlackMessenger.new(message: "🚨 CacheBrowserChannelsJsonJob V3 could not update the channels JSON. @publishers-team  🚨", channel: SlackMessenger::ALERTS)
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not update the channels JSON.")
    end

    cache_totals
  end

  def cache_totals
    results = JSON.parse(@channels_json)

    # This generates a list of the prefixes for channels ["youtube#channel:", "twitter#channel:", "twitch#:channel:"]
    prefixes = [
      TwitchChannelDetails,
      GithubChannelDetails,
      YoutubeChannelDetails,
      SiteChannelDetails,
      RedditChannelDetails,
      VimeoChannelDetails,
      TwitterChannelDetails
    ].map { |x| x.const_get('PREFIX') if x.const_defined?('PREFIX') }.compact

    counts = { site: {}, all_channels: {} }
    prefixes.each { |x| counts[x] = {} }

    results.each do |channel|
      status = channel.second
      next unless status.present?

      entry = channel.first.split(':').first + ":"
      entry = :site if counts.keys.exclude?(entry)

      # Essentially this is doing the following
      # counts["reddit#channel:"]["connected"] = 0
      # counts["github#channel:"]["verified"] = 0
      counts[entry][status] = 0 if counts[entry][status].blank?

      counts[entry][status] += 1

      # Initialize if nil
      counts[:all_channels][status] = 0 if counts[:all_channels][status].blank?
      counts[:all_channels][status] += 1
    end

    # Remove the #channel: from youtube#channel: so it's formatted in a readable way
    statistical_totals = counts.transform_keys { |k| k.to_s.split('#')[0] }

    # Generates the totals for each property
    statistical_totals.keys.map do |k|
      statistical_totals[k][:total] = statistical_totals[k].sum { |k, v| v }
    end

    loop do
      result = Rails.cache.write(TOTALS_CACHE_KEY, statistical_totals.to_json)
      break if result || retry_count > MAX_RETRY

      retry_count += 1
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not write to totals: #{result}. Retrying: #{retry_count}/#{MAX_RETRY}")
    end
  end

  def gather_channels(staging_channels_list, production_channels_list)
    existing_channels = {}
    staging_channels_list.each do |staging_channel|
      existing_channels[staging_channel[0]] = 1
    end
    production_channels_list.each do |production_channel|
      staging_channels_list.append(production_channel) unless production_channel[0].in?(existing_channels)
    end
    staging_channels_list
  end

  def staging_info
    JSON.parse(JsonBuilders::ChannelsJsonBuilderV3.new.build)
  end

  def production_info
    response = Faraday.get("https://publishers-distro.basicattentiontoken.org/api/v3/public/channels") do |req|
      req.headers['Accept-Encoding'] = 'gzip'
    end
    JSON.parse(ActiveSupport::Gzip.decompress(response.body))
  end
end
<commit_msg>Specify fake channels for testing<commit_after>class CacheBrowserChannelsJsonJobV3 < ApplicationJob
  queue_as :heavy

  MAX_RETRY = 10
  TOTALS_CACHE_KEY = 'browser_channels_json_v3_totals'
  LAST_WRITTEN_AT_KEY = "CacheBrowserChannelsJsonJobV3_last_written_at".freeze

  def perform
    last_written_at = Rails.cache.fetch(LAST_WRITTEN_AT_KEY)
    return if last_written_at.present? && last_written_at > 2.hours.ago

    if ENV["RAILS_ENV"].in?(["staging", "development"])
      @channels_json = gather_channels(staging_info, production_info).to_json
    else
      @channels_json = JsonBuilders::ChannelsJsonBuilderV3.new.build
    end
    retry_count = 0
    result = nil

    loop do
      result = Rails.cache.write(Api::V3::Public::ChannelsController::REDIS_KEY, @channels_json)
      break if result || retry_count > MAX_RETRY

      retry_count += 1
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not write to Redis result: #{result}. Retrying: #{retry_count}/#{MAX_RETRY}")
    end

    if result
      Rails.cache.write(LAST_WRITTEN_AT_KEY, Time.now)
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 updated the cached browser channels json.")
    else
      SlackMessenger.new(message: "🚨 CacheBrowserChannelsJsonJob V3 could not update the channels JSON. @publishers-team  🚨", channel: SlackMessenger::ALERTS)
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not update the channels JSON.")
    end

    cache_totals
  end

  def cache_totals
    results = JSON.parse(@channels_json)

    # This generates a list of the prefixes for channels ["youtube#channel:", "twitter#channel:", "twitch#:channel:"]
    prefixes = [
      TwitchChannelDetails,
      GithubChannelDetails,
      YoutubeChannelDetails,
      SiteChannelDetails,
      RedditChannelDetails,
      VimeoChannelDetails,
      TwitterChannelDetails
    ].map { |x| x.const_get('PREFIX') if x.const_defined?('PREFIX') }.compact

    counts = { site: {}, all_channels: {} }
    prefixes.each { |x| counts[x] = {} }

    results.each do |channel|
      status = channel.second
      next unless status.present?

      entry = channel.first.split(':').first + ":"
      entry = :site if counts.keys.exclude?(entry)

      # Essentially this is doing the following
      # counts["reddit#channel:"]["connected"] = 0
      # counts["github#channel:"]["verified"] = 0
      counts[entry][status] = 0 if counts[entry][status].blank?

      counts[entry][status] += 1

      # Initialize if nil
      counts[:all_channels][status] = 0 if counts[:all_channels][status].blank?
      counts[:all_channels][status] += 1
    end

    # Remove the #channel: from youtube#channel: so it's formatted in a readable way
    statistical_totals = counts.transform_keys { |k| k.to_s.split('#')[0] }

    # Generates the totals for each property
    statistical_totals.keys.map do |k|
      statistical_totals[k][:total] = statistical_totals[k].sum { |k, v| v }
    end

    loop do
      result = Rails.cache.write(TOTALS_CACHE_KEY, statistical_totals.to_json)
      break if result || retry_count > MAX_RETRY

      retry_count += 1
      Rails.logger.info("CacheBrowserChannelsJsonJob V3 could not write to totals: #{result}. Retrying: #{retry_count}/#{MAX_RETRY}")
    end
  end

  def gather_channels(staging_channels_list, production_channels_list)
    existing_channels = {}
    staging_channels_list.each do |staging_channel|
      existing_channels[staging_channel[0]] = 1
    end
    production_channels_list.each do |production_channel|
      production_channel[0] = production_channel[0] + "fake"
      staging_channels_list.append(production_channel) unless production_channel[0].in?(existing_channels)
    end
    staging_channels_list
  end

  def staging_info
    JSON.parse(JsonBuilders::ChannelsJsonBuilderV3.new.build)
  end

  def production_info
    response = Faraday.get("https://publishers-distro.basicattentiontoken.org/api/v3/public/channels") do |req|
      req.headers['Accept-Encoding'] = 'gzip'
    end
    JSON.parse(ActiveSupport::Gzip.decompress(response.body))
  end
end

{'fraction_non_alphanumeric': 0.10630164015291652, 'fraction_numerical': 0.00715254655321248, 'mean_word_length': 3.856287425149701, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6123380', 'n_tokens_mistral': 2617, 'n_tokens_neox': 2540, 'n_words': 531}