<commit_before>$LOAD_PATH << '..'
require 'musikbot'

module CategoryCounter

  def self.run
    @mb = MusikBot::Session.new(inspect)

    # First determine which datasets need updating based on the granularity.
    categories = @mb.config
    to_update = []
    categories.each do |entry|
      if should_refresh_data?(entry)
        to_update << entry
      end
    end

    date = @mb.now.strftime('%Y-%m-%d')
    sizes = fetch_category_sizes(to_update)
    to_update.each do |entry|
      category_title = 'Category:' + entry['category'].gsub(/^Category:/, '')
      data = JSON.parse(@mb.get(entry['dataset']))

      if entry['cutoff'].present?
        data = trim_old_data(data, entry['cutoff'])
      end

      data << {
        'date' => date,
        'value' => sizes[category_title]
      }

      @mb.edit(entry['dataset'],
        content: data,
        summary: "Adding size of [[#{category_title}]] as of #{date} ([[User:MusikBot/ALSChart|more info]])"
      )
    end
  rescue => e
    @mb.report_error('Fatal error', e)
  end

  # Remove all data points dated before the present day - 'cutoff' days
  def self.trim_old_data(data, cutoff)
    data.map do |datum|
      @mb.parse_date(datum['date']) > (@mb.today - cutoff.to_i)
    end
  end

  # Get the size of each category
  def self.fetch_category_sizes(categories)
    sizes = {}

    # Batched API requests to 500 per request.
    categories.each_slice(500).to_a.each do |chunk|
      titles = chunk.map { |c| c['category'] }

      pages = @mb.gateway.custom_query(
        titles: titles,
        prop: 'categoryinfo',
        formatversion: 2
      ).elements['pages'].to_a

      pages.each do |page|
        if page['categoryinfo']
          sizes[page['title']] = page['categoryinfo']['size']
        end
      end
    end

    sizes
  end

  def self.should_refresh_data?(entry)
    ret = entry[:category].present? && entry[:granularity].present? && entry[:dataset].present?
    ret && case granularity
      when :daily
        true
      when :weekly
        @mb.now.wday == 0
      when :monthly
        @mb.now.mday == 1
    end
  end

end

CategoryCounter.run<commit_msg>CategoryCounter: finishing touches; working task<commit_after>$LOAD_PATH << '..'
require 'musikbot'

module CategoryCounter

  def self.run
    @mb = MusikBot::Session.new(inspect)

    # First determine which datasets need updating based on the granularity.
    categories = @mb.config
    to_update = []
    categories.each_with_index do |entry, i|
      if should_refresh_data?(entry)
        # Normalize category title
        entry[:category] = 'Category:' + entry[:category].gsub(/^Category:/, '')

        to_update << entry
      end
    end

    date = @mb.now.strftime('%Y-%m-%d')
    sizes = fetch_category_sizes(to_update)

    to_update.each do |entry|
      next unless entry[:dataset] == 'Template:Orphaned articles chart/data'
      data = JSON.parse(@mb.get(entry[:dataset]))

      if entry[:cutoff].present?
        data = trim_old_data(data, entry[:cutoff])
      end

      data << {
        'date' => date,
        'value' => sizes[entry[:category]]
      }

      binding.pry
      @mb.edit(entry[:dataset],
        content: data.to_json,
        summary: "Adding size of [[:#{entry[:category]}]] as of #{date} ([[User:MusikBot/CategoryCounter|more info]])"
      )
    end
  rescue => e
    @mb.report_error('Fatal error', e)
  end

  # Remove all data points dated before the present day - 'cutoff' days
  def self.trim_old_data(data, cutoff)
    data.select do |datum|
      @mb.parse_date(datum['date']) > (@mb.today - cutoff.to_i)
    end
  end

  # Get the size of each category
  def self.fetch_category_sizes(categories)
    sizes = {}

    # Batched API requests to 500 per request.
    categories.each_slice(500).to_a.each do |chunk|
      titles = chunk.map { |c| c[:category] }

      pages = @mb.gateway.custom_query(
        titles: titles.join('|'),
        prop: 'categoryinfo',
        formatversion: 2
      ).elements['pages'].to_a

      pages.each do |page|
        if page.elements['categoryinfo']
          sizes[page['title']] = page.elements['categoryinfo']['size'].to_i
        end
      end
    end

    sizes
  end

  def self.should_refresh_data?(entry)
    ret = entry[:category].present? && entry[:granularity].present? && entry[:dataset].present?
    ret && case entry[:granularity].to_sym
      when :daily
        true
      when :weekly
        @mb.now.wday == 4
      when :monthly
        @mb.now.mday == 1
    end
  end

end

CategoryCounter.run
{'fraction_non_alphanumeric': 0.13415168785620343, 'fraction_numerical': 0.00394563787812363, 'mean_word_length': 2.5929133858267717, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741850', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1449, 'n_words': 342}