<commit_before>module Analytics::Export
  def self.included(base)
    base.extend(ClassMethods)
    base.class_eval do
      attr_accessor :data
    end
  end

  def initialize(loaded_data)
    self.data = loaded_data
  end

  def records
    data[self.class.rows]
  end

  # {
  #   :username => ["blah", "blah2", "blah3"],
  #   :role => ["admin", "owner", "owner"], ...
  # }
  def schema_records
    Hash.new { |hash, key| hash[key] = [] }.tap do |h|
      self.class.schema.each do |column, value|
        h[column] = self.records.each_with_index.map do |record, i|
          if record.respond_to? value
            record.send(value)
          else
            self.send(value, record, i)
          end
        end
      end
    end
  end

  def generate_csv(path, file_name=nil)
    unless File.exists?(path) && File.directory?(path)
      Dir.mkdir(path)
    end
    file_name ||= "#{self.class.name.underscore}.csv"<commit_msg>Added filter method and default values to analytics exports.<commit_after>module Analytics::Export
  def self.included(base)
    base.extend(ClassMethods)
    base.class_eval do
      attr_accessor :data
    end
  end

  def initialize(loaded_data)
    self.data = loaded_data
  end

  # No op
  def filter(rows)
    rows
  end

  def records
    self.filter data[self.class.rows]
  end

  # {
  #   :username => ["blah", "blah2", "blah3"],
  #   :role => ["admin", "owner", "owner"], ...
  # }
  def schema_records
    Hash.new { |hash, key| hash[key] = [] }.tap do |h|
      self.class.schema.each do |column, value|
        h[column] = self.records.each_with_index.map do |record, i|
          if record.respond_to? value
            record.send(value)
          else
            self.send(value, record, i)
          end
        end
      end
    end
  end

  def generate_csv(path, file_name=nil)
    unless File.exists?(path) && File.directory?(path)
      Dir.mkdir(path)
    end
    file_name ||= "#{self.class.name.underscore}.csv"
{'fraction_non_alphanumeric': 0.13006072874493926, 'fraction_numerical': 0.0020242914979757085, 'mean_word_length': 2.3622448979591835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7177603', 'n_tokens_mistral': 679, 'n_tokens_neox': 663, 'n_words': 152}