<commit_before>require 'digest/sha1'
require 'digest/md5'
require 'digest/sha2'

module Dap
module Filter

class FilterCopy
  include Base

  def process(doc)
    self.opts.each_pair do |k,v|
      if doc.has_key?(k)
        doc[v] = doc[k]
      end
    end
   [ doc ]
  end
end

class FilterRename
  include Base

  def initialize(args)
    super
    missing_rename = self.opts.select { |k, v| v.nil? }.keys
    unless missing_rename.empty?
      fail "Missing new name for renames of #{missing_rename.join(',')}"
    end
  end

  def process(doc)
    self.opts.each_pair do |k,v|
      if doc.has_key?(k)
        doc[v] = doc[k]
        doc.delete(k)
      end
    end
   [ doc ]
  end
end

# Example below replaces periods with underscores in the names of all keys
# one level below 'my_key'
# rename_subkey_match my_key '.' '_'
class FilterRenameSubkeyMatch
  attr_accessor :name, :opts

  def initialize(args)
    self.name = Dap::Factory.name_from_class(self.class)
    self.opts = {}
    fail "Expected 3 arguments to '#{self.name}' but got #{args.size}" unless args.size == 3
    self.opts = args
  end

  def process(doc)
    temp_field = {}
    field, original, updated = self.opts
    return [ doc ] unless doc[field].is_a?(::Hash)
    doc[field].each_key do |k|
      new_k = k.gsub(original, updated)
      temp_field[new_k] = doc[field][k]
    end
    doc[field] = temp_field
    [ doc ]
  end
end

class FilterMatchRemove
  include Base
  def process(doc)
    self.opts.each_pair do |re,_|
      doc.each_key do |k|
        if k.match(re)
          doc.delete(k)
        end
      end
    end
   [ doc ]<commit_msg>Use Base for common functionality<commit_after>require 'digest/sha1'
require 'digest/md5'
require 'digest/sha2'

module Dap
module Filter

class FilterCopy
  include Base

  def process(doc)
    self.opts.each_pair do |k,v|
      if doc.has_key?(k)
        doc[v] = doc[k]
      end
    end
   [ doc ]
  end
end

class FilterRename
  include Base

  def initialize(args)
    super
    missing_rename = self.opts.select { |k, v| v.nil? }.keys
    unless missing_rename.empty?
      fail "Missing new name for renames of #{missing_rename.join(',')}"
    end
  end

  def process(doc)
    self.opts.each_pair do |k,v|
      if doc.has_key?(k)
        doc[v] = doc[k]
        doc.delete(k)
      end
    end
   [ doc ]
  end
end

# Example below replaces periods with underscores in the names of all keys
# one level below 'my_key'
# rename_subkey_match my_key '.' '_'
class FilterRenameSubkeyMatch
  include Base

  def initialize(args)
    super
    fail "Expected 3 arguments to '#{self.name}' but got #{args.size}" unless args.size == 3
    self.opts = args
  end

  def process(doc)
    temp_field = {}
    field, original, updated = self.opts
    return [ doc ] unless doc[field].is_a?(::Hash)
    doc[field].each_key do |k|
      new_k = k.gsub(original, updated)
      temp_field[new_k] = doc[field][k]
    end
    doc[field] = temp_field
    [ doc ]
  end
end

class FilterMatchRemove
  include Base
  def process(doc)
    self.opts.each_pair do |re,_|
      doc.each_key do |k|
        if k.match(re)
          doc.delete(k)
        end
      end
    end
   [ doc ]
{'fraction_non_alphanumeric': 0.10774515927545285, 'fraction_numerical': 0.003123048094940662, 'mean_word_length': 2.5549389567147616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4977193', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1128, 'n_words': 311}