<commit_before>class Metric
  include Redised

  def self.all(start=0, max = 100)
    return redis.lrange "metrics", 0, max-1
  end
  
  def self.refresh()
    metrics = get_metrics_list
    redis.del "metrics"
    puts "Caching #{metrics.length} metrics into redis"

    redis.pipelined do
      metrics.each do |metric|
        prefix = ""
        keywords = metric.split(".")
        for i in 0..keywords.size()-2
          redis.sadd("metrics:folder:#{prefix}",keywords[i])
          if prefix != ""
            prefix << "."
          end
          prefix << keywords[i]
        end
        redis.sadd("metrics:key:#{prefix}", keywords[-1])
      end
    end
    
    metrics
  end

  def self.find_folders(match)
    redis.smembers("metrics:folder:#{match}").sort()
  end

  def self.find_metrics(match)
    redis.smembers("metrics:key:#{match}").sort()
  end

  def self.find_keywords(match, max=100)
    return [] if !use_index_by_keywords

    match = match.strip
    keywords = redis.keys("metrics:*#{match}*")
    keywords = keywords.map { |key| key.gsub("metrics:", "") }
    keywords = self.sort_prefer_prefix(keywords, match)[0, max]
  end

  def self.find(match, max = 100)<commit_msg>When refreshing metrics, batch redis pipeline into groups of 100 For really large sets, we can run out of memory.<commit_after>class Metric
  include Redised

  def self.all(start=0, max = 100)
    return redis.lrange "metrics", 0, max-1
  end
  
  def self.refresh()
    metrics = get_metrics_list
    redis.del "metrics"
    puts "Caching #{metrics.length} metrics into redis"

    metrics.each_slice(100) do |batch|
      redis.pipelined do
        batch.each do |metric|
          prefix = ""
          keywords = metric.split(".")
          for i in 0..keywords.size()-2
            redis.sadd("metrics:folder:#{prefix}",keywords[i])
            if prefix != ""
              prefix << "."
            end
            prefix << keywords[i]
          end
          redis.sadd("metrics:key:#{prefix}", keywords[-1])
        end
      end
    end
    
    metrics
  end

  def self.find_folders(match)
    redis.smembers("metrics:folder:#{match}").sort()
  end

  def self.find_metrics(match)
    redis.smembers("metrics:key:#{match}").sort()
  end

  def self.find_keywords(match, max=100)
    return [] if !use_index_by_keywords

    match = match.strip
    keywords = redis.keys("metrics:*#{match}*")
    keywords = keywords.map { |key| key.gsub("metrics:", "") }
    keywords = self.sort_prefer_prefix(keywords, match)[0, max]
  end

  def self.find(match, max = 100)
{'fraction_non_alphanumeric': 0.1352506801399145, 'fraction_numerical': 0.014768752429071124, 'mean_word_length': 2.5552486187845305, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702203', 'n_tokens_mistral': 874, 'n_tokens_neox': 822, 'n_words': 188}