<commit_before># EasyRedis is a simple ruby framework designed to make using Redis as a database simpler.
#
# Redis is a very fast key-value store that supports data structures like lists, (sorted) sets, and hashes, but because of its simplicity, using Redis to store traditional database data can be somewhat tedious. EasyRedis streamlines this process.
#
# Author:: <NAME> (mailto:<EMAIL>)


require 'redis'
require 'set'
#require 'active_support/inflector'


# main EasyRedis module which
# holds all classes and helper methods
module EasyRedis
  
  # generate a 'score' for a string
  # used for storing it in a sorted set
  #
  # This method effectively turns a string into a base 27 floating point number,
  # where 0 corresponds to no letter, 1 to A, 2 to B, etc.
  def self.string_score(str)
    str = str.downcase
    mult = 1.0
    scr = 0.0
    str.each_byte do |b|
      mult /= 27
      scr += (b-'a'.ord+1)*mult
    end
    scr
  end

  # gets a score for a generic object
  #
  # The score is determined as follows:
  # First, if the object is a string, string_score is used to get its score.
  # Otherwise, we try calling the following methods on the object in turn, returning the first that works: score, to_f, to_i.
  # If none of those work, we simply return the object itself.
  def self.score(obj)
    if obj.is_a? String
      string_score(obj)
    elsif obj.respond_to? "score"
      obj.score
    elsif obj.respond_to? "to_f"
      obj.to_f
    elsif obj.respond_to? "to_i"
      obj.to_i
    else
      obj
    end
  end

  # access the redis object
  def self.redis
    @redis
  end

  # connect to a redis server
  #
  # takes the same options that Redis#new does from the redis gem
  def self.connect(options = {})
    @redis = Redis.new(options)
  end

  # exception that indicates that the given field has not been indexed for sorting/searching
  class FieldNotSortable < RuntimeError
    def initialize(field)
      @message = "field '#{field.to_s}' not sortable"
    end
    
    def to_s
      @message
    end
  end

  # exception that indicated an unknown ordering option was encountered
  class UnknownOrderOption < RuntimeError
    def initialize(opt)
      @message = "unknown order option '#{opt}'"
    end

    def to_s
      @message
    end
  end

  # class representing a sort
  class Sort
    include Enumerable

    # initialize the sort with a specific field, ordering option, and model
    def initialize(field,order,klass)
      raise EasyRedis::FieldNotSortable, field  unless klass.sortable?(field) 
      raise EasyRedis::UnknownOrderOption, order  unless [:asc,:desc].member? order
      @field = field
      @order = order
      @klass = klass
    end

    # access elements in this sort
    #
    # Work's like an Array's [] method. It can take a specific index, a range, or an offset, amount pair.
    # This method uses the underlying access method, which handles the actual retrival.
    # Calling this method will actually query redis to retrive objects.
    def [](index,limit=nil)
      if limit
        offset = index
        self[offset...(offset+limit)]
      elsif index.is_a? Range
        access(index)
      elsif index.is_a? Integer
        self[index..index].first
      end
    end

    # helper method for []
    #
    # takes a range and returns corresponding elements
    def access(range)
      a = range.begin
      b = range.end
      b -= 1 if range.exclude_end?
      ids = []
      if @order == :asc
        ids = EasyRedis.redis.zrange(@klass.sort_prefix(@field),a,b)
      elsif @order == :desc
        ids = EasyRedis.redis.zrevrange(@klass.sort_prefix(@field),a,b)
      end
      ids.map{|i|@klass.new(i)}
    end

    # iterate through all members of this sort
    def each
      self[0..-1].each { |o| yield o }
    end

    # return the number of elements in this sort
    #
    # As of now, idential to the Model's count method.
    # This method is explicility defined here to overwrite the default one in Enumerable, which iterates through all the entries to count them, which is much slower than a ZCARD command
    def count
      EasyRedis.zcard(@klass.sort_prefix(@field))
    end

    # return the fist element of this sort, or the first n elements, if n is given
    def first(n = nil)
      if n
        self[0,n]
      else
        self[0]
      end
    end

    # return the last element of this sort, or the last n elements, if n is given
    def last(n = nil)
      if n
        self[-n..-1]
      else
        self[-1]
      end
    end

    def inspect
      "#<EasyRedis::Sort model=#{@klass.name}, field=#{@field.to_s}, order=#{@order.to_s}>"
    end

  end


  # class representing a data model
  # you want to store in redis
  class Model
    
    @@sorts = []
    @@text_searches = []

    # add a field to the model
    def self.field(name)
      @@fields ||= []
      @@fields << name.to_sym
      name = name.to_s
      getter = name
      setter = name + "="
      instance_var = '@' + name
<commit_msg>abstracted away most of the Sort class to a Collection class<commit_after># EasyRedis is a simple ruby framework designed to make using Redis as a database simpler.
#
# Redis is a very fast key-value store that supports data structures like lists, (sorted) sets, and hashes, but because of its simplicity, using Redis to store traditional database data can be somewhat tedious. EasyRedis streamlines this process.
#
# Author:: <NAME> (mailto:<EMAIL>)


require 'redis'
require 'set'


# main EasyRedis module which
# holds all classes and helper methods
module EasyRedis
  
  # generate a 'score' for a string
  # used for storing it in a sorted set
  #
  # This method effectively turns a string into a base 27 floating point number,
  # where 0 corresponds to no letter, 1 to A, 2 to B, etc.
  def self.string_score(str)
    str = str.downcase
    mult = 1.0
    scr = 0.0
    str.each_byte do |b|
      mult /= 27
      scr += (b-'a'.ord+1)*mult
    end
    scr
  end

  # gets a score for a generic object
  #
  # The score is determined as follows:
  # First, if the object is a string, string_score is used to get its score.
  # Otherwise, we try calling the following methods on the object in turn, returning the first that works: score, to_f, to_i.
  # If none of those work, we simply return the object itself.
  def self.score(obj)
    if obj.is_a? String
      string_score(obj)
    elsif obj.respond_to? "score"
      obj.score
    elsif obj.respond_to? "to_f"
      obj.to_f
    elsif obj.respond_to? "to_i"
      obj.to_i
    else
      obj
    end
  end

  # access the redis object
  def self.redis
    @redis
  end

  # connect to a redis server
  #
  # takes the same options that Redis#new does from the redis gem
  def self.connect(options = {})
    @redis = Redis.new(options)
  end

  # exception that indicates that the given field has not been indexed for sorting/searching
  class FieldNotSortable < RuntimeError
    def initialize(field)
      @message = "field '#{field.to_s}' not sortable"
    end
    
    def to_s
      @message
    end
  end

  # exception that indicated an unknown ordering option was encountered
  class UnknownOrderOption < RuntimeError
    def initialize(opt)
      @message = "unknown order option '#{opt}'"
    end

    def to_s
      @message
    end
  end

  # class representing a generic collection
  class Collection
    include Enumerable

    # access elements in this sort
    #
    # Work's like an Array's [] method. It can take a specific index, a range, or an offset and an amount/limit.
    # This method uses the underlying access method, which handles the actual retrival.
    def [](index,limit=nil)
      if limit
        offset = index
        self[offset...(offset+limit)]
      elsif index.is_a? Range
        access(index)
      elsif index.is_a? Integer
        self[index..index].first
      end
    end

    # iterate through all members of this collection
    def each
      self[0..-1].each { |o| yield o }
    end

    # return the fist element of this collection, or the first n elements, if n is given
    def first(n = nil)
      if n
        self[0,n]
      else
        self[0]
      end
    end

    # return the last element of this collection, or the last n elements, if n is given
    def last(n = nil)
      if n
        self[-n..-1]
      else
        self[-1]
      end
    end

    private

    # access the elements corresponding to the given range
    #
    # meant to be overridden in child classes
    def access(range)
      []
    end

  end

  # class representing a sort
  class Sort < Collection

    # initialize the sort with a specific field, ordering option, and model
    def initialize(field,order,klass)
      raise EasyRedis::FieldNotSortable, field  unless klass.sortable?(field) 
      raise EasyRedis::UnknownOrderOption, order  unless [:asc,:desc].member? order
      @field = field
      @order = order
      @klass = klass
    end

    # return the number of elements in this sort
    #
    # As of now, idential to the Model's count method.
    # This method is explicility defined here to overwrite the default one in Enumerable, which iterates through all the entries to count them, which is much slower than a ZCARD command
    def count
      EasyRedis.zcard(@klass.sort_prefix(@field))
    end

    def inspect
      "#<EasyRedis::Sort model=#{@klass.name}, field=#{@field.to_s}, order=#{@order.to_s}>"
    end

    private

    # takes a range and returns corresponding elements
    def access(range)
      a = range.begin
      b = range.end
      b -= 1 if range.exclude_end?
      ids = []
      if @order == :asc
        ids = EasyRedis.redis.zrange(@klass.sort_prefix(@field),a,b)
      elsif @order == :desc
        ids = EasyRedis.redis.zrevrange(@klass.sort_prefix(@field),a,b)
      end
      ids.map{|i|@klass.new(i)}
    end

  end


  # class representing a data model
  # you want to store in redis
  class Model
    
    @@sorts = []
    @@text_searches = []

    # add a field to the model
    def self.field(name)
      @@fields ||= []
      @@fields << name.to_sym
      name = name.to_s
      getter = name
      setter = name + "="
      instance_var = '@' + name

{'fraction_non_alphanumeric': 0.08129524554881627, 'fraction_numerical': 0.0037174721189591076, 'mean_word_length': 2.4890784982935155, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3049376', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3075, 'n_words': 1203}