<commit_before>require 'active_support/concern'

require 'sandstorm/records/errors'

require 'sandstorm/filters/step'

module Sandstorm

  module Filters

    module Base

      extend ActiveSupport::Concern

      attr_reader :backend

      # initial set         a Sandstorm::Record::Key object
      # associated_class    the class of the result record
      def initialize(data_backend, initial_set, associated_class)
        @backend          = data_backend
        @initial_set      = initial_set
        @associated_class = associated_class
        @steps            = []
      end

      # # TODO implement
      # # will probably need to scan and extract the last usage from steps
      # def limit(amount)
      #   @steps << Sandstorm::Filters::Step.new(:limit, {:amount => amount}, {})
      #   self
      # end

      def sort(att, opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:sort, {:key => att, :order => opts.delete(:order)}, {})
        self
      end

      def intersect(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:intersect, {}, attrs)
        self
      end

      def union(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:union, {}, attrs)
        self
      end

      def diff(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:diff, {}, attrs)
        self
      end

      def intersect_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:intersect_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      def union_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:union_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      def diff_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:diff_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      # step users
      def exists?(id)
        lock(false) { _exists?(id) }
      end

      def find_by_id(id)
        lock { _find_by_id(id) }
      end

      def find_by_id!(id)
        ret = lock { _find_by_id(id) }
        raise ::Sandstorm::Records::Errors::RecordNotFound.new(@associated_class, id) if ret.nil?
        ret
      end

      def find_by_ids(*ids)
        lock { _ids.collect {|id| _find_by_id(id) } }
      end

      def find_by_ids!(*ids)
        ret = lock { _ids.collect {|id| _find_by_id(id) } }
        unless ids.length.eql?(ret.length)
          raise ::Sandstorm::Records::Errors::RecordsNotFound.new(@associated_class, ids - ret.map(&:id))
        end
        ret
      end

      def ids
        lock(false) { _ids }
      end

      def count
        lock(false) { _count }
      end

      def empty?
        lock(false) { _count == 0 }
      end

      def all
        lock { _all }
      end

      def collect(&block)
        lock { _ids.collect {|id| block.call(_load(id))} }
      end

      def each(&block)
        lock { _ids.each {|id| block.call(_load(id)) } }
      end

      def select(&block)
        lock { _all.select {|obj| block.call(obj) } }
      end
      alias_method :find_all, :select

      def reject(&block)
        lock { _all.reject {|obj| block.call(obj)} }
      end

      def destroy_all
        lock(*@associated_class.send(:associated_classes)) { _all.each {|r| r.destroy } }
      end

      protected

      def lock(when_steps_empty = true, *klasses, &block)
        if !when_steps_empty && @steps.empty?
          return block.call
        end
        klasses += [@associated_class] if !klasses.include?(@associated_class)
        @backend.lock(*klasses, &block)
      end

      private

      def _find_by_id(id)
        if !id.nil? && _exists?(id)
          _load(id.to_s)
        else
          nil
        end
      end

      def _load(id)
        object = @associated_class.new
        object.load(id)
        object
      end

      def _all
        _ids.map {|id| _load(id) }
      end

    end

  end

end
<commit_msg>bugfix filter find_by_ids<commit_after>require 'active_support/concern'

require 'sandstorm/records/errors'

require 'sandstorm/filters/step'

module Sandstorm

  module Filters

    module Base

      extend ActiveSupport::Concern

      attr_reader :backend

      # initial set         a Sandstorm::Record::Key object
      # associated_class    the class of the result record
      def initialize(data_backend, initial_set, associated_class)
        @backend          = data_backend
        @initial_set      = initial_set
        @associated_class = associated_class
        @steps            = []
      end

      # # TODO implement
      # # will probably need to scan and extract the last usage from steps
      # def limit(amount)
      #   @steps << Sandstorm::Filters::Step.new(:limit, {:amount => amount}, {})
      #   self
      # end

      def sort(att, opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:sort, {:key => att, :order => opts.delete(:order)}, {})
        self
      end

      def intersect(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:intersect, {}, attrs)
        self
      end

      def union(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:union, {}, attrs)
        self
      end

      def diff(attrs = {})
        @steps << ::Sandstorm::Filters::Step.new(:diff, {}, attrs)
        self
      end

      def intersect_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:intersect_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      def union_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:union_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      def diff_range(start, finish, attrs_opts = {})
        @steps << ::Sandstorm::Filters::Step.new(:diff_range, {:start => start, :finish => finish,
          :order => attrs_opts.delete(:order),
          :by_score => attrs_opts.delete(:by_score)}, attrs_opts)
        self
      end

      # step users
      def exists?(e_id)
        lock(false) { _exists?(e_id) }
      end

      def find_by_id(f_id)
        lock { _find_by_id(f_id) }
      end

      def find_by_id!(f_id)
        ret = lock { _find_by_id(f_id) }
        raise ::Sandstorm::Records::Errors::RecordNotFound.new(@associated_class, f_id) if ret.nil?
        ret
      end

      def find_by_ids(*f_ids)
        lock { f_ids.collect {|f_id| _find_by_id(f_id) } }
      end

      def find_by_ids!(*f_ids)
        ret = lock { f_ids.collect {|f_id| _find_by_id(f_id) } }
        unless f_ids.length.eql?(ret.length)
          raise ::Sandstorm::Records::Errors::RecordsNotFound.new(@associated_class, f_ids - ret.map(&:id))
        end
        ret
      end

      def ids
        lock(false) { _ids }
      end

      def count
        lock(false) { _count }
      end

      def empty?
        lock(false) { _count == 0 }
      end

      def all
        lock { _all }
      end

      def collect(&block)
        lock { _ids.collect {|id| block.call(_load(id))} }
      end

      def each(&block)
        lock { _ids.each {|id| block.call(_load(id)) } }
      end

      def select(&block)
        lock { _all.select {|obj| block.call(obj) } }
      end
      alias_method :find_all, :select

      def reject(&block)
        lock { _all.reject {|obj| block.call(obj)} }
      end

      def destroy_all
        lock(*@associated_class.send(:associated_classes)) { _all.each {|r| r.destroy } }
      end

      protected

      def lock(when_steps_empty = true, *klasses, &block)
        if !when_steps_empty && @steps.empty?
          return block.call
        end
        klasses += [@associated_class] if !klasses.include?(@associated_class)
        @backend.lock(*klasses, &block)
      end

      private

      def _find_by_id(id)
        if !id.nil? && _exists?(id)
          _load(id.to_s)
        else
          nil
        end
      end

      def _load(id)
        object = @associated_class.new
        object.load(id)
        object
      end

      def _all
        _ids.map {|id| _load(id) }
      end

    end

  end

end

{'fraction_non_alphanumeric': 0.12686395962376693, 'fraction_numerical': 0.00022941041523285156, 'mean_word_length': 2.0689897923266454, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7194772', 'n_tokens_mistral': 2784, 'n_tokens_neox': 2727, 'n_words': 596}