<commit_before>require 'sandstorm/filters/base'

module Sandstorm

  module Filters

    class InfluxDBFilter

      include Sandstorm::Filters::Base

      private

      # NB not trying to handle steps yet
      def _exists?(id)
        return if id.nil?
        Sandstorm.influxdb.query("SELECT id from #{@initial_set.klass}")[@initial_set.klass].present?
      end

      def lock(when_steps_empty = true, *klasses, &block)
        # no-op
        block.call
      end

      def _ids
        resolve_steps(:ids)
      end

      def _count
        resolve_steps(:count)
      end

      def resolve_steps(result_type)
        query = case result_type
        when :ids
          "SELECT id FROM #{@initial_set.klass}"
        when :count
          "SELECT COUNT(id) FROM #{@initial_set.klass}"
        end

        unless @steps.empty?

          query += ' WHERE '
          step_count = 0

          query += ('(' * (@steps.size / 3))

          @steps.each_slice(3) do |step|
            step_type = step.first
            options   = step[1] || {}
            values    = step.last

            if step_count > 0
              case step_type
              when :intersect, :diff
                query += ' AND '
              when :union
                query += ' OR '
              else
                raise "Unhandled filter operation '#{step_type}'"
              end
            end

            case step_type
            when :intersect, :union
              query += values.collect {|k, v|
                op, value = case v
                when String
                  ["=~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
                else
                  ["=",  "'#{v}'"]
                end

               "#{k} #{op} #{value}"
              }.join(' AND ')
            when :diff
              query += values.collect {|k, v|
                op, value = case v
                when String
                  ["!~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
                else
                  ["!=",  "'#{v}'"]
                end

                "#{k} #{op} #{value}"
              }.join(' AND ')
            else
              raise "Unhandled filter operation '#{step_type}'"
            end

            query += ")"

            step_count += 3
          end

        end

        result = Sandstorm.influxdb.query(query)

        data = result[@initial_set.klass]

        return [] if data.nil?

        case result_type
        when :ids
          data.collect {|d| d['id']}
        when :count
          data.first['count']
        end
      end

    end

  end

end<commit_msg>fix chaining for #exists?<commit_after>require 'sandstorm/filters/base'

module Sandstorm

  module Filters

    class InfluxDBFilter

      include Sandstorm::Filters::Base

      private

      def _exists?(id)
        return if id.nil?
        @steps += [:intersect, {}, {:id => id}]
        resolve_steps(:count) > 0
      end

      def lock(when_steps_empty = true, *klasses, &block)
        # no-op
        block.call
      end

      def _ids
        resolve_steps(:ids)
      end

      def _count
        resolve_steps(:count)
      end

      def resolve_step(step)
        query = ''

        step_type = step.first
        options   = step[1] || {}
        values    = step.last

        case step_type
        when :intersect, :union
          query += values.collect {|k, v|
            op, value = case v
            when String
              ["=~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
            else
              ["=",  "'#{v}'"]
            end

           "#{k} #{op} #{value}"
          }.join(' AND ')
        when :diff
          query += values.collect {|k, v|
            op, value = case v
            when String
              ["!~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
            else
              ["!=",  "'#{v}'"]
            end

            "#{k} #{op} #{value}"
          }.join(' AND ')
        else
          raise "Unhandled filter operation '#{step_type}'"
        end

        query
      end

      def resolve_steps(result_type)
        query = case result_type
        when :ids
          "SELECT id FROM #{@initial_set.klass}"
        when :count
          "SELECT COUNT(id) FROM #{@initial_set.klass}"
        end

        unless @steps.empty?

          query += ' WHERE '
          step_count = 0

          query += ('(' * (@steps.size / 3))

          @steps.each_slice(3) do |step|

            step_type = step.first

            if step_count > 0
              case step_type
              when :intersect, :diff
                query += ' AND '
              when :union
                query += ' OR '
              else
                raise "Unhandled filter operation '#{step_type}'"
              end
            end

            query += resolve_step(step)

            query += ")"

            step_count += 3
          end

        end

        result = Sandstorm.influxdb.query(query)

        data = result[@initial_set.klass]

        case result_type
        when :ids
          data.nil? ? [] : data.collect {|d| d['id']}
        when :count
          data.nil? ? 0 : data.first['count']
        end
      end

    end

  end

end
{'fraction_non_alphanumeric': 0.1266450505435819, 'fraction_numerical': 0.0026702269692923898, 'mean_word_length': 1.2163989856297548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5311925', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1472, 'n_words': 368}