<commit_before>module Restforce

  module DB

    # Restforce::DB::Cleaner is responsible for culling the matching database
    # records when a Salesforce record is no longer available to synchronize
    # for a specific mapping.
    class Cleaner < Task

      # RecordsChanged is an exception which reports IDs for Salesforce records
      # that were updated externally during the cleaning process.
      class RecordsChanged < RuntimeError

        # Public: Initialize a new RecordsChanged exception.
        #
        # salesforce_model - A String name of a Salesforce object type.
        # ids              - An Array of String Salesforce IDs.
        def initialize(salesforce_model, ids)
          super <<-MESSAGE
            The following #{salesforce_model} records were updated externally
            during the cleaning phase of synchronization: #{ids.join(', ')}
          MESSAGE
        end

      end

      # Salesforce can take a few minutes to register record deletion. This
      # buffer gives us a window of time (in seconds) to look back and see
      # records which may not have been visible in previous runs.
      DELETION_READ_BUFFER = 3 * 60

      # Public: Run the database culling loop for this mapping.
      #
      # Returns nothing.
      def run(*_)
        @mapping.database_record_type.destroy_all(dropped_salesforce_ids)
      end

      private

      # Internal: Get a comprehensive list of Salesforce IDs corresponding to
      # records which should be dropped from synchronization for this mapping.
      #
      # Returns an Array of IDs.
      def dropped_salesforce_ids
        deleted_salesforce_ids + invalid_salesforce_ids
      end

      # Internal: Get the IDs of records which have been removed from Salesforce
      # for this mapping within the DELETION_BUFFER for this run.
      #
      # Returns an Array of IDs.
      def deleted_salesforce_ids
        return [] unless @runner.after

        response = Restforce::DB.client.get_deleted_between(
          @mapping.salesforce_model,
          @runner.after - DELETION_READ_BUFFER,
          @runner.before,
        )

        response.deletedRecords.map(&:id)
      end

      # Internal: Get the IDs of records which are in the larger collection
      # of Salesforce records, but which do not meet the specific conditions for
      # this mapping.
      #
      # Returns an Array of IDs.
      def invalid_salesforce_ids
        return [] if @mapping.conditions.empty? || @mapping.strategy.passive?

        # NOTE: We need to query for _valid_ records first, because, in the
        # scenario where a record is updated _between_ the two queries running,
        # the change to the SystemModstamp will prevent the record from being
        # picked up in the second query. In this situation, it's safer to omit
        # the ID from the list of aggregate IDs than it is to omit it from the
        # list of valid IDs.
        valid_ids = valid_salesforce_ids
        all_ids = all_salesforce_ids

        updated_ids = valid_ids - all_ids
        DB.logger.error(RecordsChanged.new(@mapping.salesforce_model, updated_ids)) if updated_ids

        all_ids - valid_ids
      end

      # Internal: Get the IDs of all recently-modified Salesforce records
      # corresponding to the object type for this mapping.
      #
      # Returns an Array of IDs.
      def all_salesforce_ids
        @mapping.unscoped { salesforce_ids(@mapping) }
      end

      # Internal: Get the IDs of the recently-modified Salesforce records for
      # mappings between the same object types.
      #
      # Returns an Array of IDs.
      def valid_salesforce_ids
        parallel_mappings.flat_map { |mapping| salesforce_ids(mapping) }
      end

      # Internal: Get the IDs of the recently-modified Salesforce records which
      # meet the conditions for this mapping.
      #
      # mapping - A Restforce::DB::Mapping.
      #
      # Returns an Array of IDs.
      def salesforce_ids(mapping)
        @runner.run(mapping) do |run|
          run.salesforce_instances.map(&:id)
        end
      end

      # Internal: Get a list of mappings between identical Salesforce and
      # database record types. This allows us to protect against inadvertently
      # removing records which belong to a parallel mapping on the same
      # ActiveRecord class.
      #
      # Returns an Array of Mappings.
      def parallel_mappings
        Registry[@mapping.database_model].select do |mapping|
          mapping.salesforce_model == @mapping.salesforce_model
        end
      end

    end

  end

end
<commit_msg>Fix problematic nil check on Array<commit_after>module Restforce

  module DB

    # Restforce::DB::Cleaner is responsible for culling the matching database
    # records when a Salesforce record is no longer available to synchronize
    # for a specific mapping.
    class Cleaner < Task

      # RecordsChanged is an exception which reports IDs for Salesforce records
      # that were updated externally during the cleaning process.
      class RecordsChanged < RuntimeError

        # Public: Initialize a new RecordsChanged exception.
        #
        # salesforce_model - A String name of a Salesforce object type.
        # ids              - An Array of String Salesforce IDs.
        def initialize(salesforce_model, ids)
          super <<-MESSAGE
            The following #{salesforce_model} records were updated externally
            during the cleaning phase of synchronization: #{ids.join(', ')}
          MESSAGE
        end

      end

      # Salesforce can take a few minutes to register record deletion. This
      # buffer gives us a window of time (in seconds) to look back and see
      # records which may not have been visible in previous runs.
      DELETION_READ_BUFFER = 3 * 60

      # Public: Run the database culling loop for this mapping.
      #
      # Returns nothing.
      def run(*_)
        @mapping.database_record_type.destroy_all(dropped_salesforce_ids)
      end

      private

      # Internal: Get a comprehensive list of Salesforce IDs corresponding to
      # records which should be dropped from synchronization for this mapping.
      #
      # Returns an Array of IDs.
      def dropped_salesforce_ids
        deleted_salesforce_ids + invalid_salesforce_ids
      end

      # Internal: Get the IDs of records which have been removed from Salesforce
      # for this mapping within the DELETION_BUFFER for this run.
      #
      # Returns an Array of IDs.
      def deleted_salesforce_ids
        return [] unless @runner.after

        response = Restforce::DB.client.get_deleted_between(
          @mapping.salesforce_model,
          @runner.after - DELETION_READ_BUFFER,
          @runner.before,
        )

        response.deletedRecords.map(&:id)
      end

      # Internal: Get the IDs of records which are in the larger collection
      # of Salesforce records, but which do not meet the specific conditions for
      # this mapping.
      #
      # Returns an Array of IDs.
      def invalid_salesforce_ids
        return [] if @mapping.conditions.empty? || @mapping.strategy.passive?

        # NOTE: We need to query for _valid_ records first, because, in the
        # scenario where a record is updated _between_ the two queries running,
        # the change to the SystemModstamp will prevent the record from being
        # picked up in the second query. In this situation, it's safer to omit
        # the ID from the list of aggregate IDs than it is to omit it from the
        # list of valid IDs.
        valid_ids = valid_salesforce_ids
        all_ids = all_salesforce_ids

        updated_ids = valid_ids - all_ids
        DB.logger.error(RecordsChanged.new(@mapping.salesforce_model, updated_ids)) unless updated_ids.empty?

        all_ids - valid_ids
      end

      # Internal: Get the IDs of all recently-modified Salesforce records
      # corresponding to the object type for this mapping.
      #
      # Returns an Array of IDs.
      def all_salesforce_ids
        @mapping.unscoped { salesforce_ids(@mapping) }
      end

      # Internal: Get the IDs of the recently-modified Salesforce records for
      # mappings between the same object types.
      #
      # Returns an Array of IDs.
      def valid_salesforce_ids
        parallel_mappings.flat_map { |mapping| salesforce_ids(mapping) }
      end

      # Internal: Get the IDs of the recently-modified Salesforce records which
      # meet the conditions for this mapping.
      #
      # mapping - A Restforce::DB::Mapping.
      #
      # Returns an Array of IDs.
      def salesforce_ids(mapping)
        @runner.run(mapping) do |run|
          run.salesforce_instances.map(&:id)
        end
      end

      # Internal: Get a list of mappings between identical Salesforce and
      # database record types. This allows us to protect against inadvertently
      # removing records which belong to a parallel mapping on the same
      # ActiveRecord class.
      #
      # Returns an Array of Mappings.
      def parallel_mappings
        Registry[@mapping.database_model].select do |mapping|
          mapping.salesforce_model == @mapping.salesforce_model
        end
      end

    end

  end

end

{'fraction_non_alphanumeric': 0.052449816533563565, 'fraction_numerical': 0.0006475285991797971, 'mean_word_length': 2.3588256614715477, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3367164', 'n_tokens_mistral': 2444, 'n_tokens_neox': 2300, 'n_words': 949}