<commit_before>module GitHub
  module Replicator
    # Load replicants in a streaming fashion.
    #
    # The Loader reads [type, id, attributes] replicant tuples and creates
    # objects in the current environment.
    #
    # Objects are expected to arrive in order such that a record referenced via
    # foreign key always precedes the referencing record. The Loader maintains a
    # mapping of primary keys from the dump system to the current environment.
    # This mapping is used to properly establish new foreign key values on all
    # records inserted.
    class Loader
      attr_reader :stats

      def initialize
        fail "not ready for production" if RAILS_ENV == 'production'
        @keymap = {}
        @warned = {}
        @foreign_key_map = {}
        @filters = []
        @stats = Hash.new { |hash, k| hash[k] = 0 }

        if block_given?
          yield self
          complete
        end
      end

      # Register a load filter to be called for each loaded object with the
      # type, id, attributes, object structure. Filters are executed in the
      # reverse order of which they were registered. Filters registered later
      # modify the view of filters registered earlier.
      #
      # p - An optional Proc object. Must respond to call.
      # block - An optional block.
      #
      # Returns nothing.
      def filter(p=nil, &block)
        @filters.unshift p if p
        @filters.unshift block if block
      end

      # Sugar for creating a filter with an object instance. Instances of the
      # class must respond to call(type, id, attrs, object).
      #
      # klass - The class to create. Must respond to new.
      # args  - Arguments to pass to klass#new in addition to self.
      #
      # Returns the object created.
      def use(klass, *args, &block)
        instance = klass.new(self, *args, &block)
        filter instance
        instance
      end

      # Notify all filters that processing is complete.
      def complete
        @filters.each { |f| f.complete if f.respond_to?(:complete) }
      end

      # Register a filter to write status information to the given stream. By
      # default, a single line is used to report object counts while the dump is
      # in progress; dump counts for each class are written when complete. The
      # verbose and quiet options can be used to increase or decrease
      # verbository.
      #
      # out - An IO object to write to, like stderr.
      # verbose - Whether verbose output should be enabled.
      # quiet - Whether quiet output should be enabled.
      #
      # Returns the Replicator::Status object.
      def log_to(out=$stderr, verbose=false, quiet=false)
        use Replicator::Status, 'load', out, verbose, quiet
      end

      # Feed a single replicant tuple into the loader.
      #
      # type  - The class to create. Must respond to load_replicant.
      # id    - The remote system's id for this object.
      # attrs - Hash of primitively typed objects.
      #
      # Returns the need object resulting from the load operation.
      def feed(type, id, attrs)
        object = load(type, id, attrs)
        @stats[type] += 1
        @filters.each { |filter| filter.call(type, id, attrs, object) }
        object
      end

      # Read multiple [type, id, attrs] replicant tuples from io and call the
      # feed method to load and filter the object.
      def read(io)
        begin
          while object = Marshal.load(io)
            type, id, attrs = object
            feed type, id, attrs
          end
        rescue EOFError
        end
      end

      # Load an individual replicant into the underlying datastore.
      #
      # type  - Model class name as a String.
      # id    - Primary key id of the record on the dump system. This must be
      #         translated to the local system and stored in the keymap.
      # attrs - Hash of attributes to set on the new record.
      #
      # Returns the new object instance.
      def load(type, id, attributes)
        model_class = type.constantize
        translate_ids attributes
        begin
          new_id, instance = model_class.load_replicant(type, id, attributes)
        rescue => boom
          warn "error: loading #{type} #{id} #{boom.class} #{boom}"
        end
        register_id instance, type, id, new_id
        instance
      end

      # Translate remote system id references in the attributes hash to their
      # local system id values. The attributes hash may include special id
      # values like this:<commit_msg>fail fast<commit_after>module GitHub
  module Replicator
    # Load replicants in a streaming fashion.
    #
    # The Loader reads [type, id, attributes] replicant tuples and creates
    # objects in the current environment.
    #
    # Objects are expected to arrive in order such that a record referenced via
    # foreign key always precedes the referencing record. The Loader maintains a
    # mapping of primary keys from the dump system to the current environment.
    # This mapping is used to properly establish new foreign key values on all
    # records inserted.
    class Loader
      attr_reader :stats

      def initialize
        fail "not ready for production" if RAILS_ENV == 'production'
        @keymap = {}
        @warned = {}
        @foreign_key_map = {}
        @filters = []
        @stats = Hash.new { |hash, k| hash[k] = 0 }

        if block_given?
          yield self
          complete
        end
      end

      # Register a load filter to be called for each loaded object with the
      # type, id, attributes, object structure. Filters are executed in the
      # reverse order of which they were registered. Filters registered later
      # modify the view of filters registered earlier.
      #
      # p - An optional Proc object. Must respond to call.
      # block - An optional block.
      #
      # Returns nothing.
      def filter(p=nil, &block)
        @filters.unshift p if p
        @filters.unshift block if block
      end

      # Sugar for creating a filter with an object instance. Instances of the
      # class must respond to call(type, id, attrs, object).
      #
      # klass - The class to create. Must respond to new.
      # args  - Arguments to pass to klass#new in addition to self.
      #
      # Returns the object created.
      def use(klass, *args, &block)
        instance = klass.new(self, *args, &block)
        filter instance
        instance
      end

      # Notify all filters that processing is complete.
      def complete
        @filters.each { |f| f.complete if f.respond_to?(:complete) }
      end

      # Register a filter to write status information to the given stream. By
      # default, a single line is used to report object counts while the dump is
      # in progress; dump counts for each class are written when complete. The
      # verbose and quiet options can be used to increase or decrease
      # verbository.
      #
      # out - An IO object to write to, like stderr.
      # verbose - Whether verbose output should be enabled.
      # quiet - Whether quiet output should be enabled.
      #
      # Returns the Replicator::Status object.
      def log_to(out=$stderr, verbose=false, quiet=false)
        use Replicator::Status, 'load', out, verbose, quiet
      end

      # Feed a single replicant tuple into the loader.
      #
      # type  - The class to create. Must respond to load_replicant.
      # id    - The remote system's id for this object.
      # attrs - Hash of primitively typed objects.
      #
      # Returns the need object resulting from the load operation.
      def feed(type, id, attrs)
        object = load(type, id, attrs)
        @stats[type] += 1
        @filters.each { |filter| filter.call(type, id, attrs, object) }
        object
      end

      # Read multiple [type, id, attrs] replicant tuples from io and call the
      # feed method to load and filter the object.
      def read(io)
        begin
          while object = Marshal.load(io)
            type, id, attrs = object
            feed type, id, attrs
          end
        rescue EOFError
        end
      end

      # Load an individual replicant into the underlying datastore.
      #
      # type  - Model class name as a String.
      # id    - Primary key id of the record on the dump system. This must be
      #         translated to the local system and stored in the keymap.
      # attrs - Hash of attributes to set on the new record.
      #
      # Returns the new object instance.
      def load(type, id, attributes)
        model_class = type.constantize
        translate_ids attributes
        begin
          new_id, instance = model_class.load_replicant(type, id, attributes)
        rescue => boom
          warn "error: loading #{type} #{id} #{boom.class} #{boom}"
          raise
        end
        register_id instance, type, id, new_id
        instance
      end

      # Translate remote system id references in the attributes hash to their
      # local system id values. The attributes hash may include special id
      # values like this:
{'fraction_non_alphanumeric': 0.06453730037190987, 'fraction_numerical': 0.0004375410194705754, 'mean_word_length': 2.077415011780545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6488597', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2366, 'n_words': 1079}