<commit_before>module Geminabox
  module Store
    class S3
      attr_reader :logger

      SPLICEABLE_GZIPPED_FILES = %w[
        specs.4.8.gz
        latest_specs.4.8.gz
        prerelease_specs.4.8.gz
      ]
      SPLICEABLE_TEXT_FILES = %w[
        yaml
        Marshal.4.8
        specs.4.8
        latest_specs.4.8
        prerelease_specs.4.8
      ]

      def initialize(bucket: nil, lock_manager: lock_manager, file_store: Geminabox::GemStore, logger: Logger.new(STDOUT))
        @bucket = bucket
        @file_store = file_store
        @lock_manager = lock_manager
        @logger = logger
      end

      def create(gem, overwrite = false)
        @file_store.create gem, overwrite

        object_name = gem_object_name("/gems/" + gem.name)

        logger.info "Gem: local -> S3 #{object_name}"

        @bucket
          .objects[object_name]
          .write gem.gem_data
        update_metadata
      end

      # Note: deleting doesn't make much sense in this case anyway, as
      # other instances  will continue  serving cached copies  of this
      # gem (there's no way to notify them that gem has been deleted)
      #
      # Do consider using Geminabox.allow_delete = false
      #
      def delete(path_info)
        @file_store.delete path_info

        @bucket.objects[gem_object_name(path_info)].delete
      end

      def update_local_file(path_info)
        gem_file = @file_store.local_path path_info

        unless File.exists? gem_file
          gem_object = @bucket.objects[gem_object_name(path_info)]
          if gem_object.exists?
            # Note: this will load the entire contents of the gem into
            # memory  We  might  switch   to  using  streaming  IO  or
            # temporary files  if this  proves to  be critical  in our
            # environment
            io = StringIO.new gem_object.read
            incoming_gem = Geminabox::IncomingGem.new io
            @file_store.create incoming_gem

            update_metadata
          end
        end

        @file_store.update_local_file path_info
      end

      def update_local_metadata_file(path_info)
        file_name = File.basename path_info
        pull_file file_name do |local, remote|
          if file_name =~ /\.gz$/
            merge_gzipped local, remote
          else
            merge_text local, remote
          end
        end
      end

      def reindex &block
        FileUtils.mkpath @file_store.local_path "gems"
        @bucket.objects.with_prefix("gems/").each do |object|
          path_info = "/" + object.key
          local_file_path = @file_store.local_path path_info

          file_does_not_exist = !File.exists?(local_file_path)
          file_has_different_size = object.content_length != File.size(local_file_path)
          if file_does_not_exist || file_has_different_size
            logger.info "Gem: S3 -> local #{local_file_path}"
            File.write local_file_path, object.read
          end
        end

        @file_store.reindex(&block)
      end

      private

      def update_metadata
        @lock_manager.lock ".metadata" do
          push_files SPLICEABLE_GZIPPED_FILES do |local_contents, remote_contents|
            merge_gzipped local_contents, remote_contents
          end

          push_files SPLICEABLE_TEXT_FILES do |local_contents, remote_contents|
            merge_text local_contents, remote_contents<commit_msg>[master] Proper handling of gems missing in the local cache when re-indexing.<commit_after>module Geminabox
  module Store
    class S3
      attr_reader :logger

      SPLICEABLE_GZIPPED_FILES = %w[
        specs.4.8.gz
        latest_specs.4.8.gz
        prerelease_specs.4.8.gz
      ]
      SPLICEABLE_TEXT_FILES = %w[
        yaml
        Marshal.4.8
        specs.4.8
        latest_specs.4.8
        prerelease_specs.4.8
      ]

      def initialize(bucket: nil, lock_manager: lock_manager, file_store: Geminabox::GemStore, logger: Logger.new(STDOUT))
        @bucket = bucket
        @file_store = file_store
        @lock_manager = lock_manager
        @logger = logger
      end

      def create(gem, overwrite = false)
        @file_store.create gem, overwrite

        object_name = gem_object_name("/gems/" + gem.name)

        logger.info "Gem: local -> S3 #{object_name}"

        @bucket
          .objects[object_name]
          .write gem.gem_data
        update_metadata
      end

      # Note: deleting doesn't make much sense in this case anyway, as
      # other instances  will continue  serving cached copies  of this
      # gem (there's no way to notify them that gem has been deleted)
      #
      # Do consider using Geminabox.allow_delete = false
      #
      def delete(path_info)
        @file_store.delete path_info

        @bucket.objects[gem_object_name(path_info)].delete
      end

      def update_local_file(path_info)
        gem_file = @file_store.local_path path_info

        unless File.exists? gem_file
          gem_object = @bucket.objects[gem_object_name(path_info)]
          if gem_object.exists?
            # Note: this will load the entire contents of the gem into
            # memory  We  might  switch   to  using  streaming  IO  or
            # temporary files  if this  proves to  be critical  in our
            # environment
            io = StringIO.new gem_object.read
            incoming_gem = Geminabox::IncomingGem.new io
            @file_store.create incoming_gem

            update_metadata
          end
        end

        @file_store.update_local_file path_info
      end

      def update_local_metadata_file(path_info)
        file_name = File.basename path_info
        pull_file file_name do |local, remote|
          if file_name =~ /\.gz$/
            merge_gzipped local, remote
          else
            merge_text local, remote
          end
        end
      end

      def reindex &block
        FileUtils.mkpath @file_store.local_path "gems"
        @bucket.objects.with_prefix("gems/").each do |object|
          path_info = "/" + object.key
          local_file_path = @file_store.local_path path_info

          file_does_not_exist = !File.exists?(local_file_path)

          # File.size raises an exception if file does not exist
          file_size = file_does_not_exists ? 0 : File.size(local_file_path)
          file_has_different_size = object.content_length != file_size

          if file_does_not_exist || file_has_different_size
            logger.info "Gem: S3 -> local #{local_file_path}"
            File.write local_file_path, object.read
          end
        end

        @file_store.reindex(&block)
      end

      private

      def update_metadata
        @lock_manager.lock ".metadata" do
          push_files SPLICEABLE_GZIPPED_FILES do |local_contents, remote_contents|
            merge_gzipped local_contents, remote_contents
          end

          push_files SPLICEABLE_TEXT_FILES do |local_contents, remote_contents|
            merge_text local_contents, remote_contents
{'fraction_non_alphanumeric': 0.06583833190516995, 'fraction_numerical': 0.004998571836618109, 'mean_word_length': 2.0607517482517483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042531', 'n_tokens_mistral': 2117, 'n_tokens_neox': 2035, 'n_words': 501}