<commit_before>module DC
  module Store

    # An implementation of an AssetStore.
    module S3Store

      BUCKET_NAME     = Rails.env.production? ? 's3.documentcloud.org' : "dcloud_#{Rails.env}"

      AUTH_PERIOD     = 5.minutes

      IMAGE_EXT       = /\.(gif|png|jpe?g)\Z/

      DEFAULT_ACCESS  = DC::Access::PUBLIC

      # 60 seconds for persistent connections.
      S3_PARAMS       = {:connection_lifetime => 60}

      ACCESS_TO_ACL   = Hash.new('private')
      ACCESS_TO_ACL[DC::Access::PUBLIC] = 'public-read'

      module ClassMethods
        def asset_root
          Rails.env.production? ? "http://s3.documentcloud.org" : "http://s3.amazonaws.com/#{BUCKET_NAME}"
        end
        def web_root
          Thread.current[:ssl] ? "https://s3.amazonaws.com/#{BUCKET_NAME}" : asset_root
        end
      end

      def initialize
        @key, @secret = SECRETS['aws_access_key'], SECRETS['aws_secret_key']
      end

      def read(path)
        bucket.get(path)
      end

      def authorized_url(path)
        interface = Thread.current[:ssl] ? secure_s3.interface : s3.interface
        interface.generate_link 'GET', {:url => "#{BUCKET_NAME}/#{CGI::escape(path)}"}, AUTH_PERIOD
      end

      def list(path)
        bucket.keys(:prefix => path).map {|key| key.name }
      end

      def save_pdf(document, pdf_path, access=DEFAULT_ACCESS)
        save_file(pdf_path, document.pdf_path, access)
      end

      def save_insert_pdf(document, pdf_path, pdf_name, access=DEFAULT_ACCESS)
        path = File.join(document.path, 'inserts', pdf_name)
        save_file(pdf_path, path, access)
      end

      def delete_insert_pdfs(document)
        path = File.join(document.pdf_path, 'inserts')
        bucket.delete_folder(path)
      end

      def save_full_text(document, access=DEFAULT_ACCESS)
        save_file(document.text, document.full_text_path, access, :string => true)
      end

      def save_rdf(document, rdf, access=DEFAULT_ACCESS)
        save_file(rdf, document.rdf_path, DC::Access::PRIVATE, :string => true)
      end

      def save_page_images(document, page_number, images, access=DEFAULT_ACCESS)
        Page::IMAGE_SIZES.keys.each do |size|
          save_file(images[size], document.page_image_path(page_number, size), access) if images[size]
        end
      end

      def delete_page_images(document, page_number)
        Page::IMAGE_SIZES.keys.eacy do |size|
          remove_file(document.page_image_path(page_number, size))
        end
      end

      def save_page_text(document, page_number, text, access=DEFAULT_ACCESS)
        save_file(text, document.page_text_path(page_number), access, :string => true)
      end

      def delete_page_text(document, page_number)
        remove_file(document.page_text_path(page_number))
      end

      def save_database_backup(name, path)
        bucket.put("backups/#{name}/#{Date.today}.dump", File.open(path))
      end

      # This is going to be *extremely* expensive. We can thread it, but
      # there must be a better way somehow. (running in the background for now)
      def set_access(document, access)
        save_permissions(document.pdf_path, access)
        save_permissions(document.full_text_path, access)
        document.pages.each do |page|
          save_permissions(document.page_text_path(page.page_number), access)<commit_msg>mo. typo.<commit_after>module DC
  module Store

    # An implementation of an AssetStore.
    module S3Store

      BUCKET_NAME     = Rails.env.production? ? 's3.documentcloud.org' : "dcloud_#{Rails.env}"

      AUTH_PERIOD     = 5.minutes

      IMAGE_EXT       = /\.(gif|png|jpe?g)\Z/

      DEFAULT_ACCESS  = DC::Access::PUBLIC

      # 60 seconds for persistent connections.
      S3_PARAMS       = {:connection_lifetime => 60}

      ACCESS_TO_ACL   = Hash.new('private')
      ACCESS_TO_ACL[DC::Access::PUBLIC] = 'public-read'

      module ClassMethods
        def asset_root
          Rails.env.production? ? "http://s3.documentcloud.org" : "http://s3.amazonaws.com/#{BUCKET_NAME}"
        end
        def web_root
          Thread.current[:ssl] ? "https://s3.amazonaws.com/#{BUCKET_NAME}" : asset_root
        end
      end

      def initialize
        @key, @secret = SECRETS['aws_access_key'], SECRETS['aws_secret_key']
      end

      def read(path)
        bucket.get(path)
      end

      def authorized_url(path)
        interface = Thread.current[:ssl] ? secure_s3.interface : s3.interface
        interface.generate_link 'GET', {:url => "#{BUCKET_NAME}/#{CGI::escape(path)}"}, AUTH_PERIOD
      end

      def list(path)
        bucket.keys(:prefix => path).map {|key| key.name }
      end

      def save_pdf(document, pdf_path, access=DEFAULT_ACCESS)
        save_file(pdf_path, document.pdf_path, access)
      end

      def save_insert_pdf(document, pdf_path, pdf_name, access=DEFAULT_ACCESS)
        path = File.join(document.path, 'inserts', pdf_name)
        save_file(pdf_path, path, access)
      end

      def delete_insert_pdfs(document)
        path = File.join(document.pdf_path, 'inserts')
        bucket.delete_folder(path)
      end

      def save_full_text(document, access=DEFAULT_ACCESS)
        save_file(document.text, document.full_text_path, access, :string => true)
      end

      def save_rdf(document, rdf, access=DEFAULT_ACCESS)
        save_file(rdf, document.rdf_path, DC::Access::PRIVATE, :string => true)
      end

      def save_page_images(document, page_number, images, access=DEFAULT_ACCESS)
        Page::IMAGE_SIZES.keys.each do |size|
          save_file(images[size], document.page_image_path(page_number, size), access) if images[size]
        end
      end

      def delete_page_images(document, page_number)
        Page::IMAGE_SIZES.keys.each do |size|
          remove_file(document.page_image_path(page_number, size))
        end
      end

      def save_page_text(document, page_number, text, access=DEFAULT_ACCESS)
        save_file(text, document.page_text_path(page_number), access, :string => true)
      end

      def delete_page_text(document, page_number)
        remove_file(document.page_text_path(page_number))
      end

      def save_database_backup(name, path)
        bucket.put("backups/#{name}/#{Date.today}.dump", File.open(path))
      end

      # This is going to be *extremely* expensive. We can thread it, but
      # there must be a better way somehow. (running in the background for now)
      def set_access(document, access)
        save_permissions(document.pdf_path, access)
        save_permissions(document.full_text_path, access)
        document.pages.each do |page|
          save_permissions(document.page_text_path(page.page_number), access)
{'fraction_non_alphanumeric': 0.10760059612518628, 'fraction_numerical': 0.0038748137108792846, 'mean_word_length': 2.843642611683849, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '888553', 'n_tokens_mistral': 2163, 'n_tokens_neox': 2097, 'n_words': 420}