<commit_before>require 'openssl'
require 'base64'
require 'digest/sha1'
require 'addressable/uri'
require 'shellwords'

module Travis
  module Build
    class Script
      module DirectoryCache
        class S3
          # TODO: Switch to different branch from master?
          CASHER_URL = "https://raw.github.com/travis-ci/casher/master/bin/casher"

          attr_accessor :fetch_timeout, :push_timeout, :bucket, :secret_access_key, :access_key_id, :uri_parser, :host, :scheme, :slug, :repository, :start

          def initialize(options, repository, slug, start = Time.now)
            @fetch_timeout     = options.fetch(:fetch_timeout)
            @push_timeout      = options.fetch(:push_timeout)
            @bucket            = options[:s3].fetch(:bucket)
            @secret_access_key = options[:s3].fetch(:secret_access_key)
            @access_key_id     = options[:s3].fetch(:access_key_id)
            @scheme            = options[:s3][:scheme] || "https"
            @host              = options[:s3][:host]   || "s3.amazonaws.com"
            @slug              = slug
            @repository        = repository
            @start             = start
          end

          def install(sh)
            sh.cmd "export CASHER_DIR=$HOME/.casher", log: false, echo: false
            sh.cmd "mkdir -p $CASHER_DIR/bin", log: false, echo: false
            sh.cmd "curl #{CASHER_URL} -o #{binary}", echo: false
            sh.cmd "chmod +x #{binary}", log: false, echo: false
          end

          def add(sh, path)
            run(sh, "add", path) if path
          end

          def fetch(sh)
            run(sh, "fetch", Shellwords.escape(fetch_url.to_s))
          end

          def push(sh)
            run(sh, "push", Shellwords.escape(push_url.to_s))
          end

          def fetch_url
            url("GET", prefixed, expires: start + fetch_timeout)
          end

          def push_url
            url("PUT", prefixed, expires: start + push_timeout)
          end

          private

            def prefixed
              File.join(repository.fetch(:github_id).to_s, slug) << ".tbz"
            end

            def url(verb, path, options = {})
              path    = File.join("/", bucket, path)
              expires = Integer(options[:expires])
              string  = [ verb, options[:md5], options[:content_type], expires, path ].join("\n")
              hmac    = OpenSSL::HMAC.digest('sha1', secret_access_key, string)
              Addressable::URI.new(host: host, scheme: scheme, path: path, query_values: {
                AWSAccessKeyId: access_key_id, Expires: expires, Signature: Base64.encode64(hmac).strip })
            end

            def binary
              "$CASHER_DIR/bin/casher"
            end

            def run(sh, command, argument)
              sh.cmd("#{binary} #{command} #{argument}", echo: false)
            end
        end

        class Dummy
          def initialize(*)
          end

          def method_missing(*)
            self
          end
        end

        def directory_cache
          @directory_cache ||= cache_class.new(data.cache_options, data.repository, cache_slug)
        end

        def cache_class
          type = data.cache_options[:type].to_s.capitalize
          type = "Dummy" if type.empty? or !use_directory_cache?
          raise ArgumentError, "unknown caching mode %p" % type unless DirectoryCache.const_defined?(type)
          DirectoryCache.const_get(type)
        end

        def use_directory_cache?
          data.cache?(:directories)
        end

        def setup_directory_cache<commit_msg>local only cont check<commit_after>require 'openssl'
require 'base64'
require 'digest/sha1'
require 'addressable/uri'
require 'shellwords'

module Travis
  module Build
    class Script
      module DirectoryCache
        class S3
          # TODO: Switch to different branch from master?
          CASHER_URL = "https://raw.github.com/travis-ci/casher/master/bin/casher"

          attr_accessor :fetch_timeout, :push_timeout, :bucket, :secret_access_key, :access_key_id, :uri_parser, :host, :scheme, :slug, :repository, :start

          def initialize(options, repository, slug, start = Time.now)
            @fetch_timeout     = options.fetch(:fetch_timeout)
            @push_timeout      = options.fetch(:push_timeout)
            @bucket            = options[:s3].fetch(:bucket)
            @secret_access_key = options[:s3].fetch(:secret_access_key)
            @access_key_id     = options[:s3].fetch(:access_key_id)
            @scheme            = options[:s3][:scheme] || "https"
            @host              = options[:s3][:host]   || "s3.amazonaws.com"
            @slug              = slug
            @repository        = repository
            @start             = start
          end

          def install(sh)
            sh.cmd "export CASHER_DIR=$HOME/.casher", log: false, echo: false
            sh.cmd "mkdir -p $CASHER_DIR/bin", log: false, echo: false
            sh.cmd "curl #{CASHER_URL} -o #{binary}", echo: false
            sh.cmd "chmod +x #{binary}", log: false, echo: false
          end

          def add(sh, path)
            run(sh, "add", path) if path
          end

          def fetch(sh)
            run(sh, "fetch", Shellwords.escape(fetch_url.to_s))
          end

          def push(sh)
            run(sh, "push", Shellwords.escape(push_url.to_s))
          end

          def fetch_url
            url("GET", prefixed, expires: start + fetch_timeout)
          end

          def push_url
            url("PUT", prefixed, expires: start + push_timeout)
          end

          private

            def prefixed
              File.join(repository.fetch(:github_id).to_s, slug) << ".tbz"
            end

            def url(verb, path, options = {})
              path    = File.join("/", bucket, path)
              expires = Integer(options[:expires])
              string  = [ verb, options[:md5], options[:content_type], expires, path ].join("\n")
              hmac    = OpenSSL::HMAC.digest('sha1', secret_access_key, string)
              Addressable::URI.new(host: host, scheme: scheme, path: path, query_values: {
                AWSAccessKeyId: access_key_id, Expires: expires, Signature: Base64.encode64(hmac).strip })
            end

            def binary
              "$CASHER_DIR/bin/casher"
            end

            def run(sh, command, argument)
              sh.cmd("#{binary} #{command} #{argument}", echo: false)
            end
        end

        class Dummy
          def initialize(*)
          end

          def method_missing(*)
            self
          end
        end

        def directory_cache
          @directory_cache ||= cache_class.new(data.cache_options, data.repository, cache_slug)
        end

        def cache_class
          type = data.cache_options[:type].to_s.capitalize
          type = "Dummy" if type.empty? or !use_directory_cache?
          raise ArgumentError, "unknown caching mode %p" % type unless DirectoryCache.const_defined?(type, false)
          DirectoryCache.const_get(type)
        end

        def use_directory_cache?
          data.cache?(:directories)
        end

        def setup_directory_cache
{'fraction_non_alphanumeric': 0.1073649302197043, 'fraction_numerical': 0.00442172170789001, 'mean_word_length': 1.8174386920980927, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6635184', 'n_tokens_mistral': 2098, 'n_tokens_neox': 2027, 'n_words': 485}