<commit_before>require 'rbbt/util/open'
require 'rbbt/util/log'
require 'rbbt/resource/path'
require 'net/http'
require 'set'

 
module Resource

  class << self
    attr_accessor :lock_dir
    
    def lock_dir
      @lock_dir ||= Rbbt.tmp.produce_locks.find
    end
  end

  def self.remote_servers
    @remote_servers = Rbbt.etc.file_servers.exists? ? Rbbt.etc.file_servers.yaml : {}
  end

  def self.extended(base)
    base.pkgdir = 'rbbt'
    base.subdir = ''
    base.resources = {}
    base.rake_dirs = {}
    base.search_paths = Path::SEARCH_PATHS.dup
    base.remote_server = Resource.remote_servers[base.to_s] || Resource.remote_servers["*"]
    base
  end

  attr_accessor :pkgdir, :subdir, :resources, :rake_dirs, :remote_server, :search_paths

  def set_libdir(value = nil)
    _libdir = value || Path.caller_lib_dir
    search_paths.merge!(:lib => File.join(_libdir, '{TOPLEVEL}', '{SUBPATH}'))
  end

  def root
    Path.setup @subdir || "", @pkgdir, self, @search_paths
  end

  def method_missing(name, prev = nil, *args)
    if prev.nil?
      root.send(name, *args)
    else
      root.send(name, prev, *args)
    end
  end

  def [](file = nil)
    if file.nil?
      root
    else
      root.send(file)
    end
  end

  def claim(path, type, content = nil, &block)
    if type == :rake
      @rake_dirs[path] = content
    else
      @resources[path] = [type, content || block]

      if type == :install
        software_dir = path.resource.root.software
        set_software_env(software_dir) unless $set_software_env
        $set_software_env = true
      end
    end
  end

  attr_accessor :server_missing_resource_cache
  def get_from_server(path, final_path, remote_server = nil)
    remote_server ||= self.remote_server
    remote_server = "http://" + remote_server unless remote_server =~ /^[a-z]+:\/\//
    url = File.join(remote_server, '/resource/', self.to_s, 'get_file')
    url << "?" << Misc.hash2GET_params(:file => path, :create => false)

    begin
      @server_missing_resource_cache ||= Set.new
      raise "Resource Not Found" if @server_missing_resource_cache.include? url
      
      #lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
      
      lock_filename = nil # it seems like this was locked already.

      Misc.lock lock_filename do
        uri = URI(url)

        http = Net::HTTP.new(uri.host, uri.port)
        request = Net::HTTP::Get.new(uri.request_uri)
        timeout = 60 * 10
        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
          http.request request do |response|
            filename = if response["Content-Disposition"] 
                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
                       else
                         nil
                       end

            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
              extension = filename.split(".").last
              final_path += '.' + extension
            end
            case response
            when Net::HTTPSuccess, Net::HTTPOK
              Misc.sensiblewrite(final_path) do |file|
                response.read_body do |chunk|
                  file.write chunk
                end
              end
            when Net::HTTPRedirection, Net::HTTPFound
              location = response['location']
              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
              FileUtils.mkdir_p final_path unless File.exist? final_path
              TmpFile.with_file do |tmp_dir|
                Misc.in_dir tmp_dir do
                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
                end
                FileUtils.mv tmp_dir, final_path
              end
            when Net::HTTPInternalServerError
              @server_missing_resource_cache << url
              raise "Resource Not Found"
            else
              raise "Response not understood: #{response.inspect}"
            end
          end
        end
      end
    rescue
      Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
      Log.error $!.message
      FileUtils.rm_rf final_path if File.exist? final_path
      return false
    end
  end

  def produce(path, force = false)
    case
    when @resources.include?(path)
      type, content = @resources[path]
    when (Path === path && @resources.include?(path.original))
      type, content = @resources[path.original]
    when has_rake(path)
      type = :rake
      rake_dir, content = rake_for(path)
      rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
    else
      begin<commit_msg>Fix issue with redirects and directories in file_server<commit_after>require 'rbbt/util/open'
require 'rbbt/util/log'
require 'rbbt/resource/path'
require 'net/http'
require 'set'

 
module Resource

  class << self
    attr_accessor :lock_dir
    
    def lock_dir
      @lock_dir ||= Rbbt.tmp.produce_locks.find
    end
  end

  def self.remote_servers
    @remote_servers = Rbbt.etc.file_servers.exists? ? Rbbt.etc.file_servers.yaml : {}
  end

  def self.extended(base)
    base.pkgdir = 'rbbt'
    base.subdir = ''
    base.resources = {}
    base.rake_dirs = {}
    base.search_paths = Path::SEARCH_PATHS.dup
    base.remote_server = Resource.remote_servers[base.to_s] || Resource.remote_servers["*"]
    base
  end

  attr_accessor :pkgdir, :subdir, :resources, :rake_dirs, :remote_server, :search_paths

  def set_libdir(value = nil)
    _libdir = value || Path.caller_lib_dir
    search_paths.merge!(:lib => File.join(_libdir, '{TOPLEVEL}', '{SUBPATH}'))
  end

  def root
    Path.setup @subdir || "", @pkgdir, self, @search_paths
  end

  def method_missing(name, prev = nil, *args)
    if prev.nil?
      root.send(name, *args)
    else
      root.send(name, prev, *args)
    end
  end

  def [](file = nil)
    if file.nil?
      root
    else
      root.send(file)
    end
  end

  def claim(path, type, content = nil, &block)
    if type == :rake
      @rake_dirs[path] = content
    else
      @resources[path] = [type, content || block]

      if type == :install
        software_dir = path.resource.root.software
        set_software_env(software_dir) unless $set_software_env
        $set_software_env = true
      end
    end
  end

  attr_accessor :server_missing_resource_cache
  def get_from_server(path, final_path, remote_server = nil)
    remote_server ||= self.remote_server
    remote_server = "http://" + remote_server unless remote_server =~ /^[a-z]+:\/\//
    url = File.join(remote_server, '/resource/', self.to_s, 'get_file')
    url << "?" << Misc.hash2GET_params(:file => path, :create => false)

    begin
      @server_missing_resource_cache ||= Set.new
      raise "Resource Not Found" if @server_missing_resource_cache.include? url
      
      #lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
      
      lock_filename = nil # it seems like this was locked already.

      Misc.lock lock_filename do
        uri = URI(url)

        http = Net::HTTP.new(uri.host, uri.port)
        request = Net::HTTP::Get.new(uri.request_uri)
        timeout = 60 * 10
        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
          http.request request do |response|
            filename = if response["Content-Disposition"] 
                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
                       else
                         nil
                       end

            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
              extension = filename.split(".").last
              final_path += '.' + extension
            end
            case response
            when Net::HTTPSuccess, Net::HTTPOK
              Misc.sensiblewrite(final_path) do |file|
                response.read_body do |chunk|
                  file.write chunk
                end
              end
            when Net::HTTPRedirection, Net::HTTPFound
              location = response['location']
              if location.include? 'get_directory'
                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
                FileUtils.mkdir_p final_path unless File.exist? final_path
                TmpFile.with_file do |tmp_dir|
                  Misc.in_dir tmp_dir do
                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
                  end
                  FileUtils.mv tmp_dir, final_path
                end
              else
                Open.open(location, :nocache => true) do |s|
                  Misc.sensiblewrite(final_path, s)
                end
              end
            when Net::HTTPInternalServerError
              @server_missing_resource_cache << url
              raise "Resource Not Found"
            else
              raise "Response not understood: #{response.inspect}"
            end
          end
        end
      end
    rescue
      Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
      Log.error $!.message
      FileUtils.rm_rf final_path if File.exist? final_path
      return false
    end
  end

  def produce(path, force = false)
    case
    when @resources.include?(path)
      type, content = @resources[path]
    when (Path === path && @resources.include?(path.original))
      type, content = @resources[path.original]
    when has_rake(path)
      type = :rake
      rake_dir, content = rake_for(path)
      rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
    else
      begin
{'fraction_non_alphanumeric': 0.11577879249112126, 'fraction_numerical': 0.0010147133434804667, 'mean_word_length': 2.17628101836932, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272069', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2989, 'n_words': 687}