<commit_before>require 'open-uri'
require 'vendor/multi_json'

class AbstractDownloadStrategy
  def initialize name, package
    @url = package.url
    specs = package.specs
    @spec, @ref = specs.dup.shift unless specs.empty?
  end

  def expand_safe_system_args args
    args = args.dup
    args.each_with_index do |arg, ii|
      if arg.is_a? Hash
        unless ARGV.verbose?
          args[ii] = arg[:quiet_flag]
        else
          args.delete_at ii
        end
        return args
      end
    end
    # 2 as default because commands are eg. svn up, git pull
    args.insert(2, '-q') unless ARGV.verbose?
    args
  end

  def quiet_safe_system *args
    safe_system(*expand_safe_system_args(args))
  end
end

class CurlDownloadStrategy < AbstractDownloadStrategy
  attr_reader :tarball_path
  attr_accessor :local_bottle_path

  def initialize name, package
    super

    if name.to_s.empty? || name == '__UNKNOWN__'
      @tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{File.basename(@url)}")
    else
      @tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{name}-#{package.version}#{ext}")
    end

    @mirrors = package.mirrors
    @temporary_path = Pathname.new("#@tarball_path.incomplete")
    @local_bottle_path = nil
  end

  def cached_location
    @tarball_path
  end

  def downloaded_size
    @temporary_path.size? or 0
  end

  # Private method, can be overridden if needed.
  def _fetch
    curl @url, '-C', downloaded_size, '-o', @temporary_path
  end

  def fetch
    if @local_bottle_path
      @tarball_path = @local_bottle_path
      return @local_bottle_path
    end

    ohai "Downloading #{@url}"
    unless @tarball_path.exist?
      begin
        _fetch
      rescue ErrorDuringExecution
        raise CurlDownloadStrategyError, "Download failed: #{@url}"
      end
      ignore_interrupts { @temporary_path.rename(@tarball_path) }
    else<commit_msg>Retry downloads if a server doesn't support byte ranges<commit_after>require 'open-uri'
require 'vendor/multi_json'

class AbstractDownloadStrategy
  def initialize name, package
    @url = package.url
    specs = package.specs
    @spec, @ref = specs.dup.shift unless specs.empty?
  end

  def expand_safe_system_args args
    args = args.dup
    args.each_with_index do |arg, ii|
      if arg.is_a? Hash
        unless ARGV.verbose?
          args[ii] = arg[:quiet_flag]
        else
          args.delete_at ii
        end
        return args
      end
    end
    # 2 as default because commands are eg. svn up, git pull
    args.insert(2, '-q') unless ARGV.verbose?
    args
  end

  def quiet_safe_system *args
    safe_system(*expand_safe_system_args(args))
  end
end

class CurlDownloadStrategy < AbstractDownloadStrategy
  attr_reader :tarball_path
  attr_accessor :local_bottle_path

  def initialize name, package
    super

    if name.to_s.empty? || name == '__UNKNOWN__'
      @tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{File.basename(@url)}")
    else
      @tarball_path = Pathname.new("#{HOMEBREW_CACHE}/#{name}-#{package.version}#{ext}")
    end

    @mirrors = package.mirrors
    @temporary_path = Pathname.new("#@tarball_path.incomplete")
    @local_bottle_path = nil
  end

  def cached_location
    @tarball_path
  end

  def downloaded_size
    @temporary_path.size? or 0
  end

  # Private method, can be overridden if needed.
  def _fetch
    curl @url, '-C', downloaded_size, '-o', @temporary_path
  end

  def fetch
    if @local_bottle_path
      @tarball_path = @local_bottle_path
      return @local_bottle_path
    end

    ohai "Downloading #{@url}"
    unless @tarball_path.exist?
      had_incomplete_download = @temporary_path.exist?
      begin
        _fetch
      rescue ErrorDuringExecution
        # 33 == range not supported
        # try wiping the incomplete download and retrying once
        if $?.exitstatus == 33 && had_incomplete_download
          ohai "Trying a full download"
          @temporary_path.unlink
          had_incomplete_download = false
          retry
        else
          raise CurlDownloadStrategyError, "Download failed: #{@url}"
        end
      end
      ignore_interrupts { @temporary_path.rename(@tarball_path) }
    else
{'fraction_non_alphanumeric': 0.08794089609151573, 'fraction_numerical': 0.0023832221163012394, 'mean_word_length': 2.8575367647058822, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5984861', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1346, 'n_words': 334}