<commit_before>class Downloader
  LOCAL_DIR = '/tmp/file_downloads'

  class << self
    # Download the url to local and return an open file. nil will be returned if failed.
    def download_to_local(url, object, logger, file_name = nil)
      if path = $url_mapper.get_file_path(url)
        return File.open(path) if File.exist?(path)
      end

      dir = download_dir(object)
      FileUtils.mkdir_p(dir) unless File.exist?(dir)

      file_name ||= name_from_url(url)
      local_file_path = File.join(dir, file_name)
      return File.open(local_file_path) if File.exist?(local_file_path)

      local_file = File.open(local_file_path, 'wb')
      tries = 10
      begin
        open(url, 'rb') do |read_file|
          local_file.write(read_file.read)
        end
        # Set the read count to the begin of the file
        local_file.rewind
      rescue StandardError => e
        tries -= 1
        if tries > 0
          retry
        else
          logger.log "Download #{object.class} #{object.primary_key_value} failed, error: #{e.inspect}"
          local_file.close
          File.delete(local_file_path)
          local_file_path = nil
          local_file = nil
        end
      end

      $url_mapper.set_file_path(url, local_file_path) if local_file_path

      local_file
    end

    # @param [Proc] download_proc, the proc which downloads the file, if url is not found in
    #   cache, file will be downloaded.
    def url_to_attachment_reference(url, download_proc, options = {})
      name = options[:name] || name_from_url(url)
      hash = $url_mapper.get_hash(url)
      reference = nil
      if hash && attachment = ::Attachment.find_by(name: hash)
        reference = ::AttachmentReference.new(
          attachment: attachment,
          name: name,
        )
      elsif local_file = download_proc.call
        attachment = ::Attachment.find_or_initialize_by(file: local_file)
        if attachment.new_record?
          attachment.created_at = options[:created_at]
          attachment.updated_at = options[:updated_at]
          attachment.save!
        end

        local_file.close unless local_file.closed?
        reference = ::AttachmentReference.new(attachment: attachment)
        reference.name = name

        $url_mapper.set_hash(url, reference.attachment.name)
      end

      reference.creator_id = options[:creator_id]
      reference.updater_id = options[:creator_id]
      reference.created_at = options[:created_at]
      reference.updated_at = options[:updated_at]
      reference
    end

    private

    def download_dir(object)
      File.join(LOCAL_DIR, object.class.table_name, object.primary_key_value.to_s)
    end
<commit_msg>Handle the nil case of the reference<commit_after>class Downloader
  LOCAL_DIR = '/tmp/file_downloads'

  class << self
    # Download the url to local and return an open file. nil will be returned if failed.
    def download_to_local(url, object, logger, file_name = nil)
      if path = $url_mapper.get_file_path(url)
        return File.open(path) if File.exist?(path)
      end

      dir = download_dir(object)
      FileUtils.mkdir_p(dir) unless File.exist?(dir)

      file_name ||= name_from_url(url)
      local_file_path = File.join(dir, file_name)
      return File.open(local_file_path) if File.exist?(local_file_path)

      local_file = File.open(local_file_path, 'wb')
      tries = 10
      begin
        open(url, 'rb') do |read_file|
          local_file.write(read_file.read)
        end
        # Set the read count to the begin of the file
        local_file.rewind
      rescue StandardError => e
        tries -= 1
        if tries > 0
          retry
        else
          logger.log "Download #{object.class} #{object.primary_key_value} failed, error: #{e.inspect}"
          local_file.close
          File.delete(local_file_path)
          local_file_path = nil
          local_file = nil
        end
      end

      $url_mapper.set_file_path(url, local_file_path) if local_file_path

      local_file
    end

    # @param [Proc] download_proc, the proc which downloads the file, if url is not found in
    #   cache, file will be downloaded.
    def url_to_attachment_reference(url, download_proc, options = {})
      name = options[:name] || name_from_url(url)
      hash = $url_mapper.get_hash(url)
      reference = nil
      if hash && attachment = ::Attachment.find_by(name: hash)
        reference = ::AttachmentReference.new(
          attachment: attachment,
          name: name,
        )
      elsif local_file = download_proc.call
        attachment = ::Attachment.find_or_initialize_by(file: local_file)
        if attachment.new_record?
          attachment.created_at = options[:created_at]
          attachment.updated_at = options[:updated_at]
          attachment.save!
        end

        local_file.close unless local_file.closed?
        reference = ::AttachmentReference.new(attachment: attachment)
        reference.name = name

        $url_mapper.set_hash(url, reference.attachment.name)
      end

      if reference
        reference.creator_id = options[:creator_id]
        reference.updater_id = options[:creator_id]
        reference.created_at = options[:created_at]
        reference.updated_at = options[:updated_at]
      end

      reference
    end

    private

    def download_dir(object)
      File.join(LOCAL_DIR, object.class.table_name, object.primary_key_value.to_s)
    end

{'fraction_non_alphanumeric': 0.08598199522322249, 'fraction_numerical': 0.0014697776961234614, 'mean_word_length': 2.5373619233268356, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4465990', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1627, 'n_words': 385}