<commit_before>require 'file_cache/version'
require 'open-uri'
require 'digest/sha2'

module FileCache
  extend self

  # Public: save a file to the cache
  #
  # url - The URL to the file
  #
  # Example:
  #
  #   FileCache.put('http://example.com/test123.jpg')
  #   #=> nil
  #
  # Returns: nil
  def put(url)
    FileUtils.mkdir_p path(url)
    File.open(path(url) + filename(url), "wb") { |f| f << open(url).read }
    nil
  end

  # Public: get a file from the cache
  #
  # url - The URL to the file
  #
  # Examples:
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => #<File:9ae274d94c34542ddd1b64667c1d4e392211ff67>
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => nil
  #
  # Returns the File or nil
  def get(url)
    File.open(path(url) + filename(url), "rb")
  rescue Errno::ENOENT
    nil
  end

  # Public: get a file and save it to the cache if it isn't there already
  #
  # url - The URL to the file
  #
  # Examples:
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => #<File:9ae274d94c34542ddd1b64667c1d4e392211ff67>
  #
  # Returns a File
  def pget(url)
    f = get(url)
    return f unless f.nil?

    put(url)
    get(url)
  end

  # Public: delete a file from the cache
  #
  # url - The URL to the file
  #
  # Example:<commit_msg>don't save a file if it's already cached<commit_after>require 'file_cache/version'
require 'open-uri'
require 'digest/sha2'

module FileCache
  extend self

  # Public: save a file to the cache
  #
  # url - The URL to the file
  #
  # Example:
  #
  #   FileCache.put('http://example.com/test123.jpg')
  #   #=> nil
  #
  # Returns: nil
  def put(url)
    p = path(url)
    f = filename(url)
    return nil if File.exist?(p + f)

    FileUtils.mkdir_p p
    File.open(p + f, "wb") { |f| f << open(url).read }
    nil
  end

  # Public: get a file from the cache
  #
  # url - The URL to the file
  #
  # Examples:
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => #<File:9ae274d94c34542ddd1b64667c1d4e392211ff67>
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => nil
  #
  # Returns the File or nil
  def get(url)
    File.open(path(url) + filename(url), "rb")
  rescue Errno::ENOENT
    nil
  end

  # Public: get a file and save it to the cache if it isn't there already
  #
  # url - The URL to the file
  #
  # Examples:
  #
  #   FileCache.get('http://example.com/test123.jpg')
  #   # => #<File:9ae274d94c34542ddd1b64667c1d4e392211ff67>
  #
  # Returns a File
  def pget(url)
    put(url)
    get(url)
  end

  # Public: delete a file from the cache
  #
  # url - The URL to the file
  #
  # Example:
{'fraction_non_alphanumeric': 0.13544018058690746, 'fraction_numerical': 0.05041384499623777, 'mean_word_length': 2.5691275167785235, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '760523', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1048, 'n_words': 257}