<commit_before>require "down"

require "fileutils"
require "pathname"

class Shrine
  module Storage
    # The FileSystem storage handles uploads to the filesystem, and it is
    # most commonly initialized with a "base" folder and a "subdirectory":
    #
    #     storage = Shrine::Storage::FileSystem.new("public", subdirectory: "uploads")
    #     storage.url("image.jpg") #=> "/uploads/image.jpg"
    #
    # This storage will upload all files to "public/uploads", and the URLs
    # of the uploaded files will start with "/uploads/*". This way you can
    # use FileSystem for both cache and store, one having subdirectory
    # "uploads/cache" and other "uploads/store".
    #
    # You can also initialize the storage just with the "base" directory, and
    # then the FileSystem storage will generate absolute URLs to files:
    #
    #     storage = Shrine::Storage::FileSystem.new(Dir.tmpdir)
    #     storage.url("image.jpg") #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/image.jpg"
    #
    # ## Host
    #
    # It's generally a good idea to serve your files via a CDN, so an
    # additional `:host` option can be provided:
    #
    #     storage = Shrine::Storage::FileSystem.new("public",
    #       subdirectory: "uploads", host: "http://abc123.cloudfront.net")
    #     storage.url("image.jpg") #=> "http://abc123.cloudfront.net/uploads/image.jpg"
    #
    # If you're not using a CDN, it's recommended that you still set `:host` to
    # your application's domain (at least in production).
    #
    # The `:host` option can also be used wihout `:subdirectory`, and is
    # useful if you for example have files located on another server:
    #
    #     storage = Shrine::Storage::FileSystem.new("files", host: "http://943.23.43.1")
    #     storage.url("image.jpg") #=> "http://943.23.43.1/files/image.jpg"
    #
    # ## Clearing cache
    #
    # If you're using FileSystem as cache, you will probably want to
    # periodically delete old files which aren't used anymore. You can put
    # the following in a periodic Rake task:
    #
    #     file_system = Shrine.storages[:cache]
    #     file_system.clear!(older_than: 1.week.ago) # adjust the time
    #
    # ## Permissions
    #
    # If you want your files and folders to have certain permissions, you can
    # pass the `:permissions` option:
    #
    #     Shrine::Storage::FileSystem.new("directory", permissions: 0755)
    class FileSystem
      attr_reader :directory, :subdirectory, :host, :permissions

      # Initializes a storage for uploading to the filesystem.
      #
      # :subdirectory
      # :  The directory relative to `directory` to which files will be stored,
      #    and it is included in the URL.
      #
      # :host
      # :  URLs will by default be relative if `:subdirectory` is set, and you
      #    can use this option to set a CDN host (e.g. `//abc123.cloudfront.net`).
      #
      # :permissions
      # :  The generated files and folders will have default UNIX permissions,
      #    but if you want specific ones you can use this option (e.g. `0755`).
      #
      # :clean
      # :  By default empty folders inside the directory are automatically
      #    deleted, but if it happens that it causes too much load on the<commit_msg>Add section about FileSystem and Heroku<commit_after>require "down"

require "fileutils"
require "pathname"

class Shrine
  module Storage
    # The FileSystem storage handles uploads to the filesystem, and it is
    # most commonly initialized with a "base" folder and a "subdirectory":
    #
    #     storage = Shrine::Storage::FileSystem.new("public", subdirectory: "uploads")
    #     storage.url("image.jpg") #=> "/uploads/image.jpg"
    #
    # This storage will upload all files to "public/uploads", and the URLs
    # of the uploaded files will start with "/uploads/*". This way you can
    # use FileSystem for both cache and store, one having subdirectory
    # "uploads/cache" and other "uploads/store".
    #
    # You can also initialize the storage just with the "base" directory, and
    # then the FileSystem storage will generate absolute URLs to files:
    #
    #     storage = Shrine::Storage::FileSystem.new(Dir.tmpdir)
    #     storage.url("image.jpg") #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/image.jpg"
    #
    # ## Host
    #
    # It's generally a good idea to serve your files via a CDN, so an
    # additional `:host` option can be provided:
    #
    #     storage = Shrine::Storage::FileSystem.new("public",
    #       subdirectory: "uploads", host: "http://abc123.cloudfront.net")
    #     storage.url("image.jpg") #=> "http://abc123.cloudfront.net/uploads/image.jpg"
    #
    # If you're not using a CDN, it's recommended that you still set `:host` to
    # your application's domain (at least in production).
    #
    # The `:host` option can also be used wihout `:subdirectory`, and is
    # useful if you for example have files located on another server:
    #
    #     storage = Shrine::Storage::FileSystem.new("files", host: "http://943.23.43.1")
    #     storage.url("image.jpg") #=> "http://943.23.43.1/files/image.jpg"
    #
    # ## Clearing cache
    #
    # If you're using FileSystem as cache, you will probably want to
    # periodically delete old files which aren't used anymore. You can put
    # the following in a periodic Rake task:
    #
    #     file_system = Shrine.storages[:cache]
    #     file_system.clear!(older_than: 1.week.ago) # adjust the time
    #
    # ## Permissions
    #
    # If you want your files and folders to have certain permissions, you can
    # pass the `:permissions` option:
    #
    #     Shrine::Storage::FileSystem.new("directory", permissions: 0755)
    #
    # ## Heroku
    #
    # Note that Heroku has a read-only filesystem, and doesn't allow you to
    # upload your files to the "public" directory, you can however upload to
    # "tmp" directory:
    #
    #     Shrine::Storage::FileSystem.new("tmp/uploads")
    #
    # Note that this approach has a couple of downsides. For example, you can
    # only use it for cache, since Heroku wipes this directory between app
    # restarts. This also means that deploying the app can cancel someone's
    # uploading if you're using backgrounding. Also, you cannot display a link
    # to the cached file before it is promoted to store, since it won't be in
    # the "public" directory.
    class FileSystem
      attr_reader :directory, :subdirectory, :host, :permissions

      # Initializes a storage for uploading to the filesystem.
      #
      # :subdirectory
      # :  The directory relative to `directory` to which files will be stored,
      #    and it is included in the URL.
      #
      # :host
      # :  URLs will by default be relative if `:subdirectory` is set, and you
      #    can use this option to set a CDN host (e.g. `//abc123.cloudfront.net`).
      #
      # :permissions
      # :  The generated files and folders will have default UNIX permissions,
      #    but if you want specific ones you can use this option (e.g. `0755`).
      #
      # :clean
      # :  By default empty folders inside the directory are automatically
      #    deleted, but if it happens that it causes too much load on the
{'fraction_non_alphanumeric': 0.10265219183729558, 'fraction_numerical': 0.012917411021025147, 'mean_word_length': 2.7361396303901437, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5195805', 'n_tokens_mistral': 2227, 'n_tokens_neox': 2110, 'n_words': 782}