<commit_before>require 'aws-sdk'
require 'mini_magick'
class FileProcessor
  def self.upload(file, args = {})
    return if file == "" || file.nil?
    size = args[:size] || args["size"]
    path = args[:path] || args["path"] || DateTime.now.strftime('%Y-%m-%d')
    priv = args[:private] || args["private"] || false
    filename = args[:name] || args["name"]
    
    if file.is_a?(String)
      return if name.nil?
      s3 = Aws::S3::Resource.new()
      
      name = "#{path}/#{filename}"
      obj = s3.bucket(ENV['AWS_BUCKET']).object(name)
      if priv
        obj.put(body: file)
      else
        obj.put(body: file, acl: "public-read")
      end

      {
        name: filename,
        url: obj.public_url
      }
    else
      t_file = file.tempfile
      if size && file.content_type.match(/image\/.*/) && !file.content_type.include?("svg")
        parseFile = MiniMagick::Image.new(t_file.path).resize size
        ap parseFile
        t_file = parseFile
      end

      s3 = Aws::S3::Resource.new()
      obj_name = filename || file.original_filename
      name = "#{path}/#{SecureRandom.uuid}--#{obj_name}"
      obj = s3.bucket(ENV['AWS_BUCKET']).object(name)
      File.open(t_file.path) do |f|
        if priv
          obj.put(body: f)
        else
          obj.put(body: f, acl: "public-read")
        end
      end

      {
        name: obj_name,
        url: obj.public_url
      }
    end
  end

  def self.delete(file)
    s3 = Aws::S3::Resource.new()
    aws_file = file.sub("https://#{ENV['AWS_BUCKET']}.s3-us-west-2.amazonaws.com/", "")
    obj = s3.bucket(ENV['AWS_BUCKET']).object(aws_file)
    obj.delete
  end

  def self.read(file)
    s3 = Aws::S3::Resource.new()
    aws_file = file.sub("https://#{ENV['AWS_BUCKET']}.s3-us-west-2.amazonaws.com/", "")
    obj = s3.bucket(ENV['AWS_BUCKET']).object(aws_file)
    result = obj.get
    result.body
  end
end
<commit_msg>Create file_processor.rb<commit_after>require 'aws-sdk'
require 'mini_magick'
class FileProcessor
  def self.upload(file, args = {})
    return if file == "" || file.nil?
    size = args[:size] || args["size"]
    path = args[:path] || args["path"] || DateTime.now.strftime('%Y-%m-%d')
    priv = args[:private] || args["private"] || false
    filename = args[:name] || args["name"]
    
    if file.is_a?(String)
      return if filename.nil?
      s3 = Aws::S3::Resource.new()
      
      name = "#{path}/#{filename}"
      obj = s3.bucket(ENV['AWS_BUCKET']).object(name)
      if priv
        obj.put(body: file)
      else
        obj.put(body: file, acl: "public-read")
      end

      {
        name: filename,
        url: obj.public_url
      }
    else
      t_file = file.tempfile
      if size && file.content_type.match(/image\/.*/) && !file.content_type.include?("svg")
        parseFile = MiniMagick::Image.new(t_file.path).resize size
        ap parseFile
        t_file = parseFile
      end

      s3 = Aws::S3::Resource.new()
      obj_name = filename || file.original_filename
      name = "#{path}/#{SecureRandom.uuid}--#{obj_name}"
      obj = s3.bucket(ENV['AWS_BUCKET']).object(name)
      File.open(t_file.path) do |f|
        if priv
          obj.put(body: f)
        else
          obj.put(body: f, acl: "public-read")
        end
      end

      {
        name: obj_name,
        url: obj.public_url
      }
    end
  end

  def self.delete(file)
    s3 = Aws::S3::Resource.new()
    aws_file = file.sub("https://#{ENV['AWS_BUCKET']}.s3-us-west-2.amazonaws.com/", "")
    obj = s3.bucket(ENV['AWS_BUCKET']).object(aws_file)
    obj.delete
  end

  def self.read(file)
    s3 = Aws::S3::Resource.new()
    aws_file = file.sub("https://#{ENV['AWS_BUCKET']}.s3-us-west-2.amazonaws.com/", "")
    obj = s3.bucket(ENV['AWS_BUCKET']).object(aws_file)
    result = obj.get
    result.body
  end
end

{'fraction_non_alphanumeric': 0.1720177499347429, 'fraction_numerical': 0.00835291046724093, 'mean_word_length': 2.364354697102722, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754263', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1399, 'n_words': 239}