<commit_before>require 'zip'

module FileHelper
  extend LogHelper
  extend TimeoutHelper

  def check_mime_against_list! (file, expect, type_list)
    fm = FileMagic.new(FileMagic::MAGIC_MIME)

    mime_type = fm.file(file)

    # check mime is correct before uploading
    if not mime_type.start_with?(*type_list)
      error!({"error" => "File given is not a #{expect} file - detected #{mime_type}"}, 403)
    end
  end

  #
  # Test if a file should be accepted based on an expected kind
  # - file is passed the file uploaded to Doubtfire (a hash with all relevant data about the file)
  #
  def accept_file(file, name, kind)
    logger.debug "FileHelper is accepting file: filename=#{file.filename}, name=#{name}, kind=#{kind}"

    fm = FileMagic.new(FileMagic::MAGIC_MIME)
    mime = fm.file file.tempfile.path
    logger.debug "#{name} has MIME type: #{mime}"

    valid = true

    case kind
    when 'image'
      accept = ["image/png", "image/gif", "image/bmp", "image/tiff", "image/jpeg", "image/x-ms-bmp"]
    when 'code'
      accept = ["text/x-pascal", "text/x-c", "text/x-c++", "text/plain", "text/", "application/javascript, text/html",
                "text/css", "text/x-ruby", "text/coffeescript", "text/x-scss", "application/json", "text/xml", "application/xml",
                "text/x-yaml", "application/xml", "text/x-typescript"]
    when 'document'
      accept = [ # -- one day"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                 # --"application/msword",
                 "application/pdf" ]
      valid = pdf_valid? file.tempfile.path
    else
      logger.error "Unknown type '#{kind}' provided for '#{name}'"
      return false
    end

    # result is true when...
    mime.start_with?(*accept) && valid
  end


  #
  # Sanitize the passed in paths, and ensure each part is valid
  # Will kill things like ../ etc or spaces in paths
  #
  def sanitized_path(*paths)
    safe_paths = paths.map do | path_name |
      path_name.strip.tap do |name|
        # Finally, replace all non alphanumeric, underscore
        # or periods with underscore
        name.gsub! /[^\w\-]/, '_'
      end
    end

    File.join(safe_paths)
  end

  #
  # Sanitize the passed in filename -- should not include any path details
  #
  def sanitized_filename(filename)
    filename.strip.tap do |name|
      # NOTE: File.basename doesn't work right with Windows paths on Unix
      # get only the filename, not the whole path
      name.sub! /\A.*(\\|\/)/, ''
      # Finally, replace all non alphanumeric, underscore
      # or periods with underscore
      name.gsub! /[^\w\.\-]/, '_'
    end
  end

  def task_file_dir_for_unit(unit, create = true)
    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/" # trust the server config and passed in type for paths
    dst << sanitized_path("#{unit.code}-#{unit.id}","TaskFiles") << "/"

    if create and not Dir.exists? dst
      FileUtils.mkdir_p dst
    end

    dst
  end

  def student_group_work_dir(type, group_submission, task=nil, create=false)
    return nil unless group_submission

    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/" # trust the server config and passed in type for paths

    group = group_submission.group
    return nil unless group
    unit = group.unit

    if type == :pdf
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}") << "/"
    elsif type == :done
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}", "#{group_submission.id}") << "/"
    elsif type == :plagarism
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}", "#{group_submission.id}") << "/"
    else  # new and in_process -- just have task id -- will link to group when done etc.
      # Add task id to dst if we want task
      if task.nil?
        raise 'Unable to locate file!'
      end
      dst << "#{type}/#{task.id}/"
    end

    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  #
  # Generates a path for storing student work
  # type = [:new, :in_process, :done, :pdf, :plagarism]
  #
  def student_work_dir(type = nil, task = nil, create = true)
    if task && task.group_task?
      dst = student_group_work_dir type, task.group_submission, task
    else
      file_server = Doubtfire::Application.config.student_work_dir
      dst = "#{file_server}/" # trust the server config and passed in type for paths

      if not (type.nil? || task.nil?)
        if type == :pdf
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}") << "/"
        elsif type == :done
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}", "#{task.id}") << "/"
        elsif type == :plagarism
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}", "#{task.id}") << "/"
        else  # new and in_process -- just have task id
          # Add task id to dst if we want task
          dst << "#{type}/#{task.id}/"
        end
      elsif (not type.nil?)
        if [:in_process, :new].include? type
          # Add task id to dst if we want task
          dst << "#{type}/"
        else
          raise "Error in request to student work directory"
        end
      end
    end

    # Create current dst directory should it not exist
    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  #
  # Generates a path for storing student portfolios
  #
  def student_portfolio_dir(project, create = true)
    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/portfolio/" # trust the server config and passed in type for paths

    dst << sanitized_path("#{project.unit.code}-#{project.unit.id}", "#{project.student.username}" )

    # Create current dst directory should it not exist
    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  def compress_image(path)
    return true if File.size?(path) < 1000000

    tmp_file = File.join( Dir.tmpdir, 'doubtfire', 'compress', "#{File.dirname(path).split(File::Separator).last}-file#{File.extname(path)}" )
    logger.debug "File helper has started compressing #{path} to #{tmp_file}..."

    begin
      exec = "#{Rails.root.join('lib', 'shell', 'timeout.sh')} -t 30 nice -n 10 convert \"#{path}\" -resize 1024x1024 \"#{tmp_file}\" >>/dev/null 2>>/dev/null"
      # puts exec

      # try with convert
      did_compress = false
      try_within 40, "compressing image" do
        did_compress = system exec
      end

      if did_compress
        FileUtils.mv tmp_file, path
      end
    ensure
      if File.exists? tmp_file
        FileUtils.rm tmp_file
      end
    end<commit_msg>FIX: Correct issue with compressing images<commit_after>require 'zip'

module FileHelper
  extend LogHelper
  extend TimeoutHelper

  def check_mime_against_list! (file, expect, type_list)
    fm = FileMagic.new(FileMagic::MAGIC_MIME)

    mime_type = fm.file(file)

    # check mime is correct before uploading
    if not mime_type.start_with?(*type_list)
      error!({"error" => "File given is not a #{expect} file - detected #{mime_type}"}, 403)
    end
  end

  #
  # Test if a file should be accepted based on an expected kind
  # - file is passed the file uploaded to Doubtfire (a hash with all relevant data about the file)
  #
  def accept_file(file, name, kind)
    logger.debug "FileHelper is accepting file: filename=#{file.filename}, name=#{name}, kind=#{kind}"

    fm = FileMagic.new(FileMagic::MAGIC_MIME)
    mime = fm.file file.tempfile.path
    logger.debug "#{name} has MIME type: #{mime}"

    valid = true

    case kind
    when 'image'
      accept = ["image/png", "image/gif", "image/bmp", "image/tiff", "image/jpeg", "image/x-ms-bmp"]
    when 'code'
      accept = ["text/x-pascal", "text/x-c", "text/x-c++", "text/plain", "text/", "application/javascript, text/html",
                "text/css", "text/x-ruby", "text/coffeescript", "text/x-scss", "application/json", "text/xml", "application/xml",
                "text/x-yaml", "application/xml", "text/x-typescript"]
    when 'document'
      accept = [ # -- one day"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                 # --"application/msword",
                 "application/pdf" ]
      valid = pdf_valid? file.tempfile.path
    else
      logger.error "Unknown type '#{kind}' provided for '#{name}'"
      return false
    end

    # result is true when...
    mime.start_with?(*accept) && valid
  end


  #
  # Sanitize the passed in paths, and ensure each part is valid
  # Will kill things like ../ etc or spaces in paths
  #
  def sanitized_path(*paths)
    safe_paths = paths.map do | path_name |
      path_name.strip.tap do |name|
        # Finally, replace all non alphanumeric, underscore
        # or periods with underscore
        name.gsub! /[^\w\-]/, '_'
      end
    end

    File.join(safe_paths)
  end

  #
  # Sanitize the passed in filename -- should not include any path details
  #
  def sanitized_filename(filename)
    filename.strip.tap do |name|
      # NOTE: File.basename doesn't work right with Windows paths on Unix
      # get only the filename, not the whole path
      name.sub! /\A.*(\\|\/)/, ''
      # Finally, replace all non alphanumeric, underscore
      # or periods with underscore
      name.gsub! /[^\w\.\-]/, '_'
    end
  end

  def task_file_dir_for_unit(unit, create = true)
    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/" # trust the server config and passed in type for paths
    dst << sanitized_path("#{unit.code}-#{unit.id}","TaskFiles") << "/"

    if create and not Dir.exists? dst
      FileUtils.mkdir_p dst
    end

    dst
  end

  def student_group_work_dir(type, group_submission, task=nil, create=false)
    return nil unless group_submission

    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/" # trust the server config and passed in type for paths

    group = group_submission.group
    return nil unless group
    unit = group.unit

    if type == :pdf
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}") << "/"
    elsif type == :done
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}", "#{group_submission.id}") << "/"
    elsif type == :plagarism
      dst << sanitized_path("#{unit.code}-#{unit.id}","Group-#{group.id}", "#{type}", "#{group_submission.id}") << "/"
    else  # new and in_process -- just have task id -- will link to group when done etc.
      # Add task id to dst if we want task
      if task.nil?
        raise 'Unable to locate file!'
      end
      dst << "#{type}/#{task.id}/"
    end

    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  #
  # Generates a path for storing student work
  # type = [:new, :in_process, :done, :pdf, :plagarism]
  #
  def student_work_dir(type = nil, task = nil, create = true)
    if task && task.group_task?
      dst = student_group_work_dir type, task.group_submission, task
    else
      file_server = Doubtfire::Application.config.student_work_dir
      dst = "#{file_server}/" # trust the server config and passed in type for paths

      if not (type.nil? || task.nil?)
        if type == :pdf
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}") << "/"
        elsif type == :done
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}", "#{task.id}") << "/"
        elsif type == :plagarism
          dst << sanitized_path("#{task.project.unit.code}-#{task.project.unit.id}","#{task.project.student.username}", "#{type}", "#{task.id}") << "/"
        else  # new and in_process -- just have task id
          # Add task id to dst if we want task
          dst << "#{type}/#{task.id}/"
        end
      elsif (not type.nil?)
        if [:in_process, :new].include? type
          # Add task id to dst if we want task
          dst << "#{type}/"
        else
          raise "Error in request to student work directory"
        end
      end
    end

    # Create current dst directory should it not exist
    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  #
  # Generates a path for storing student portfolios
  #
  def student_portfolio_dir(project, create = true)
    file_server = Doubtfire::Application.config.student_work_dir
    dst = "#{file_server}/portfolio/" # trust the server config and passed in type for paths

    dst << sanitized_path("#{project.unit.code}-#{project.unit.id}", "#{project.student.username}" )

    # Create current dst directory should it not exist
    if create
      FileUtils.mkdir_p(dst)
    end
    dst
  end

  def compress_image(path)
    return true if File.size?(path) < 1000000

    compress_folder = File.join(Dir.tmpdir, 'doubtfire', 'compress')

    FileUtils.mkdir compress_folder unless Directory.exists? compress_folder

    tmp_file = File.join( compress_folder, "#{File.dirname(path).split(File::Separator).last}-file#{File.extname(path)}" )
    logger.debug "File helper has started compressing #{path} to #{tmp_file}..."

    begin
      exec = "#{Rails.root.join('lib', 'shell', 'timeout.sh')} -t 30 nice -n 10 convert \"#{path}\" -resize 1024x1024 \"#{tmp_file}\" >>/dev/null 2>>/dev/null"
      # puts exec

      # try with convert
      did_compress = false
      try_within 40, "compressing image" do
        did_compress = system exec
      end

      if did_compress
        FileUtils.mv tmp_file, path
      end
    ensure
      if File.exists? tmp_file
        FileUtils.rm tmp_file
      end
    end
{'fraction_non_alphanumeric': 0.13871175754101883, 'fraction_numerical': 0.0035824317546750734, 'mean_word_length': 2.751142166084386, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4861378', 'n_tokens_mistral': 4560, 'n_tokens_neox': 4408, 'n_words': 1235}