<commit_before>require 'tmpdir'
require 'fileutils'

def mv_r(src, dest, options = {})
  if File.directory? src
    d = File.directory?(dest) ? File.join(dest, File.basename(src)) : dest
    if File.directory? d
      Dir.glob(File.join(src, '*')).each do |s|
        mv_r(s, d, options)
      end
    else
      FileUtils.mv(src, dest, options)
    end
  else
    FileUtils.mv(src, dest, options)
  end
end

def extract(file, target, options = {})
  fail unless File.directory?(target)
  
  # create a temporary folder used to extract files there
  tmpdir = File.expand_path(File.join(Dir.tmpdir, "extract_sandbox_#{$$}"))
  FileUtils.mkpath(tmpdir) unless File.exist?(tmpdir)

  # based on filetypes, extract the intermediate file into the temporary folder
  case file
    # tar.z, tar.gz and tar.bz2 contains .tar files inside, extract into 
    # temp first
    when /(^.+\.tar)\.z$/, /(^.+\.tar)\.gz$/, /(^.+\.tar)\.bz2$/, /(^.+\.tar)\.lzma$/
      seven_zip tmpdir, file
      seven_zip target, File.join(tmpdir, File.basename($1))
    when /(^.+)\.tgz$/
      seven_zip tmpdir, file
      seven_zip target, File.join(tmpdir, "#{File.basename($1)}.tar")
    when /(^.+\.zip$)/
      seven_zip(target, $1)
    else
      raise "Unknown file extension! (for file #{file})"
  end

  # after extraction, lookup for a folder that contains '-' (version number or datestamp)
  folders_in_target = []
  Dir.chdir(target) { folders_in_target = Dir.glob('*') }

  # the package was created inside another folder
  # exclude folders packagename-X.Y.Z or packagename-DATE
  # move all the folders within that into target directly.
  folders_in_target.each do |folder|
    next unless File.directory?(File.join(target, folder)) && folder =~ /\-/

    # take the folder contents out!, now!
    contents = []
    Dir.chdir(File.join(target, folder)) { contents = Dir.glob('*') }

    contents.each do |c|
      puts "** Moving out #{c} from #{folder} and drop into #{target}" if Rake.application.options.trace
      mv_r File.join(target, folder, c), target
    end
    
    # remove the now empty folder
    puts "** Removing #{folder}" if Rake.application.options.trace
    FileUtils.rm_rf File.join(target, folder)
  end

  # remove the temporary directory
  puts "** Removing #{tmpdir}" if Rake.application.options.trace
  FileUtils.rm_rf tmpdir
end

def seven_zip(target, file)
  puts "** Extracting #{file} into #{target}" if Rake.application.options.trace
  sh "\"#{File.expand_path(File.join('sandbox/extract_utils', '7za.exe'))}\" x -y -o\"#{target}\" \"#{file}\" > NUL"
end<commit_msg>teach extracttask to 7-Zip verify and archive<commit_after>require 'tmpdir'
require 'fileutils'

def mv_r(src, dest, options = {})
  if File.directory? src
    d = File.directory?(dest) ? File.join(dest, File.basename(src)) : dest
    if File.directory? d
      Dir.glob(File.join(src, '*')).each do |s|
        mv_r(s, d, options)
      end
    else
      FileUtils.mv(src, dest, options)
    end
  else
    FileUtils.mv(src, dest, options)
  end
end

def extract(file, target, options = {})
  fail unless File.directory?(target)
  
  # create a temporary folder used to extract files there
  tmpdir = File.expand_path(File.join(Dir.tmpdir, "extract_sandbox_#{$$}"))
  FileUtils.mkpath(tmpdir) unless File.exist?(tmpdir)

  # based on filetypes, extract the intermediate file into the temporary folder
  case file
    # tar.z, tar.gz and tar.bz2 contains .tar files inside, extract into 
    # temp first
    when /(^.+\.tar)\.z$/, /(^.+\.tar)\.gz$/, /(^.+\.tar)\.bz2$/, /(^.+\.tar)\.lzma$/
      seven_zip tmpdir, file
      seven_zip target, File.join(tmpdir, File.basename($1))
    when /(^.+)\.tgz$/
      seven_zip tmpdir, file
      seven_zip target, File.join(tmpdir, "#{File.basename($1)}.tar")
    when /(^.+\.zip$)/
      seven_zip(target, $1)
    else
      raise "Unknown file extension! (for file #{file})"
  end

  # after extraction, lookup for a folder that contains '-' (version number or datestamp)
  folders_in_target = []
  Dir.chdir(target) { folders_in_target = Dir.glob('*') }

  # the package was created inside another folder
  # exclude folders packagename-X.Y.Z or packagename-DATE
  # move all the folders within that into target directly.
  folders_in_target.each do |folder|
    next unless File.directory?(File.join(target, folder)) && folder =~ /\-/

    # take the folder contents out!, now!
    contents = []
    Dir.chdir(File.join(target, folder)) { contents = Dir.glob('*') }

    contents.each do |c|
      puts "** Moving out #{c} from #{folder} and drop into #{target}" if Rake.application.options.trace
      mv_r File.join(target, folder, c), target
    end
    
    # remove the now empty folder
    puts "** Removing #{folder}" if Rake.application.options.trace
    FileUtils.rm_rf File.join(target, folder)
  end

  # remove the temporary directory
  puts "** Removing #{tmpdir}" if Rake.application.options.trace
  FileUtils.rm_rf tmpdir
end

def seven_zip(target, file)
  puts "** Extracting #{file} into #{target}" if Rake.application.options.trace
  sh "\"#{File.expand_path(File.join('sandbox/extract_utils', '7za.exe'))}\" x -y -o\"#{target}\" \"#{file}\" > NUL"
end

#TODO confirm function returns false upon failing 7-Zip integrity test
def seven_zip_valid?(target)
  puts "** 7-Zip integrity checking '#{target}'" if Rake.application.options.trace
  sh "\"#{File.expand_path(File.join('sandbox/extract_utils', '7za.exe'))}\" t \"#{target}\" > NUL"
end

def seven_zip_build(source, target)
  puts "** Building 7-Zip archive from '#{source}'" if Rake.application.options.trace
  sh "\"#{File.expand_path(File.join('sandbox/extract_utils', '7za.exe'))}\" a \"#{target}\" \"#{source}\"> NUL"
end
{'fraction_non_alphanumeric': 0.15491974877878575, 'fraction_numerical': 0.00314026517794836, 'mean_word_length': 3.393103448275862, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157576', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1938, 'n_words': 532}