<commit_before>require 'zip'

include Rake::DSL

private

class ZipEntry
  def initialize(source)
    @_source = source
    @_target = nil
  end

  def self.process_file(zipfile, source, target)
    targetName = target || File.basename(source)
    puts "Adding file to zip archive: #{source}, #{target}"
    zipfile.add(targetName, source)
  end

  def as(target)
    @_target = target
  end

  def _zip(zipfile)
    if !File.exists?(@_source)
      raise "File or directory does not exist: #{@_source}"
    end

    if File.directory?(@_source)
      base = @_target || File.basename(@_source)
      spn = Pathname.new(@_source)
      self._sources().each do |f|
        pn = Pathname.new(f)
        relative = pn.relative_path_from(spn)
        puts "---- base=#{base}"
        puts "---- relative=#{relative}"
        ZipEntry.process_file(zipfile, f, File.join(base, relative))
      end
    else
      ZipEntry.process_file(zipfile, @_source, @_target)
    end
  end

  def _sources()
    if File.directory?(@_source)
      return Dir[File.join(@_source, '**', '**')]
    else
      return [@_source]
    end
  end
end

class ZipGlobEntry
  def initialize(glob)
    @_glob = glob
    @_target = nil
  end

  def into(target)
    @_target = target
  end

  def _zip(zip)
    self._sources().each do |source|
      targetName = @_target != nil ? File.join(@_target, source) : source
      ZipFileEntry.zip(zipfile, source, targetName)
    end
  end

  def _sources()
    return Dir[@_glob]
  end
end

class ZipTask

  attr_reader :_archive, :_entries, :_ignore_deps

  def initialize(archive)
    @_archive = archive
    @_entries = []
  end

  def add(path, &block)
    is_glob = path.include?('*')

    if is_glob
      globEntry = ZipGlobEntry.new(path)
      globEntry.instance_eval(&block) if block
      @_entries.push(globEntry)
    else
      entry = ZipEntry.new(path)
      entry.instance_eval(&block) if block
      @_entries.push(entry)
    end
  end

  def no_deps(flag)
    @_ignore_deps = flag
  end

  def _execute()
    Zip.continue_on_exists_proc = true
    Zip::File.open(@_archive, Zip::File::CREATE) do |zipfile|
      @_entries.each do |entry|
        puts "-- entry: #{entry.inspect}"
        entry._zip(zipfile)
      end
    end
  end

  def _sources()
    sources = []
    @_entries.each do |entry|
      sources += entry._sources()
    end
    return sources
  end
end


public

def zip(archive, &block)
  task = ZipTask.new(archive)
  task.instance_eval(&block) if block

  if task._ignore_deps
    file task._archive do
      task._execute()
    end
  else
    file task._archive => task._sources() do
      task._execute()
    end
  end
end

def unzip(archive, &block)
end


# folder = "./data"
# input_filenames = ['bla.txt']

# zipfile_name = "./archive.zip"

# Zip::File.open(zipfile_name, Zip::File::CREATE) do |zipfile|
#   input_filenames.each do |filename|
#     # Two arguments:
#     # - The name of the file as it will appear in the archive
#     # - The original file, including the path to find it
#     zipfile.add('data/'+filename, folder + '/' + filename)
#   end
#   zipfile.get_output_stream("myFile") { |os| os.write "myFile contains just this" }
# end<commit_msg>Clean up.<commit_after>require 'zip'

include Rake::DSL

private

class ZipEntry
  def initialize(source)
    @_source = source
    @_target = nil
  end

  def self.process_file(zipfile, source, target)
    targetName = target || File.basename(source)
    puts "Adding file to zip archive: #{source}, #{target}"
    zipfile.add(targetName, source)
  end

  def as(target)
    @_target = target
  end

  def _zip(zipfile)
    if !File.exists?(@_source)
      raise "File or directory does not exist: #{@_source}"
    end

    if File.directory?(@_source)
      base = @_target || File.basename(@_source)
      spn = Pathname.new(@_source)
      self._sources().each do |f|
        pn = Pathname.new(f)
        relative = pn.relative_path_from(spn)
        ZipEntry.process_file(zipfile, f, File.join(base, relative))
      end
    else
      ZipEntry.process_file(zipfile, @_source, @_target)
    end
  end

  def _sources()
    if File.directory?(@_source)
      return Dir[File.join(@_source, '**', '**')]
    else
      return [@_source]
    end
  end
end

class ZipGlobEntry
  def initialize(glob)
    @_glob = glob
    @_target = nil
  end

  def into(target)
    @_target = target
  end

  def _zip(zip)
    self._sources().each do |source|
      targetName = @_target != nil ? File.join(@_target, source) : source
      ZipFileEntry.zip(zipfile, source, targetName)
    end
  end

  def _sources()
    return Dir[@_glob]
  end
end

class ZipTask

  attr_reader :_archive, :_entries, :_ignore_deps

  def initialize(archive)
    @_archive = archive
    @_entries = []
  end

  def add(path, &block)
    is_glob = path.include?('*')

    if is_glob
      globEntry = ZipGlobEntry.new(path)
      globEntry.instance_eval(&block) if block
      @_entries.push(globEntry)
    else
      entry = ZipEntry.new(path)
      entry.instance_eval(&block) if block
      @_entries.push(entry)
    end
  end

  def no_deps(flag)
    @_ignore_deps = flag
  end

  def _execute()
    Zip.continue_on_exists_proc = true
    Zip::File.open(@_archive, Zip::File::CREATE) do |zipfile|
      @_entries.each do |entry|
        entry._zip(zipfile)
      end
    end
  end

  def _sources()
    sources = []
    @_entries.each do |entry|
      sources += entry._sources()
    end
    return sources
  end
end


public

def zip(archive, &block)
  task = ZipTask.new(archive)
  task.instance_eval(&block) if block

  if task._ignore_deps
    file task._archive do
      task._execute()
    end
  else
    file task._archive => task._sources() do
      task._execute()
    end
  end
end

def unzip(archive, &block)
end
{'fraction_non_alphanumeric': 0.11837577426015142, 'fraction_numerical': 0.0, 'mean_word_length': 2.7503225806451614, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5176274', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1969, 'n_words': 493}