<commit_before>module Dnatural
  VERSION = 'Dnatural/0.17'
  class Dir < ::Dir
    include Namaste::Mixin
    include LockIt::Mixin
    
    def self.mkdir path, integer=0777, args = {}
      super path, integer
      d = Dir.new path
      d.type = Dnatural::VERSION

      ::Dir.chdir(d.path)  do
        ::Dir.mkdir 'admin'
        ::Dir.mkdir 'consumer'
	::Dir.mkdir 'producer'
	::Dir.mkdir 'system'
      end

      d
    end

    def list
      glob('**/*')
    end

    def add src, dest, options = {}
      FileUtils.cp src, File.join(path, dest), options
    end

    def remove list, options = {}
      list = [list] if list.instance_of? String 
      FileUtils.rm list.map { |x| File.join(path, x) }, options
    end
  end
end<commit_msg>add manifest updates to add/rm<commit_after>module Dnatural
  VERSION = 'Dnatural/0.17'
  class Dir < ::Dir
    include Namaste::Mixin
    include LockIt::Mixin
    
    def self.mkdir path, integer=0777, args = {}
      super path, integer
      d = Dir.new path
      d.type = Dnatural::VERSION

      ::Dir.chdir(d.path)  do
        ::Dir.mkdir 'admin'
        ::Dir.mkdir 'consumer'
	::Dir.mkdir 'producer'
	::Dir.mkdir 'system'
      end

      d
    end

    def manifest
      @manifest ||= Checkm::Manifest.new open(File.join(path, 'manifest.txt')).read 
    end

    def manifest!
      @manifest = nil
      manifest
    end

    def list
      glob('**/*')
    end

    def add src, dest, options = {}
      FileUtils.cp src, File.join(path, dest), options

      manifest!

      lock
      m = manifest.add dest, :base => path
      File.open(File.join(path, 'manifest.txt'), 'w') do |f|
        f.write(m.to_s)
      end

      unlock
    end

    def remove list, options = {}
      list = [list] if list.instance_of? String 
      FileUtils.rm list.map { |x| File.join(path, x) }, options

      m = manifest!
      lock

      list.each do |l|
        m = m.remove l, :base => path
      end

      File.open(File.join(path, 'manifest.txt'), 'w') do |f|
        f.write(m.to_s)
      end

      unlock
    end

  end
end
{'fraction_non_alphanumeric': 0.12799616490891658, 'fraction_numerical': 0.006711409395973154, 'mean_word_length': 1.994261119081779, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6562898', 'n_tokens_mistral': 739, 'n_tokens_neox': 692, 'n_words': 191}