<commit_before>require 'time'

module Checkm
  CHUNK_SIZE = 8*1024*1024
  def self.checksum file, alg
      digest_alg = case alg
        when nil
          return true
        when /md5/
           Digest::MD5.new if alg == 'md5'
        when /sha1/
        when /sha256/
        when /dir/
          return File.directory? file
        else 
          return false      
      end

      while not file.eof? and chunk = file.readpartial(CHUNK_SIZE)
        digest_alg << chunk
      end
      digest_alg.hexdigest
  end

  class Manifest
    def self.parse str, args = {}
      Manifest.new str, args
    end

    attr_reader :version
    attr_reader :entries
    attr_reader :fields
    attr_reader :path

    def initialize checkm, args = {}
      @args = args
      @version = nil
      @checkm = checkm
      @lines = checkm.split "\n"
      @entries = []
      @eof = false
      @fields= nil
      @path = args[:path]
      @path ||= Dir.pwd
      parse_lines 
      # xxx error on empty entries?
      @lines.unshift('#%checkm_0.7') and @version = '0.7' if @version.nil?

    end

    def valid?
      @entries.map { |e| e.valid? }.any? { |b| b == false }<commit_msg>valid if checkm empty<commit_after>require 'time'

module Checkm
  CHUNK_SIZE = 8*1024*1024
  def self.checksum file, alg
      digest_alg = case alg
        when nil
          return true
        when /md5/
           Digest::MD5.new if alg == 'md5'
        when /sha1/
        when /sha256/
        when /dir/
          return File.directory? file
        else 
          return false      
      end

      while not file.eof? and chunk = file.readpartial(CHUNK_SIZE)
        digest_alg << chunk
      end
      digest_alg.hexdigest
  end

  class Manifest
    def self.parse str, args = {}
      Manifest.new str, args
    end

    attr_reader :version
    attr_reader :entries
    attr_reader :fields
    attr_reader :path

    def initialize checkm, args = {}
      @args = args
      @version = nil
      @checkm = checkm
      @lines = checkm.split "\n"
      @entries = []
      @eof = false
      @fields= nil
      @path = args[:path]
      @path ||= Dir.pwd
      parse_lines 
      # xxx error on empty entries?
      @lines.unshift('#%checkm_0.7') and @version = '0.7' if @version.nil?

    end

    def valid?
      return true if @entries.empty?
      @entries.map { |e| e.valid? }.any? { |b| b == false }
{'fraction_non_alphanumeric': 0.09970796829370046, 'fraction_numerical': 0.016687526074259492, 'mean_word_length': 1.7531572904707233, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3545408', 'n_tokens_mistral': 803, 'n_tokens_neox': 768, 'n_words': 225}