<commit_before>#!/usr/bin/env ruby

require "ddv/version"
require 'fileutils'

module Ddv
  class DirVisitor
    def initialize(output=NodePrinter.new)
      @output = output
    end

    def tree(parent_dir, max_detailed_files_num=nil, level=0)
      @output.output_dir(File.basename(parent_dir), level)
      nodes = Dir.entries(parent_dir) - [".", ".."]
      dirs = nodes.select {|n| dir?(parent_dir, n) }.sort
      files = (nodes - dirs).sort
      @output.output_files(files, max_detailed_files_num, level)
      dirs.each do |dir|
        tree(File.join(parent_dir, dir), max_detailed_files_num, level + 1)
      end
    end
    
    def dir?(parent_dir, node)
      File.directory?(File.join(parent_dir, node))
    end<commit_msg>refactoring DirVisitor#tree<commit_after>#!/usr/bin/env ruby

require "ddv/version"
require 'fileutils'

module Ddv
  class DirVisitor
    def initialize(output=NodePrinter.new)
      @output = output
    end

    def tree(parent_dir, max_detailed_files_num=nil, level=0)
      @output.output_dir(File.basename(parent_dir), level)
      nodes = Dir.entries(parent_dir) - [".", ".."]
      dirs, files = nodes.partition {|n| dir?(parent_dir, n) }.map(&:sort)
      @output.output_files(files, max_detailed_files_num, level)
      dirs.each do |dir|
        tree(File.join(parent_dir, dir), max_detailed_files_num, level + 1)
      end
    end
    
    def dir?(parent_dir, node)
      File.directory?(File.join(parent_dir, node))
    end
{'fraction_non_alphanumeric': 0.128135593220339, 'fraction_numerical': 0.002711864406779661, 'mean_word_length': 3.3157894736842106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2670836', 'n_tokens_mistral': 524, 'n_tokens_neox': 519, 'n_words': 100}