<commit_before>require 'find'

# = Wriggle
#
# A simple directory crawler DSL.
#
# == Usage
#
#   require 'wriggle'
#
#   wriggle '/path/to/files' do |w|
#
#     # Build an array of Ruby code files
#     ruby_files = []
#     w.file :rb do |path|
#       ruby_files << path
#     end
#
#     # Build an array of video files
#     video_files = []
#     w.file %w(mpg mpeg wmv avi mkv) do |path|
#       video_files << path
#     end
#
#     # Delete directories that are empty
#     w.directory do |path|
#       Dir.rmdir(path) unless Dir.entries(path).length > 2
#     end
#   end
module Wriggle
  # Crawl the given +path+
  #
  # @raise ArgumentError Given path does not exist or is not a directory
  def wriggle(path, &block)
    raise ArgumentError, "#{path} does not exist or is not a directory" unless File.directory?(path)

    Wriggle.new(path, &block)
  end

  class Wriggle
    attr_accessor :root, :file_blocks, :directory_blocks

    def initialize(root, &block)
      @root             = root
      @file_blocks      = []
      @directory_blocks = []

      crawl(&block)
    end

    def crawl(&block)
      Find.find(root) do |current|
        yield self

        if File.file?(current)
          dispatch_file(current)
        elsif File.directory?(current)
          dispatch_directory(current)
        end
      end
    end

    # Define a block to be called when a file is encountered
    #
    # Provide one or more extensions to limit the files yielded.
    #
    # == Examples:
    #   file :rb { |file| ... }
    #   file :rb, :rdoc { |file| ... }
    #   file %w(mpeg mpeg wmv avi mkv) { |file| ... }
    #
    # @raise ArgumentError When no block provided
    def file(*extensions, &block)
      raise ArgumentError, "a block is required" unless block_given?<commit_msg>Fix a major logic error<commit_after>require 'find'

# = Wriggle
#
# A simple directory crawler DSL.
#
# == Usage
#
#   require 'wriggle'
#
#   wriggle '/path/to/files' do |w|
#
#     # Build an array of Ruby code files
#     ruby_files = []
#     w.file :rb do |path|
#       ruby_files << path
#     end
#
#     # Build an array of video files
#     video_files = []
#     w.file %w(mpg mpeg wmv avi mkv) do |path|
#       video_files << path
#     end
#
#     # Delete directories that are empty
#     w.directory do |path|
#       Dir.rmdir(path) unless Dir.entries(path).length > 2
#     end
#   end
module Wriggle
  # Crawl the given +path+
  #
  # @raise ArgumentError Given path does not exist or is not a directory
  def wriggle(path, &block)
    raise ArgumentError, "#{path} does not exist or is not a directory" unless File.directory?(path)

    Wriggle.new(path, &block)
  end

  class Wriggle
    attr_accessor :root, :file_blocks, :directory_blocks

    def initialize(root, &block)
      @root             = root
      @file_blocks      = []
      @directory_blocks = []

      crawl(&block)
    end

    def crawl(&block)
      yield self

      Find.find(root) do |current|
        if File.file?(current)
          dispatch_file(current)
        elsif File.directory?(current)
          dispatch_directory(current)
        end
      end
    end

    # Define a block to be called when a file is encountered
    #
    # Provide one or more extensions to limit the files yielded.
    #
    # == Examples:
    #   file :rb { |file| ... }
    #   file :rb, :rdoc { |file| ... }
    #   file %w(mpeg mpeg wmv avi mkv) { |file| ... }
    #
    # @raise ArgumentError When no block provided
    def file(*extensions, &block)
      raise ArgumentError, "a block is required" unless block_given?
{'fraction_non_alphanumeric': 0.11, 'fraction_numerical': 0.0005555555555555556, 'mean_word_length': 2.2588235294117647, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3340458', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1164, 'n_words': 369}