<commit_before>require 'fileutils'
require 'filesize'

class FileManager
  attr_accessor :path
  attr_reader :excludes
  def initialize(options = {})
    @path = options[:path] || './'
    @excludes = options[:excludes] || ''
    resolve_path(@path)
    options.each do |k,v|
      if k != :path && k != :excludes
        instance_variable_set('@' + k.to_s, v)
        add_accessor(k)
      end
    end
  end

  def excludes=(value)
    if value.nil? || value.class == Integer
      raise FileManagementErrors.new('Invalid value for @excludes')
    else
      @excludes = value
    end
  end

  def pwd(file)
    "#{path}/#{file}"
  end

  def format_excludes
    return excludes.split(' ') if excludes.class == String
    excludes
  end

  def all_files(all = false)
    file_list = Dir.entries(path).select { |f| File.file?("#{path}/#{f}") }
    file_list - (%w(. .. .DS_Store file_management.rb) + format_excludes)
  end

  def file_exists?(file)
    all_files.include?(file)
  end

  def stats(file, request)<commit_msg>Create file_manager.rb<commit_after>require 'fileutils'
require 'filesize'

class FileManager
  attr_accessor :path
  attr_reader :excludes
  def initialize(options = {})
    @path = options[:path] || './'
    @excludes = options[:excludes] || ''
    resolve_path(@path)
    options.each do |k,v|
      if k != :path && k != :excludes
        instance_variable_set('@' + k.to_s, v)
        add_accessor(k)
      end
    end
  end

  def excludes=(value)
    if value.nil? || value.class == Integer
      raise FileManagementErrors.new('Invalid value for @excludes')
    else
      @excludes = value
    end
  end

  def pwd(file)
    "#{path}/#{file}"
  end

  def format_excludes
    return excludes.split(' ') if excludes.class == String
    excludes
  end

  def all_files
    file_list = Dir.entries(path).select { |f| File.file?("#{path}/#{f}") }
    file_list - (%w(. .. .DS_Store file_management.rb) + format_excludes)
  end

  def file_exists?(file)
    all_files.include?(file)
  end

  def stats(file, request)
{'fraction_non_alphanumeric': 0.13894324853228962, 'fraction_numerical': 0.0, 'mean_word_length': 2.955512572533849, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '955644', 'n_tokens_mistral': 721, 'n_tokens_neox': 697, 'n_words': 161}