<commit_before>class Dir
  include Enumerable

  GLOB_VERBOSE = 1 << (4 * 8 - 1) # HACK
  GLOB_RECURSIVE = -2

  def self.[](pattern)
    glob(pattern, 0)
  end

  def self.old_glob(pattern, flags = 0)
    matches = []

    glob_brace_expand pattern.to_str, flags & ~GLOB_VERBOSE, matches

    matches
  end

  def self.glob(pattern, flags=0)
    return [] if pattern.empty?

    # The new glob impl doesn't handle flags or curly expansion, so
    # use the old glob if those are used.
    if flags != 0 or pattern.include? "{"
      old_glob(pattern, flags)
    else
      node = Dir::Glob.compile pattern
      Dir::Glob.run node
    end
  end

  def self.glob0(pattern, flags, matches)
    root = pattern.dup
    path = ''

    if root[0] == ?/ then # HACK windows
      root = root[1..-1]
      path = '/'
    end

    list = glob_pattern root, flags

    glob_helper path, false, :unknown, :unknown, list, 0, 1, flags, matches
  end

  def self.find_dirsep(pattern, flags)
    escape = (flags & File::FNM_NOESCAPE) == 0
    open = false

    chars = pattern.split ''

    chars.each_with_index do |char, i|
      case char
      when '[' then
        open = true
      when ']' then
        open = false
      when '/' then
        return i unless open
      when '\\' then
        return i if escape and (i + 1 == chars.length)
      end
    end

    chars.length
  end

  def self.glob_brace_expand(pattern, flags, matches)
    escape = (flags & File::FNM_NOESCAPE) == 0

    rbrace = nil
    lbrace = nil
    nest = 0

    chars = pattern.split ''
    skip = false

    chars.each_with_index do |char, i|
      if skip then
        skip = false
        next
      end

      if char == '{' and nest == 0 then
        lbrace = i
        nest += 1
      end

      if char == '}' and nest - 1 <= 0 then
        rbrace = i
        nest -= 1
      end

      skip = true if char == '\\' and escape
    end

    if lbrace and rbrace then
      pos = lbrace
      front = pattern[0...lbrace]
      back = pattern[(rbrace + 1)..-1]

      while pos < rbrace do
        nest = 0
        pos += 1
        last = pos

        while pos < rbrace and not (chars[pos] == ',' and nest == 0) do
          nest += 1 if chars[pos] == '{'
          nest -= 1 if chars[pos] == '}'

          if chars[pos] == '\\' and escape then
            pos += 1
            break if pos == rbrace
          end

          pos += 1
        end

        brace_pattern = "#{front}#{pattern[last...pos]}#{back}"

        glob_brace_expand brace_pattern, flags, matches
      end

    else
      glob0 pattern, flags, matches
    end
  end

  ##
  # +dirsep+:: Should '/' be placed before appending child's entry name to
  #            +path+?
  # +exist+:: Does 'path' indicate an existing entry?
  # +isdir+:: Does 'path' indicate a directory or a symlink to a directory?

  def self.glob_helper(path, dirsep, exist, isdir, pattern, start, stop, flags, matches)
    status = nil
    plain = magic = recursive = match_all = match_dir = false
    escape = (flags & File::FNM_NOESCAPE) == 0

    last_type = nil

    pattern[start, stop].each_with_index do |part, i|
      if part[1] == :recursive then
        recursive = true
        part = pattern[i + 1]
      end

      case part[1]
      when :magic then
        magic = true
      when :match_all then
        match_all = true
      when :match_dir then
        match_dir = true
      when :plain then
        plain = true
      when :recursive then
        raise "continuous RECURSIVEs"
      end
    end

    unless path.empty? then
      if match_all and exist == :unknown then
        if stat = File.stat(path) rescue nil then
          exist = :yes
          isdir = if stat.directory? then
                    :yes
                  elsif stat.symlink? then
                    :unknown
                  else
                    :no
                  end
        else
          exist = isdir = :no
        end
      end

      if match_dir and isdir == :unknown then
        if stat = File.stat(path) rescue nil then
          exist = :yes
          isdir = stat.directory? ? :yes : :no
        else
          exist = isdir = :no
        end
      end

      matches << path if match_all and exist == :yes

      if match_dir and isdir == :yes then
        return if path.empty? and not dirsep
        tmp = join_path path, '', dirsep
        matches << tmp
      end
    end

    return if exist == :no or isdir == :no

    if magic or recursive then
      dir = path.empty? ? '.' : path

      return unless File.directory? dir

      begin
        Dir.foreach dir do |entry|
          buf = join_path path, entry, dirsep
          new_isdir = :unknown
          new_pattern = []<commit_msg>Use File::Stat.stat to remove the exception overhead<commit_after>class Dir
  include Enumerable

  GLOB_VERBOSE = 1 << (4 * 8 - 1) # HACK
  GLOB_RECURSIVE = -2

  def self.[](pattern)
    glob(pattern, 0)
  end

  def self.old_glob(pattern, flags = 0)
    matches = []

    glob_brace_expand pattern.to_str, flags & ~GLOB_VERBOSE, matches

    matches
  end

  def self.glob(pattern, flags=0)
    return [] if pattern.empty?

    # The new glob impl doesn't handle flags or curly expansion, so
    # use the old glob if those are used.
    if flags != 0 or pattern.include? "{"
      old_glob(pattern, flags)
    else
      node = Dir::Glob.compile pattern
      Dir::Glob.run node
    end
  end

  def self.glob0(pattern, flags, matches)
    root = pattern.dup
    path = ''

    if root[0] == ?/ then # HACK windows
      root = root[1..-1]
      path = '/'
    end

    list = glob_pattern root, flags

    glob_helper path, false, :unknown, :unknown, list, 0, 1, flags, matches
  end

  def self.find_dirsep(pattern, flags)
    escape = (flags & File::FNM_NOESCAPE) == 0
    open = false

    chars = pattern.split ''

    chars.each_with_index do |char, i|
      case char
      when '[' then
        open = true
      when ']' then
        open = false
      when '/' then
        return i unless open
      when '\\' then
        return i if escape and (i + 1 == chars.length)
      end
    end

    chars.length
  end

  def self.glob_brace_expand(pattern, flags, matches)
    escape = (flags & File::FNM_NOESCAPE) == 0

    rbrace = nil
    lbrace = nil
    nest = 0

    chars = pattern.split ''
    skip = false

    chars.each_with_index do |char, i|
      if skip then
        skip = false
        next
      end

      if char == '{' and nest == 0 then
        lbrace = i
        nest += 1
      end

      if char == '}' and nest - 1 <= 0 then
        rbrace = i
        nest -= 1
      end

      skip = true if char == '\\' and escape
    end

    if lbrace and rbrace then
      pos = lbrace
      front = pattern[0...lbrace]
      back = pattern[(rbrace + 1)..-1]

      while pos < rbrace do
        nest = 0
        pos += 1
        last = pos

        while pos < rbrace and not (chars[pos] == ',' and nest == 0) do
          nest += 1 if chars[pos] == '{'
          nest -= 1 if chars[pos] == '}'

          if chars[pos] == '\\' and escape then
            pos += 1
            break if pos == rbrace
          end

          pos += 1
        end

        brace_pattern = "#{front}#{pattern[last...pos]}#{back}"

        glob_brace_expand brace_pattern, flags, matches
      end

    else
      glob0 pattern, flags, matches
    end
  end

  ##
  # +dirsep+:: Should '/' be placed before appending child's entry name to
  #            +path+?
  # +exist+:: Does 'path' indicate an existing entry?
  # +isdir+:: Does 'path' indicate a directory or a symlink to a directory?

  def self.glob_helper(path, dirsep, exist, isdir, pattern, start, stop, flags, matches)
    status = nil
    plain = magic = recursive = match_all = match_dir = false
    escape = (flags & File::FNM_NOESCAPE) == 0

    last_type = nil

    pattern[start, stop].each_with_index do |part, i|
      if part[1] == :recursive then
        recursive = true
        part = pattern[i + 1]
      end

      case part[1]
      when :magic then
        magic = true
      when :match_all then
        match_all = true
      when :match_dir then
        match_dir = true
      when :plain then
        plain = true
      when :recursive then
        raise "continuous RECURSIVEs"
      end
    end

    unless path.empty? then
      if match_all and exist == :unknown then
        if stat = File::Stat.stat(path)
          exist = :yes
          isdir = if stat.directory? then
                    :yes
                  elsif stat.symlink? then
                    :unknown
                  else
                    :no
                  end
        else
          exist = isdir = :no
        end
      end

      if match_dir and isdir == :unknown then
        if stat = File::Stat.stat(path)
          exist = :yes
          isdir = stat.directory? ? :yes : :no
        else
          exist = isdir = :no
        end
      end

      matches << path if match_all and exist == :yes

      if match_dir and isdir == :yes then
        return if path.empty? and not dirsep
        tmp = join_path path, '', dirsep
        matches << tmp
      end
    end

    return if exist == :no or isdir == :no

    if magic or recursive then
      dir = path.empty? ? '.' : path

      return unless File.directory? dir

      begin
        Dir.foreach dir do |entry|
          buf = join_path path, entry, dirsep
          new_isdir = :unknown
          new_pattern = []
{'fraction_non_alphanumeric': 0.09760472723435687, 'fraction_numerical': 0.00823045267489712, 'mean_word_length': 1.6819468024900963, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2444363', 'n_tokens_mistral': 3037, 'n_tokens_neox': 2828, 'n_words': 980}