<commit_before># Copyright 2014 <NAME>. All rights reserved.
# Licensed under the terms of the BSD 2-clause license.

require 'command-t/vim'
require 'command-t/scanner/file_scanner'
require 'open3'

module CommandT
  class FileScanner
    # Uses git ls-files to scan for files
    class GitFileScanner < FindFileScanner
      def paths
        @paths[@path] ||= Dir.chdir(@path) do
          prepare_paths

          command = "git ls-files --exclude-standard "
          stdin, stdout, stderr = Open3.popen3(*[
            "git",
            "ls-files",
            "--exclude-standard",
            @path
          ])

          set_wild_ignore(@wild_ignore)
          all_files = stdout.readlines.
            map { |x| x.chomp }.
            select { |x| not path_excluded?(x, prefix_len = 0) }.
            take(@max_files).
            to_a

          # either git is not available, or this is not a git repository
          # fall back to find
          return super if stderr.gets

          all_files<commit_msg>Simplify require statements<commit_after># Copyright 2014 <NAME>. All rights reserved.
# Licensed under the terms of the BSD 2-clause license.

require 'command-t/scanner/file_scanner/find_file_scanner'

module CommandT
  class FileScanner
    # Uses git ls-files to scan for files
    class GitFileScanner < FindFileScanner
      def paths
        @paths[@path] ||= Dir.chdir(@path) do
          prepare_paths

          command = "git ls-files --exclude-standard "
          stdin, stdout, stderr = Open3.popen3(*[
            "git",
            "ls-files",
            "--exclude-standard",
            @path
          ])

          set_wild_ignore(@wild_ignore)
          all_files = stdout.readlines.
            map { |x| x.chomp }.
            select { |x| not path_excluded?(x, prefix_len = 0) }.
            take(@max_files).
            to_a

          # either git is not available, or this is not a git repository
          # fall back to find
          return super if stderr.gets

          all_files
{'fraction_non_alphanumeric': 0.09184675834970531, 'fraction_numerical': 0.008349705304518664, 'mean_word_length': 1.8609550561797752, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7554774', 'n_tokens_mistral': 620, 'n_tokens_neox': 586, 'n_words': 175}