<commit_before>################################################################################
#
# searcher.rb
#
# class Searcher: executes a file search
#
################################################################################

require 'find'
require 'pathname'
require 'set'
require 'fileutil.rb'
require 'searchresult.rb'

class Searcher
  attr_accessor :results
  attr_accessor :settings

  def initialize(settings)
    @settings = settings
    validate_settings
    @fileutil = FileUtil.new
    @results = []
    @timers = {}
    @totalElapsed = 0
    @filehash = Hash.new([])
  end

  def log(message)
    puts message
  end

  def validate_settings
    raise 'Startpath not defined' unless @settings.startpath
    raise 'Startpath not found' unless Pathname.new(@settings.startpath).exist?
    raise 'No search patterns specified' unless @settings.searchpatterns
  end

  def matches_any_pattern(s, pattern_set)
    pattern_set.any? {|p| p.match(s)}
  end

  def any_matches_any_pattern(slist, pattern_set)
    slist.each do |s|
      if matches_any_pattern(s, pattern_set)
        return true
      end
    end
    return false
  end

  def is_search_dir(d)
    path_elems = d.split(File::SEPARATOR) - ['.', '..']
    if @settings.excludehidden and path_elems.any? {|p| p.start_with?('.')}
      return false
    end
    if @settings.in_dirpatterns.count > 0 and
      not any_matches_any_pattern(path_elems, @settings.in_dirpatterns)
      return false
    end
    if @settings.out_dirpatterns.count > 0 and
      any_matches_any_pattern(path_elems, @settings.out_dirpatterns)
      return false
    end
    true
  end

  def is_search_file(f)
    if @settings.excludehidden and f.start_with?('.')
      return false
    end
    if @settings.in_extensions.count > 0 and
      not @settings.in_extensions.include?(@fileutil.get_extension(f))
      return false
    end
    if @settings.out_extensions.count > 0 and
      @settings.out_extensions.include?(@fileutil.get_extension(f))
      return false
    end
    filename = Pathname.new(f).basename.to_s
    if @settings.in_filepatterns.count > 0 and
      not matches_any_pattern(filename, @settings.in_filepatterns)
      return false
    end
    if @settings.out_filepatterns.count > 0 and
      matches_any_pattern(filename, @settings.out_filepatterns)
      return false
    end
    true
  end

  def is_archive_search_file(f)
    if @settings.excludehidden and f.start_with?('.')
      return false
    end
    if @settings.in_archiveextensions.count > 0 and 
      not @settings.in_archiveextensions.include?(@fileutil.get_extension(f))
      return false
    end
    if @settings.out_archiveextensions.count > 0 and
      @settings.out_archiveextensions.include?(@fileutil.get_extension(f))
      return false
    end
    filename = Pathname.new(f).basename.to_s
    if @settings.in_archivefilepatterns.count > 0 and
      not matches_any_pattern(filename, @settings.in_archivefilepatterns)
      return false
    end
    if @settings.out_archivefilepatterns.count > 0 and
      matches_any_pattern(filename, @settings.out_archivefilepatterns)
      return false
    end
    true
  end

  def get_search_dirs
    searchdirs = []
    if FileTest.directory?(@settings.startpath)
      if @settings.recursive
        Find.find(@settings.startpath) do |f|
          if FileTest.directory?(f)
            searchdirs.push(f) if is_search_dir(f)
          end
        end
      else
        searchdirs.push(@settings.startpath) if is_search_dir(@settings.startpath)
      end
    elsif FileTest.file?(@settings.startpath)
      d = File.dirname(@settings.startpath)
      if not d
        d = '.'
      end
      searchdirs.push(d) if is_search_dir(d)
    end
    searchdirs
  end

  def get_search_files(searchdirs)
    searchfiles = []
    if FileTest.directory?(@settings.startpath)
      searchdirs.each do |d|
        all_dirs = Dir.entries(d)
        all_dirs.each do |f|
          unless FileTest.directory?(f)
            if @fileutil.is_archive_file(f) and @settings.searcharchives and is_archive_search_file(f)
              searchfiles.push(Pathname.new(d).join(f).to_s)
            elsif not @settings.archivesonly and is_search_file(f)
              searchfiles.push(Pathname.new(d).join(f).to_s)
            end
          end
        end
      end
    elsif FileTest.file?(@settings.startpath)
      searchfiles.push(@settings.startpath) if is_search_file(@settings.startpath)
    end
    searchfiles
  end

  def add_timer(name, action)
    @timers[name+':'+action] = Time.new
  end

  def start_timer(name)
    add_timer(name, 'start')
  end

  def stop_timer(name)
    add_timer(name, 'stop')
    add_elapsed(name)
  end

  def get_elapsed(name)
    start = @timers[name+':start']
    stop = @timers[name+':stop']
    stop - start
  end

  def add_elapsed(name)
    @totalElapsed += get_elapsed(name)
  end

  def print_elapsed(name)
    elapsed = get_elapsed(name) * 1000
    log("Elapsed time for #{name}: #{elapsed} ms")
  end

  def print_total_elapsed
    log("Total elapsed time: #{@totalElapsed * 1000} ms")
  end

  def search
    # get the searchdirs
    if @settings.dotiming
      start_timer('get_search_dirs')
    end
    searchdirs = get_search_dirs()
    if @settings.dotiming
      stop_timer('get_search_dirs')
      if @settings.printresults
        print_elapsed('get_search_dirs')
      end
    end
    if @settings.verbose<commit_msg>Added filter_file method to ruby<commit_after>################################################################################
#
# searcher.rb
#
# class Searcher: executes a file search
#
################################################################################

require 'find'
require 'pathname'
require 'set'
require 'fileutil.rb'
require 'searchresult.rb'

class Searcher
  attr_accessor :results
  attr_accessor :settings

  def initialize(settings)
    @settings = settings
    validate_settings
    @fileutil = FileUtil.new
    @results = []
    @timers = {}
    @totalElapsed = 0
    @filehash = Hash.new([])
  end

  def log(message)
    puts message
  end

  def validate_settings
    raise 'Startpath not defined' unless @settings.startpath
    raise 'Startpath not found' unless Pathname.new(@settings.startpath).exist?
    raise 'No search patterns specified' unless @settings.searchpatterns
  end

  def matches_any_pattern(s, pattern_set)
    pattern_set.any? {|p| p.match(s)}
  end

  def any_matches_any_pattern(slist, pattern_set)
    slist.each do |s|
      if matches_any_pattern(s, pattern_set)
        return true
      end
    end
    return false
  end

  def is_search_dir(d)
    path_elems = d.split(File::SEPARATOR) - ['.', '..']
    if @settings.excludehidden and path_elems.any? {|p| p.start_with?('.')}
      return false
    end
    if @settings.in_dirpatterns.count > 0 and
      not any_matches_any_pattern(path_elems, @settings.in_dirpatterns)
      return false
    end
    if @settings.out_dirpatterns.count > 0 and
      any_matches_any_pattern(path_elems, @settings.out_dirpatterns)
      return false
    end
    true
  end

  def is_search_file(f)
    if @settings.excludehidden and f.start_with?('.')
      return false
    end
    if @settings.in_extensions.count > 0 and
      not @settings.in_extensions.include?(@fileutil.get_extension(f))
      return false
    end
    if @settings.out_extensions.count > 0 and
      @settings.out_extensions.include?(@fileutil.get_extension(f))
      return false
    end
    filename = Pathname.new(f).basename.to_s
    if @settings.in_filepatterns.count > 0 and
      not matches_any_pattern(filename, @settings.in_filepatterns)
      return false
    end
    if @settings.out_filepatterns.count > 0 and
      matches_any_pattern(filename, @settings.out_filepatterns)
      return false
    end
    true
  end

  def is_archive_search_file(f)
    if @settings.excludehidden and f.start_with?('.')
      return false
    end
    if @settings.in_archiveextensions.count > 0 and 
      not @settings.in_archiveextensions.include?(@fileutil.get_extension(f))
      return false
    end
    if @settings.out_archiveextensions.count > 0 and
      @settings.out_archiveextensions.include?(@fileutil.get_extension(f))
      return false
    end
    filename = Pathname.new(f).basename.to_s
    if @settings.in_archivefilepatterns.count > 0 and
      not matches_any_pattern(filename, @settings.in_archivefilepatterns)
      return false
    end
    if @settings.out_archivefilepatterns.count > 0 and
      matches_any_pattern(filename, @settings.out_archivefilepatterns)
      return false
    end
    true
  end

  def get_search_dirs
    searchdirs = []
    if FileTest.directory?(@settings.startpath)
      if @settings.recursive
        Find.find(@settings.startpath) do |f|
          if FileTest.directory?(f)
            searchdirs.push(f) if is_search_dir(f)
          end
        end
      else
        searchdirs.push(@settings.startpath) if is_search_dir(@settings.startpath)
      end
    elsif FileTest.file?(@settings.startpath)
      d = File.dirname(@settings.startpath)
      if not d
        d = '.'
      end
      searchdirs.push(d) if is_search_dir(d)
    end
    searchdirs
  end

  def filter_file(f)
    if @fileutil.is_archive_file(f) and @settings.searcharchives and is_archive_search_file(f)
      true
    elsif not @settings.archivesonly and is_search_file(f)
      true
    else
      false
    end
  end

  def get_search_files(searchdirs)
    searchfiles = []
    if FileTest.directory?(@settings.startpath)
      searchdirs.each do |d|
        all_dirs = Dir.entries(d)
        all_dirs.each do |f|
          unless FileTest.directory?(f)
            if filter_file(f)
              searchfiles.push(Pathname.new(d).join(f).to_s)
            end
          end
        end
      end
    elsif FileTest.file?(@settings.startpath)
      searchfiles.push(@settings.startpath) if is_search_file(@settings.startpath)
    end
    searchfiles
  end

  def add_timer(name, action)
    @timers[name+':'+action] = Time.new
  end

  def start_timer(name)
    add_timer(name, 'start')
  end

  def stop_timer(name)
    add_timer(name, 'stop')
    add_elapsed(name)
  end

  def get_elapsed(name)
    start = @timers[name+':start']
    stop = @timers[name+':stop']
    stop - start
  end

  def add_elapsed(name)
    @totalElapsed += get_elapsed(name)
  end

  def print_elapsed(name)
    elapsed = get_elapsed(name) * 1000
    log("Elapsed time for #{name}: #{elapsed} ms")
  end

  def print_total_elapsed
    log("Total elapsed time: #{@totalElapsed * 1000} ms")
  end

  def search
    # get the searchdirs
    if @settings.dotiming
      start_timer('get_search_dirs')
    end
    searchdirs = get_search_dirs
    if @settings.dotiming
      stop_timer('get_search_dirs')
      if @settings.printresults
        print_elapsed('get_search_dirs')
      end
    end
    if @settings.verbose
{'fraction_non_alphanumeric': 0.12018659105460533, 'fraction_numerical': 0.0034757157230403367, 'mean_word_length': 3.0154241645244215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6312300', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3489, 'n_words': 752}