<commit_before>require 'listen/file'
require 'listen/directory'

module Listen
  # TODO: rename to Snapshot
  class Change
    # TODO: test this class for coverage
    class Config
      def initialize(queue, silencer)
        @queue = queue
        @silencer = silencer
      end

      def silenced?(path, type)
        @silencer.silenced?(Pathname(path), type)
      end

      def queue(*args)
        @queue << args
      end
    end

    attr_reader :record

    def initialize(config, record)
      @config = config
      @record = record
    end

    # Invalidate some part of the snapshot/record (dir, file, subtree, etc.)
    def invalidate(type, rel_path, options)
      watched_dir = Pathname.new(record.root)

      change = options[:change]
      cookie = options[:cookie]

      if !cookie && config.silenced?(rel_path, type)
        Listen::Logger.debug {  "(silenced): #{rel_path.inspect}" }
        return
      end

      path = watched_dir + rel_path

      Listen::Logger.debug do
        log_details = options[:silence] && 'recording' || change || 'unknown'
        "#{log_details}: #{type}:#{path} (#{options.inspect})"
      end

      if change
        # TODO: move this to Listener to avoid overhead
        # from caller
        options = cookie ? { cookie: cookie } : {}
        config.queue(type, change, watched_dir, rel_path, options)
      else
        if type == :dir
          # NOTE: POSSIBLE RECURSION
          # TODO: fix - use a queue instead
          Directory.scan(self, rel_path, options)
        else
          change = File.change(record, rel_path)
          return if !change || options[:silence]<commit_msg>remove TODO<commit_after>require 'listen/file'
require 'listen/directory'

module Listen
  # TODO: rename to Snapshot
  class Change
    # TODO: test this class for coverage
    class Config
      def initialize(queue, silencer)
        @queue = queue
        @silencer = silencer
      end

      def silenced?(path, type)
        @silencer.silenced?(Pathname(path), type)
      end

      def queue(*args)
        @queue << args
      end
    end

    attr_reader :record

    def initialize(config, record)
      @config = config
      @record = record
    end

    # Invalidate some part of the snapshot/record (dir, file, subtree, etc.)
    def invalidate(type, rel_path, options)
      watched_dir = Pathname.new(record.root)

      change = options[:change]
      cookie = options[:cookie]

      if !cookie && config.silenced?(rel_path, type)
        Listen::Logger.debug {  "(silenced): #{rel_path.inspect}" }
        return
      end

      path = watched_dir + rel_path

      Listen::Logger.debug do
        log_details = options[:silence] && 'recording' || change || 'unknown'
        "#{log_details}: #{type}:#{path} (#{options.inspect})"
      end

      if change
        options = cookie ? { cookie: cookie } : {}
        config.queue(type, change, watched_dir, rel_path, options)
      else
        if type == :dir
          # NOTE: POSSIBLE RECURSION
          # TODO: fix - use a queue instead
          Directory.scan(self, rel_path, options)
        else
          change = File.change(record, rel_path)
          return if !change || options[:silence]
{'fraction_non_alphanumeric': 0.10625774473358117, 'fraction_numerical': 0.0, 'mean_word_length': 2.2815040650406506, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6233228', 'n_tokens_mistral': 941, 'n_tokens_neox': 924, 'n_words': 266}