<commit_before>module RDig
  
  module UrlFilters

    class FilterChain
      def initialize(chain_config)
        @filters = []
        chain_config.each { |filter|
          case filter
          when Hash
            filter.each_pair { |f, args|
              add(f, args)
            }
          when Array
            args = filter
            filter = args.shift
            add(filter, args)
          else
            add(filter)
          end
        }
      end

      # add a filter and it's args to the chain
      # when args is a symbol, it is treated as a configuration key
      def add(filter, args=nil)
        args = RDig.config.crawler.send(args) if args.is_a? Symbol
        case filter
        when Symbol
          if args.nil?
            @filters << lambda { |document|
              UrlFilters.send(filter, document)
            }
          else
            @filters << lambda { |document|
              UrlFilters.send(filter, document, args)
            }
          end
        when Class
          if args.nil?
            if filter.respond_to?(:instance)
              filter_instance = filter.instance
            else
              filter_instance = filter.new
            end
          else
            filter_instance = filter.new(args)
          end
          @filters << lambda { |document|
            filter_instance.apply(document)
          }
        end
      end

      def apply(document)
        @filters.each { |filter|
          return nil unless filter.call(document)
        }
        return document
      end
    end

    # takes care of a list of all Urls visited during a crawl, to avoid
    # indexing pages more than once
    # implemented as a thread safe singleton as it has to be shared
    # between all crawler threads
    class VisitedUrlFilter
      include MonitorMixin, Singleton
      def initialize
        @visited_urls = Set.new
        super
      end

      # return document if this document's url has not been visited yet,
      # nil otherwise
      def apply(document)
        synchronize do
          @visited_urls.add?(document.uri.to_s) ? document : nil 
        end
      end
    end


    # base class for url inclusion / exclusion filters
    class PatternFilter
      # takes an Array of Regexps, or nil to disable the filter
      def initialize(args=nil)
        unless args.nil?
          @patterns = []
          if args.respond_to? :each
            args.each { |pattern| 
              # cloning because unsure if regexps are thread safe...
              @patterns << pattern.clone
            }
          else
            @patterns << args.clone
          end
        end
      end
    end
    class UrlExclusionFilter < PatternFilter
      # returns nil if any of the patterns matches it's URI,
      # the document itself otherwise
      def apply(document)
        return document unless @patterns
        @patterns.each { |p|
          return nil if document.uri.to_s =~ p
        }
        return document
      end
    end
    class UrlInclusionFilter < PatternFilter
      # returns the document if any of the patterns matches it's URI,
      # nil otherwise
      def apply(document)
        return document unless @patterns
        @patterns.each { |p|
          return document if document.uri.to_s =~ p<commit_msg>clone raising security errors when applied to a literal regexp<commit_after>module RDig
  
  module UrlFilters

    class FilterChain
      def initialize(chain_config)
        @filters = []
        chain_config.each { |filter|
          case filter
          when Hash
            filter.each_pair { |f, args|
              add(f, args)
            }
          when Array
            args = filter
            filter = args.shift
            add(filter, args)
          else
            add(filter)
          end
        }
      end

      # add a filter and it's args to the chain
      # when args is a symbol, it is treated as a configuration key
      def add(filter, args=nil)
        args = RDig.config.crawler.send(args) if args.is_a? Symbol
        case filter
        when Symbol
          if args.nil?
            @filters << lambda { |document|
              UrlFilters.send(filter, document)
            }
          else
            @filters << lambda { |document|
              UrlFilters.send(filter, document, args)
            }
          end
        when Class
          if args.nil?
            if filter.respond_to?(:instance)
              filter_instance = filter.instance
            else
              filter_instance = filter.new
            end
          else
            filter_instance = filter.new(args)
          end
          @filters << lambda { |document|
            filter_instance.apply(document)
          }
        end
      end

      def apply(document)
        @filters.each { |filter|
          return nil unless filter.call(document)
        }
        return document
      end
    end

    # takes care of a list of all Urls visited during a crawl, to avoid
    # indexing pages more than once
    # implemented as a thread safe singleton as it has to be shared
    # between all crawler threads
    class VisitedUrlFilter
      include MonitorMixin, Singleton
      def initialize
        @visited_urls = Set.new
        super
      end

      # return document if this document's url has not been visited yet,
      # nil otherwise
      def apply(document)
        synchronize do
          @visited_urls.add?(document.uri.to_s) ? document : nil 
        end
      end
    end


    # base class for url inclusion / exclusion filters
    class PatternFilter
      # takes an Array of Regexps, or nil to disable the filter
      def initialize(args=nil)
        unless args.nil?
          @patterns = []
          if args.respond_to? :each
            args.each { |pattern| 
              # cloning because unsure if regexps are thread safe ?
              @patterns << pattern #.clone
            }
          else
            @patterns << args #.clone
          end
        end
      end
    end
    class UrlExclusionFilter < PatternFilter
      # returns nil if any of the patterns matches it's URI,
      # the document itself otherwise
      def apply(document)
        return document unless @patterns
        @patterns.each { |p|
          return nil if document.uri.to_s =~ p
        }
        return document
      end
    end
    class UrlInclusionFilter < PatternFilter
      # returns the document if any of the patterns matches it's URI,
      # nil otherwise
      def apply(document)
        return document unless @patterns
        @patterns.each { |p|
          return document if document.uri.to_s =~ p
{'fraction_non_alphanumeric': 0.05989817310572028, 'fraction_numerical': 0.0, 'mean_word_length': 1.5356871678056188, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '625947', 'n_tokens_mistral': 1719, 'n_tokens_neox': 1688, 'n_words': 596}