<commit_before>module CMSScanner
  module Finders
    class Finder
      # Module to provide an easy way to enumerate items such as plugins, themes etc
      module Enumerator
        # @param [ Hash ] opts
        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
        # @option opts [ Regexp ] :exclude_content
        #
        # @yield [ Typhoeus::Response, String ]
        def enumerate(opts = {})
          targets = target_urls(opts)
          bar     = progress_bar(targets.size) if opts[:show_progression]

          targets.each do |url, id|
            request = browser.forge_request(url, request_params)

            request.on_complete do |res|
              bar.progress += 1 if opts[:show_progression]

              next if target.homepage_or_404?(res)
              next if opts[:exclude_content] && res.body.match(opts[:exclude_content])

              yield res, id
            end

            hydra.queue(request)
          end

          hydra.run
        end

        # @param [ Hash ] opts
        #
        # @return [ Hash ]
        def target_urls(_opts = {})
          fail NotImplementedError
        end

        # @param [ Integer ] total
        #
        # @return [ ProgressBar ]
        # :nocov:
        def progress_bar(total)
          ProgressBar.create(
            format: '%t %a <%B> (%c / %C) %P%% %e',
            title: ' ', # Used to craete a left margin
            total: total
          )
        end
        # :nocov:

        # @return [ CMSScanner::Browser ]
        def browser
          @browser ||= NS::Browser.instance
        end

        def request_params
          # disabling the cache, as it causes a 'stack level too deep' exception
          # with a large number of requests :/
          # See https://github.com/typhoeus/typhoeus/issues/408
          { cache_ttl: 0 }
        end

        # @return [ Typhoeus::Hydra ]
        def hydra
          @hydra ||= browser.hydra
        end
      end
    end
  end
end<commit_msg>Allows instances using the Enumerator module to set a custom progressbar title<commit_after>module CMSScanner
  module Finders
    class Finder
      # Module to provide an easy way to enumerate items such as plugins, themes etc
      module Enumerator
        # @param [ Hash ] opts
        # @option opts [ Boolean ] :show_progression Wether or not to display the progress bar
        # @option opts [ Regexp ] :exclude_content
        #
        # @yield [ Typhoeus::Response, String ]
        def enumerate(opts = {})
          targets = target_urls(opts)
          bar     = progress_bar(targets.size) if opts[:show_progression]

          targets.each do |url, id|
            request = browser.forge_request(url, request_params)

            request.on_complete do |res|
              bar.progress += 1 if opts[:show_progression]

              next if target.homepage_or_404?(res)
              next if opts[:exclude_content] && res.body.match(opts[:exclude_content])

              yield res, id
            end

            hydra.queue(request)
          end

          hydra.run
        end

        # @param [ Hash ] opts
        #
        # @return [ Hash ]
        def target_urls(_opts = {})
          fail NotImplementedError
        end

        # @param [ Integer ] total
        #
        # @return [ ProgressBar ]
        # :nocov:
        def progress_bar(total)
          ProgressBar.create(
            format: '%t %a <%B> (%c / %C) %P%% %e',
            title: progress_bar_title,
            total: total
          )
        end
        # :nocov:

        # Progress Bar title to use, allow instance using this module
        # to display a custom title if needed
        #
        # @return [ String ]
        def progress_bar_title
          ' ' # Used to craete a left margin
        end

        # @return [ CMSScanner::Browser ]
        def browser
          @browser ||= NS::Browser.instance
        end

        def request_params
          # disabling the cache, as it causes a 'stack level too deep' exception
          # with a large number of requests :/
          # See https://github.com/typhoeus/typhoeus/issues/408
          { cache_ttl: 0 }
        end

        # @return [ Typhoeus::Hydra ]
        def hydra
          @hydra ||= browser.hydra
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.08541905855338691, 'fraction_numerical': 0.0036739380022962113, 'mean_word_length': 1.6177884615384615, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7381720', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1195, 'n_words': 357}