<commit_before>module CMSScanner
  module Controller
    # CLI Options for the Core Controller
    class Core < Base
      def cli_options
        formats = NS::Formatter.availables

        [
          OptURL.new(['-u', '--url URL'], required: true, default_protocol: 'http'),
          OptBoolean.new(%w(-v --verbose)),
          OptFilePath.new(['-o', '--output FILE', 'Output to FILE'], writable: true, exists: false),
          OptChoice.new(['-f', '--format FORMAT',
                         "Available formats: #{formats.join(', ')}"], choices: formats),
          OptChoice.new(['--detection-mode MODE', 'Modes: mixed (default), passive, aggressive'],
                        choices: %w(mixed passive aggressive),
                        normalize: :to_sym,
                        default: :mixed),
          OptArray.new(['--scope DOMAINS',
                        'Coma separated (sub-)domains to consider in scope. ' \
                        'Wildcard(s) allowed in the trd of valid domains, e.g: *.target.tld'])
        ] + cli_browser_options
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_options
        [
          OptString.new(['--user-agent VALUE', '--ua']),
          OptBoolean.new(['--random-user-agent', '--rua',
                          'Use a random user-agent for each scan']),
          OptFilePath.new(['--user-agents-list FILE-PATH',
                           'List of agents to use with --random-user-agent'], exists: true),
          OptCredentials.new(['--http-auth login:password']),
          OptPositiveInteger.new(['--max-threads VALUE', '-t', 'The max threads to use'],
                                 default: 5),
          OptPositiveInteger.new(['--request-timeout SECONDS', 'The request timeout in seconds']),
          OptPositiveInteger.new(['--connect-timeout SECONDS',
                                  'The connection timeout in seconds'])
        ] + cli_browser_proxy_options + cli_browser_cookies_options + cli_browser_cache_options
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_proxy_options
        [
          OptProxy.new(['--proxy protocol://IP:port',
                        'Supported protocols depend on the cURL installed']),
          OptCredentials.new(['--proxy-auth login:password'])
        ]
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_cookies_options
        [
          OptString.new(['--cookie-string COOKIE',
                         'Cookie string to use in requests, ' \
                         'format: cookie1=value1[; cookie2=value2]']),
          OptFilePath.new(['--cookie-jar FILE-PATH', 'File to read and write cookies'],
                          writable: true,
                          exists: false,
                          default: '/tmp/cms_scanner/cookie_jar.txt')
        ]
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_cache_options
        [
          OptInteger.new(['--cache-ttl TIME_TO_LIVE'], default: 600),
          OptBoolean.new(['--clear-cache', 'Clear the cache before the scan']),
          OptDirectoryPath.new(['--cache-dir PATH'],
                               readable: true,
                               writable: true,
                               default: '/tmp/cms_scanner/cache/')
        ]
      end
    end
  end
end
<commit_msg>Sets default timeout values for the rquest and connect, to avoid requests getting stuck too long<commit_after>module CMSScanner
  module Controller
    # CLI Options for the Core Controller
    class Core < Base
      def cli_options
        formats = NS::Formatter.availables

        [
          OptURL.new(['-u', '--url URL'], required: true, default_protocol: 'http'),
          OptBoolean.new(%w(-v --verbose)),
          OptFilePath.new(['-o', '--output FILE', 'Output to FILE'], writable: true, exists: false),
          OptChoice.new(['-f', '--format FORMAT',
                         "Available formats: #{formats.join(', ')}"], choices: formats),
          OptChoice.new(['--detection-mode MODE', 'Modes: mixed (default), passive, aggressive'],
                        choices: %w(mixed passive aggressive),
                        normalize: :to_sym,
                        default: :mixed),
          OptArray.new(['--scope DOMAINS',
                        'Coma separated (sub-)domains to consider in scope. ' \
                        'Wildcard(s) allowed in the trd of valid domains, e.g: *.target.tld'])
        ] + cli_browser_options
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_options
        [
          OptString.new(['--user-agent VALUE', '--ua']),
          OptBoolean.new(['--random-user-agent', '--rua',
                          'Use a random user-agent for each scan']),
          OptFilePath.new(['--user-agents-list FILE-PATH',
                           'List of agents to use with --random-user-agent'], exists: true),
          OptCredentials.new(['--http-auth login:password']),
          OptPositiveInteger.new(['--max-threads VALUE', '-t', 'The max threads to use'],
                                 default: 5),
          OptPositiveInteger.new(['--request-timeout SECONDS', 'The request timeout in seconds'],
                                 default: 4),
          OptPositiveInteger.new(['--connect-timeout SECONDS', 'The connection timeout in seconds'],
                                 default: 2)
        ] + cli_browser_proxy_options + cli_browser_cookies_options + cli_browser_cache_options
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_proxy_options
        [
          OptProxy.new(['--proxy protocol://IP:port',
                        'Supported protocols depend on the cURL installed']),
          OptCredentials.new(['--proxy-auth login:password'])
        ]
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_cookies_options
        [
          OptString.new(['--cookie-string COOKIE',
                         'Cookie string to use in requests, ' \
                         'format: cookie1=value1[; cookie2=value2]']),
          OptFilePath.new(['--cookie-jar FILE-PATH', 'File to read and write cookies'],
                          writable: true,
                          exists: false,
                          default: '/tmp/cms_scanner/cookie_jar.txt')
        ]
      end

      # @return [ Array<OptParseValidator::OptBase> ]
      def cli_browser_cache_options
        [
          OptInteger.new(['--cache-ttl TIME_TO_LIVE'], default: 600),
          OptBoolean.new(['--clear-cache', 'Clear the cache before the scan']),
          OptDirectoryPath.new(['--cache-dir PATH'],
                               readable: true,
                               writable: true,
                               default: '/tmp/cms_scanner/cache/')
        ]
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12602150537634407, 'fraction_numerical': 0.0025806451612903226, 'mean_word_length': 1.7165109034267914, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5982880', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1805, 'n_words': 439}