<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "websearch_configuration.h"

#include <iostream>

namespace seeks_plugins
{

#define hash_lang                   3579134231ul /* "search-langage" */
#define hash_n                       578814699ul /* "search-results-page" */  
#define hash_se                     1635576913ul /* "search-engine" */
#define hash_qcd                    4118649627ul /* "query-context-delay" */
#define hash_thumbs                  793242781ul /* "enable-thumbs" */
#define hash_js                     3171705030ul /* "enable-js" */
#define hash_content_analysis       1483831511ul /* "enable-content-analysis" */
#define hash_se_transfer_timeout    2056038060ul /* "se-transfer-timeout" */
#define hash_se_connect_timeout     1026838950ul /* "se-connect-timeout" */
#define hash_ct_transfer_timeout    3371661146ul /* "ct-transfer-timeout" */
#define hash_ct_connect_timeout     3817701526ul /* "ct-connect-timeout" */
#define hash_clustering             2382120344ul /* "enable-clustering" */
#define hash_max_expansions         3838821776ul /* "max-expansions" */
   
   websearch_configuration::websearch_configuration(const std::string &filename)
     :configuration_spec(filename)
       {
	  _se_enabled = std::bitset<NSEs>(0);
	  load_config();
       }
   
   websearch_configuration::~websearch_configuration()
     {
     }
   
   void websearch_configuration::set_default_config()
     {
	_lang = "auto";
	_N = 10;
	_thumbs = false;
	_se_enabled.set(); // all engines is default.
	_query_context_delay = 300; // in seconds, 5 minutes.
	_js = false; // default is no javascript, this may change later on.
	_content_analysis = false;
	_clustering = false;
	_se_connect_timeout = 3;  // in seconds.
	_se_transfer_timeout = 5; // in seconds.
	_ct_connect_timeout = 1; // in seconds.
	_ct_transfer_timeout = 3; // in seconds.
	_max_expansions = 100;
     }
   
   void websearch_configuration::handle_config_cmd(char *cmd, const uint32_t &cmd_hash, char *arg,
						   char *buf, const unsigned long &linenum)
     {
	switch(cmd_hash)
	  {
	   case hash_lang :
	     _lang = std::string(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Websearch language");
	     break;
	     
	   case hash_n :
	       _N = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Number of websearch results per page");
	     break;
	     
	   case hash_se :
	     if (_se_enabled.count() == NSEs) // all bits set is default, so now reset to 0.
	       _se_enabled.reset();
	     
	     if (strcasecmp(arg,"google") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_GOOGLE);
	     else if (strcasecmp(arg,"cuil") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_CUIL);
	     else if (strcasecmp(arg,"bing") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_BING);
	     else if (strcasecmp(arg,"yahoo") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_YAHOO);
	     else if (strcasecmp(arg,"exalead") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_EXALEAD);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enabled search engine");
	     break;
	     
	   case hash_thumbs:
	      _thumbs = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable search results webpage thumbnails");
	     break;
	   
	   case hash_qcd :
	     _query_context_delay = strtod(arg,NULL);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Delay in seconds before deletion of cached websearches and results");
	     break;
	   
	   case hash_js:
	     _js = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable javascript use on the websearch result page");
	     break;
	     
	   case hash_content_analysis:
	     _content_analysis = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable the background download of webpages pointed to by websearch results and content analysis");
	     break;
	     
	   case hash_se_transfer_timeout:
	     _se_transfer_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the transfer timeout in seconds for connections to a search engine");
	     break;
	     
	   case hash_se_connect_timeout:
	     _se_connect_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the connection timeout in seconds for connections to a search engine");
	     break;
	     
	   case hash_ct_transfer_timeout:
	     _ct_transfer_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the transfer timeout in seconds when fetching content for analysis and caching");
	     break;
	     
	   case hash_ct_connect_timeout:
	     _ct_connect_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the connection timeout in seconds when fetching content for analysis and caching");
	     break;
	     
	   case hash_clustering:
	     _clustering = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enables the clustering from the UI");
	     break;
	     
	   case hash_max_expansions:
	     _max_expansions = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the maximum number of query expansions");
	     break;
	     
	   default :
	     break;
	     
	  } // end of switch.
     }
   
   void websearch_configuration::finalize_configuration()
     {
     }
   
} /* end of namespace. */<commit_msg>added highlight of most discriminative words in snippets<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "websearch_configuration.h"

#include <iostream>

namespace seeks_plugins
{

#define hash_lang                   3579134231ul /* "search-langage" */
#define hash_n                       578814699ul /* "search-results-page" */  
#define hash_se                     1635576913ul /* "search-engine" */
#define hash_qcd                    4118649627ul /* "query-context-delay" */
#define hash_thumbs                  793242781ul /* "enable-thumbs" */
#define hash_js                     3171705030ul /* "enable-js" */
#define hash_content_analysis       1483831511ul /* "enable-content-analysis" */
#define hash_se_transfer_timeout    2056038060ul /* "se-transfer-timeout" */
#define hash_se_connect_timeout     1026838950ul /* "se-connect-timeout" */
#define hash_ct_transfer_timeout    3371661146ul /* "ct-transfer-timeout" */
#define hash_ct_connect_timeout     3817701526ul /* "ct-connect-timeout" */
#define hash_clustering             2382120344ul /* "enable-clustering" */
#define hash_max_expansions         3838821776ul /* "max-expansions" */
#define hash_extended_highlight     2722091897ul /* "extended-highlight" */   
   
   websearch_configuration::websearch_configuration(const std::string &filename)
     :configuration_spec(filename)
       {
	  _se_enabled = std::bitset<NSEs>(0);
	  load_config();
       }
   
   websearch_configuration::~websearch_configuration()
     {
     }
   
   void websearch_configuration::set_default_config()
     {
	_lang = "auto";
	_N = 10;
	_thumbs = false;
	_se_enabled.set(); // all engines is default.
	_query_context_delay = 300; // in seconds, 5 minutes.
	_js = false; // default is no javascript, this may change later on.
	_content_analysis = false;
	_clustering = false;
	_se_connect_timeout = 3;  // in seconds.
	_se_transfer_timeout = 5; // in seconds.
	_ct_connect_timeout = 1; // in seconds.
	_ct_transfer_timeout = 3; // in seconds.
	_max_expansions = 100;
	_extended_highlight = false; // experimental.
     }
   
   void websearch_configuration::handle_config_cmd(char *cmd, const uint32_t &cmd_hash, char *arg,
						   char *buf, const unsigned long &linenum)
     {
	switch(cmd_hash)
	  {
	   case hash_lang :
	     _lang = std::string(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Websearch language");
	     break;
	     
	   case hash_n :
	       _N = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Number of websearch results per page");
	     break;
	     
	   case hash_se :
	     if (_se_enabled.count() == NSEs) // all bits set is default, so now reset to 0.
	       _se_enabled.reset();
	     
	     if (strcasecmp(arg,"google") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_GOOGLE);
	     else if (strcasecmp(arg,"cuil") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_CUIL);
	     else if (strcasecmp(arg,"bing") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_BING);
	     else if (strcasecmp(arg,"yahoo") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_YAHOO);
	     else if (strcasecmp(arg,"exalead") == 0)
	       _se_enabled |= std::bitset<NSEs>(SE_EXALEAD);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enabled search engine");
	     break;
	     
	   case hash_thumbs:
	      _thumbs = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable search results webpage thumbnails");
	     break;
	   
	   case hash_qcd :
	     _query_context_delay = strtod(arg,NULL);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Delay in seconds before deletion of cached websearches and results");
	     break;
	   
	   case hash_js:
	     _js = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable javascript use on the websearch result page");
	     break;
	     
	   case hash_content_analysis:
	     _content_analysis = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enable the background download of webpages pointed to by websearch results and content analysis");
	     break;
	     
	   case hash_se_transfer_timeout:
	     _se_transfer_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the transfer timeout in seconds for connections to a search engine");
	     break;
	     
	   case hash_se_connect_timeout:
	     _se_connect_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the connection timeout in seconds for connections to a search engine");
	     break;
	     
	   case hash_ct_transfer_timeout:
	     _ct_transfer_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the transfer timeout in seconds when fetching content for analysis and caching");
	     break;
	     
	   case hash_ct_connect_timeout:
	     _ct_connect_timeout = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the connection timeout in seconds when fetching content for analysis and caching");
	     break;
	     
	   case hash_clustering:
	     _clustering = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enables the clustering from the UI");
	     break;
	     
	   case hash_max_expansions:
	     _max_expansions = atoi(arg);
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Sets the maximum number of query expansions");
	     break;

	   case hash_extended_highlight:
	     _extended_highlight = static_cast<bool>(atoi(arg));
	     configuration_spec::html_table_row(_config_args,cmd,arg,
						"Enables a more discriminative word highlight scheme");
	     break;
	     
	   default:
	     break;
	     
	  } // end of switch.
     }
   
   void websearch_configuration::finalize_configuration()
     {
     }
   
} /* end of namespace. */
{'fraction_non_alphanumeric': 0.09745923397800531, 'fraction_numerical': 0.02427000379218809, 'mean_word_length': 3.1141965678627144, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5929908', 'n_tokens_mistral': 4569, 'n_tokens_neox': 3981, 'n_words': 1013}