<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 **/
 
#include "search_snippet.h"
#include "websearch.h" // for configuration.
#include "mem_utils.h"
#include "miscutil.h"
#include "encode.h"
#include "loaders.h"
#include "urlmatch.h"
#include "plugin_manager.h" // for _plugin_repository.

#ifndef FEATURE_EXTENDED_HOST_PATTERNS
#include "proxy_dts.h" // for http_request.
#endif

#include "mrf.h"

#include <iostream>

using sp::miscutil;
using sp::encode;
using sp::loaders;
using sp::urlmatch;
using sp::plugin_manager;
using sp::http_request;
using lsh::mrf;

namespace seeks_plugins
{
   // loaded tagging patterns.
   std::vector<url_spec*> search_snippet::_pdf_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_file_doc_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_audio_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_video_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_forum_pos_patterns = std::vector<url_spec*>();
   
   search_snippet::search_snippet()
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(0),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL)
       {
       }
   
   search_snippet::search_snippet(const short &rank)
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(rank),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL)
       {
       }
      
   search_snippet::~search_snippet()
     {
	if (_cached_content)
	  free_const(_cached_content);
	if (_features)
	  delete _features;
	if (_features_tfidf)
	  delete _features_tfidf;
	if (_bag_of_words)
	  delete _bag_of_words;
     }

   void search_snippet::highlight_query(std::vector<std::string> &words,
					std::string &str)
     {
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// surround every of those words appearing within the
	// argument string with <b> </b> for html
	// bold format. TODO: green ?
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  std::string bold_str = "<b>" + words.at(i) + "</b>";
		  miscutil::ci_replace_in_string(str,words.at(i),bold_str);
	       }
	  }
     }
      
   std::ostream& search_snippet::print(std::ostream &output)
     {
	output << "-----------------------------------\n";
	output << "- seeks rank: " << _seeks_rank << std::endl;
	output << "- rank: " << _rank << std::endl;
	output << "- title: " << _title << std::endl;
	output << "- url: " << _url << std::endl;
	output << "- cite: " << _cite << std::endl;
	output << "- cached: " << _cached << std::endl;
	output << "- summary: " << _summary << std::endl;
	output << "- file format: " << _file_format << std::endl;
	output << "- date: " << _date << std::endl;
	output << "- lang: " << _lang << std::endl;
	if (_doc_type == FORUM)
	  output << "- forum thread info: " << _forum_thread_info << std::endl;
	output << "-----------------------------------\n";
	
	return output;
     }
     
   std::string search_snippet::to_html()
     {
	std::vector<std::string> words;
	return to_html_with_highlight(words);
     }

   std::string search_snippet::to_html_with_highlight(std::vector<std::string> &words)
     {
	static std::string se_icon = "<span class=\"search_engine icon\">&nbsp;</span>";
	std::string html_content = "<li class=\"search_snippet\"";
/*	if ( websearch::_wconfig->_thumbs )	
		html_content += " onmouseover=\"snippet_focus(this, 'on');\" onmouseout=\"snippet_focus(this, 'off');\""; */
	html_content += ">";
	if ( websearch::_wconfig->_thumbs ){
		html_content += "<img class=\"preview\" src=\"http://open.thumbshots.org/image.pxf?url=";
		html_content += _url;
		html_content += "\" />";
	}
	html_content += "<h3><a href=\"";
	html_content += _url;
	html_content += "\">";
	
	const char *title_enc = encode::html_encode(_title.c_str());
	html_content += title_enc;
	free_const(title_enc);
	html_content += "</a>";
	
	if (_engine.to_ulong()&SE_GOOGLE)
	  {
	     std::string ggle_se_icon = se_icon;
	     miscutil::replace_in_string(ggle_se_icon,"icon","search_engine_google");
	     miscutil::replace_in_string(ggle_se_icon,"hspace=\"2\"","hspace=\"5\"");
	     html_content += ggle_se_icon;
	  }
	if (_engine.to_ulong()&SE_CUIL)
	  {
	     std::string cuil_se_icon = se_icon;
	     miscutil::replace_in_string(cuil_se_icon,"icon","search_engine_cuil");
	     html_content += cuil_se_icon;
	  }
	if (_engine.to_ulong()&SE_BING)
	  {
	     std::string bing_se_icon = se_icon;
	     miscutil::replace_in_string(bing_se_icon,"icon","search_engine_bing");
	     html_content += bing_se_icon;
	  }
	if (_engine.to_ulong()&SE_YAHOO)
	  {
	     std::string yahoo_se_icon = se_icon;
	     miscutil::replace_in_string(yahoo_se_icon,"icon","search_engine_yahoo");
	     html_content += yahoo_se_icon;
	  }
		
	html_content += "</h3>";
		
	if (_summary != "")
	  {
	     html_content += "<div>";
	     std::string summary = _summary;
	     search_snippet::highlight_query(words,summary);
	     html_content += summary;
	  }
	else html_content += "<div>";
	
	const char *cite_enc = NULL;
	if (_cite != "")
	  {
	     cite_enc = encode::html_encode(_cite.c_str());
	  }
	else 
	  {
	     cite_enc = encode::html_encode(_url.c_str());
	  }
	html_content += "<br><cite>";<commit_msg>added name of search engine over icon in results<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 **/
 
#include "search_snippet.h"
#include "websearch.h" // for configuration.
#include "mem_utils.h"
#include "miscutil.h"
#include "encode.h"
#include "loaders.h"
#include "urlmatch.h"
#include "plugin_manager.h" // for _plugin_repository.

#ifndef FEATURE_EXTENDED_HOST_PATTERNS
#include "proxy_dts.h" // for http_request.
#endif

#include "mrf.h"

#include <iostream>

using sp::miscutil;
using sp::encode;
using sp::loaders;
using sp::urlmatch;
using sp::plugin_manager;
using sp::http_request;
using lsh::mrf;

namespace seeks_plugins
{
   // loaded tagging patterns.
   std::vector<url_spec*> search_snippet::_pdf_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_file_doc_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_audio_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_video_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_forum_pos_patterns = std::vector<url_spec*>();
   
   search_snippet::search_snippet()
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(0),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL)
       {
       }
   
   search_snippet::search_snippet(const short &rank)
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(rank),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL)
       {
       }
      
   search_snippet::~search_snippet()
     {
	if (_cached_content)
	  free_const(_cached_content);
	if (_features)
	  delete _features;
	if (_features_tfidf)
	  delete _features_tfidf;
	if (_bag_of_words)
	  delete _bag_of_words;
     }

   void search_snippet::highlight_query(std::vector<std::string> &words,
					std::string &str)
     {
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// surround every of those words appearing within the
	// argument string with <b> </b> for html
	// bold format. TODO: green ?
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  std::string bold_str = "<b>" + words.at(i) + "</b>";
		  miscutil::ci_replace_in_string(str,words.at(i),bold_str);
	       }
	  }
     }
      
   std::ostream& search_snippet::print(std::ostream &output)
     {
	output << "-----------------------------------\n";
	output << "- seeks rank: " << _seeks_rank << std::endl;
	output << "- rank: " << _rank << std::endl;
	output << "- title: " << _title << std::endl;
	output << "- url: " << _url << std::endl;
	output << "- cite: " << _cite << std::endl;
	output << "- cached: " << _cached << std::endl;
	output << "- summary: " << _summary << std::endl;
	output << "- file format: " << _file_format << std::endl;
	output << "- date: " << _date << std::endl;
	output << "- lang: " << _lang << std::endl;
	if (_doc_type == FORUM)
	  output << "- forum thread info: " << _forum_thread_info << std::endl;
	output << "-----------------------------------\n";
	
	return output;
     }
     
   std::string search_snippet::to_html()
     {
	std::vector<std::string> words;
	return to_html_with_highlight(words);
     }

   std::string search_snippet::to_html_with_highlight(std::vector<std::string> &words)
     {
	static std::string se_icon = "<span class=\"search_engine icon\" title=\"setitle\">&nbsp;</span>";
	std::string html_content = "<li class=\"search_snippet\"";
/*	if ( websearch::_wconfig->_thumbs )	
		html_content += " onmouseover=\"snippet_focus(this, 'on');\" onmouseout=\"snippet_focus(this, 'off');\""; */
	html_content += ">";
	if ( websearch::_wconfig->_thumbs ){
		html_content += "<img class=\"preview\" src=\"http://open.thumbshots.org/image.pxf?url=";
		html_content += _url;
		html_content += "\" />";
	}
	html_content += "<h3><a href=\"";
	html_content += _url;
	html_content += "\">";
	
	const char *title_enc = encode::html_encode(_title.c_str());
	html_content += title_enc;
	free_const(title_enc);
	html_content += "</a>";
	
	if (_engine.to_ulong()&SE_GOOGLE)
	  {
	     std::string ggle_se_icon = se_icon;
	     miscutil::replace_in_string(ggle_se_icon,"icon","search_engine_google");
	     miscutil::replace_in_string(ggle_se_icon,"setitle","Google");
	     html_content += ggle_se_icon;
	  }
	if (_engine.to_ulong()&SE_CUIL)
	  {
	     std::string cuil_se_icon = se_icon;
	     miscutil::replace_in_string(cuil_se_icon,"icon","search_engine_cuil");
	     miscutil::replace_in_string(cuil_se_icon,"setitle","Cuil");
	     html_content += cuil_se_icon;
	  }
	if (_engine.to_ulong()&SE_BING)
	  {
	     std::string bing_se_icon = se_icon;
	     miscutil::replace_in_string(bing_se_icon,"icon","search_engine_bing");
	     miscutil::replace_in_string(bing_se_icon,"setitle","Bing");
	     html_content += bing_se_icon;
	  }
	if (_engine.to_ulong()&SE_YAHOO)
	  {
	     std::string yahoo_se_icon = se_icon;
	     miscutil::replace_in_string(yahoo_se_icon,"icon","search_engine_yahoo");
	     miscutil::replace_in_string(yahoo_se_icon,"setitle","Yahoo!");
	     html_content += yahoo_se_icon;
	  }
		
	html_content += "</h3>";
		
	if (_summary != "")
	  {
	     html_content += "<div>";
	     std::string summary = _summary;
	     search_snippet::highlight_query(words,summary);
	     html_content += summary;
	  }
	else html_content += "<div>";
	
	const char *cite_enc = NULL;
	if (_cite != "")
	  {
	     cite_enc = encode::html_encode(_cite.c_str());
	  }
	else 
	  {
	     cite_enc = encode::html_encode(_url.c_str());
	  }
	html_content += "<br><cite>";
{'fraction_non_alphanumeric': 0.16886930983847284, 'fraction_numerical': 0.004791714970244995, 'mean_word_length': 4.044834307992203, 'pattern_counts': {'":': 0, '<': 211, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3857023', 'n_tokens_mistral': 4677, 'n_tokens_neox': 4440, 'n_words': 889}