<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009, 2010 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "search_snippet.h"
#include "websearch.h" // for configuration.
#include "mem_utils.h"
#include "miscutil.h"
#include "encode.h"
#include "loaders.h"
#include "urlmatch.h"
#include "plugin_manager.h" // for _plugin_repository.
#include "seeks_proxy.h" // for _datadir.

#ifndef FEATURE_EXTENDED_HOST_PATTERNS
#include "proxy_dts.h" // for http_request.
#endif

#include "mrf.h"

#include <iostream>

using sp::miscutil;
using sp::encode;
using sp::loaders;
using sp::urlmatch;
using sp::plugin_manager;
using sp::seeks_proxy;
using sp::http_request;
using lsh::mrf;

namespace seeks_plugins
{
   // loaded tagging patterns.
   std::vector<url_spec*> search_snippet::_pdf_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_file_doc_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_audio_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_video_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_forum_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_reject_pos_patterns = std::vector<url_spec*>();
   
   search_snippet::search_snippet()
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(0),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL),_safe(true)
       {
       }
   
   search_snippet::search_snippet(const short &rank)
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(rank),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL),_safe(true)
       {
       }
      
   search_snippet::~search_snippet()
     {
	if (_cached_content)
	  free_const(_cached_content);
	if (_features)
	  delete _features;
	if (_features_tfidf)
	  delete _features_tfidf;
	if (_bag_of_words)
	  delete _bag_of_words;
     }

   void search_snippet::highlight_query(std::vector<std::string> &words,
					std::string &str)
     {
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// surround every of those words appearing within the
	// argument string with <b> </b> for html
	// bold format. TODO: green ?
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  std::string bold_str = "<b>" + words.at(i) + "</b>";
		  miscutil::ci_replace_in_string(str,words.at(i),bold_str);
	       }
	  }
     }
      
   void search_snippet::highlight_discr(std::string &str, const std::string &base_url_str,
					const std::vector<std::string> &query_words)
     {
	static int max_highlights = 3; // ad-hoc default.
	
	if (!_features_tfidf)
	  return;
	
	std::vector<std::string> words;
	words.reserve(max_highlights);
	std::map<float,uint32_t,std::greater<float> > f_tfidf;
	
	// sort features in decreasing tf-idf order.
	hash_map<uint32_t,float,id_hash_uint>::const_iterator fit
	  = _features_tfidf->begin();
	while(fit!=_features_tfidf->end())
	  {
	     f_tfidf.insert(std::pair<float,uint32_t>((*fit).second,(*fit).first));
	     ++fit;
	  }
	
	size_t nqw = query_words.size();
	int i = 0;
	std::map<float,uint32_t,std::greater<float> >::const_iterator mit = f_tfidf.begin();
	while(mit!=f_tfidf.end())
	  {
	     hash_map<uint32_t,std::string,id_hash_uint>::const_iterator bit;
	     if ((bit=_bag_of_words->find((*mit).second))!=_bag_of_words->end())
	       {
		  bool add = true;
		  for (size_t j=0;j<nqw;j++)
		    if (query_words.at(j) == (*bit).second)
		      add = false;
		  if (add)
		    words.push_back((*bit).second);
		  i++;
		  if (i>=max_highlights)
		    break;
	       }
	     ++mit;
	  }
	
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// highlighting.
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  char *wenc = encode::url_encode(words.at(i).c_str());
		  std::string rword = " " + words.at(i) + " ";
		  std::string bold_str = "<span class=\"highlight\"><a href=\"" + base_url_str + "/search?q=" + _qc->_url_enc_query + "+" + std::string(wenc) + "&page=1&expansion=1&action=expand\">" + rword + "</a></span>";
		  free(wenc);
		  miscutil::ci_replace_in_string(str,rword,bold_str);
	       }
	  }
     }
      
   std::ostream& search_snippet::print(std::ostream &output)
     {
	output << "-----------------------------------\n";
	output << "- seeks rank: " << _seeks_rank << std::endl;
	output << "- rank: " << _rank << std::endl;
	output << "- title: " << _title << std::endl;
	output << "- url: " << _url << std::endl;
	output << "- cite: " << _cite << std::endl;
	output << "- cached: " << _cached << std::endl;
	output << "- summary: " << _summary << std::endl;
	output << "- file format: " << _file_format << std::endl;
	output << "- date: " << _date << std::endl;
	output << "- lang: " << _lang << std::endl;
	if (_doc_type == FORUM)
	  output << "- forum thread info: " << _forum_thread_info << std::endl;
	output << "-----------------------------------\n";
	
	return output;
     }
     
   std::string search_snippet::to_json(const bool &thumbs)
     {
	std::string json_str;
	json_str += "{";
	json_str += "\"id\":" + miscutil::to_string(_id) + ",";
	std::string title = _title;
	miscutil::replace_in_string(title,"\"","\\\"");
	json_str += "\"title\":\"" + title + "\",";
	std::string url = _url;
	miscutil::replace_in_string(url,"\"","\\\"");
	json_str += "\"url\":\"" + url + "\",";
	std::string summary = _summary_noenc;
	miscutil::replace_in_string(summary,"\"","\\\"");
	json_str += "\"summary\":\"" + summary + "\",";
	json_str += "\"seeks_score\":" + miscutil::to_string(_seeks_rank) + ",";
	double rank = _rank / static_cast<double>(_engine.count());
	json_str += "\"rank\":" + miscutil::to_string(rank) + ",";
	json_str += "\"cite\":\"";
	if (!_cite.empty())
	  {
	     std::string cite = _cite;
	     miscutil::replace_in_string(cite,"\"","\\\"");
	     json_str += cite + "\",";
	  }
	else json_str += url + "\",";
	if (!_cached.empty())
	  {
	     std::string cached = _cached;
	     miscutil::replace_in_string(cached,"\"","\\\"");
	     json_str += "\"cached\":\"" + cached + "\","; // XXX: cached might be malformed without preprocessing.<commit_msg>fix to escape characters in served content that seem to be misunderstood by some JSON parsers<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009, 2010 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "search_snippet.h"
#include "websearch.h" // for configuration.
#include "mem_utils.h"
#include "miscutil.h"
#include "encode.h"
#include "loaders.h"
#include "urlmatch.h"
#include "plugin_manager.h" // for _plugin_repository.
#include "seeks_proxy.h" // for _datadir.

#ifndef FEATURE_EXTENDED_HOST_PATTERNS
#include "proxy_dts.h" // for http_request.
#endif

#include "mrf.h"

#include <iostream>

using sp::miscutil;
using sp::encode;
using sp::loaders;
using sp::urlmatch;
using sp::plugin_manager;
using sp::seeks_proxy;
using sp::http_request;
using lsh::mrf;

namespace seeks_plugins
{
   // loaded tagging patterns.
   std::vector<url_spec*> search_snippet::_pdf_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_file_doc_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_audio_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_video_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_forum_pos_patterns = std::vector<url_spec*>();
   std::vector<url_spec*> search_snippet::_reject_pos_patterns = std::vector<url_spec*>();
   
   search_snippet::search_snippet()
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(0),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL),_safe(true)
       {
       }
   
   search_snippet::search_snippet(const short &rank)
     :_qc(NULL),_new(true),_id(0),_sim_back(false),_rank(rank),_seeks_ir(0.0),_seeks_rank(0),_doc_type(WEBPAGE),
      _cached_content(NULL),_features(NULL),_features_tfidf(NULL),_bag_of_words(NULL),_safe(true)
       {
       }
      
   search_snippet::~search_snippet()
     {
	if (_cached_content)
	  free_const(_cached_content);
	if (_features)
	  delete _features;
	if (_features_tfidf)
	  delete _features_tfidf;
	if (_bag_of_words)
	  delete _bag_of_words;
     }

   void search_snippet::highlight_query(std::vector<std::string> &words,
					std::string &str)
     {
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// surround every of those words appearing within the
	// argument string with <b> </b> for html
	// bold format. TODO: green ?
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  std::string bold_str = "<b>" + words.at(i) + "</b>";
		  miscutil::ci_replace_in_string(str,words.at(i),bold_str);
	       }
	  }
     }
      
   void search_snippet::highlight_discr(std::string &str, const std::string &base_url_str,
					const std::vector<std::string> &query_words)
     {
	static int max_highlights = 3; // ad-hoc default.
	
	if (!_features_tfidf)
	  return;
	
	std::vector<std::string> words;
	words.reserve(max_highlights);
	std::map<float,uint32_t,std::greater<float> > f_tfidf;
	
	// sort features in decreasing tf-idf order.
	hash_map<uint32_t,float,id_hash_uint>::const_iterator fit
	  = _features_tfidf->begin();
	while(fit!=_features_tfidf->end())
	  {
	     f_tfidf.insert(std::pair<float,uint32_t>((*fit).second,(*fit).first));
	     ++fit;
	  }
	
	size_t nqw = query_words.size();
	int i = 0;
	std::map<float,uint32_t,std::greater<float> >::const_iterator mit = f_tfidf.begin();
	while(mit!=f_tfidf.end())
	  {
	     hash_map<uint32_t,std::string,id_hash_uint>::const_iterator bit;
	     if ((bit=_bag_of_words->find((*mit).second))!=_bag_of_words->end())
	       {
		  bool add = true;
		  for (size_t j=0;j<nqw;j++)
		    if (query_words.at(j) == (*bit).second)
		      add = false;
		  if (add)
		    words.push_back((*bit).second);
		  i++;
		  if (i>=max_highlights)
		    break;
	       }
	     ++mit;
	  }
	
	if (words.empty())
	  return;
	
	// sort words by size.
	std::sort(words.begin(),words.end(),std::greater<std::string>());
	
	// highlighting.
	for (size_t i=0;i<words.size();i++)
	  {
	     if (words.at(i).length() > 2)
	       {
		  char *wenc = encode::url_encode(words.at(i).c_str());
		  std::string rword = " " + words.at(i) + " ";
		  std::string bold_str = "<span class=\"highlight\"><a href=\"" + base_url_str + "/search?q=" + _qc->_url_enc_query + "+" + std::string(wenc) + "&page=1&expansion=1&action=expand\">" + rword + "</a></span>";
		  free(wenc);
		  miscutil::ci_replace_in_string(str,rword,bold_str);
	       }
	  }
     }
      
   std::ostream& search_snippet::print(std::ostream &output)
     {
	output << "-----------------------------------\n";
	output << "- seeks rank: " << _seeks_rank << std::endl;
	output << "- rank: " << _rank << std::endl;
	output << "- title: " << _title << std::endl;
	output << "- url: " << _url << std::endl;
	output << "- cite: " << _cite << std::endl;
	output << "- cached: " << _cached << std::endl;
	output << "- summary: " << _summary << std::endl;
	output << "- file format: " << _file_format << std::endl;
	output << "- date: " << _date << std::endl;
	output << "- lang: " << _lang << std::endl;
	if (_doc_type == FORUM)
	  output << "- forum thread info: " << _forum_thread_info << std::endl;
	output << "-----------------------------------\n";
	
	return output;
     }
     
   std::string search_snippet::to_json(const bool &thumbs)
     {
	std::string json_str;
	json_str += "{";
	json_str += "\"id\":" + miscutil::to_string(_id) + ",";
	std::string title = _title;
	miscutil::replace_in_string(title,"\\","");
	miscutil::replace_in_string(title,"\"","\\\"");
	json_str += "\"title\":\"" + title + "\",";
	std::string url = _url;
	miscutil::replace_in_string(url,"\"","\\\"");
	json_str += "\"url\":\"" + url + "\",";
	std::string summary = _summary_noenc;
	miscutil::replace_in_string(title,"\\","");
	miscutil::replace_in_string(summary,"\"","\\\"");
	json_str += "\"summary\":\"" + summary + "\",";
	json_str += "\"seeks_score\":" + miscutil::to_string(_seeks_rank) + ",";
	double rank = _rank / static_cast<double>(_engine.count());
	json_str += "\"rank\":" + miscutil::to_string(rank) + ",";
	json_str += "\"cite\":\"";
	if (!_cite.empty())
	  {
	     std::string cite = _cite;
	     miscutil::replace_in_string(cite,"\"","\\\"");
	     json_str += cite + "\",";
	  }
	else json_str += url + "\",";
	if (!_cached.empty())
	  {
	     std::string cached = _cached;
	     miscutil::replace_in_string(cached,"\"","\\\"");
	     json_str += "\"cached\":\"" + cached + "\","; // XXX: cached might be malformed without preprocessing.
{'fraction_non_alphanumeric': 0.1918109370706238, 'fraction_numerical': 0.005083814234679857, 'mean_word_length': 3.8362126245847175, 'pattern_counts': {'":': 16, '<': 223, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1676898', 'n_tokens_mistral': 5452, 'n_tokens_neox': 5153, 'n_words': 1010}