<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 **/

#include "se_parser_ggle.h"

#include "miscutil.h"

#include <strings.h>
#include <iostream>

using sp::miscutil;

namespace seeks_plugins
{
   std::string se_parser_ggle::_sr_string_en = "Search Results";
   std::string se_parser_ggle::_sr_string_fr = "Résultats de recherche";
   std::string se_parser_ggle::_cached_string = "Cached";
   
   se_parser_ggle::se_parser_ggle()
     :se_parser(),_body_flag(false),_h2_flag(false),_h2_sr_flag(false),
      _ol_flag(false),_li_flag(false),_h3_flag(false),
      _div_flag_summary(false),_div_flag_forum(false),_sum_flag(false),
      _link_flag(false),_cite_flag(false),
      _cached_flag(false),_span_cached_flag(false),
      _ff_flag(false),_new_link_flag(false),_spell_flag(false),_sgg_spell_flag(false),
      _end_sgg_spell_flag(false),_rt_flag(false)
       {
       }
   
   se_parser_ggle::~se_parser_ggle()
     {
     }
   
   void se_parser_ggle::start_element(parser_context *pc,
				      const xmlChar *name,
				      const xmlChar **attributes)
     {
	const char *tag = (const char*)name;
	
	if (strcasecmp(tag, "body") == 0)
	  {
	     _body_flag = true;
	  }
	// check for h3 flag -> snippet's title.
	else if (_h2_sr_flag && _li_flag && strcasecmp(tag,"h3") == 0)
	  {
	     _h3_flag = true;
	     _new_link_flag = true;
	  }
	// check for h2 flag -> search results 'title'.
	else if (_body_flag && !_h2_sr_flag && strcasecmp(tag,"h2") == 0)
	  {
	     _h2_flag = true;
	     const char *a_class = se_parser::get_attribute((const char**)attributes,"class");
	     if (a_class && strcasecmp(a_class,"hd") == 0)
	       _h2_sr_flag = true;
	  }
	// real time results avoidance.
	else if (_h2_sr_flag && strcasecmp(tag,"span") == 0)
	  {
	     const char *a_id = se_parser::get_attribute((const char**)attributes,"id");
	     if (a_id && strcasecmp(a_id,"rth") == 0)
	       {
		  _rt_flag = true;
		  pc->_current_snippet->_title = "";  // scheduled for removal.
		  _li_flag = false;
		  _h3_flag = false;
	       }
	  }
	else if (_h3_flag && strcasecmp(tag,"a") == 0)
	  {
	     const char *a_link = se_parser::get_attribute((const char**)attributes,"href");
	     
	     if (a_link)
	       {
		  if (_spell_flag)  // the spell section provides links as queries.
		    {
		       std::string a_link_str = std::string(a_link);
		       miscutil::replace_in_string(a_link_str,"/url?q=",""); // remove query form
		       size_t pos = a_link_str.find("&");
		       a_link_str = a_link_str.substr(0,pos);
		       pc->_current_snippet->set_url(a_link_str);
		    }
		  else pc->_current_snippet->set_url(a_link);
	       }
	  }
	else if (_h2_sr_flag && strcasecmp(tag,"ol") == 0)
	  {
	     _ol_flag = true;
	  }
	else if (_h2_sr_flag && strcasecmp(tag,"li") == 0)
	  {
	     const char *a_class = se_parser::get_attribute((const char**)attributes,"class");
	     
	     if (!a_class || strcasecmp(a_class,"g") != 0)<commit_msg>url parsing fix<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 <NAME>, <EMAIL>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 **/

#include "se_parser_ggle.h"

#include "miscutil.h"

#include <strings.h>
#include <iostream>

using sp::miscutil;

namespace seeks_plugins
{
   std::string se_parser_ggle::_sr_string_en = "Search Results";
   std::string se_parser_ggle::_sr_string_fr = "Résultats de recherche";
   std::string se_parser_ggle::_cached_string = "Cached";
   
   se_parser_ggle::se_parser_ggle()
     :se_parser(),_body_flag(false),_h2_flag(false),_h2_sr_flag(false),
      _ol_flag(false),_li_flag(false),_h3_flag(false),
      _div_flag_summary(false),_div_flag_forum(false),_sum_flag(false),
      _link_flag(false),_cite_flag(false),
      _cached_flag(false),_span_cached_flag(false),
      _ff_flag(false),_new_link_flag(false),_spell_flag(false),_sgg_spell_flag(false),
      _end_sgg_spell_flag(false),_rt_flag(false)
       {
       }
   
   se_parser_ggle::~se_parser_ggle()
     {
     }
   
   void se_parser_ggle::start_element(parser_context *pc,
				      const xmlChar *name,
				      const xmlChar **attributes)
     {
	const char *tag = (const char*)name;
	
	if (strcasecmp(tag, "body") == 0)
	  {
	     _body_flag = true;
	  }
	// check for h3 flag -> snippet's title.
	else if (_h2_sr_flag && _li_flag && strcasecmp(tag,"h3") == 0)
	  {
	     _h3_flag = true;
	     _new_link_flag = true;
	  }
	// check for h2 flag -> search results 'title'.
	else if (_body_flag && !_h2_sr_flag && strcasecmp(tag,"h2") == 0)
	  {
	     _h2_flag = true;
	     const char *a_class = se_parser::get_attribute((const char**)attributes,"class");
	     if (a_class && strcasecmp(a_class,"hd") == 0)
	       _h2_sr_flag = true;
	  }
	// real time results avoidance.
	else if (_h2_sr_flag && strcasecmp(tag,"span") == 0)
	  {
	     const char *a_id = se_parser::get_attribute((const char**)attributes,"id");
	     if (a_id && strcasecmp(a_id,"rth") == 0)
	       {
		  _rt_flag = true;
		  pc->_current_snippet->_title = "";  // scheduled for removal.
		  _li_flag = false;
		  _h3_flag = false;
	       }
	  }
	else if (_h3_flag && strcasecmp(tag,"a") == 0)
	  {
	     const char *a_link = se_parser::get_attribute((const char**)attributes,"href");
	     
	     if (a_link)
	       {
		  /* if (_spell_flag)  // the spell section provides links as queries.
		    { */
		  std::string a_link_str = std::string(a_link);
		  size_t p = miscutil::replace_in_string(a_link_str,"/url?q=",""); // remove query form
		  if (p != 0)
		    {
		       size_t pos = a_link_str.find("&");
		       a_link_str = a_link_str.substr(0,pos);
		    }
		  pc->_current_snippet->set_url(a_link_str);
	       //}
	     //	  else pc->_current_snippet->set_url(a_link);
	       }
	  }
	else if (_h2_sr_flag && strcasecmp(tag,"ol") == 0)
	  {
	     _ol_flag = true;
	  }
	else if (_h2_sr_flag && strcasecmp(tag,"li") == 0)
	  {
	     const char *a_class = se_parser::get_attribute((const char**)attributes,"class");
	     
	     if (!a_class || strcasecmp(a_class,"g") != 0)
{'fraction_non_alphanumeric': 0.12840774397471355, 'fraction_numerical': 0.011984722770973264, 'mean_word_length': 3.354357798165138, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5818525', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2600, 'n_words': 634}