<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 - 2011 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "se_parser_bing.h"
#include "miscutil.h"

#include <strings.h>
#include <iostream>

using sp::miscutil;

namespace seeks_plugins
{
  std::string se_parser_bing::_bing_stupid[2] =
  { "Document title", "Titre du document / Document title" };

  se_parser_bing::se_parser_bing(const std::string &url)
    :se_parser(url),_h1_sr_flag(false),_results_flag(false),_h3_flag(false),
     _link_flag(false),_p_flag(false),_cite_flag(false),_cached_flag(false)
  {
  }

  se_parser_bing::~se_parser_bing()
  {
  }

  void se_parser_bing::start_element(parser_context *pc,
                                     const xmlChar *name,
                                     const xmlChar **attributes)
  {
    const char *tag = (const char*)name;

    if (strcasecmp(tag,"h1") == 0)
      {
        _h1_sr_flag = true;
      }
    if (strcasecmp(tag,"div") == 0)
      {
        const char *a_class = se_parser::get_attribute((const char**)attributes,"class");

        if (a_class && strcasecmp(a_class,"sb_tlst") == 0)
          {
            if (pc->_snippets->empty())
              _results_flag = true;

            // assert previous snippet if any.
            if (pc->_current_snippet)
              {
                if (pc->_current_snippet->_title.empty()  // consider the parsing did fail on the snippet.
                    || pc->_current_snippet->_url.empty()
                    || pc->_current_snippet->_summary.empty()
                    || pc->_current_snippet->_cite.empty())
                  {
                    delete pc->_current_snippet;
                    pc->_current_snippet = NULL;
                    _count--;
                    pc->_snippets->pop_back();
                  }
              }

            // create new snippet.
            search_snippet *sp = new search_snippet(_count+1);
            _count++;
            sp->_engine = feeds("bing",_url);
            pc->_current_snippet = sp;
            pc->_snippets->push_back(pc->_current_snippet);

            _cached_flag = false; // in case previous snippet did not close the cached flag.
          }
      }
    else if (_results_flag && strcasecmp(tag,"h2") == 0)
      {
        _results_flag = false;
      }
    else if (_results_flag && _h1_sr_flag && strcasecmp(tag,"h3") == 0)
      {
        _h3_flag = true;
      }
    else if (_results_flag && _h1_sr_flag && _h3_flag && strcasecmp(tag,"a") == 0)
      {
        _link_flag = true;
        const char *a_link = se_parser::get_attribute((const char**)attributes,"href");

        if (a_link)<commit_msg>Bing parser fix: was failing to report all results on some queries, due to some changes in the Bing UI as it seems<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2009 - 2011 <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "se_parser_bing.h"
#include "miscutil.h"

#include <strings.h>
#include <iostream>

using sp::miscutil;

namespace seeks_plugins
{
  std::string se_parser_bing::_bing_stupid[2] =
  { "Document title", "Titre du document / Document title" };

  se_parser_bing::se_parser_bing(const std::string &url)
    :se_parser(url),_h1_sr_flag(false),_results_flag(false),_h3_flag(false),
     _link_flag(false),_p_flag(false),_cite_flag(false),_cached_flag(false)
  {
  }

  se_parser_bing::~se_parser_bing()
  {
  }

  void se_parser_bing::start_element(parser_context *pc,
                                     const xmlChar *name,
                                     const xmlChar **attributes)
  {
    const char *tag = (const char*)name;

    if (strcasecmp(tag,"h1") == 0)
      {
        _h1_sr_flag = true;
      }
    if (strcasecmp(tag,"div") == 0)
      {
        const char *a_class = se_parser::get_attribute((const char**)attributes,"class");

        if (a_class && strcasecmp(a_class,"sb_tlst") == 0)
          {
            if (pc->_snippets->empty())
              _results_flag = true;

            // create new snippet.
            search_snippet *sp = new search_snippet(_count+1);
            _count++;
            sp->_engine = feeds("bing",_url);
            pc->_current_snippet = sp;

            _cached_flag = false; // in case previous snippet did not close the cached flag.
          }
        else if (_results_flag && a_class && (strcasecmp(a_class,"sa_cc")==0
                                              || strcasecmp(a_class,"sb_pag")==0))
          {
            // assert previous snippet if any.
            if (pc->_current_snippet)
              {
                if (pc->_current_snippet->_title.empty()  // consider the parsing did fail on the snippet.
                    || pc->_current_snippet->_url.empty()
                    || pc->_current_snippet->_summary.empty()
                    || pc->_current_snippet->_cite.empty())
                  {
                    delete pc->_current_snippet;
                    pc->_current_snippet = NULL;
                    _count--;
                  }
                else pc->_snippets->push_back(pc->_current_snippet);
              }
          }
      }
    /*else if (_results_flag && strcasecmp(tag,"h2") == 0)
      {
        _results_flag = false;
    }*/
    else if (_results_flag && _h1_sr_flag && strcasecmp(tag,"h3") == 0)
      {
        _h3_flag = true;
      }
    else if (_results_flag && _h1_sr_flag && _h3_flag && strcasecmp(tag,"a") == 0)
      {
        _link_flag = true;
        const char *a_link = se_parser::get_attribute((const char**)attributes,"href");

        if (a_link)
{'fraction_non_alphanumeric': 0.11268613974799542, 'fraction_numerical': 0.008018327605956471, 'mean_word_length': 2.196796338672769, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5095136', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2021, 'n_words': 557}