<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "uri_capture.h"
#include "uc_configuration.h"
#include "db_uri_record.h"
#include "seeks_proxy.h" // for user_db.
#include "proxy_configuration.h"
#include "user_db.h"
#include "proxy_dts.h"
#include "urlmatch.h"
#include "miscutil.h"
#include "charset_conv.h"
#include "curl_mget.h"
#include "errlog.h"

#include <sys/time.h>
#include <sys/stat.h>

#include <algorithm>
#include <iterator>
#include <iostream>

using sp::seeks_proxy;
using sp::user_db;
using sp::db_record;
using sp::urlmatch;
using sp::miscutil;
using sp::charset_conv;
using sp::errlog;

namespace seeks_plugins
{

  /*- uri_db_sweepable -*/
  uri_db_sweepable::uri_db_sweepable()
    :user_db_sweepable()
  {
    // set last sweep to now.
    // sweeping is done when plugin starts.
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    _last_sweep = tv_now.tv_sec;
  }

  uri_db_sweepable::~uri_db_sweepable()
  {
  }

  bool uri_db_sweepable::sweep_me()
  {
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    if ((tv_now.tv_sec - _last_sweep)
        > uc_configuration::_config->_sweep_cycle)
      return true;
    else return false;
  }

  int uri_db_sweepable::sweep_records()
  {
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    time_t sweep_date = tv_now.tv_sec - uc_configuration::_config->_retention;
    return seeks_proxy::_user_db->prune_db("uri-capture",sweep_date);
  }

  /*- uri_capture -*/
  uri_capture::uri_capture()
    : plugin(),_nr(0)
  {
    _name = "uri-capture";
    _version_major = "0";
    _version_minor = "1";
    _configuration = NULL;
    _interceptor_plugin = new uri_capture_element(this);

    if (seeks_proxy::_datadir.empty())
      _config_filename = plugin_manager::_plugin_repository + "uri_capture/uri-capture-config";
    else
      _config_filename = seeks_proxy::_datadir + "/plugins/uri_capture/uri-capture-config";

#ifdef SEEKS_CONFIGDIR
    struct stat stFileInfo;
    if (!stat(_config_filename.c_str(), &stFileInfo)  == 0)
      {
        _config_filename = SEEKS_CONFIGDIR "/uri-capture-config";
      }
#endif

    if (uc_configuration::_config == NULL)
      uc_configuration::_config = new uc_configuration(_config_filename);
    _configuration = uc_configuration::_config;
  }

  uri_capture::~uri_capture()
  {
    uc_configuration::_config = NULL; // configuration is deleted in parent class.
  }

  void uri_capture::start()
  {
    // check for user db.
    if (!seeks_proxy::_user_db || !seeks_proxy::_user_db->_opened)
      {
        errlog::log_error(LOG_LEVEL_ERROR,"user db is not opened for URI capture plugin to work with it");
        return;
      }
    else if (seeks_proxy::_config->_user_db_startup_check)
      {
        // preventive sweep of records.
        static_cast<uri_capture_element*>(_interceptor_plugin)->_uds.sweep_records();
      }

    // get number of captured URI already in user_db.
    _nr = seeks_proxy::_user_db->number_records(_name);

    errlog::log_error(LOG_LEVEL_INFO,"uri_capture plugin: %u records",_nr);
  }

  void uri_capture::stop()
  {
  }

  sp::db_record* uri_capture::create_db_record()
  {
    return new db_uri_record();
  }

  int uri_capture::remove_all_uri_records()
  {
    return seeks_proxy::_user_db->prune_db(_name);
  }

  uc_err uri_capture::fetch_uri_html_title(const std::vector<std::string> &uris,
      std::vector<std::string> &titles,
      const long &timeout,
      const std::vector<std::list<const char*>*> *headers)
  {
    curl_mget cmg(uris.size(),timeout,0,timeout,0);
    cmg.www_mget(uris,uris.size(),headers,"",0);

    return uri_capture::parse_uri_html_title(uris,titles,cmg._outputs);
  }

  uc_err uri_capture::parse_uri_html_title(const std::vector<std::string> &uris,
      std::vector<std::string> &titles,
      std::string **outputs)
  {
    static std::string pattern_b = "<title>";
    static std::string pattern_e = "</title>";
    static std::list<const char*> cheaders; // empty, for charset conv.

    size_t err = 0;
    for (size_t i=0; i<uris.size(); i++)
      {
        std::string title;
        if (outputs[i])
          {
            std::string *pageptr = outputs[i];
            size_t pos_b = 0;
            std::string::const_iterator sit = pageptr->begin();
            if ((pos_b = miscutil::ci_find(*pageptr,pattern_b,sit))!=std::string::npos)
              {
                size_t pos_e = 0;
                if ((pos_e = miscutil::ci_find(*pageptr,pattern_e,sit))!=std::string::npos)
                  {
                    title = pageptr->substr(pos_b+pattern_b.size(),pos_e-pos_b-pattern_e.size()+1);
                  }
              }
          }
        if (title.empty())
          {
            titles.push_back("");
            errlog::log_error(LOG_LEVEL_ERROR,"Failed fetching or parsing title of uri %s",uris.at(i).c_str());
            err++;
          }
        else if (title.find("404")!=std::string::npos
                 ||title.find("400")!=std::string::npos)
          {
            titles.push_back("");
          }
        else
          {
            title = miscutil::chomp_cpp(title);
            miscutil::replace_in_string(title,"\n","");
            miscutil::replace_in_string(title,"\r","");
            std::string titlec = charset_conv::charset_check_and_conversion(title,cheaders);
            if (titlec.empty())
              {
                errlog::log_error(LOG_LEVEL_ERROR,"bad charset encoding for title %s or uri %s",
                                  title.c_str(),uris.at(i).c_str());
                titles.push_back("");
              }<commit_msg>fixed memory leak in db title fetcher. refs #417<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 <NAME>, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "uri_capture.h"
#include "uc_configuration.h"
#include "db_uri_record.h"
#include "seeks_proxy.h" // for user_db.
#include "proxy_configuration.h"
#include "user_db.h"
#include "proxy_dts.h"
#include "urlmatch.h"
#include "miscutil.h"
#include "charset_conv.h"
#include "curl_mget.h"
#include "errlog.h"

#include <sys/time.h>
#include <sys/stat.h>

#include <algorithm>
#include <iterator>
#include <iostream>

using sp::seeks_proxy;
using sp::user_db;
using sp::db_record;
using sp::urlmatch;
using sp::miscutil;
using sp::charset_conv;
using sp::errlog;

namespace seeks_plugins
{

  /*- uri_db_sweepable -*/
  uri_db_sweepable::uri_db_sweepable()
    :user_db_sweepable()
  {
    // set last sweep to now.
    // sweeping is done when plugin starts.
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    _last_sweep = tv_now.tv_sec;
  }

  uri_db_sweepable::~uri_db_sweepable()
  {
  }

  bool uri_db_sweepable::sweep_me()
  {
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    if ((tv_now.tv_sec - _last_sweep)
        > uc_configuration::_config->_sweep_cycle)
      return true;
    else return false;
  }

  int uri_db_sweepable::sweep_records()
  {
    struct timeval tv_now;
    gettimeofday(&tv_now,NULL);
    time_t sweep_date = tv_now.tv_sec - uc_configuration::_config->_retention;
    return seeks_proxy::_user_db->prune_db("uri-capture",sweep_date);
  }

  /*- uri_capture -*/
  uri_capture::uri_capture()
    : plugin(),_nr(0)
  {
    _name = "uri-capture";
    _version_major = "0";
    _version_minor = "1";
    _configuration = NULL;
    _interceptor_plugin = new uri_capture_element(this);

    if (seeks_proxy::_datadir.empty())
      _config_filename = plugin_manager::_plugin_repository + "uri_capture/uri-capture-config";
    else
      _config_filename = seeks_proxy::_datadir + "/plugins/uri_capture/uri-capture-config";

#ifdef SEEKS_CONFIGDIR
    struct stat stFileInfo;
    if (!stat(_config_filename.c_str(), &stFileInfo)  == 0)
      {
        _config_filename = SEEKS_CONFIGDIR "/uri-capture-config";
      }
#endif

    if (uc_configuration::_config == NULL)
      uc_configuration::_config = new uc_configuration(_config_filename);
    _configuration = uc_configuration::_config;
  }

  uri_capture::~uri_capture()
  {
    uc_configuration::_config = NULL; // configuration is deleted in parent class.
  }

  void uri_capture::start()
  {
    // check for user db.
    if (!seeks_proxy::_user_db || !seeks_proxy::_user_db->_opened)
      {
        errlog::log_error(LOG_LEVEL_ERROR,"user db is not opened for URI capture plugin to work with it");
        return;
      }
    else if (seeks_proxy::_config->_user_db_startup_check)
      {
        // preventive sweep of records.
        static_cast<uri_capture_element*>(_interceptor_plugin)->_uds.sweep_records();
      }

    // get number of captured URI already in user_db.
    _nr = seeks_proxy::_user_db->number_records(_name);

    errlog::log_error(LOG_LEVEL_INFO,"uri_capture plugin: %u records",_nr);
  }

  void uri_capture::stop()
  {
  }

  sp::db_record* uri_capture::create_db_record()
  {
    return new db_uri_record();
  }

  int uri_capture::remove_all_uri_records()
  {
    return seeks_proxy::_user_db->prune_db(_name);
  }

  uc_err uri_capture::fetch_uri_html_title(const std::vector<std::string> &uris,
      std::vector<std::string> &titles,
      const long &timeout,
      const std::vector<std::list<const char*>*> *headers)
  {
    curl_mget cmg(uris.size(),timeout,0,timeout,0);
    cmg.www_mget(uris,uris.size(),headers,"",0);

    uc_err err = uri_capture::parse_uri_html_title(uris,titles,cmg._outputs);
    delete[] cmg._outputs;
    return err;
  }

  uc_err uri_capture::parse_uri_html_title(const std::vector<std::string> &uris,
      std::vector<std::string> &titles,
      std::string **outputs)
  {
    static std::string pattern_b = "<title>";
    static std::string pattern_e = "</title>";
    static std::list<const char*> cheaders; // empty, for charset conv.

    size_t err = 0;
    for (size_t i=0; i<uris.size(); i++)
      {
        std::string title;
        if (outputs[i])
          {
            std::string *pageptr = outputs[i];
            size_t pos_b = 0;
            std::string::const_iterator sit = pageptr->begin();
            if ((pos_b = miscutil::ci_find(*pageptr,pattern_b,sit))!=std::string::npos)
              {
                size_t pos_e = 0;
                if ((pos_e = miscutil::ci_find(*pageptr,pattern_e,sit))!=std::string::npos)
                  {
                    title = pageptr->substr(pos_b+pattern_b.size(),pos_e-pos_b-pattern_e.size()+1);
                  }
              }
            delete outputs[i];
          }
        if (title.empty())
          {
            titles.push_back("");
            errlog::log_error(LOG_LEVEL_ERROR,"Failed fetching or parsing title of uri %s",uris.at(i).c_str());
            err++;
          }
        else if (title.find("404")!=std::string::npos
                 ||title.find("400")!=std::string::npos)
          {
            titles.push_back("");
          }
        else
          {
            title = miscutil::chomp_cpp(title);
            miscutil::replace_in_string(title,"\n","");
            miscutil::replace_in_string(title,"\r","");
            std::string titlec = charset_conv::charset_check_and_conversion(title,cheaders);
            if (titlec.empty())
              {
                errlog::log_error(LOG_LEVEL_ERROR,"bad charset encoding for title %s or uri %s",
                                  title.c_str(),uris.at(i).c_str());
                titles.push_back("");
              }
{'fraction_non_alphanumeric': 0.12894471582538575, 'fraction_numerical': 0.004419632472668062, 'mean_word_length': 3.00186161960906, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4426972', 'n_tokens_mistral': 4361, 'n_tokens_neox': 4218, 'n_words': 897}