<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 <NAME>, <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "httpserv.h"
#include "httpserv_configuration.h"
#include "seeks_proxy.h"
#include "plugin_manager.h"
#include "websearch.h"
#include "websearch_api_compat.h"
#include "cgisimple.h"
#include "sweeper.h"
#include "miscutil.h"
#include "encode.h"
#include "errlog.h"
#include "cgi.h"

#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
#include "img_websearch.h"
#endif

#if defined(PROTOBUF) && defined(TC)
#include "query_capture.h"
#include "cf.h"
#include "udb_service.h"
#endif

#include <unistd.h>
#include <sys/stat.h>
#include <sys/times.h>
#include <iostream>

using namespace sp;

namespace seeks_plugins
{

  httpserv::httpserv(const std::string &address,
                     const u_short &port)
    : plugin(),_address(address),_port(port)
  {
    /* initializing and starting server. */
    _evbase = event_base_new();
    _srv = evhttp_new(_evbase);
    evhttp_bind_socket(_srv, _address.c_str(), _port); // server binding to the address & port.

    errlog::log_error(LOG_LEVEL_INFO,"Seeks HTTP server plugin listening on %s:%u",
                      _address.c_str(),_port);

    init_callbacks();
    event_base_dispatch(_evbase); // XXX: for debug.
  }

  httpserv::httpserv()
    : plugin()
  {
    _name = "httpserv";
    _version_major = "0";
    _version_minor = "1";

    if (seeks_proxy::_datadir.empty())
      _config_filename = plugin_manager::_plugin_repository + "httpserv/httpserv-config";
    else
      _config_filename = seeks_proxy::_datadir + "/plugins/httpserv/httpserv-config";

#ifdef SEEKS_CONFIGDIR
    struct stat stFileInfo;
    if (!stat(_config_filename.c_str(), &stFileInfo)  == 0)
      {
        _config_filename = SEEKS_CONFIGDIR "/httpserv-config";
      }
#endif

    if (httpserv_configuration::_hconfig == NULL)
      httpserv_configuration::_hconfig = new httpserv_configuration(_config_filename);
    _configuration = httpserv_configuration::_hconfig;

    /* sets address & port. */
    _address = httpserv_configuration::_hconfig->_host;
    _port = httpserv_configuration::_hconfig->_port;
  }

  httpserv::~httpserv()
  {
  }

  void httpserv::start()
  {
    /* initializing and starting server. */
    _evbase = event_base_new();
    _srv = evhttp_new(_evbase);
    evhttp_bind_socket(_srv, _address.c_str(), _port); // server binding to the address & port.

    errlog::log_error(LOG_LEVEL_INFO,"Seeks HTTP server plugin listening on %s:%u",
                      _address.c_str(),_port);

    init_callbacks();

    int err = pthread_create(&_server_thread,NULL,
                             (void*(*)(void*))&event_base_dispatch,_evbase);
    seeks_proxy::_httpserv_thread = &_server_thread;
    if (seeks_proxy::_run_proxy)
      pthread_detach(_server_thread); // detach if proxy is running, else no (join).
  }

  void httpserv::stop()
  {
    event_base_loopbreak(_evbase);
    evhttp_free(_srv);
    event_base_free(_evbase);
  }

  void httpserv::init_callbacks()
  {
    evhttp_set_cb(_srv,"/search/txt",&httpserv::websearch,NULL);
    evhttp_set_cb(_srv,"/search",&httpserv::websearch,NULL); // compatibility API.
#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
    evhttp_set_cb(_srv,"/search_img",&httpserv::img_websearch,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/seeks_img_search.css",&httpserv::seeks_img_search_css,NULL);
#endif
    evhttp_set_cb(_srv,"/",&httpserv::websearch_hp,NULL);
    evhttp_set_cb(_srv,"/websearch-hp",&httpserv::websearch_hp,NULL);
    evhttp_set_cb(_srv,"/seeks_hp_search.css",&httpserv::seeks_hp_css,NULL);
    evhttp_set_cb(_srv,"/seeks_search.css",&httpserv::seeks_search_css,NULL);
    evhttp_set_cb(_srv,"/opensearch.xml",&httpserv::opensearch_xml,NULL);
    evhttp_set_cb(_srv,"/info",&httpserv::node_info,NULL);
#if defined(PROTOBUF) && defined(TC)
    evhttp_set_cb(_srv,"/qc_redir",&httpserv::qc_redir,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/tbd",&httpserv::tbd,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/find_dbr",&httpserv::find_dbr,NULL);
    evhttp_set_cb(_srv,"/find_bqc",&httpserv::find_bqc,NULL);
    evhttp_set_cb(_srv,"/peers",&httpserv::peers,NULL);
#endif

    //evhttp_set_gencb(_srv,&httpserv::unknown_path,NULL); /* 404: catches all other resources. */
    evhttp_set_gencb(_srv,&httpserv::api_route,NULL);
  }

  void httpserv::reply_with_redirect_302(struct evhttp_request *r,
                                         const char *url)
  {
    evhttp_add_header(r->output_headers,"Location",url);
    evhttp_send_reply(r, HTTP_MOVETEMP, "OK", NULL);
  }

  void httpserv::reply_with_error_400(struct evhttp_request *r)
  {
    evhttp_send_reply(r, HTTP_BADREQUEST, "BAD REQUEST", NULL);
  }

  void httpserv::reply_with_error(struct evhttp_request *r,
                                  const int &http_code,
                                  const char *message,
                                  const std::string &error_message)
  {
    /* error output. */
    errlog::log_error(LOG_LEVEL_ERROR,"httpserv error: code %d %s",
                      http_code,error_message.c_str());

    // body.
    struct evbuffer *buffer;
    buffer = evbuffer_new();

    char msg[error_message.length()];
    for (size_t i=0; i<error_message.length(); i++)
      msg[i] = error_message[i];
    evbuffer_add(buffer,msg,sizeof(msg));

    evhttp_send_reply(r, http_code, message, buffer);
    evbuffer_free(buffer);
  }

  void httpserv::reply_with_empty_body(struct evhttp_request *r,
                                       const int &http_code,
                                       const char *message)
  {
    evhttp_send_reply(r, http_code, message, NULL);
  }

  void httpserv::reply_with_body(struct evhttp_request *r,
                                 const int &http_code,
                                 const char *message,
                                 const std::string &content,
                                 const std::string &content_type)
  {
    /* headers. */
    evhttp_add_header(r->output_headers,"Content-Type",content_type.c_str());

    /* body. */
    struct evbuffer *buffer;
    buffer = evbuffer_new();

    char msg[content.length()];
    for (size_t i=0; i<content.length(); i++)
      msg[i] = content[i];
    evbuffer_add(buffer,msg,sizeof(msg));

    evhttp_send_reply(r, http_code, message, buffer);
    evbuffer_free(buffer);
  }

  void httpserv::websearch_hp(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_hp(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content);

    /* run the sweeper, for timed out query contexts. */
    sweeper::sweep();
  }

  void httpserv::seeks_hp_css(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_search_hp_css(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"text/css");
  }

  void httpserv::seeks_search_css(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;
    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_search_css(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"text/css");
  }

  void httpserv::opensearch_xml(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_opensearch_xml(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"application/opensearchdescription+xml");
  }

  void httpserv::api_route(struct evhttp_request *r, void *arg)
  {
    // route to API resources.
    const char *uri_str = r->uri;
    std::string uri;
    if (uri_str)
      {
        uri = r->uri;
      }
    miscutil::to_lower(uri);
    if (uri.substr(0,12)=="/search/txt/")
      httpserv::websearch(r,arg);
#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
    else if (uri.substr(0,12)=="/search/img/")
      httpserv::img_websearch(r,arg);
#endif
    // XXX: all websearch plugin calls use the same callback,
    // but we check here for fast error response, if resource
    // is unknown.
    else if (uri.substr(0,7)=="/words/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,16)=="/recent/queries/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,15)=="/cluster/types/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,14)=="/cluster/auto/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,13)=="/similar/txt/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,11)=="/cache/txt/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,12)=="/suggestion/")
      httpserv::suggestion(r,arg);
    else if (uri.substr(0,16)=="/recommendation/")
      httpserv::recommendation(r,arg);

    // if unknown resource, trigger file service.
    else httpserv::file_service(r,arg);
  }

  void httpserv::node_info(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters = NULL;

    /* parse query. */
    const char *uri_str = r->uri;
    if (uri_str)
      {
        std::string uri = std::string(r->uri);
        parameters = httpserv::parse_query(uri);
      }

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested information. */
    sp_err serr = websearch::cgi_websearch_node_info(&csp,&rsp,parameters);
    miscutil::free_map(parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,500,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"application/json"); // this call is JSON only.
  }

  void httpserv::file_service(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters
    = new hash_map<const char*,const char*,hash<const char*>,eqstr>();

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    std::string uri_str = std::string(r->uri);

    /* XXX: truely, this is a hack, we're routing websearch file service. */
    std::string ct;  // content-type.
    sp_err serr = SP_ERR_OK;
    if (miscutil::strncmpic(uri_str.c_str(),"/plugins",8)==0)
      {
        uri_str = uri_str.substr(9);
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1);
        serr = cgisimple::cgi_plugin_file_server(&csp,&rsp,parameters);
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/public",7)==0)
      {
        uri_str = uri_str.substr(8);
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1);
        serr = cgisimple::cgi_file_server(&csp,&rsp,parameters);
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/websearch-hp",13)==0) // to catch some errors.
      {
        miscutil::free_map(parameters);
        miscutil::list_remove_all(&csp._headers);
        httpserv::websearch_hp(r,arg);
        return;
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/robots.txt",11)==0)
      {
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1); // returns public/robots.txt
        serr = cgisimple::cgi_file_server(&csp,&rsp,parameters);
        ct = "text/plain";
      }

    // XXX: other services can be routed here.
    miscutil::free_map(parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    if (ct.empty())
      {
        std::list<const char*>::const_iterator lit = rsp._headers.begin();
        while (lit!=rsp._headers.end())
          {
            if (miscutil::strncmpic((*lit),"content-type:",13) == 0)
              {
                ct = std::string((*lit));
                ct = ct.substr(14);
                break;
              }
            ++lit;
          }
      }
    std::string content = std::string(rsp._body,rsp._content_length);
    httpserv::reply_with_body(r,200,"OK",content,ct);
  }

  void httpserv::websearch(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters = NULL;

    /* parse query. */
    const char *uri_str = r->uri;
    std::string uri;
    if (uri_str)
      {
        uri = std::string(r->uri);
        parameters = httpserv::parse_query(uri);
      }
    if (!parameters || !uri_str)
      {
        // send 400 error response.
        if (parameters)
          miscutil::free_map(parameters);
        httpserv::reply_with_error_400(r);
        return;
      }

    /* fill up csp headers. */
    const char *rheader = evhttp_find_header(r->input_headers, "accept-language");
    if (rheader)
      miscutil::enlist_unique_header(&csp._headers,"accept-language",rheader);

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);
    const char *ua = evhttp_find_header(r->input_headers, "user-agent");
    if (ua)
      miscutil::enlist_unique_header(&csp._headers,"user-agent",ua);

    /* perform websearch. */
    const char *output = miscutil::lookup(parameters,"output");
    if (!output)
      output = "html";
    sp_err serr = SP_ERR_OK;
    csp._http._path = strdup(uri.c_str());
    std::string http_method = httpserv::get_method(r);
    csp._http._gpc = strdup(http_method.c_str());
    if (uri.substr(0,12)=="/search/txt/")
      serr = websearch::cgi_websearch_search(&csp,&rsp,parameters);
    else if (uri.substr(0,7)=="/words/")
      serr = websearch::cgi_websearch_words(&csp,&rsp,parameters);
    else if (uri.substr(0,16)=="/recent/queries/")
      serr = websearch::cgi_websearch_recent_queries(&csp,&rsp,parameters);
    else if (uri.substr(0,15)=="/cluster/types/")
      serr = websearch::cgi_websearch_clustered_types(&csp,&rsp,parameters);
    else if (uri.substr(0,14)=="/cluster/auto/")
      serr = websearch::cgi_websearch_clusterize(&csp,&rsp,parameters);
    else if (uri.substr(0,13)=="/similar/txt/")
      serr = websearch::cgi_websearch_similarity(&csp,&rsp,parameters);<commit_msg>fixed callback handler for /recent/queries resource in HTTP server plugin<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 <NAME>, <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "httpserv.h"
#include "httpserv_configuration.h"
#include "seeks_proxy.h"
#include "plugin_manager.h"
#include "websearch.h"
#include "websearch_api_compat.h"
#include "cgisimple.h"
#include "sweeper.h"
#include "miscutil.h"
#include "encode.h"
#include "errlog.h"
#include "cgi.h"

#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
#include "img_websearch.h"
#endif

#if defined(PROTOBUF) && defined(TC)
#include "query_capture.h"
#include "cf.h"
#include "udb_service.h"
#endif

#include <unistd.h>
#include <sys/stat.h>
#include <sys/times.h>
#include <iostream>

using namespace sp;

namespace seeks_plugins
{

  httpserv::httpserv(const std::string &address,
                     const u_short &port)
    : plugin(),_address(address),_port(port)
  {
    /* initializing and starting server. */
    _evbase = event_base_new();
    _srv = evhttp_new(_evbase);
    evhttp_bind_socket(_srv, _address.c_str(), _port); // server binding to the address & port.

    errlog::log_error(LOG_LEVEL_INFO,"Seeks HTTP server plugin listening on %s:%u",
                      _address.c_str(),_port);

    init_callbacks();
    event_base_dispatch(_evbase); // XXX: for debug.
  }

  httpserv::httpserv()
    : plugin()
  {
    _name = "httpserv";
    _version_major = "0";
    _version_minor = "1";

    if (seeks_proxy::_datadir.empty())
      _config_filename = plugin_manager::_plugin_repository + "httpserv/httpserv-config";
    else
      _config_filename = seeks_proxy::_datadir + "/plugins/httpserv/httpserv-config";

#ifdef SEEKS_CONFIGDIR
    struct stat stFileInfo;
    if (!stat(_config_filename.c_str(), &stFileInfo)  == 0)
      {
        _config_filename = SEEKS_CONFIGDIR "/httpserv-config";
      }
#endif

    if (httpserv_configuration::_hconfig == NULL)
      httpserv_configuration::_hconfig = new httpserv_configuration(_config_filename);
    _configuration = httpserv_configuration::_hconfig;

    /* sets address & port. */
    _address = httpserv_configuration::_hconfig->_host;
    _port = httpserv_configuration::_hconfig->_port;
  }

  httpserv::~httpserv()
  {
  }

  void httpserv::start()
  {
    /* initializing and starting server. */
    _evbase = event_base_new();
    _srv = evhttp_new(_evbase);
    evhttp_bind_socket(_srv, _address.c_str(), _port); // server binding to the address & port.

    errlog::log_error(LOG_LEVEL_INFO,"Seeks HTTP server plugin listening on %s:%u",
                      _address.c_str(),_port);

    init_callbacks();

    int err = pthread_create(&_server_thread,NULL,
                             (void*(*)(void*))&event_base_dispatch,_evbase);
    seeks_proxy::_httpserv_thread = &_server_thread;
    if (seeks_proxy::_run_proxy)
      pthread_detach(_server_thread); // detach if proxy is running, else no (join).
  }

  void httpserv::stop()
  {
    event_base_loopbreak(_evbase);
    evhttp_free(_srv);
    event_base_free(_evbase);
  }

  void httpserv::init_callbacks()
  {
    evhttp_set_cb(_srv,"/search/txt",&httpserv::websearch,NULL);
    evhttp_set_cb(_srv,"/search",&httpserv::websearch,NULL); // compatibility API.
#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
    evhttp_set_cb(_srv,"/search_img",&httpserv::img_websearch,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/seeks_img_search.css",&httpserv::seeks_img_search_css,NULL);
#endif
    evhttp_set_cb(_srv,"/",&httpserv::websearch_hp,NULL);
    evhttp_set_cb(_srv,"/websearch-hp",&httpserv::websearch_hp,NULL);
    evhttp_set_cb(_srv,"/seeks_hp_search.css",&httpserv::seeks_hp_css,NULL);
    evhttp_set_cb(_srv,"/seeks_search.css",&httpserv::seeks_search_css,NULL);
    evhttp_set_cb(_srv,"/opensearch.xml",&httpserv::opensearch_xml,NULL);
    evhttp_set_cb(_srv,"/info",&httpserv::node_info,NULL);
#if defined(PROTOBUF) && defined(TC)
    evhttp_set_cb(_srv,"/qc_redir",&httpserv::qc_redir,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/tbd",&httpserv::tbd,NULL); // compatibility API.
    evhttp_set_cb(_srv,"/find_dbr",&httpserv::find_dbr,NULL);
    evhttp_set_cb(_srv,"/find_bqc",&httpserv::find_bqc,NULL);
    evhttp_set_cb(_srv,"/peers",&httpserv::peers,NULL);
#endif

    //evhttp_set_gencb(_srv,&httpserv::unknown_path,NULL); /* 404: catches all other resources. */
    evhttp_set_gencb(_srv,&httpserv::api_route,NULL);
  }

  void httpserv::reply_with_redirect_302(struct evhttp_request *r,
                                         const char *url)
  {
    evhttp_add_header(r->output_headers,"Location",url);
    evhttp_send_reply(r, HTTP_MOVETEMP, "OK", NULL);
  }

  void httpserv::reply_with_error_400(struct evhttp_request *r)
  {
    evhttp_send_reply(r, HTTP_BADREQUEST, "BAD REQUEST", NULL);
  }

  void httpserv::reply_with_error(struct evhttp_request *r,
                                  const int &http_code,
                                  const char *message,
                                  const std::string &error_message)
  {
    /* error output. */
    errlog::log_error(LOG_LEVEL_ERROR,"httpserv error: code %d %s",
                      http_code,error_message.c_str());

    // body.
    struct evbuffer *buffer;
    buffer = evbuffer_new();

    char msg[error_message.length()];
    for (size_t i=0; i<error_message.length(); i++)
      msg[i] = error_message[i];
    evbuffer_add(buffer,msg,sizeof(msg));

    evhttp_send_reply(r, http_code, message, buffer);
    evbuffer_free(buffer);
  }

  void httpserv::reply_with_empty_body(struct evhttp_request *r,
                                       const int &http_code,
                                       const char *message)
  {
    evhttp_send_reply(r, http_code, message, NULL);
  }

  void httpserv::reply_with_body(struct evhttp_request *r,
                                 const int &http_code,
                                 const char *message,
                                 const std::string &content,
                                 const std::string &content_type)
  {
    /* headers. */
    evhttp_add_header(r->output_headers,"Content-Type",content_type.c_str());

    /* body. */
    struct evbuffer *buffer;
    buffer = evbuffer_new();

    char msg[content.length()];
    for (size_t i=0; i<content.length(); i++)
      msg[i] = content[i];
    evbuffer_add(buffer,msg,sizeof(msg));

    evhttp_send_reply(r, http_code, message, buffer);
    evbuffer_free(buffer);
  }

  void httpserv::websearch_hp(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_hp(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content);

    /* run the sweeper, for timed out query contexts. */
    sweeper::sweep();
  }

  void httpserv::seeks_hp_css(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_search_hp_css(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"text/css");
  }

  void httpserv::seeks_search_css(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;
    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_search_css(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"text/css");
  }

  void httpserv::opensearch_xml(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> parameters;

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    sp_err serr = websearch::cgi_websearch_opensearch_xml(&csp,&rsp,&parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"application/opensearchdescription+xml");
  }

  void httpserv::api_route(struct evhttp_request *r, void *arg)
  {
    // route to API resources.
    const char *uri_str = r->uri;
    std::string uri;
    if (uri_str)
      {
        uri = r->uri;
      }
    miscutil::to_lower(uri);
    if (uri.substr(0,12)=="/search/txt/")
      httpserv::websearch(r,arg);
#ifdef FEATURE_IMG_WEBSEARCH_PLUGIN
    else if (uri.substr(0,12)=="/search/img/")
      httpserv::img_websearch(r,arg);
#endif
    // XXX: all websearch plugin calls use the same callback,
    // but we check here for fast error response, if resource
    // is unknown.
    else if (uri.substr(0,7)=="/words/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,15)=="/recent/queries")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,15)=="/cluster/types/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,14)=="/cluster/auto/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,13)=="/similar/txt/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,11)=="/cache/txt/")
      httpserv::websearch(r,arg);
    else if (uri.substr(0,12)=="/suggestion/")
      httpserv::suggestion(r,arg);
    else if (uri.substr(0,16)=="/recommendation/")
      httpserv::recommendation(r,arg);

    // if unknown resource, trigger file service.
    else httpserv::file_service(r,arg);
  }

  void httpserv::node_info(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters = NULL;

    /* parse query. */
    const char *uri_str = r->uri;
    if (uri_str)
      {
        std::string uri = std::string(r->uri);
        parameters = httpserv::parse_query(uri);
      }

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested information. */
    sp_err serr = websearch::cgi_websearch_node_info(&csp,&rsp,parameters);
    miscutil::free_map(parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,500,"ERROR");
        return;
      }

    /* fill up response. */
    std::string content = std::string(rsp._body);
    httpserv::reply_with_body(r,200,"OK",content,"application/json"); // this call is JSON only.
  }

  void httpserv::file_service(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters
    = new hash_map<const char*,const char*,hash<const char*>,eqstr>();

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);

    /* return requested file. */
    std::string uri_str = std::string(r->uri);

    /* XXX: truely, this is a hack, we're routing websearch file service. */
    std::string ct;  // content-type.
    sp_err serr = SP_ERR_OK;
    if (miscutil::strncmpic(uri_str.c_str(),"/plugins",8)==0)
      {
        uri_str = uri_str.substr(9);
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1);
        serr = cgisimple::cgi_plugin_file_server(&csp,&rsp,parameters);
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/public",7)==0)
      {
        uri_str = uri_str.substr(8);
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1);
        serr = cgisimple::cgi_file_server(&csp,&rsp,parameters);
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/websearch-hp",13)==0) // to catch some errors.
      {
        miscutil::free_map(parameters);
        miscutil::list_remove_all(&csp._headers);
        httpserv::websearch_hp(r,arg);
        return;
      }
    else if (miscutil::strncmpic(uri_str.c_str(),"/robots.txt",11)==0)
      {
        miscutil::add_map_entry(parameters,"file",1,uri_str.c_str(),1); // returns public/robots.txt
        serr = cgisimple::cgi_file_server(&csp,&rsp,parameters);
        ct = "text/plain";
      }

    // XXX: other services can be routed here.
    miscutil::free_map(parameters);
    miscutil::list_remove_all(&csp._headers);
    if (serr != SP_ERR_OK)
      {
        httpserv::reply_with_empty_body(r,404,"ERROR");
        return;
      }

    /* fill up response. */
    if (ct.empty())
      {
        std::list<const char*>::const_iterator lit = rsp._headers.begin();
        while (lit!=rsp._headers.end())
          {
            if (miscutil::strncmpic((*lit),"content-type:",13) == 0)
              {
                ct = std::string((*lit));
                ct = ct.substr(14);
                break;
              }
            ++lit;
          }
      }
    std::string content = std::string(rsp._body,rsp._content_length);
    httpserv::reply_with_body(r,200,"OK",content,ct);
  }

  void httpserv::websearch(struct evhttp_request *r, void *arg)
  {
    client_state csp;
    csp._config = seeks_proxy::_config;
    http_response rsp;
    hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters = NULL;

    /* parse query. */
    const char *uri_str = r->uri;
    std::string uri;
    if (uri_str)
      {
        uri = std::string(r->uri);
        parameters = httpserv::parse_query(uri);
      }
    if (!parameters || !uri_str)
      {
        // send 400 error response.
        if (parameters)
          miscutil::free_map(parameters);
        httpserv::reply_with_error_400(r);
        return;
      }

    /* fill up csp headers. */
    const char *rheader = evhttp_find_header(r->input_headers, "accept-language");
    if (rheader)
      miscutil::enlist_unique_header(&csp._headers,"accept-language",rheader);

    const char *host = evhttp_find_header(r->input_headers, "host");
    if (host)
      miscutil::enlist_unique_header(&csp._headers,"host",host);
    const char *baseurl = evhttp_find_header(r->input_headers, "seeks-remote-location");
    if (baseurl)
      miscutil::enlist_unique_header(&csp._headers,"seeks-remote-location",baseurl);
    const char *ua = evhttp_find_header(r->input_headers, "user-agent");
    if (ua)
      miscutil::enlist_unique_header(&csp._headers,"user-agent",ua);

    /* perform websearch. */
    const char *output = miscutil::lookup(parameters,"output");
    if (!output)
      output = "html";
    sp_err serr = SP_ERR_OK;
    csp._http._path = strdup(uri.c_str());
    std::string http_method = httpserv::get_method(r);
    csp._http._gpc = strdup(http_method.c_str());
    if (uri.substr(0,12)=="/search/txt/")
      serr = websearch::cgi_websearch_search(&csp,&rsp,parameters);
    else if (uri.substr(0,7)=="/words/")
      serr = websearch::cgi_websearch_words(&csp,&rsp,parameters);
    else if (uri.substr(0,15)=="/recent/queries")
      serr = websearch::cgi_websearch_recent_queries(&csp,&rsp,parameters);
    else if (uri.substr(0,15)=="/cluster/types/")
      serr = websearch::cgi_websearch_clustered_types(&csp,&rsp,parameters);
    else if (uri.substr(0,14)=="/cluster/auto/")
      serr = websearch::cgi_websearch_clusterize(&csp,&rsp,parameters);
    else if (uri.substr(0,13)=="/similar/txt/")
      serr = websearch::cgi_websearch_similarity(&csp,&rsp,parameters);
{'fraction_non_alphanumeric': 0.1439906898011003, 'fraction_numerical': 0.007088446889547186, 'mean_word_length': 3.4702057223930005, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7466524', 'n_tokens_mistral': 13189, 'n_tokens_neox': 12769, 'n_words': 2218}