<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010, 2011 <NAME>, <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "rank_estimators.h"
#include "query_recommender.h"
#include "cf.h"
#include "cf_configuration.h"
#include "qprocess.h"
#include "query_capture.h"
#include "uri_capture.h"
#include "db_uri_record.h"
#include "udb_client.h"
#include "mrf.h"
#include "urlmatch.h"
#include "miscutil.h"
#include "errlog.h"

#include <assert.h>
#include <math.h>
#include <iostream>

using lsh::qprocess;
using lsh::mrf;
using lsh::str_chain;
using lsh::stopwordlist;
using sp::urlmatch;
using sp::miscutil;
using sp::errlog;

namespace seeks_plugins
{

  /*- rank_estimator -*/
  cr_store rank_estimator::_store;
  sp_mutex_t rank_estimator::_est_mutex;

  rank_estimator::rank_estimator()
  {
    mutex_init(&_est_mutex);
  }

  // threaded call to personalization.
  void rank_estimator::peers_personalize(const std::string &query,
                                         const std::string &lang,
                                         const uint32_t &expansion,
                                         uint32_t &npeers,
                                         std::vector<search_snippet*> &snippets,
                                         std::multimap<double,std::string,std::less<double> > &related_queries,
                                         hash_map<uint32_t,search_snippet*,id_hash_uint> &reco_snippets,
                                         query_context *qc)
  {
    std::vector<pthread_t> perso_threads;
    std::vector<perso_thread_arg*> perso_args;

    bool have_peers = npeers > 0 ? true : false;

    // thread for local db.
    threaded_personalize(perso_args,perso_threads,
                         query,lang,expansion,&snippets,&related_queries,&reco_snippets,NULL,qc);

    // one thread per remote peer, to handle the IO.
    hash_map<const char*,peer*,hash<const char*>,eqstr>::const_iterator hit
    = cf_configuration::_config->_pl._peers.begin();
    while(hit!=cf_configuration::_config->_pl._peers.end())
      {
        threaded_personalize(perso_args,perso_threads,
                             query,lang,expansion,&snippets,&related_queries,&reco_snippets,
                             (*hit).second);
        ++hit;
      }

    // join.
    for (size_t i=0; i<perso_threads.size(); i++)
      {
        if (perso_threads.at(i)!=0)
          pthread_join(perso_threads.at(i),NULL);
      }

    // clear.
    for (size_t i=0; i<perso_args.size(); i++)
      {
        if (perso_args.at(i))
          {
            if (!have_peers && perso_args.at(i)->_err == SP_ERR_OK)
              npeers++;
            delete perso_args.at(i);
          }<commit_msg>fixing bad query context parameter to threaded personnalization function. #refs 413<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010, 2011 <NAME>, <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "rank_estimators.h"
#include "query_recommender.h"
#include "cf.h"
#include "cf_configuration.h"
#include "qprocess.h"
#include "query_capture.h"
#include "uri_capture.h"
#include "db_uri_record.h"
#include "udb_client.h"
#include "mrf.h"
#include "urlmatch.h"
#include "miscutil.h"
#include "errlog.h"

#include <assert.h>
#include <math.h>
#include <iostream>

using lsh::qprocess;
using lsh::mrf;
using lsh::str_chain;
using lsh::stopwordlist;
using sp::urlmatch;
using sp::miscutil;
using sp::errlog;

namespace seeks_plugins
{

  /*- rank_estimator -*/
  cr_store rank_estimator::_store;
  sp_mutex_t rank_estimator::_est_mutex;

  rank_estimator::rank_estimator()
  {
    mutex_init(&_est_mutex);
  }

  // threaded call to personalization.
  void rank_estimator::peers_personalize(const std::string &query,
                                         const std::string &lang,
                                         const uint32_t &expansion,
                                         uint32_t &npeers,
                                         std::vector<search_snippet*> &snippets,
                                         std::multimap<double,std::string,std::less<double> > &related_queries,
                                         hash_map<uint32_t,search_snippet*,id_hash_uint> &reco_snippets,
                                         query_context *qc)
  {
    std::vector<pthread_t> perso_threads;
    std::vector<perso_thread_arg*> perso_args;

    bool have_peers = npeers > 0 ? true : false;

    // thread for local db.
    threaded_personalize(perso_args,perso_threads,
                         query,lang,expansion,&snippets,&related_queries,&reco_snippets,NULL,qc);

    // one thread per remote peer, to handle the IO.
    hash_map<const char*,peer*,hash<const char*>,eqstr>::const_iterator hit
    = cf_configuration::_config->_pl._peers.begin();
    while(hit!=cf_configuration::_config->_pl._peers.end())
      {
        threaded_personalize(perso_args,perso_threads,
                             query,lang,expansion,&snippets,&related_queries,&reco_snippets,
                             (*hit).second,qc);
        ++hit;
      }

    // join.
    for (size_t i=0; i<perso_threads.size(); i++)
      {
        if (perso_threads.at(i)!=0)
          pthread_join(perso_threads.at(i),NULL);
      }

    // clear.
    for (size_t i=0; i<perso_args.size(); i++)
      {
        if (perso_args.at(i))
          {
            if (!have_peers && perso_args.at(i)->_err == SP_ERR_OK)
              npeers++;
            delete perso_args.at(i);
          }
{'fraction_non_alphanumeric': 0.11062654455589475, 'fraction_numerical': 0.005960168629161215, 'mean_word_length': 2.5318275154004106, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3499506', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2082, 'n_words': 528}