<commit_before>#include "QueryIntentHelper.h"
#include "Classifier.h"

#include <util/driver/Value.h>
#include <util/driver/Request.h>
#include <util/driver/Response.h>
#include <common/Keys.h>

#include <limits>
#include <glog/logging.h>

namespace sf1r
{
using driver::Keys;
using namespace izenelib::driver;
using namespace NQI;

void refineRequest(izenelib::driver::Request& request,
             izenelib::driver::Response& response,
             WMVContainer& wmvs)
{
    if (wmvs.empty())
        return;
    izenelib::driver::Value& queryIntents = response["query_intent"];
    WMVCIterator array;
    WMVIterator item;
    for (array = wmvs.begin(); array != wmvs.end(); array++)
    {
        if (array->second.empty())
            continue;
        if ("group_label" == array->first.op_)
        {
            izenelib::driver::Value& groupLabels = request[Keys::search][Keys::group_label];
            izenelib::driver::Value& groupLabel = groupLabels();
            groupLabel[Keys::property] = array->first.name_;
            int operands = array->first.operands_;
            if (-1 == operands)
                operands = std::numeric_limits<int>::max();
            izenelib::driver::Value& values = groupLabel[Keys::value];
            izenelib::driver::Value& queryIntent = queryIntents();
            izenelib::driver::Value& queryIntentValues = queryIntent[array->first.name_];
            int i = 0;
            for (item = array->second.begin(); item != array->second.end(); i++, item++)
            {
                if (i >= operands)
                    break;
                std::string vs = item->first;
                std::size_t pos = 0;
                while (true)
                {
                    std::size_t found = vs.find_first_of('>', pos);
                    if (std::string::npos == found)
                    {
                        values() = vs.substr(pos);
                        LOG(INFO)<<vs.substr(pos);
                        break;
                    }
                    values() = vs.substr(pos, found - pos);
                    LOG(INFO)<<vs.substr(pos, found - pos);
                    pos = found + 1;
                }
                queryIntentValues() = item->first;
            }
        }
        else if ("conditions" == array->first.op_)
        {
            izenelib::driver::Value& groupLabels = request[Keys::conditions];
            izenelib::driver::Value& groupLabel = groupLabels();
            groupLabel[Keys::property] = array->first.name_;
            int operands = array->first.operands_;
            if (-1 == operands)
                operands = std::numeric_limits<int>::max();
            izenelib::driver::Value& values = groupLabel[Keys::value];
            izenelib::driver::Value& queryIntent = queryIntents();
            izenelib::driver::Value& queryIntentValues = queryIntent[array->first.name_];
            int i = 0;
            for (item = array->second.begin(); item != array->second.end(); i++, item++)
            {
                if (i >= operands)
                    break;
                std::string vs = item->first;
                std::size_t pos = 0;
                while (true)
                {
                    std::size_t found = vs.find_first_of('>', pos);
                    if (std::string::npos == found)
                    {
                        values() = vs.substr(pos);
                        LOG(INFO)<<vs.substr(pos);
                        break;
                    }
                    values() = vs.substr(pos, found - pos);
                    LOG(INFO)<<vs.substr(pos, found - pos);
                    pos = found + 1;
                }
                queryIntentValues() = item->first;
            }
        }
    }
}

}
<commit_msg>Tiny for Source.<commit_after>#include "QueryIntentHelper.h"
#include "Classifier.h"

#include <util/driver/Value.h>
#include <util/driver/Request.h>
#include <util/driver/Response.h>
#include <common/Keys.h>

#include <limits>
#include <glog/logging.h>

namespace sf1r
{
using driver::Keys;
using namespace izenelib::driver;
using namespace NQI;

void refineRequest(izenelib::driver::Request& request,
             izenelib::driver::Response& response,
             WMVContainer& wmvs)
{
    if (wmvs.empty())
        return;
    izenelib::driver::Value& queryIntents = response["query_intent"];
    WMVCIterator array;
    WMVIterator item;
    for (array = wmvs.begin(); array != wmvs.end(); array++)
    {
        if (array->second.empty())
            continue;
        if ("group_label" == array->first.op_)
        {
            izenelib::driver::Value& groupLabels = request[Keys::search][Keys::group_label];
            izenelib::driver::Value& groupLabel = groupLabels();
            groupLabel[Keys::property] = array->first.name_;
            int operands = array->first.operands_;
            if (-1 == operands)
                operands = std::numeric_limits<int>::max();
            izenelib::driver::Value& values = groupLabel[Keys::value];
            izenelib::driver::Value& queryIntent = queryIntents();
            izenelib::driver::Value& queryIntentValues = queryIntent[array->first.name_];
            int i = 0;
            for (item = array->second.begin(); item != array->second.end(); i++, item++)
            {
                if (i >= operands)
                    break;
                std::string vs = item->first;
                std::size_t pos = 0;
                while (true)
                {
                    std::size_t found = vs.find_first_of('>', pos);
                    if (std::string::npos == found)
                    {
                        values() = vs.substr(pos);
                        LOG(INFO)<<vs.substr(pos);
                        break;
                    }
                    values() = vs.substr(pos, found - pos);
                    LOG(INFO)<<vs.substr(pos, found - pos);
                    pos = found + 1;
                }
                queryIntentValues() = item->first;
            }
        }
        else if ("conditions" == array->first.op_)
        {
            izenelib::driver::Value& groupLabels = request[Keys::conditions];
            izenelib::driver::Value& groupLabel = groupLabels();
            groupLabel[Keys::property] = array->first.name_;
            groupLabel["operator"] = "=";
            int operands = 1;//array->first.operands_;
            //if (-1 == operands)
            //    operands = std::numeric_limits<int>::max();
            izenelib::driver::Value& values = groupLabel[Keys::value];
            izenelib::driver::Value& queryIntent = queryIntents();
            izenelib::driver::Value& queryIntentValues = queryIntent[array->first.name_];
            int i = 0;
            for (item = array->second.begin(); item != array->second.end(); i++, item++)
            {
                if (i >= operands)
                    break;
                values() = item->first;
                LOG(INFO)<<item->first;
                queryIntentValues() = item->first;
            }
        }
    }
}

}

{'fraction_non_alphanumeric': 0.1402842268186295, 'fraction_numerical': 0.002392007879555368, 'mean_word_length': 1.6502609992542878, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2402676', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1912, 'n_words': 375}