<commit_before>/**
 * @file process/parsers/SearchParser.cpp
 * @author <NAME>
 * @date Created <2010-06-11 17:12:27>
 */
#include "SearchParser.h"

#include <common/BundleSchemaHelpers.h>
#include <common/Keys.h>

#include <boost/algorithm/string/case_conv.hpp>

namespace sf1r {

using driver::Keys;

/**
 * @class SearchParser
 *
 * The @b search field is an object. It specifies a full text search in
 * documents. The search object has following fields:
 *
 * - @b keywords* (@c String): Keywords used in the search.
 * - @b USERID (@c String): USERID is used to get the user info, which assists
 *   to get personalized search result. This item is not required to set. if it
 *   is not set, the result is the normal ranking search result.
 * - @b in (@c Array): Which properties the search is performed in. Every item
 *   can be an Object or an String. If it is an Object, the \b property field is
 *   used as the property name, otherwise, the String itself is used as property
 *   name. If this is not specified, all indexed properties are used. Valid
 *   properties can be check though schema/get (see SchemaController::get() ).
 *   All index names (the @b name field in every index) can be used.
 * - @b count (@c Array): Which properties the COUNT() is performed in. Every item
 *   can be an Object or an String. If it is an Object, the \b property field is
 *   used as the property name, otherwise, the String itself is used as property
 *   name. Valid properties should have numeric type.
 * - @b taxonomy_label (@c String): Only get documents in the specified
 *   label. It cannot be used with @b name_entity_type and @b name_entity_item
 *   together.
 * - @b name_entity_type (@c String): Only get documents in the specified name
 *   entity item. Must be used with @b name_entity_item together and cannot be
 *   used with @b taxonomy_label.
 * - @b name_entity_item (@c Object): Only get documents in the specified name
 *   entity item. Must be used with @b name_entity_type together and cannot be
 *   used with @b taxonomy_label.
 * - @b group_label (@c Array): Get documents in the specified group labels. Each label consists of a property name and value.@n
 *   You could specify multiple labels, for example, there are 4 labels named as "A1", "A2", "B1", "B2".@n
 *   If the property name of label "A1" and "A2" is "A", while the property name of label "B1" and "B2" is "B",@n
 *   then the documents returned would belong to the labels with the condition of ("A1" or "A2") and ("B1" or "B2").
 *   - @b property* (@c String): the property name.
 *   - @b value* (@c Array): the label value.@n
 *   For the property type of string, it's an array of the path from root to leaf node.
 *   Each is a @c String of node value. Only those documents belonging to the leaf node
 *   would be returned.@n
 *   For the property type of int or float, you could either specify a property value
 *   or a range in @b value[0]. If it is a range, you could not specify the same @b property
 *   in request["group"].@n
 *   Regarding the range format, you could specify it in the form of "101-200",
 *   meaning all values between 101 and 200, including both boundary values,
 *   or in the form of "-200", meaning all values not larger than 200,
 *   or in the form of "101-", meaning all values not less than 101.@n
 *   For the property type of datetime, in @b value[0], you could specify a string of year, month or day.@n
 *   For example, "2012" means the year of 2012, "2012-07" means the month of 2012 July, "2012-07-06" means the day of 2012 July 6th.
 *   - @b auto_select_limit (@c Uint = 1): when @b value is an empty array, the top labels would be selected automatically.@n
 *   You could specify this parameter as how many top labels need to be selected automatically.@n
 *   The selected top labels are returned as @b top_group_label in search response.@n
 *   This feature only supports the property type of string.
 * - @b attr_label (@c Array): Get documents in the specified attribute labels. Each label consists of an attribute name and value.@n
 *   You could specify multiple labels, for example, there are 4 labels named as "A1", "A2", "B1", "B2".@n
 *   If the attribute name of label "A1" and "A2" is "A", while the attribute name of label "B1" and "B2" is "B",@n
 *   then the documents returned would belong to the labels with the condition of ("A1" or "A2") and ("B1" or "B2").
 *   - @b attr_name* (@c String): the attribute name
 *   - @b attr_value* (@c String): the attribute value
 * - @b ranking_model (@c String): How to rank the search result. Result with
 *   high ranking score is considered has high relevance. Now SF1 supports
 *   following ranking models.
 *   - @b plm
 *   - @b bm25
 *   - @b kl.@n
 *   If this is omitted, A default ranking model specified in configuration file
 *   is used (In Collection node attribute).
 * - @b searching_mode (@c Object): Searching mode options
 *   We can choose the searching mode to combine our
 *   query parsed term for supporting different search in different applications.
 *   following searching modes are available
 *   - @b and
 *   - @b or
 *   - @b wand
 *   - @b verbose
 *   - @b knn: TODO@n
 *   If this is omitted, @b and searching mode is used as the default value.
 * - @b log_keywords (@c Bool = @c true): Whether the keywords should be
 *   logged.
 * - @b analyzer (@c Object): Keywords analyzer options
 *   - @b apply_la (@c Bool = @c true): TODO
 *   - @b use_synonym_extension (@c Bool = @c false): TODO
 *   - @b use_original_keyword (@c Bool = @c false): TODO
 * - @b is_random_rank (@c Bool = @c false): If true, the search results would
 *   be randomly ordered.@n
 * - @b is_require_related (@c Bool = @c false): If true, the search results would
 *   be contain related queries.@n
 *   In order to enable this feature, you need also configure a non-zero weight
 *   for <ProductRanking><Score type="random"> in collection config file.
 * - @b query_source (@c String): Where does the query come from, used to decide
 *   the categories to boost in product ranking.
 * - @b boost_group_label (@c Array): The group labels to boost product rankings.@n
 *   It's an array of group labels, each element is an array of the label path
 *   started from root node. As an example, two labels could be written as
 *   [["手机数码", "手机通讯", "手机"], ["电脑办公", "电脑整机"]].@n
 *   In order to enable this feature, in collection config file
 *   <ProductRanking><Score type="category">, you need to configure the property
 *   name of these group labels, and also a non-zero weight.
 *
 * Example<commit_msg>adjust the API document on search parameter "is_require_related".<commit_after>/**
 * @file process/parsers/SearchParser.cpp
 * @author <NAME>
 * @date Created <2010-06-11 17:12:27>
 */
#include "SearchParser.h"

#include <common/BundleSchemaHelpers.h>
#include <common/Keys.h>

#include <boost/algorithm/string/case_conv.hpp>

namespace sf1r {

using driver::Keys;

/**
 * @class SearchParser
 *
 * The @b search field is an object. It specifies a full text search in
 * documents. The search object has following fields:
 *
 * - @b keywords* (@c String): Keywords used in the search.
 * - @b USERID (@c String): USERID is used to get the user info, which assists
 *   to get personalized search result. This item is not required to set. if it
 *   is not set, the result is the normal ranking search result.
 * - @b in (@c Array): Which properties the search is performed in. Every item
 *   can be an Object or an String. If it is an Object, the \b property field is
 *   used as the property name, otherwise, the String itself is used as property
 *   name. If this is not specified, all indexed properties are used. Valid
 *   properties can be check though schema/get (see SchemaController::get() ).
 *   All index names (the @b name field in every index) can be used.
 * - @b count (@c Array): Which properties the COUNT() is performed in. Every item
 *   can be an Object or an String. If it is an Object, the \b property field is
 *   used as the property name, otherwise, the String itself is used as property
 *   name. Valid properties should have numeric type.
 * - @b taxonomy_label (@c String): Only get documents in the specified
 *   label. It cannot be used with @b name_entity_type and @b name_entity_item
 *   together.
 * - @b name_entity_type (@c String): Only get documents in the specified name
 *   entity item. Must be used with @b name_entity_item together and cannot be
 *   used with @b taxonomy_label.
 * - @b name_entity_item (@c Object): Only get documents in the specified name
 *   entity item. Must be used with @b name_entity_type together and cannot be
 *   used with @b taxonomy_label.
 * - @b group_label (@c Array): Get documents in the specified group labels. Each label consists of a property name and value.@n
 *   You could specify multiple labels, for example, there are 4 labels named as "A1", "A2", "B1", "B2".@n
 *   If the property name of label "A1" and "A2" is "A", while the property name of label "B1" and "B2" is "B",@n
 *   then the documents returned would belong to the labels with the condition of ("A1" or "A2") and ("B1" or "B2").
 *   - @b property* (@c String): the property name.
 *   - @b value* (@c Array): the label value.@n
 *   For the property type of string, it's an array of the path from root to leaf node.
 *   Each is a @c String of node value. Only those documents belonging to the leaf node
 *   would be returned.@n
 *   For the property type of int or float, you could either specify a property value
 *   or a range in @b value[0]. If it is a range, you could not specify the same @b property
 *   in request["group"].@n
 *   Regarding the range format, you could specify it in the form of "101-200",
 *   meaning all values between 101 and 200, including both boundary values,
 *   or in the form of "-200", meaning all values not larger than 200,
 *   or in the form of "101-", meaning all values not less than 101.@n
 *   For the property type of datetime, in @b value[0], you could specify a string of year, month or day.@n
 *   For example, "2012" means the year of 2012, "2012-07" means the month of 2012 July, "2012-07-06" means the day of 2012 July 6th.
 *   - @b auto_select_limit (@c Uint = 1): when @b value is an empty array, the top labels would be selected automatically.@n
 *   You could specify this parameter as how many top labels need to be selected automatically.@n
 *   The selected top labels are returned as @b top_group_label in search response.@n
 *   This feature only supports the property type of string.
 * - @b attr_label (@c Array): Get documents in the specified attribute labels. Each label consists of an attribute name and value.@n
 *   You could specify multiple labels, for example, there are 4 labels named as "A1", "A2", "B1", "B2".@n
 *   If the attribute name of label "A1" and "A2" is "A", while the attribute name of label "B1" and "B2" is "B",@n
 *   then the documents returned would belong to the labels with the condition of ("A1" or "A2") and ("B1" or "B2").
 *   - @b attr_name* (@c String): the attribute name
 *   - @b attr_value* (@c String): the attribute value
 * - @b ranking_model (@c String): How to rank the search result. Result with
 *   high ranking score is considered has high relevance. Now SF1 supports
 *   following ranking models.
 *   - @b plm
 *   - @b bm25
 *   - @b kl.@n
 *   If this is omitted, A default ranking model specified in configuration file
 *   is used (In Collection node attribute).
 * - @b searching_mode (@c Object): Searching mode options
 *   We can choose the searching mode to combine our
 *   query parsed term for supporting different search in different applications.
 *   following searching modes are available
 *   - @b and
 *   - @b or
 *   - @b wand
 *   - @b verbose
 *   - @b knn: TODO@n
 *   If this is omitted, @b and searching mode is used as the default value.
 * - @b log_keywords (@c Bool = @c true): Whether the keywords should be
 *   logged.
 * - @b analyzer (@c Object): Keywords analyzer options
 *   - @b apply_la (@c Bool = @c true): TODO
 *   - @b use_synonym_extension (@c Bool = @c false): TODO
 *   - @b use_original_keyword (@c Bool = @c false): TODO
 * - @b is_random_rank (@c Bool = @c false): If true, the search results would
 *   be randomly ordered.@n
 *   In order to enable this feature, you need also configure a non-zero weight
 *   for <ProductRanking><Score type="random"> in collection config file.
 * - @b is_require_related (@c Bool = @c false): If true, the search results would
 *   be contain related queries.
 * - @b query_source (@c String): Where does the query come from, used to decide
 *   the categories to boost in product ranking.
 * - @b boost_group_label (@c Array): The group labels to boost product rankings.@n
 *   It's an array of group labels, each element is an array of the label path
 *   started from root node. As an example, two labels could be written as
 *   [["手机数码", "手机通讯", "手机"], ["电脑办公", "电脑整机"]].@n
 *   In order to enable this feature, in collection config file
 *   <ProductRanking><Score type="category">, you need to configure the property
 *   name of these group labels, and also a non-zero weight.
 *
 * Example
{'fraction_non_alphanumeric': 0.09441349194398434, 'fraction_numerical': 0.015359132660743863, 'mean_word_length': 3.399801258694932, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3855722', 'n_tokens_mistral': 3966, 'n_tokens_neox': 3829, 'n_words': 1990}