<commit_before>//! Parses "multi_match" queries

use rustc_serialize::json::Json;

use search::term::Term;
use search::analysis::Analyzer;

use search::query::Query;
use search::query::term_matcher::TermMatcher;
use search::query::term_scorer::TermScorer;
use search::query::parser::{QueryParseContext, QueryParseError};
use search::query::parser::utils::{parse_string, parse_float, Operator, parse_operator, parse_field_and_boost};


pub fn parse(context: &QueryParseContext, json: &Json) -> Result<Query, QueryParseError> {
    let object = try!(json.as_object().ok_or(QueryParseError::ExpectedObject));

    // Get configuration
    let mut fields_with_boosts = Vec::new();
    let mut query = String::new();
    let mut boost = 1.0f64;
    let mut operator = Operator::Or;

    let mut has_fields_key = false;
    let mut has_query_key = false;

    for (key, val) in object.iter() {
        match key.as_ref() {
            "fields" => {
                has_fields_key = true;

                match *val {
                    Json::Array(ref array) => {
                        for field in array.iter() {
                            fields_with_boosts.push(try!(parse_field_and_boost(field)));
                        }
                    }
                    _ => return Err(QueryParseError::ExpectedArray)
                }
            }
            "query" => {
                has_query_key = true;
                query = try!(parse_string(val));
            }
            "boost" => {
                boost = try!(parse_float(val));
            }
            "operator" => {
                operator = try!(parse_operator(val))
            }
            _ => return Err(QueryParseError::UnrecognisedKey(key.clone()))
        }
    }

    if !has_fields_key {
        return Err(QueryParseError::ExpectedKey("fields"))
    }

    if !has_query_key {
        return Err(QueryParseError::ExpectedKey("query"))
    }

    // Convert query string into term query objects
    let mut field_queries = Vec::new();
    for (field_name, field_boost) in fields_with_boosts {
        let mut term_queries = Vec::new();
        for token in Analyzer::Standard.run(query.clone()) {
            term_queries.push(Query::MatchTerm {
                field: field_name.clone(),
                term: token.term,
                matcher: TermMatcher::Exact,
                scorer: TermScorer::default(),
            });
        }

        let mut field_query = match operator {
            Operator::Or => {
                Query::new_disjunction(term_queries)
            }
            Operator::And => {
                Query::new_conjunction(term_queries)
            }<commit_msg>Make multi-match query use correct query analyzer for field<commit_after>//! Parses "multi_match" queries

use rustc_serialize::json::Json;

use search::term::Term;
use search::analysis::Analyzer;
use search::mapping::FieldMapping;

use search::query::Query;
use search::query::term_matcher::TermMatcher;
use search::query::term_scorer::TermScorer;
use search::query::parser::{QueryParseContext, QueryParseError};
use search::query::parser::utils::{parse_string, parse_float, Operator, parse_operator, parse_field_and_boost};


pub fn parse(context: &QueryParseContext, json: &Json) -> Result<Query, QueryParseError> {
    let object = try!(json.as_object().ok_or(QueryParseError::ExpectedObject));

    // Get configuration
    let mut fields_with_boosts = Vec::new();
    let mut query = String::new();
    let mut boost = 1.0f64;
    let mut operator = Operator::Or;

    let mut has_fields_key = false;
    let mut has_query_key = false;

    for (key, val) in object.iter() {
        match key.as_ref() {
            "fields" => {
                has_fields_key = true;

                match *val {
                    Json::Array(ref array) => {
                        for field in array.iter() {
                            fields_with_boosts.push(try!(parse_field_and_boost(field)));
                        }
                    }
                    _ => return Err(QueryParseError::ExpectedArray)
                }
            }
            "query" => {
                has_query_key = true;
                query = try!(parse_string(val));
            }
            "boost" => {
                boost = try!(parse_float(val));
            }
            "operator" => {
                operator = try!(parse_operator(val))
            }
            _ => return Err(QueryParseError::UnrecognisedKey(key.clone()))
        }
    }

    if !has_fields_key {
        return Err(QueryParseError::ExpectedKey("fields"))
    }

    if !has_query_key {
        return Err(QueryParseError::ExpectedKey("query"))
    }

    // Convert query string into term query objects
    let mut field_queries = Vec::new();
    for (field_name, field_boost) in fields_with_boosts {
        // Get mapping for field
        let field_mapping = match context.mappings {
            Some(mappings) => mappings.get_field(&field_name),
            None => None,
        };

        // Tokenise query string
        let tokens = match field_mapping {
            Some(ref field_mapping) => {
                field_mapping.process_value_for_query(Json::String(query.clone()))
            }
            None => {
                // TODO: Raise error?
                warn!("Unknown field: {}", field_name);

                FieldMapping::default().process_value_for_query(Json::String(query.clone()))
            }
        };

        let tokens = match tokens {
            Some(tokens) => tokens,
            None => {
                // Couldn't convert the passed in value into tokens
                // TODO: Raise error
                warn!("Unprocessable query: {}", query);

                vec![]
            }
        };

        let mut term_queries = Vec::new();
        for token in tokens {
            term_queries.push(Query::MatchTerm {
                field: field_name.clone(),
                term: token.term,
                matcher: TermMatcher::Exact,
                scorer: TermScorer::default(),
            });
        }

        let mut field_query = match operator {
            Operator::Or => {
                Query::new_disjunction(term_queries)
            }
            Operator::And => {
                Query::new_conjunction(term_queries)
            }
{'fraction_non_alphanumeric': 0.11660110149488592, 'fraction_numerical': 0.0012588512981904013, 'mean_word_length': 1.8851566046300499, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2225706', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1691, 'n_words': 354}