<commit_before>use super::bit;
use super::error::{Error, ErrorKind};
use super::query::Query;
use super::result::Result;
use super::utf8::{COMMA, CR, HT, LF, RIGHT_BRACE, SPACE};
use fnv::{FnvHashMap, FnvHashSet};

#[inline]
pub fn basic_parse<'a>(
    rec: &'a [u8],
    index: &[Vec<u64>],
    queries: &mut FnvHashMap<&[u8], Query>,
    start: usize,
    end: usize,
    level: usize,
    stats: &mut Vec<FnvHashSet<usize>>,
    set_stats: bool,
    results: &mut Vec<Option<&'a [u8]>>,
    b_quote: &[u64],
    colon_positions: &mut Vec<Vec<usize>>,
) -> Result<()> {
    generate_colon_positions(index, start, end, level, colon_positions);

    let query_num = queries.len();
    let mut found_num = 0;
    let mut vei = end;
    let cp_len = colon_positions[level].len();
    for i in (0..cp_len).rev() {
        let (fsi, fei) = search_pre_field_indices(
            b_quote,
            if i > 0 {
                colon_positions[level][i - 1]
            } else {
                start
            },
            colon_positions[level][i],
        )?;
        let field = &rec[fsi + 1..fei];
        if let Some(query) = queries.get_mut(field) {
            let (vsi, vei) = search_post_value_indices(
                rec,
                colon_positions[level][i] + 1,
                vei,
                if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
            )?;
            found_num += 1;
            if set_stats && !stats[query.i].contains(&i) {
                stats[query.i].insert(i);
            }
            if let Some(ref mut children) = query.children {
                basic_parse(
                    rec,
                    index,
                    children,
                    vsi,
                    vei,
                    level + 1,
                    stats,
                    set_stats,
                    results,
                    b_quote,
                    colon_positions,
                )?;
            }
            if query.target {
                results[query.ri] = Some(&rec[vsi..vei + 1]);
            }
            if found_num == query_num {
                return Ok(());
            }
        }
        vei = fsi - 1;
    }
    Ok(())
}

#[inline]
pub fn speculative_parse<'a>(rec: &'a [u8], index: &[Vec<u64>], queries: &FnvHashMap<&[u8], Query>, start: usize, end: usize, level: usize, stats: &[FnvHashSet<usize>], results: &mut Vec<Option<&'a [u8]>>, b_quote: &[u64], colon_positions: &mut Vec<Vec<usize>>) -> Result<bool> {
    generate_colon_positions(index, start, end, level, colon_positions);

    for (&s, q) in queries {
        let mut found = false;
        for &i in &stats[q.i] {
            let cp_len = colon_positions[level].len();
            if i >= cp_len {
                continue;
            }<commit_msg>refactor(parser): remove unnecessary `mut` flag from arguments<commit_after>use super::bit;
use super::error::{Error, ErrorKind};
use super::query::Query;
use super::result::Result;
use super::utf8::{COMMA, CR, HT, LF, RIGHT_BRACE, SPACE};
use fnv::{FnvHashMap, FnvHashSet};

#[inline]
pub fn basic_parse<'a>(
    rec: &'a [u8],
    index: &[Vec<u64>],
    queries: &FnvHashMap<&[u8], Query>,
    start: usize,
    end: usize,
    level: usize,
    stats: &mut Vec<FnvHashSet<usize>>,
    set_stats: bool,
    results: &mut Vec<Option<&'a [u8]>>,
    b_quote: &[u64],
    colon_positions: &mut Vec<Vec<usize>>,
) -> Result<()> {
    generate_colon_positions(index, start, end, level, colon_positions);

    let mut found_num = 0;
    let mut vei = end;
    let cp_len = colon_positions[level].len();
    for i in (0..cp_len).rev() {
        let (fsi, fei) = search_pre_field_indices(
            b_quote,
            if i > 0 {
                colon_positions[level][i - 1]
            } else {
                start
            },
            colon_positions[level][i],
        )?;
        let field = &rec[fsi + 1..fei];
        if let Some(query) = queries.get(field) {
            let (vsi, vei) = search_post_value_indices(
                rec,
                colon_positions[level][i] + 1,
                vei,
                if i == cp_len - 1 { RIGHT_BRACE } else { COMMA },
            )?;
            found_num += 1;
            if set_stats && !stats[query.i].contains(&i) {
                stats[query.i].insert(i);
            }
            if let Some(ref children) = query.children {
                basic_parse(
                    rec,
                    index,
                    children,
                    vsi,
                    vei,
                    level + 1,
                    stats,
                    set_stats,
                    results,
                    b_quote,
                    colon_positions,
                )?;
            }
            if query.target {
                results[query.ri] = Some(&rec[vsi..vei + 1]);
            }
            if found_num == queries.len() {
                return Ok(());
            }
        }
        vei = fsi - 1;
    }
    Ok(())
}

#[inline]
pub fn speculative_parse<'a>(rec: &'a [u8], index: &[Vec<u64>], queries: &FnvHashMap<&[u8], Query>, start: usize, end: usize, level: usize, stats: &[FnvHashSet<usize>], results: &mut Vec<Option<&'a [u8]>>, b_quote: &[u64], colon_positions: &mut Vec<Vec<usize>>) -> Result<bool> {
    generate_colon_positions(index, start, end, level, colon_positions);

    for (&s, q) in queries {
        let mut found = false;
        for &i in &stats[q.i] {
            let cp_len = colon_positions[level].len();
            if i >= cp_len {
                continue;
            }
{'fraction_non_alphanumeric': 0.14069186875891584, 'fraction_numerical': 0.009272467902995721, 'mean_word_length': 1.602784222737819, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7268506', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1787, 'n_words': 411}