<commit_before>use {Token, SyntaxKind, TextUnit};
use super::{Event};
use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
use tree::EOF;


pub(crate) struct Parser<'t> {
    #[allow(unused)]
    text: &'t str,
    #[allow(unused)]
    start_offsets: Vec<TextUnit>,
    tokens: Vec<Token>, // non-whitespace tokens

    pos: usize,
    events: Vec<Event>,

    curly_level: i32,
    curly_limit: Option<i32>,
}

impl<'t> Parser<'t> {
    pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
        let mut tokens = Vec::new();
        let mut start_offsets = Vec::new();
        let mut len = TextUnit::new(0);
        for &token in raw_tokens.iter() {
            if !is_insignificant(token.kind) {
                tokens.push(token);
                start_offsets.push(len);
            }
            len += token.len;
        }

        Parser {
            text,
            start_offsets,
            tokens,

            pos: 0,
            events: Vec::new(),
            curly_level: 0,
            curly_limit: None,
        }
    }

    pub(crate) fn pos(&self) -> usize {
        self.pos
    }

    pub(crate) fn into_events(self) -> Vec<Event> {
        assert!(self.curly_limit.is_none());<commit_msg>Encapsulate position<commit_after>use {Token, SyntaxKind, TextUnit};
use super::{Event};
use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
use tree::EOF;


pub(crate) struct Parser<'t> {
    #[allow(unused)]
    text: &'t str,
    #[allow(unused)]
    start_offsets: Vec<TextUnit>,
    tokens: Vec<Token>, // non-whitespace tokens

    pos: usize,
    events: Vec<Event>,

    curly_level: i32,
    curly_limit: Option<i32>,
}

#[derive(Debug, Clone, Copy,PartialEq, Eq)]
pub(crate) struct Pos(u32);

impl<'t> Parser<'t> {
    pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
        let mut tokens = Vec::new();
        let mut start_offsets = Vec::new();
        let mut len = TextUnit::new(0);
        for &token in raw_tokens.iter() {
            if !is_insignificant(token.kind) {
                tokens.push(token);
                start_offsets.push(len);
            }
            len += token.len;
        }

        Parser {
            text,
            start_offsets,
            tokens,

            pos: 0,
            events: Vec::new(),
            curly_level: 0,
            curly_limit: None,
        }
    }

    pub(crate) fn pos(&self) -> Pos {
        Pos(self.pos as u32)
    }

    pub(crate) fn into_events(self) -> Vec<Event> {
        assert!(self.curly_limit.is_none());
{'fraction_non_alphanumeric': 0.14732824427480917, 'fraction_numerical': 0.006870229007633588, 'mean_word_length': 2.1128266033254155, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4208132', 'n_tokens_mistral': 912, 'n_tokens_neox': 911, 'n_words': 191}