<commit_before>use core::{BytePos, ByteRange};

/// Type of the string
#[derive(Clone, Copy, Debug)]
enum StrStyle {
    /// normal string starts with "
    Cooked,
    /// Raw(n) => raw string started with n #s
    Raw(usize),
}

#[derive(Clone, Copy)]
enum State {
    Code,
    Comment,
    CommentBlock,
    String(StrStyle),
    Char,
    Finished,
}

#[derive(Clone, Copy)]
pub struct CodeIndicesIter<'a> {
    src: &'a str,
    pos: BytePos,
    state: State,
}

impl<'a> Iterator for CodeIndicesIter<'a> {
    type Item = ByteRange;

    fn next(&mut self) -> Option<ByteRange> {
        match self.state {
            State::Code => Some(self.code()),
            State::Comment => Some(self.comment()),
            State::CommentBlock => Some(self.comment_block()),
            State::String(style) => Some(self.string(style)),
            State::Char => Some(self.char()),
            State::Finished => None,
        }
    }
}

impl<'a> CodeIndicesIter<'a> {
    fn code(&mut self) -> ByteRange {
        let mut pos = self.pos;
        let start = match self.state {
            State::String(_) | State::Char => pos.decrement(), // include quote
            _ => pos,
        };
        let src_bytes = self.src.as_bytes();
        for &b in &src_bytes[pos.0..] {
            pos = pos.increment();
            match b {
                b'/' if src_bytes.len() > pos.0 => match src_bytes[pos.0] {
                    b'/' => {
                        self.state = State::Comment;
                        self.pos = pos.increment();
                        return ByteRange::new(start, pos.decrement());
                    }
                    b'*' => {
                        self.state = State::CommentBlock;
                        self.pos = pos.increment();
                        return ByteRange::new(start, pos.decrement());
                    }
                    _ => {}
                },
                b'"' => {
                    // "
                    let str_type = self.detect_str_type(pos);
                    self.state = State::String(str_type);
                    self.pos = pos;
                    return ByteRange::new(start, pos); // include dblquotes
                }
                b'\'' => {
                    // single quotes are also used for lifetimes, so we need to
                    // be confident that this is not a lifetime.
                    // Look for backslash starting the escape, or a closing quote:
                    if src_bytes.len() > pos.increment().0
                        && (src_bytes[pos.0] == b'\\' || src_bytes[pos.increment().0] == b'\'')
                    {
                        self.state = State::Char;
                        self.pos = pos;
                        return ByteRange::new(start, pos); // include single quote
                    }
                }
                _ => {}
            }
        }

        self.state = State::Finished;
        ByteRange::new(start, self.src.len().into())
    }

    fn comment(&mut self) -> ByteRange {
        let mut pos = self.pos;
        let src_bytes = self.src.as_bytes();
        for &b in &src_bytes[pos.0..] {
            pos = pos.increment();
            if b == b'\n' {
                if pos.0 + 2 <= src_bytes.len() && src_bytes[pos.0..pos.0 + 2] == [b'/', b'/'] {
                    continue;
                }
                break;
            }
        }
        self.pos = pos;
        self.code()
    }

    fn comment_block(&mut self) -> ByteRange {
        let mut nesting_level = 0usize;
        let mut prev = b' ';
        let mut pos = self.pos;
        for &b in &self.src.as_bytes()[pos.0..] {
            pos = pos.increment();
            match b {
                b'/' if prev == b'*' => {
                    if nesting_level == 0 {
                        break;
                    } else {
                        nesting_level -= 1;
                    }
                }
                b'*' if prev == b'/' => {
                    nesting_level += 1;
                }
                _ => {
                    prev = b;<commit_msg>When parsing a block comment, set `prev` to `b' '`  after a '/*' or a '*/' has been found.<commit_after>use core::{BytePos, ByteRange};

/// Type of the string
#[derive(Clone, Copy, Debug)]
enum StrStyle {
    /// normal string starts with "
    Cooked,
    /// Raw(n) => raw string started with n #s
    Raw(usize),
}

#[derive(Clone, Copy)]
enum State {
    Code,
    Comment,
    CommentBlock,
    String(StrStyle),
    Char,
    Finished,
}

#[derive(Clone, Copy)]
pub struct CodeIndicesIter<'a> {
    src: &'a str,
    pos: BytePos,
    state: State,
}

impl<'a> Iterator for CodeIndicesIter<'a> {
    type Item = ByteRange;

    fn next(&mut self) -> Option<ByteRange> {
        match self.state {
            State::Code => Some(self.code()),
            State::Comment => Some(self.comment()),
            State::CommentBlock => Some(self.comment_block()),
            State::String(style) => Some(self.string(style)),
            State::Char => Some(self.char()),
            State::Finished => None,
        }
    }
}

impl<'a> CodeIndicesIter<'a> {
    fn code(&mut self) -> ByteRange {
        let mut pos = self.pos;
        let start = match self.state {
            State::String(_) | State::Char => pos.decrement(), // include quote
            _ => pos,
        };
        let src_bytes = self.src.as_bytes();
        for &b in &src_bytes[pos.0..] {
            pos = pos.increment();
            match b {
                b'/' if src_bytes.len() > pos.0 => match src_bytes[pos.0] {
                    b'/' => {
                        self.state = State::Comment;
                        self.pos = pos.increment();
                        return ByteRange::new(start, pos.decrement());
                    }
                    b'*' => {
                        self.state = State::CommentBlock;
                        self.pos = pos.increment();
                        return ByteRange::new(start, pos.decrement());
                    }
                    _ => {}
                },
                b'"' => {
                    // "
                    let str_type = self.detect_str_type(pos);
                    self.state = State::String(str_type);
                    self.pos = pos;
                    return ByteRange::new(start, pos); // include dblquotes
                }
                b'\'' => {
                    // single quotes are also used for lifetimes, so we need to
                    // be confident that this is not a lifetime.
                    // Look for backslash starting the escape, or a closing quote:
                    if src_bytes.len() > pos.increment().0
                        && (src_bytes[pos.0] == b'\\' || src_bytes[pos.increment().0] == b'\'')
                    {
                        self.state = State::Char;
                        self.pos = pos;
                        return ByteRange::new(start, pos); // include single quote
                    }
                }
                _ => {}
            }
        }

        self.state = State::Finished;
        ByteRange::new(start, self.src.len().into())
    }

    fn comment(&mut self) -> ByteRange {
        let mut pos = self.pos;
        let src_bytes = self.src.as_bytes();
        for &b in &src_bytes[pos.0..] {
            pos = pos.increment();
            if b == b'\n' {
                if pos.0 + 2 <= src_bytes.len() && src_bytes[pos.0..pos.0 + 2] == [b'/', b'/'] {
                    continue;
                }
                break;
            }
        }
        self.pos = pos;
        self.code()
    }

    fn comment_block(&mut self) -> ByteRange {
        let mut nesting_level = 0usize;
        let mut prev = b' ';
        let mut pos = self.pos;
        for &b in &self.src.as_bytes()[pos.0..] {
            pos = pos.increment();
            match b {
                b'/' if prev == b'*' => {
                    prev = b' ';
                    if nesting_level == 0 {
                        break;
                    } else {
                        nesting_level -= 1;
                    }
                }
                b'*' if prev == b'/' => {
                    prev = b' ';
                    nesting_level += 1;
                }
                _ => {
                    prev = b;
{'fraction_non_alphanumeric': 0.13654235672584297, 'fraction_numerical': 0.00405099487668295, 'mean_word_length': 1.14955185659411, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6311044', 'n_tokens_mistral': 2393, 'n_tokens_neox': 2304, 'n_words': 548}