<commit_before>use crate::parse::{ParseBuffer, ParseStream};
use proc_macro2::TokenStream;
use std::iter;

pub fn between<'a>(begin: ParseBuffer<'a>, end: ParseStream<'a>) -> TokenStream {
    let end = end.cursor();
    let mut cursor = begin.cursor();
    let mut tokens = TokenStream::new();
    while cursor != end {
        let (tt, next) = cursor.token_tree().unwrap();
        tokens.extend(iter::once(tt));
        cursor = next;
    }
    tokens
}<commit_msg>Handle the case where verbatim enters a None group<commit_after>use crate::parse::{ParseBuffer, ParseStream};
use proc_macro2::{Delimiter, TokenStream};
use std::iter;

pub fn between<'a>(begin: ParseBuffer<'a>, end: ParseStream<'a>) -> TokenStream {
    let end = end.cursor();
    let mut cursor = begin.cursor();
    let mut tokens = TokenStream::new();
    while cursor != end {
        let (tt, next) = cursor.token_tree().unwrap();

        if next > end {
            // In some edge cases, a syntax node can actually cross the border
            // of a None-delimited group, due to such groups being transparent
            // to the parser in most cases. In the cases that this can occur,
            // the presence of the group is known to be semantically irrelevant,
            // so we should just ignore the presence of the group. (Issue #1235)
            if let Some((inside, _span, after)) = cursor.group(Delimiter::None) {
                assert!(next == after);
                cursor = inside;
                continue;
            } else {
                panic!("verbatim end must not be inside a delimited group");
            }
        }

        tokens.extend(iter::once(tt));
        cursor = next;
    }
    tokens
}
{'fraction_non_alphanumeric': 0.12200817279626386, 'fraction_numerical': 0.0035026269702276708, 'mean_word_length': 2.4211576846307383, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3463048', 'n_tokens_mistral': 475, 'n_tokens_neox': 470, 'n_words': 163}