<commit_before>//! This module defines a cheaply-copyable cursor into a TokenStream's data.
//!
//! It does this by copying the data into a stably-addressed structured buffer,
//! and holding raw pointers into that buffer to allow walking through delimited
//! sequences cheaply.
//!
//! This module is heavily commented as it contains the only unsafe code in
//! `syn`, and caution should be made when editing it. It provides a safe
//! interface, but is fragile internally.

use proc_macro2::*;

use std::ptr;
use std::fmt;
use std::marker::PhantomData;

/// Internal type which is used instead of `TokenTree` to represent a single
/// `TokenTree` within a `SynomBuffer`.
#[derive(Debug)]
enum Entry {
    /// Mimicing types from proc-macro.
    Group(Span, Delimiter, SynomBuffer),
    Term(Span, Term),
    Op(Span, char, Spacing),
    Literal(Span, Literal),
    /// End entries contain a raw pointer to the entry from the containing
    /// TokenTree.
    End(*const Entry),
}

/// A buffer of data which contains a structured representation of the input
/// `TokenStream` object.
#[derive(Debug)]
pub struct SynomBuffer {
    // NOTE: Do not derive clone on this - there are raw pointers inside which
    // will be messed up. Moving the `SynomBuffer` itself is safe as the actual
    // backing slices won't be moved.
    data: Box<[Entry]>,
}

impl SynomBuffer {
    // NOTE: DO NOT MUTATE THE `Vec` RETURNED FROM THIS FUNCTION ONCE IT
    // RETURNS, THE ADDRESS OF ITS BACKING MEMORY MUST REMAIN STABLE.
    fn inner_new(stream: TokenStream, up: *const Entry) -> SynomBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut seqs = Vec::new();
        for tt in stream.into_iter() {
            match tt.kind {
                TokenNode::Term(sym) => {
                    entries.push(Entry::Term(tt.span, sym));
                }
                TokenNode::Op(chr, ok) => {
                    entries.push(Entry::Op(tt.span, chr, ok));
                }
                TokenNode::Literal(lit) => {
                    entries.push(Entry::Literal(tt.span, lit));
                }
                TokenNode::Group(delim, seq_stream) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarially.
                    seqs.push((entries.len(), tt.span, delim, seq_stream));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));<commit_msg>Resolve clippy lint in cursor impl<commit_after>//! This module defines a cheaply-copyable cursor into a TokenStream's data.
//!
//! It does this by copying the data into a stably-addressed structured buffer,
//! and holding raw pointers into that buffer to allow walking through delimited
//! sequences cheaply.
//!
//! This module is heavily commented as it contains the only unsafe code in
//! `syn`, and caution should be made when editing it. It provides a safe
//! interface, but is fragile internally.

use proc_macro2::*;

use std::ptr;
use std::fmt;
use std::marker::PhantomData;

/// Internal type which is used instead of `TokenTree` to represent a single
/// `TokenTree` within a `SynomBuffer`.
#[derive(Debug)]
enum Entry {
    /// Mimicing types from proc-macro.
    Group(Span, Delimiter, SynomBuffer),
    Term(Span, Term),
    Op(Span, char, Spacing),
    Literal(Span, Literal),
    /// End entries contain a raw pointer to the entry from the containing
    /// TokenTree.
    End(*const Entry),
}

/// A buffer of data which contains a structured representation of the input
/// `TokenStream` object.
#[derive(Debug)]
pub struct SynomBuffer {
    // NOTE: Do not derive clone on this - there are raw pointers inside which
    // will be messed up. Moving the `SynomBuffer` itself is safe as the actual
    // backing slices won't be moved.
    data: Box<[Entry]>,
}

impl SynomBuffer {
    // NOTE: DO NOT MUTATE THE `Vec` RETURNED FROM THIS FUNCTION ONCE IT
    // RETURNS, THE ADDRESS OF ITS BACKING MEMORY MUST REMAIN STABLE.
    fn inner_new(stream: TokenStream, up: *const Entry) -> SynomBuffer {
        // Build up the entries list, recording the locations of any Groups
        // in the list to be processed later.
        let mut entries = Vec::new();
        let mut seqs = Vec::new();
        for tt in stream {
            match tt.kind {
                TokenNode::Term(sym) => {
                    entries.push(Entry::Term(tt.span, sym));
                }
                TokenNode::Op(chr, ok) => {
                    entries.push(Entry::Op(tt.span, chr, ok));
                }
                TokenNode::Literal(lit) => {
                    entries.push(Entry::Literal(tt.span, lit));
                }
                TokenNode::Group(delim, seq_stream) => {
                    // Record the index of the interesting entry, and store an
                    // `End(null)` there temporarially.
                    seqs.push((entries.len(), tt.span, delim, seq_stream));
                    entries.push(Entry::End(ptr::null()));
                }
            }
        }
        // Add an `End` entry to the end with a reference to the enclosing token
        // stream which was passed in.
        entries.push(Entry::End(up));
{'fraction_non_alphanumeric': 0.11010667148797686, 'fraction_numerical': 0.0003615982643283312, 'mean_word_length': 2.1995373048004625, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302216', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1471, 'n_words': 582}