<commit_before>use std::fmt;
use std::ops;
use std::slice;
use std::vec;

use features::Features;

/// A sentence with the CoNLL-X annotation layers.
///
/// This data type is a small wrapper around `Vec<Token>` that provides some
/// extra convenience. For example, the implementation of the `Display` trait
/// outputs the sentence in CoNLL-X format.
#[derive(Clone, Debug, PartialEq)]
pub struct Sentence {
    tokens: Vec<Token>,
}

impl Sentence {
    /// Create a new `Sentence` from a vector of tokens.
    pub fn new(tokens: Vec<Token>) -> Sentence {
        Sentence { tokens: tokens }
    }

    /// Get the sentence tokens as a slice.
    pub fn as_tokens(&self) -> &[Token] {
        &self.tokens
    }

    /// Get the sentence tokens as a mutable slice.
    pub fn as_tokens_mut(&mut self) -> &mut [Token] {
        &mut self.tokens
    }

    /// Get an iterator over the sentence tokens.
    pub fn iter(&self) -> slice::Iter<Token> {
        self.tokens.iter()
    }

    /// Get a mutable iterator over the sentence tokens.
    pub fn iter_mut(&mut self) -> slice::IterMut<Token> {
        self.tokens.iter_mut()
    }
}

impl fmt::Display for Sentence {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let last_id = self.tokens.len() - 1;

        for (id, token) in self.iter().enumerate() {
            if id == last_id {
                write!(f, "{}\t{}", id + 1, token)?
            } else {
                write!(f, "{}\t{}\n", id + 1, token)?
            }
        }

        Ok(())
    }
}

impl ops::Index<usize> for Sentence {<commit_msg>Implement From<Sentence> for Vec<Token>.<commit_after>use std::fmt;
use std::ops;
use std::slice;
use std::vec;

use features::Features;

/// A sentence with the CoNLL-X annotation layers.
///
/// This data type is a small wrapper around `Vec<Token>` that provides some
/// extra convenience. For example, the implementation of the `Display` trait
/// outputs the sentence in CoNLL-X format.
#[derive(Clone, Debug, PartialEq)]
pub struct Sentence {
    tokens: Vec<Token>,
}

impl Sentence {
    /// Create a new `Sentence` from a vector of tokens.
    pub fn new(tokens: Vec<Token>) -> Sentence {
        Sentence { tokens: tokens }
    }

    /// Get the sentence tokens as a slice.
    pub fn as_tokens(&self) -> &[Token] {
        &self.tokens
    }

    /// Get the sentence tokens as a mutable slice.
    pub fn as_tokens_mut(&mut self) -> &mut [Token] {
        &mut self.tokens
    }

    /// Get an iterator over the sentence tokens.
    pub fn iter(&self) -> slice::Iter<Token> {
        self.tokens.iter()
    }

    /// Get a mutable iterator over the sentence tokens.
    pub fn iter_mut(&mut self) -> slice::IterMut<Token> {
        self.tokens.iter_mut()
    }
}

impl fmt::Display for Sentence {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let last_id = self.tokens.len() - 1;

        for (id, token) in self.iter().enumerate() {
            if id == last_id {
                write!(f, "{}\t{}", id + 1, token)?
            } else {
                write!(f, "{}\t{}\n", id + 1, token)?
            }
        }

        Ok(())
    }
}

impl From<Sentence> for Vec<Token> {
    fn from(sentence: Sentence) -> Self {
        sentence.tokens
    }
}

impl ops::Index<usize> for Sentence {
{'fraction_non_alphanumeric': 0.14458194989435558, 'fraction_numerical': 0.0018110473890733474, 'mean_word_length': 2.230019493177388, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591228', 'n_tokens_mistral': 1060, 'n_tokens_neox': 1026, 'n_words': 340}