<commit_before>#![deny(missing_docs)]

//! Meta parsing and encoding for data oriented design

extern crate read_token;
extern crate range;

pub use whitespace::{ Whitespace, WHITESPACE };
pub use parse_error_handler::{ ParseErrorHandler, ParseStdErr };
pub use parse_error::ParseError;
pub use ty::Type;
pub use token::Token;
pub use select::Select;
pub use parameter::Parameter;
pub use optional::Optional;
pub use until_any_or_whitespace::UntilAnyOrWhitespace;
pub use text::Text;
pub use number::Number;
pub use rule::Rule;
pub use meta_reader::MetaReader;

mod parse_error;
mod parse_error_handler;
mod ty;
mod token;
mod whitespace;
mod select;
mod parameter;
mod optional;
mod until_any_or_whitespace;
mod text;
mod number;
mod rule;
mod meta_reader;

/// Implemented by meta writers.
pub trait MetaWriter {
    /// Starts encoding a node.
    fn start_node(&mut self, name: &str);
    /// Ends encoding a node.
    fn end_node(&mut self, name: &str);
    /// Get bool property.
    fn get_as_bool(&mut self, name: &str) -> Option<bool>;
    /// Get str property.
    fn get_as_str<F, U>(&mut self, name: &str, f: F) -> Option<U>
        where F: FnOnce(&str) -> U;
    /// Get f64 property.
    fn get_as_f64(&mut self, name: &str) -> Option<f64>;
}
<commit_msg>Added `MetaData`<commit_after>#![deny(missing_docs)]

//! Meta parsing and encoding for data oriented design

extern crate read_token;
extern crate range;

pub use whitespace::{ Whitespace, WHITESPACE };
pub use parse_error_handler::{ ParseErrorHandler, ParseStdErr };
pub use parse_error::ParseError;
pub use ty::Type;
pub use token::Token;
pub use select::Select;
pub use parameter::Parameter;
pub use optional::Optional;
pub use until_any_or_whitespace::UntilAnyOrWhitespace;
pub use text::Text;
pub use number::Number;
pub use rule::Rule;
pub use meta_reader::MetaReader;

use std::rc::Rc;
use std::cell::RefCell;

mod parse_error;
mod parse_error_handler;
mod ty;
mod token;
mod whitespace;
mod select;
mod parameter;
mod optional;
mod until_any_or_whitespace;
mod text;
mod number;
mod rule;
mod meta_reader;

/// Represents meta data.
pub enum MetaData<'a> {
    /// Starts node.
    StartNode(&'a str),
    /// Ends node.
    EndNode,
    /// Sets bool property.
    Bool(&'a str, bool),
    /// Sets f64 property.
    F64(&'a str, f64),
    /// Sets string property.
    String(&'a str, String),
}

/// Implemented by meta writers.
pub trait MetaWriter {
    /// Starts encoding a node.
    fn start_node(&mut self, name: &str);
    /// Ends encoding a node.
    fn end_node(&mut self, name: &str);
    /// Get bool property.
    fn get_as_bool(&mut self, name: &str) -> Option<bool>;
    /// Get str property.
    fn get_as_str<F, U>(&mut self, name: &str, f: F) -> Option<U>
        where F: FnOnce(&str) -> U;
    /// Get f64 property.
    fn get_as_f64(&mut self, name: &str) -> Option<f64>;
}

{'fraction_non_alphanumeric': 0.1310392770246785, 'fraction_numerical': 0.006256517205422315, 'mean_word_length': 3.8532883642495785, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1239427', 'n_tokens_mistral': 984, 'n_tokens_neox': 967, 'n_words': 322}