<commit_before>extern crate rustc_serialize;

pub use self::Ast::*;

use lexer::Lexer;
use lexer::Token;
use self::rustc_serialize::json::{Json};

use std::iter::Peekable;

/// Parses a JMESPath expression into an AST
pub fn parse(expr: &str) -> Result<Ast, ParseError> {
    Parser::new(expr).parse()
}

/// Represents the abstract syntax tree of a JMESPath expression.
#[derive(Clone, PartialEq, Debug)]
pub enum Ast {
    Comparison(Comparator, Box<Ast>, Box<Ast>),
    CurrentNode,
    Expref(Box<Ast>),
    Flatten(Box<Ast>),
    Function(char, Vec<Box<Ast>>),
    Identifier(String),
    Index(i32),
    Literal(Json),
    MultiList(Vec<Box<Ast>>),
    MultiHash(Vec<KeyValuePair>),
    ArrayProjection(Box<Ast>, Box<Ast>),
    ObjectProjection(Box<Ast>, Box<Ast>),
    Or(Box<Ast>, Box<Ast>),
    Slice(Option<i32>, Option<i32>, Option<i32>),
    Subexpr(Box<Ast>, Box<Ast>),
    WildcardIndex,
    WildcardValues,
}

/// Represents a key value pair in a multi-hash
#[derive(Clone, PartialEq, Debug)]
pub struct KeyValuePair {
    key: Box<Ast>,
    value: Box<Ast>
}

/// Comparators (i.e., less than, greater than, etc.)
#[derive(Clone, PartialEq, Debug)]
pub enum Comparator { Eq, Lt, Le, Ne, Ge, Gt }
<commit_msg>Removing unused nodes from AST<commit_after>extern crate rustc_serialize;

pub use self::Ast::*;

use lexer::Lexer;
use lexer::Token;
use self::rustc_serialize::json::{Json};

use std::iter::Peekable;

/// Parses a JMESPath expression into an AST
pub fn parse(expr: &str) -> Result<Ast, ParseError> {
    Parser::new(expr).parse()
}

/// Represents the abstract syntax tree of a JMESPath expression.
#[derive(Clone, PartialEq, Debug)]
pub enum Ast {
    Comparison(Comparator, Box<Ast>, Box<Ast>),
    CurrentNode,
    Expref(Box<Ast>),
    Flatten(Box<Ast>),
    Function(char, Vec<Box<Ast>>),
    Identifier(String),
    Index(i32),
    Literal(Json),
    MultiList(Vec<Box<Ast>>),
    MultiHash(Vec<KeyValuePair>),
    ArrayProjection(Box<Ast>, Box<Ast>),
    ObjectProjection(Box<Ast>, Box<Ast>),
    Or(Box<Ast>, Box<Ast>),
    Slice(Option<i32>, Option<i32>, Option<i32>),
    Subexpr(Box<Ast>, Box<Ast>),
}

/// Represents a key value pair in a multi-hash
#[derive(Clone, PartialEq, Debug)]
pub struct KeyValuePair {
    key: Box<Ast>,
    value: Box<Ast>
}

/// Comparators (i.e., less than, greater than, etc.)
#[derive(Clone, PartialEq, Debug)]
pub enum Comparator { Eq, Lt, Le, Ne, Ge, Gt }

{'fraction_non_alphanumeric': 0.1665292662819456, 'fraction_numerical': 0.006595218466611707, 'mean_word_length': 4.453932584269663, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793721', 'n_tokens_mistral': 917, 'n_tokens_neox': 871, 'n_words': 214}