<commit_before>// pest. The Elegant Parser
// Copyright (c) 2018 <NAME>
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

extern crate pest;
extern crate pest_meta;

use std::collections::HashMap;

use pest::{Atomicity, Error, ParserState, Position};
use pest::iterators::Pairs;

use pest_meta::ast::{Expr, Rule, RuleType};

pub struct Vm {
    rules: HashMap<String, Rule>
}

impl Vm {
    pub fn new(rules: Vec<Rule>) -> Vm {
        let rules = rules.into_iter().map(|r| (r.name.clone(), r)).collect();
        Vm { rules }
    }

    pub fn parse<'i>(
        &self,
        rule: &str,
        input: &'i str
    ) -> Option<Result<Pairs<'i, &str>, Error<'i, &str>>> {
        if let Some(rule) = self.rules.get(rule) {
            let result = pest::state(input, |mut state, pos| {
                self.parse_rule(input, rule, pos, &mut state)
            });

            Some(result)
        } else {
            None
        }
    }

    fn parse_rule<'a, 'i>(
        &self,
        input: &'i str,
        rule: &'a Rule,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        let is_atomic = (rule.ty == RuleType::Atomic || rule.ty == RuleType::CompoundAtomic)
            || &rule.name == "whitespace" || &rule.name == "comment";

        match rule.ty {
            RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                self.parse_expr(input, &rule.expr, is_atomic, pos, state)
            }),
            RuleType::Silent => self.parse_expr(input, &rule.expr, is_atomic, pos, state),
            RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                state.atomic(Atomicity::Atomic, move |state| {
                    self.parse_expr(input, &rule.expr, is_atomic, pos, state)
                })
            }),
            RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                state.rule(&rule.name, pos, |state, pos| {
                    self.parse_expr(input, &rule.expr, is_atomic, pos, state)
                })
            }),
            RuleType::NonAtomic => state.atomic(Atomicity::NonAtomic, move |state| {
                state.rule(&rule.name, pos, |state, pos| {
                    self.parse_expr(input, &rule.expr, is_atomic, pos, state)
                })
            })
        }
    }

    fn parse_expr<'i>(
        &self,
        input: &'i str,
        expr: &Expr,
        is_atomic: bool,
        pos: Position,
        state: &mut ParserState<'i, &str>
    ) -> Result<Position<'i>, Position<'i>> {
        unimplemented!()
    }
}<commit_msg>Added skip to VM.<commit_after>// pest. The Elegant Parser
// Copyright (c) 2018 <NAME>
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

extern crate pest;
extern crate pest_meta;

use std::collections::HashMap;

use pest::{Atomicity, Error, ParserState, Position};
use pest::iterators::Pairs;

use pest_meta::ast::{Expr, Rule, RuleType};

pub struct Vm {
    rules: HashMap<String, Rule>
}

impl Vm {
    pub fn new(rules: Vec<Rule>) -> Vm {
        let rules = rules.into_iter().map(|r| (r.name.clone(), r)).collect();
        Vm { rules }
    }

    pub fn parse<'i>(
        &self,
        rule: &str,
        input: &'i str
    ) -> Option<Result<Pairs<'i, &str>, Error<'i, &str>>> {
        if let Some(rule) = self.rules.get(rule) {
            let result = pest::state(input, |mut state, pos| {
                self.parse_rule(rule, pos, &mut state)
            });

            Some(result)
        } else {
            None
        }
    }

    fn parse_rule<'a, 'i>(
        &self,
        rule: &'a Rule,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        if &rule.name == "whitespace" || &rule.name == "comment" {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::Silent => state.atomic(Atomicity::Atomic, move |state| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::Atomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        } else {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Silent => self.parse_expr(&rule.expr, pos, state),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::NonAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        }
    }

    fn parse_expr<'i>(
        &self,
        expr: &Expr,
        pos: Position<'i>,
        state: &mut ParserState<'i, &str>
    ) -> Result<Position<'i>, Position<'i>> {
        unimplemented!()
    }

    fn skip<'a, 'i>(
        &'a self,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        match (self.rules.get("whitespace"), self.rules.get("comment")) {
            (None, None) => Ok(pos),
            (Some(whitespace), None) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(whitespace, pos, state)
                })
            } else {
                Ok(pos)
            },
            (None, Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(comment, pos, state)
                })
            } else {
                Ok(pos)
            },
            (Some(whitespace), Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                state.sequence(move |state| {
                    pos.sequence(|pos| {
                        pos.repeat(|pos| {
                            self.parse_rule(whitespace, pos, state)
                        }).and_then(|pos| {
                            pos.repeat(|pos| {
                                state.sequence(move |state| {
                                    pos.sequence(|pos| {
                                        self.parse_rule(comment, pos, state).and_then(|pos| {
                                            pos.repeat(|pos| {
                                                self.parse_rule(whitespace, pos, state)
                                            })
                                        })
                                    })
                                })
                            })
                        })
                    })
                })
            } else {
                Ok(pos)
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.14153079710144928, 'fraction_numerical': 0.0018115942028985507, 'mean_word_length': 1.474922947604371, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1401633', 'n_tokens_mistral': 2601, 'n_tokens_neox': 2522, 'n_words': 549}