<commit_before>// pest. The Elegant Parser
// Copyright (C) 2017  <NAME>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use ast::*;

pub fn optimize(rules: Vec<Rule>) -> Vec<Rule> {
    rules.into_iter().map(|rule| {
        let rotate_right = |expr| {
            // TODO: Use box syntax when it gets stabilized.
            match expr {
                Expr::Seq(lhs, rhs) => {
                    let lhs = *lhs;
                    match lhs {
                        Expr::Seq(ll, lr) => {
                            Expr::Seq(
                                ll,
                                Box::new(Expr::Seq(lr, rhs))
                            )
                        }
                        lhs => Expr::Seq(Box::new(lhs), rhs)
                    }
                }
                Expr::Choice(lhs, rhs) => {
                    let lhs = *lhs;
                    match lhs {
                        Expr::Choice(ll, lr) => {
                            Expr::Choice(
                                ll,
                                Box::new(Expr::Choice(lr, rhs))
                            )
                        }
                        lhs => Expr::Choice(Box::new(lhs), rhs)
                    }
                }
                expr => expr
            }
        };

        let unpack_repetitions = |expr| {
            match expr {
                Expr::RepOnce(expr) => {
                    Expr::Seq(
                        expr.clone(),
                        Box::new(Expr::Rep(expr))
                    )
                }
                Expr::RepExact(expr, num) => {
                    (1..num + 1).map(|_| {
                                    *expr.clone()
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMin(expr, min) => {
                    (1..min + 2).map(|i| {
                                    if i <= min {
                                        *expr.clone()
                                    } else {
                                        Expr::Rep(expr.clone())
                                    }
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMax(expr, max) => {
                    (1..max + 1).map(|_| {
                                    Expr::Opt(expr.clone())
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMinMax(expr, min, max) => {
                    (1..max + 1).map(|i| {
                                    if i <= min {
                                        *expr.clone()
                                    } else {
                                        Expr::Opt(expr.clone())
                                    }
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                expr => expr
            }
        };

        match rule {
            Rule { name, ty, expr } => Rule {
                name,
                ty,
                expr: expr.map_bottom_up(rotate_right)
                          .map_bottom_up(unpack_repetitions)
                          .map_bottom_up(|expr| {
                              if ty == RuleType::Atomic {
                                  // TODO: Use box syntax when it gets stabilized.
                                  match expr {
                                      Expr::Seq(lhs, rhs) => {
                                          match (*lhs, *rhs) {
                                              (Expr::Str(lhs), Expr::Str(rhs)) => {
                                                  Expr::Str(lhs + &rhs)
                                              }
                                              (Expr::Insens(lhs), Expr::Insens(rhs)) => {
                                                  Expr::Insens(lhs + &rhs)
                                              }
                                              (lhs, rhs) => Expr::Seq(Box::new(lhs), Box::new(rhs))
                                          }
                                      }
                                      expr => expr
                                  }
                              } else {
                                  expr
                              }
                          })
                          .map_top_down(|expr| {<commit_msg>Renamed closure.<commit_after>// pest. The Elegant Parser
// Copyright (C) 2017  <NAME>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use ast::*;

pub fn optimize(rules: Vec<Rule>) -> Vec<Rule> {
    rules.into_iter().map(|rule| {
        let rotate_right = |expr| {
            // TODO: Use box syntax when it gets stabilized.
            match expr {
                Expr::Seq(lhs, rhs) => {
                    let lhs = *lhs;
                    match lhs {
                        Expr::Seq(ll, lr) => {
                            Expr::Seq(
                                ll,
                                Box::new(Expr::Seq(lr, rhs))
                            )
                        }
                        lhs => Expr::Seq(Box::new(lhs), rhs)
                    }
                }
                Expr::Choice(lhs, rhs) => {
                    let lhs = *lhs;
                    match lhs {
                        Expr::Choice(ll, lr) => {
                            Expr::Choice(
                                ll,
                                Box::new(Expr::Choice(lr, rhs))
                            )
                        }
                        lhs => Expr::Choice(Box::new(lhs), rhs)
                    }
                }
                expr => expr
            }
        };

        let unroll_loops = |expr| {
            match expr {
                Expr::RepOnce(expr) => {
                    Expr::Seq(
                        expr.clone(),
                        Box::new(Expr::Rep(expr))
                    )
                }
                Expr::RepExact(expr, num) => {
                    (1..num + 1).map(|_| {
                                    *expr.clone()
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMin(expr, min) => {
                    (1..min + 2).map(|i| {
                                    if i <= min {
                                        *expr.clone()
                                    } else {
                                        Expr::Rep(expr.clone())
                                    }
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMax(expr, max) => {
                    (1..max + 1).map(|_| {
                                    Expr::Opt(expr.clone())
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                Expr::RepMinMax(expr, min, max) => {
                    (1..max + 1).map(|i| {
                                    if i <= min {
                                        *expr.clone()
                                    } else {
                                        Expr::Opt(expr.clone())
                                    }
                                })
                                .rev()
                                .fold(None, |rep, expr| {
                                    match rep {
                                        None => Some(expr),
                                        Some(rep) => {
                                            Some(
                                                Expr::Seq(
                                                    Box::new(expr),
                                                    Box::new(rep)
                                                )
                                            )
                                        }
                                    }
                                })
                                .unwrap()
                }
                expr => expr
            }
        };

        match rule {
            Rule { name, ty, expr } => Rule {
                name,
                ty,
                expr: expr.map_bottom_up(rotate_right)
                          .map_bottom_up(unroll_loops)
                          .map_bottom_up(|expr| {
                              if ty == RuleType::Atomic {
                                  // TODO: Use box syntax when it gets stabilized.
                                  match expr {
                                      Expr::Seq(lhs, rhs) => {
                                          match (*lhs, *rhs) {
                                              (Expr::Str(lhs), Expr::Str(rhs)) => {
                                                  Expr::Str(lhs + &rhs)
                                              }
                                              (Expr::Insens(lhs), Expr::Insens(rhs)) => {
                                                  Expr::Insens(lhs + &rhs)
                                              }
                                              (lhs, rhs) => Expr::Seq(Box::new(lhs), Box::new(rhs))
                                          }
                                      }
                                      expr => expr
                                  }
                              } else {
                                  expr
                              }
                          })
                          .map_top_down(|expr| {
{'fraction_non_alphanumeric': 0.08347278934332954, 'fraction_numerical': 0.0021420443135417365, 'mean_word_length': 0.3763242745278673, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 222, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1978492', 'n_tokens_mistral': 2770, 'n_tokens_neox': 2549, 'n_words': 489}