<commit_before>use std::borrow::ToOwned;
use std::collections::HashMap;
use quote::Tokens;
pub use self::Expr::*;

#[derive(Debug)]
pub struct Error {
	message: String,
}

impl From<Error> for String {
	fn from(e: Error) -> String {
		e.message
	}
}

fn raw(s: &str) -> Tokens {
	let mut t = Tokens::new();
	t.append(s);
	t
}

pub struct Grammar {
	pub imports: Vec<String>,
	pub rules: Vec<Rule>,
	pub templates: HashMap<String, Template>,
	pub args: Vec<(String, String)>,
}

impl Grammar {
	pub fn from_ast(items: Vec<Item>) -> Result<Grammar, Error> {
		let mut imports = Vec::new();
		let mut rules = Vec::new();
		let mut templates = HashMap::new();

		let mut grammar_args = None;

		for item in items {
			match item {
				Item::Use(u) => { imports.push(u); }
				Item::Rule(rule) => { rules.push(rule); }
				Item::Template(template) => { templates.insert(template.name.clone(), template); }
				Item::GrammarArgs(args) => {
					if grammar_args.is_none() {
						grammar_args = Some(args);
					} else {
						Err(Error{ message: format!("Multiple #![grammar_args] attributes") })?
					}
				}
			}
		}
		Ok(Grammar{ imports:imports, rules:rules, templates:templates, args: grammar_args.unwrap_or(vec![]) })
	}

	fn find_rule(&self, name: &str) -> Option<&Rule> {
		self.rules.iter().find(|rule| rule.name == name)
	}

	fn extra_args_def(&self) -> Tokens {
		let args: Vec<Tokens> = self.args.iter().map(|&(ref name, ref tp)| {
			let name = raw(name);
			let tp = raw(tp);
			quote!(, #name: #tp)
		}).collect();
		quote!(#(#args)*)
	}

	fn extra_args_call(&self) -> Tokens {
		let args: Vec<Tokens> = self.args.iter().map(|&(ref name, _)| {
			let name = raw(name);
			quote!(, #name)
		}).collect();
		quote!(#(#args)*)
	}

	fn check_name(&self, name: &str) -> Result<(), Error> {
		if name.starts_with("__") {
			Err(Error{ message: format!("Capture variable `{}` begins with `__`, which is reserved for use by the parser generator", name) })?
		}

		if self.args.iter().any(|x| x.0 == name) {
			Err(Error{ message: format!("Capture variable `{}` shadows grammar argument", name) })?
		}

		Ok(())<commit_msg>Temporarily revert backwards-incompatible part of 4b7a7fd6<commit_after>use std::borrow::ToOwned;
use std::collections::HashMap;
use quote::Tokens;
pub use self::Expr::*;

#[derive(Debug)]
pub struct Error {
	message: String,
}

impl From<Error> for String {
	fn from(e: Error) -> String {
		e.message
	}
}

fn raw(s: &str) -> Tokens {
	let mut t = Tokens::new();
	t.append(s);
	t
}

pub struct Grammar {
	pub imports: Vec<String>,
	pub rules: Vec<Rule>,
	pub templates: HashMap<String, Template>,
	pub args: Vec<(String, String)>,
}

impl Grammar {
	pub fn from_ast(items: Vec<Item>) -> Result<Grammar, Error> {
		let mut imports = Vec::new();
		let mut rules = Vec::new();
		let mut templates = HashMap::new();

		let mut grammar_args = None;

		for item in items {
			match item {
				Item::Use(u) => { imports.push(u); }
				Item::Rule(rule) => { rules.push(rule); }
				Item::Template(template) => { templates.insert(template.name.clone(), template); }
				Item::GrammarArgs(args) => {
					if grammar_args.is_none() {
						grammar_args = Some(args);
					} else {
						Err(Error{ message: format!("Multiple #![grammar_args] attributes") })?
					}
				}
			}
		}
		Ok(Grammar{ imports:imports, rules:rules, templates:templates, args: grammar_args.unwrap_or(vec![]) })
	}

	fn find_rule(&self, name: &str) -> Option<&Rule> {
		self.rules.iter().find(|rule| rule.name == name)
	}

	fn extra_args_def(&self) -> Tokens {
		let args: Vec<Tokens> = self.args.iter().map(|&(ref name, ref tp)| {
			let name = raw(name);
			let tp = raw(tp);
			quote!(, #name: #tp)
		}).collect();
		quote!(#(#args)*)
	}

	fn extra_args_call(&self) -> Tokens {
		let args: Vec<Tokens> = self.args.iter().map(|&(ref name, _)| {
			let name = raw(name);
			quote!(, #name)
		}).collect();
		quote!(#(#args)*)
	}

	fn check_name(&self, name: &str) -> Result<(), Error> {
		if self.args.iter().any(|x| x.0 == name) {
			Err(Error{ message: format!("Capture variable `{}` shadows grammar argument", name) })?
		}

		Ok(())
{'fraction_non_alphanumeric': 0.19316529326223947, 'fraction_numerical': 0.001454192922927775, 'mean_word_length': 3.8495887191539366, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6767334', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1511, 'n_words': 358}