<commit_before>extern crate regex;

fn main() {
	println!("!!!!");
	let input = "(begin (define r 10) (* pi (* r r)))";
	let parsed_code = parse(input);
	println!("{}", parsed_code);
	// Execute script from the command line
}

fn parse(code: &str) -> String {
	// code.to_string()
	let left_paren_re = regex::Regex::new(r"\s*\(\s*").unwrap();
	let right_paren_re = regex::Regex::new(r"\s*\)\s*").unwrap();
	let partial_parsed_code = left_paren_re.replace_all(&code, " ( ");
	let parsed_code = right_paren_re.replace_all(&partial_parsed_code," ) ");
	// let parsed_code = code.replace(" (", "(").replace("( ", "(").replace("(", " ( ").replace()
	let final_string : String = parsed_code.split(" ").collect();
	final_string.clone()
}

// fn tokenize() {

// }

// fn read_from_tokens() {

// }

#[test]<commit_msg>changes<commit_after>extern crate regex;

fn main() {
	println!("!!!!");
	let input = "(begin (define r 10) (* pi (* r r)))";
	let parsed_code = parse(input);
	println!("{}", parsed_code);
	// Execute script from the command line
}

fn parse(code: &str) -> String {
	// code.to_string()
	let left_paren_re = regex::Regex::new(r"\s*\(\s*").unwrap();
	let right_paren_re = regex::Regex::new(r"\s*\)\s*").unwrap();
	let partial_parsed_code = left_paren_re.replace_all(&code, " ( ");
	let parsed_code = right_paren_re.replace_all(&partial_parsed_code," ) ");
	// let parsed_code = code.replace(" (", "(").replace("( ", "(").replace("(", " ( ").replace()
	let final_string : String = parsed_code.split(" ").collect();
	final_string.clone()
}

// fn tokenize() {

// }

// fn read_from_tokens() {

// }

// #[test]
{'fraction_non_alphanumeric': 0.23224212476837555, 'fraction_numerical': 0.0024706609017912293, 'mean_word_length': 4.4, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '286090', 'n_tokens_mistral': 589, 'n_tokens_neox': 606, 'n_words': 119}