<commit_before>extern crate decimal;

use decimal::*;
use std::fs::File;
use std::path::Path;
use std::io::BufRead;
use std::io::BufReader;

/// Splits the line into the first token and the rest of the line.
/// The token can be quoted in single or double quotes and it will be returned without the quotes.
/// When the rest of the line is empty it means the whole line was processed.
///
/// # Example
///
/// ```rust
/// let line = "a '1 2' b";
/// let (token, line) = split_token(line);
/// assert_eq!(token, "a");
/// assert_eq!(line, "'1 2' b");
/// let (token, line) = split_token(line);
/// assert_eq!(token, "1 2");
/// assert_eq!(line, "b");
/// let (token, line) = split_token(line);
/// assert_eq!(token, "b");
/// assert_eq!(line, "");
/// ```
fn split_token<'a>(line: &'a str) -> (&'a str, &'a str) {
    let line = line.trim_left();
    if line.starts_with("--") {
        ("", "")
    } else if line.starts_with("\"") {
        let end = line[1..].find('"').expect("Unmatched double quote");
        (&line[1..end], &line[end + 2..])
    } else if line.starts_with("'") {
        let end = line[1..].find('\'').expect("Unmatched single quote");
        (&line[1..end], &line[end + 2..])
    } else {<commit_msg>Fix tokenization bug. Quoted tokens can have their quote escaped by prefixing it with itself. For example 'aa''bb' is the token [aa'bb].<commit_after>extern crate decimal;

use decimal::*;
use std::fs::File;
use std::path::Path;
use std::io::BufRead;
use std::io::BufReader;

fn find_end_quote(s: &str, quote: char) -> Option<usize> {
    match s.find(quote) {
        None => None,
        Some(i) => {
            match s[i+1..].chars().next() {
                None => Some(i),
                Some(c) if c == quote => find_end_quote(&s[i+2..], quote).and_then(|j| { Some(i + 2 + j) }),
                Some(_) => Some(i),
            }
        }
    }
}

fn split_token<'a>(line: &'a str) -> (&'a str, &'a str) {
    let line = line.trim_left();
    if line.starts_with("--") {
        ("", "")
    } else if line.starts_with("\"") {
        let end = find_end_quote(&line[1..], '"').expect("Unmatched double quote") + 1;
        (&line[1..end], &line[end + 1..])
    } else if line.starts_with("'") {
        let end = find_end_quote(&line[1..], '\'').expect("Unmatched single quote") + 1;
        (&line[1..end], &line[end + 1..])
    } else {
{'fraction_non_alphanumeric': 0.22021052631578947, 'fraction_numerical': 0.00968421052631579, 'mean_word_length': 2.644171779141104, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5068864', 'n_tokens_mistral': 838, 'n_tokens_neox': 827, 'n_words': 229}