<commit_before>use regex::{Captures, Regex};
use analysis::namespaces::MAIN;
use env::Env;

const LANGUAGE_SEP_BEGIN : &'static str = "<!-- language=\"";
const LANGUAGE_SEP_END : &'static str = "\" -->";
const LANGUAGE_BLOCK_BEGIN : &'static str = "|[";
const LANGUAGE_BLOCK_END : &'static str = "\n]|";

pub fn reformat_doc(input: &str, env: &Env) -> String {
    code_blocks_transformation(input, env)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str,
                              env: &Env) -> String {
    let mut out = String::new();

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&replace_c_types(before, env));
                if let (before, Some(after)) = try_split(get_language(after, &mut out),
                                                         LANGUAGE_BLOCK_END) {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&replace_c_types(before, env));
                return out
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("```{}", before));
            return after
        }
    }
    out.push_str("```text");
    entry
}

lazy_static! {
    static ref REG : Regex = Regex::new(r"#?(G[dt]k)([\w]+:?:?\.?[\w-]+)").unwrap();
    static ref REG2 : Regex = Regex::new(r"@(\w*)").unwrap();
}

fn replace_c_types(entry: &str, env: &Env) -> String {
    let out = &REG.replace_all(entry, |caps: &Captures| {
        let pos = env.library.find_namespace(&caps[1]).unwrap();
        if pos == MAIN {
            format!("`{}`", &caps[2])
        } else {
            format!("`{}::{}`", &env.namespaces[pos].crate_name, &caps[2])
        }
    });
    REG2.replace_all(out, "`$1`")
}
<commit_msg>Delimit code blocks with newlines<commit_after>use regex::{Captures, Regex};
use analysis::namespaces::MAIN;
use env::Env;

const LANGUAGE_SEP_BEGIN : &'static str = "<!-- language=\"";
const LANGUAGE_SEP_END : &'static str = "\" -->";
const LANGUAGE_BLOCK_BEGIN : &'static str = "|[";
const LANGUAGE_BLOCK_END : &'static str = "\n]|";

pub fn reformat_doc(input: &str, env: &Env) -> String {
    code_blocks_transformation(input, env)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str,
                              env: &Env) -> String {
    let mut out = String::new();

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&replace_c_types(before, env));
                if let (before, Some(after)) = try_split(get_language(after, &mut out),
                                                         LANGUAGE_BLOCK_END) {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&replace_c_types(before, env));
                return out
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("\n```{}", before));
            return after
        }
    }
    out.push_str("\n```text");
    entry
}

lazy_static! {
    static ref REG : Regex = Regex::new(r"#?(G[dt]k)([\w]+:?:?\.?[\w-]+)").unwrap();
    static ref REG2 : Regex = Regex::new(r"@(\w*)").unwrap();
}

fn replace_c_types(entry: &str, env: &Env) -> String {
    let out = &REG.replace_all(entry, |caps: &Captures| {
        let pos = env.library.find_namespace(&caps[1]).unwrap();
        if pos == MAIN {
            format!("`{}`", &caps[2])
        } else {
            format!("`{}::{}`", &env.namespaces[pos].crate_name, &caps[2])
        }
    });
    REG2.replace_all(out, "`$1`")
}

{'fraction_non_alphanumeric': 0.18243243243243243, 'fraction_numerical': 0.002533783783783784, 'mean_word_length': 2.0541586073500966, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5077966', 'n_tokens_mistral': 1628, 'n_tokens_neox': 1608, 'n_words': 312}