<commit_before>// Sync available crate categories from `src/categories.txt`.
// Runs when the server is started.

use pg;
use env;
use util::errors::CargoResult;

pub fn sync() -> CargoResult<()> {
    let conn = pg::Connection::connect(&env("DATABASE_URL")[..],
                                             pg::TlsMode::None).unwrap();
    let tx = conn.transaction().unwrap();

    let categories = include_str!("./categories.txt");

    let slug_categories: Vec<_> = categories.lines().map(|c| {
        let mut parts = c.split(' ');
        let slug = parts.next().expect("No slug found!");
        let name = parts.collect::<Vec<_>>().join(" ");
        (slug, name)
    }).collect();

    let insert = slug_categories.iter().map(|&(ref slug, ref name)| {
        format!("(LOWER('{}'), '{}')", slug, name)
    }).collect::<Vec<_>>().join(",");

    let in_clause = slug_categories.iter().map(|&(slug, _)| {
        format!("LOWER('{}')", slug)
    }).collect::<Vec<_>>().join(",");

    try!(tx.batch_execute(
        &format!(" \
            INSERT INTO categories (slug, category) \
            VALUES {} \
            ON CONFLICT (slug) DO UPDATE SET category = EXCLUDED.category; \
            DELETE FROM categories \
            WHERE slug NOT IN ({});",
            insert,
            in_clause
        )[..]
    ));
    tx.set_commit();
    tx.finish().unwrap();
    Ok(())<commit_msg>Use splitn on the category slug/display name pairs<commit_after>// Sync available crate categories from `src/categories.txt`.
// Runs when the server is started.

use pg;
use env;
use util::errors::CargoResult;

pub fn sync() -> CargoResult<()> {
    let conn = pg::Connection::connect(&env("DATABASE_URL")[..],
                                             pg::TlsMode::None).unwrap();
    let tx = conn.transaction().unwrap();

    let categories = include_str!("./categories.txt");

    let slug_categories: Vec<_> = categories.lines().map(|c| {
        let mut parts = c.splitn(2, ' ');
        let slug = parts.next().expect("No slug found!");
        let name = parts.next().expect("No display name found!");
        (slug, name)
    }).collect();

    let insert = slug_categories.iter().map(|&(ref slug, ref name)| {
        format!("(LOWER('{}'), '{}')", slug, name)
    }).collect::<Vec<_>>().join(",");

    let in_clause = slug_categories.iter().map(|&(slug, _)| {
        format!("LOWER('{}')", slug)
    }).collect::<Vec<_>>().join(",");

    try!(tx.batch_execute(
        &format!(" \
            INSERT INTO categories (slug, category) \
            VALUES {} \
            ON CONFLICT (slug) DO UPDATE SET category = EXCLUDED.category; \
            DELETE FROM categories \
            WHERE slug NOT IN ({});",
            insert,
            in_clause
        )[..]
    ));
    tx.set_commit();
    tx.finish().unwrap();
    Ok(())
{'fraction_non_alphanumeric': 0.1918625043844265, 'fraction_numerical': 0.00035075412136092597, 'mean_word_length': 2.308584686774942, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6154886', 'n_tokens_mistral': 875, 'n_tokens_neox': 855, 'n_words': 201}