<commit_before>#![feature(question_mark)]

extern crate cloudfm;
extern crate chill;
extern crate dotenv;

use std::{error};
use cloudfm::{views, User, Error};
use chill::{UnreducedView, ViewRow, DocumentId, AllDocumentsViewValue};
use std::env;
use dotenv::dotenv;

pub fn main() {
    dotenv().ok();

    let db_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
    let db = chill::Client::new(&db_url).expect("DATABASE_URL must be a valid URL");

    let errors = index_all(db);
    println!("index_all done. error count: {}", errors.len());
    for error in errors {
        println!("error: {}", error);
    }
}

pub fn index_all(db: chill::Client) -> Vec<Error> {
    let mut errors: Vec<Error> = Vec::new();

    let action = match db.read_all_documents("/_users") {
        Ok(action) => action,
        Err(error) => return vec![Error::from(error)]
    };

    let res = match action.run() {
        Ok(res) => res,
        Err(error) => return vec![Error::from(error)]
    };

    let view = match res.as_unreduced() {
        Some(view) => view,
        None => unimplemented!()
    };

    for row in view.rows() {
        if let Err(e) = index_user(&db, row) {
            errors.push(e);
        }
    }

    errors
}


pub fn index_user(db: &chill::Client, row: &ViewRow<DocumentId, AllDocumentsViewValue>) -> Result<(), Error> {
    let doc = db.read_document(("/_users", row.key()))?.run()?; // TODO use include_docs
    let user: User = doc.get_content()?;
    views::apply(db, &user.db_name())?;
    Ok(())
}<commit_msg>fix(indexd): skip design docs, print success count<commit_after>#![feature(question_mark)]

extern crate cloudfm;
extern crate chill;
extern crate dotenv;

use std::{error};
use cloudfm::{views, User, Error};
use chill::{UnreducedView, ViewRow, DocumentId, AllDocumentsViewValue};
use std::env;
use dotenv::dotenv;

pub fn main() {
    dotenv().ok();

    let db_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
    let db = chill::Client::new(&db_url).expect("DATABASE_URL must be a valid URL");

    let (count, errors) = index_all(db);
    println!("index_all done. success: {}, failed: {}", count - errors.len(), errors.len());
    for error in errors {
        println!("error: {}", error);
    }
}

pub fn index_all(db: chill::Client) -> (usize, Vec<Error>) {
    let mut errors: Vec<Error> = Vec::new();

    // skips design docs
    let start_key = &DocumentId::from("org.couchdb.user:");

    let action = match db.read_all_documents("/_users") {
        Ok(action) => action.with_start_key(start_key),
        Err(error) => return (0, vec![Error::from(error)])
    };

    let res = match action.run() {
        Ok(res) => res,
        Err(error) => return (0, vec![Error::from(error)])
    };

    let view = match res.as_unreduced() {
        Some(view) => view,
        None => unimplemented!()
    };

    for row in view.rows() {
        if let Err(e) = index_user(&db, row) {
            errors.push(e);
        }
    }

    (view.rows().len(), errors)
}


pub fn index_user(db: &chill::Client, row: &ViewRow<DocumentId, AllDocumentsViewValue>) -> Result<(), Error> {
    let doc = db.read_document(("/_users", row.key()))?.run()?; // TODO use include_docs
    let user: User = doc.get_content()?;
    views::apply(db, &user.db_name())?;
    Ok(())
}
{'fraction_non_alphanumeric': 0.17874251497005988, 'fraction_numerical': 0.0005988023952095808, 'mean_word_length': 3.3054123711340204, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2719343', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1119, 'n_words': 273}