<commit_before>

extern crate cratesfyi;
#[macro_use]
extern crate log;
extern crate clap;
extern crate time;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::{Command, exit};

use cratesfyi::docbuilder::{DocBuilder, DocBuilderError, command_result};
use cratesfyi::docbuilder::crte::Crate;
use log::{LogLevel, LogLevelFilter, LogRecord, LogMetadata};
use clap::{Arg, App};
use time::now;


// I am using really simple Logger here
// I need to print everything to stdout for this program
struct Logger;

impl log::Log for Logger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= LogLevel::Info
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {

            println!("{} {} - {}", now().to_utc().rfc822z(), record.level(), record.args());
        }
    }
}


fn update_crates_io_index(path: &PathBuf) -> Result<String, String> {
    info!("Updating crates.io-index");

    if path.exists() {
        let cwd = env::current_dir().unwrap();
        env::set_current_dir(path).unwrap();
        let res = command_result(Command::new("git").arg("pull").output().unwrap());
        env::set_current_dir(cwd).unwrap();
        res
    } else {
        command_result(Command::new("git")
            .arg("clone")
            .arg("https://github.com/rust-lang/crates.io-index.git")
            .arg(path.to_str().unwrap())
            .output().unwrap())
    }
}
<commit_msg>Use rfc3339 format in logs<commit_after>

extern crate cratesfyi;
#[macro_use]
extern crate log;
extern crate clap;
extern crate time;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::{Command, exit};

use cratesfyi::docbuilder::{DocBuilder, DocBuilderError, command_result};
use cratesfyi::docbuilder::crte::Crate;
use log::{LogLevel, LogLevelFilter, LogRecord, LogMetadata};
use clap::{Arg, App};
use time::now;


// I am using really simple Logger here
// I need to print everything to stdout for this program
struct Logger;

impl log::Log for Logger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= LogLevel::Info
    }

    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {

            println!("{} {} - {}", now().to_utc().rfc3339(), record.level(), record.args());
        }
    }
}


fn update_crates_io_index(path: &PathBuf) -> Result<String, String> {
    info!("Updating crates.io-index");

    if path.exists() {
        let cwd = env::current_dir().unwrap();
        env::set_current_dir(path).unwrap();
        let res = command_result(Command::new("git").arg("pull").output().unwrap());
        env::set_current_dir(cwd).unwrap();
        res
    } else {
        command_result(Command::new("git")
            .arg("clone")
            .arg("https://github.com/rust-lang/crates.io-index.git")
            .arg(path.to_str().unwrap())
            .output().unwrap())
    }
}

{'fraction_non_alphanumeric': 0.168985408890397, 'fraction_numerical': 0.0037326094333220224, 'mean_word_length': 3.2911208151382825, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6579199', 'n_tokens_mistral': 954, 'n_tokens_neox': 954, 'n_words': 210}