<commit_before>#![deny(warnings)]

extern crate cargo_registry;
extern crate postgres;
extern crate semver;
extern crate time;

use std::env;
use std::collections::HashMap;
use time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 1000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse::<u32>().unwrap());
    loop {
        let conn = postgres::Connection::connect(&env("DATABASE_URL")[..],
                                                 &postgres::SslMode::None).unwrap();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep_ms(sleep.unwrap() * 1000);
        } else {
            break
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn update(conn: &postgres::GenericConnection) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        let tx = try!(conn.transaction());
        {
            let stmt = try!(tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2"));
            let mut rows = try!(stmt.query(&[&max, &LIMIT]));
            match try!(collect(&tx, &mut rows)) {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        try!(tx.finish());
    }
    Ok(())
}

fn collect(tx: &postgres::Transaction,
           rows: &mut postgres::rows::Rows) -> postgres::Result<Option<i32>> {

    // Anything older than 24 hours ago will be frozen and will not be queried<commit_msg>Fix compile on nightly<commit_after>#![deny(warnings)]

extern crate cargo_registry;
extern crate postgres;
extern crate semver;
extern crate time;

use std::env;
use std::collections::HashMap;
use time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 1000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse::<u32>().unwrap());
    loop {
        let conn = postgres::Connection::connect(&env("DATABASE_URL")[..],
                                                 &postgres::SslMode::None).unwrap();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep_ms(sleep.unwrap() * 1000);
        } else {
            break
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn update(conn: &postgres::GenericConnection) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        // FIXME(rust-lang/rust#27401): weird declaration to make sure this
        // variable gets dropped.
        let tx; tx = try!(conn.transaction());
        {
            let stmt = try!(tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2"));
            let mut rows = try!(stmt.query(&[&max, &LIMIT]));
            match try!(collect(&tx, &mut rows)) {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        try!(tx.finish());
    }
    Ok(())
}

fn collect(tx: &postgres::Transaction,
           rows: &mut postgres::rows::Rows) -> postgres::Result<Option<i32>> {

    // Anything older than 24 hours ago will be frozen and will not be queried
{'fraction_non_alphanumeric': 0.15809669992325404, 'fraction_numerical': 0.012023535431056536, 'mean_word_length': 1.83744557329463, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5871740', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1144, 'n_words': 283}