<commit_before>#![deny(warnings)]

extern crate cargo_registry;
extern crate openssl;
extern crate postgres;
extern crate semver;
extern crate time;

use std::collections::HashMap;
use std::env;
use std::time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 1000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse().unwrap());
    loop {
        let conn = cargo_registry::db::connect_now();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep(Duration::new(sleep.unwrap(), 0));
        } else {
            break
        }
    }
}

fn update(conn: &postgres::GenericConnection) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        // FIXME(rust-lang/rust#27401): weird declaration to make sure this
        // variable gets dropped.
        let tx; tx = conn.transaction()?;
        {
            let stmt = tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2")?;
            let mut rows = stmt.query(&[&max, &LIMIT])?;
            match collect(&tx, &mut rows)? {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        tx.finish()?;
    }
    Ok(())
}

fn collect(tx: &postgres::transaction::Transaction,
           rows: &mut postgres::rows::Rows) -> postgres::Result<Option<i32>> {
    use time::Duration;

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let cutoff = time::now_utc().to_timespec();
    let cutoff = cutoff - Duration::days(1);

    let mut map = HashMap::new();
    for row in rows.iter() {
        let download: VersionDownload = Model::from_row(&row);
        assert!(map.insert(download.id, download).is_none());
    }
    println!("updating {} versions (cutoff {})", map.len(),
             time::at(cutoff).rfc822());
    if map.len() == 0 {
        return Ok(None)
    }

    let mut max = 0;
    let mut total = 0;
    for (id, download) in map.iter() {
        if *id > max {
            max = *id;
        }
        if download.date > cutoff && download.counted == download.downloads {
            continue
        }
        let amt = download.downloads - download.counted;

        // Flag this row as having been processed if we're passed the cutoff,
        // and unconditionally increment the number of counted downloads.
        tx.execute("UPDATE version_downloads
                         SET processed = $2, counted = counted + $3
                         WHERE id = $1",
                   &[id, &(download.date < cutoff), &amt])?;
        println!("{}\n{}", time::at(download.date).rfc822(),
                 time::at(cutoff).rfc822());
        total += amt as i64;

        if amt == 0 {
            continue
        }

        let crate_id = Version::find(tx, download.version_id).unwrap().crate_id;

        // Update the total number of version downloads
        tx.execute("UPDATE versions
                         SET downloads = downloads + $1
                         WHERE id = $2",
                   &[&amt, &download.version_id])?;
        // Update the total number of crate downloads
        tx.execute("UPDATE crates SET downloads = downloads + $1
                         WHERE id = $2", &[&amt, &crate_id])?;

        // Update the total number of crate downloads for today
        let cnt = tx.execute("UPDATE crate_downloads<commit_msg>Remove logging of dates from update-downloads<commit_after>#![deny(warnings)]

extern crate cargo_registry;
extern crate openssl;
extern crate postgres;
extern crate semver;
extern crate time;

use std::collections::HashMap;
use std::env;
use std::time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 1000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse().unwrap());
    loop {
        let conn = cargo_registry::db::connect_now();
        update(&conn).unwrap();
        drop(conn);
        if daemon {
            std::thread::sleep(Duration::new(sleep.unwrap(), 0));
        } else {
            break
        }
    }
}

fn update(conn: &postgres::GenericConnection) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        // FIXME(rust-lang/rust#27401): weird declaration to make sure this
        // variable gets dropped.
        let tx; tx = conn.transaction()?;
        {
            let stmt = tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2")?;
            let mut rows = stmt.query(&[&max, &LIMIT])?;
            match collect(&tx, &mut rows)? {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        tx.finish()?;
    }
    Ok(())
}

fn collect(tx: &postgres::transaction::Transaction,
           rows: &mut postgres::rows::Rows) -> postgres::Result<Option<i32>> {
    use time::Duration;

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let cutoff = time::now_utc().to_timespec();
    let cutoff = cutoff - Duration::days(1);

    let mut map = HashMap::new();
    for row in rows.iter() {
        let download: VersionDownload = Model::from_row(&row);
        assert!(map.insert(download.id, download).is_none());
    }
    println!("updating {} versions (cutoff {})", map.len(),
             time::at(cutoff).rfc822());
    if map.len() == 0 {
        return Ok(None)
    }

    let mut max = 0;
    let mut total = 0;
    for (id, download) in map.iter() {
        if *id > max {
            max = *id;
        }
        if download.date > cutoff && download.counted == download.downloads {
            continue
        }
        let amt = download.downloads - download.counted;

        // Flag this row as having been processed if we're passed the cutoff,
        // and unconditionally increment the number of counted downloads.
        tx.execute("UPDATE version_downloads
                         SET processed = $2, counted = counted + $3
                         WHERE id = $1",
                   &[id, &(download.date < cutoff), &amt])?;
        total += amt as i64;

        if amt == 0 {
            continue
        }

        let crate_id = Version::find(tx, download.version_id).unwrap().crate_id;

        // Update the total number of version downloads
        tx.execute("UPDATE versions
                         SET downloads = downloads + $1
                         WHERE id = $2",
                   &[&amt, &download.version_id])?;
        // Update the total number of crate downloads
        tx.execute("UPDATE crates SET downloads = downloads + $1
                         WHERE id = $2", &[&amt, &crate_id])?;

        // Update the total number of crate downloads for today
        let cnt = tx.execute("UPDATE crate_downloads
{'fraction_non_alphanumeric': 0.1325706594885599, 'fraction_numerical': 0.011036339165545087, 'mean_word_length': 1.8330156309569197, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7630711', 'n_tokens_mistral': 2150, 'n_tokens_neox': 2078, 'n_words': 595}