<commit_before>use std::ascii::AsciiExt;
use std::cmp;
use std::collections::HashMap;
use std::io::prelude::*;
use std::io;
use std::mem;
use std::sync::Arc;

use conduit::{Request, Response};
use conduit_router::RequestParams;
use curl::easy::Easy;
use license_exprs;
use pg::GenericConnection;
use pg::rows::Row;
use pg::types::ToSql;
use pg;
use rustc_serialize::hex::ToHex;
use rustc_serialize::json;
use semver;
use time::{Timespec, Duration};
use url::Url;

use {Model, User, Keyword, Version, Category};
use app::{App, RequestApp};
use db::RequestTransaction;
use dependency::{Dependency, EncodableDependency};
use download::{VersionDownload, EncodableVersionDownload};
use git;
use keyword::EncodableKeyword;
use category::EncodableCategory;
use upload;
use user::RequestUser;
use owner::{EncodableOwner, Owner, Rights, OwnerKind, Team, rights};
use util::errors::NotFound;
use util::{LimitErrorReader, HashingReader};
use util::{RequestUtils, CargoResult, internal, ChainError, human};
use version::EncodableVersion;

#[derive(Clone)]
pub struct Crate {
    pub id: i32,
    pub name: String,
    pub updated_at: Timespec,
    pub created_at: Timespec,
    pub downloads: i32,
    pub max_version: semver::Version,
    pub description: Option<String>,
    pub homepage: Option<String>,
    pub documentation: Option<String>,
    pub readme: Option<String>,
    pub license: Option<String>,
    pub repository: Option<String>,
    pub max_upload_size: Option<i32>,
}

#[derive(RustcEncodable, RustcDecodable)]
pub struct EncodableCrate {
    pub id: String,
    pub name: String,
    pub updated_at: String,
    pub versions: Option<Vec<i32>>,
    pub keywords: Option<Vec<String>>,
    pub categories: Option<Vec<String>>,
    pub created_at: String,
    pub downloads: i32,
    pub max_version: String,
    pub description: Option<String>,
    pub homepage: Option<String>,
    pub documentation: Option<String>,
    pub license: Option<String>,
    pub repository: Option<String>,
    pub links: CrateLinks,
}

#[derive(RustcEncodable, RustcDecodable)]
pub struct CrateLinks {
    pub version_downloads: String,
    pub versions: Option<String>,
    pub owners: Option<String>,
    pub reverse_dependencies: String,
}

impl Crate {
    pub fn find_by_name(conn: &GenericConnection,
                        name: &str) -> CargoResult<Crate> {
        let stmt = try!(conn.prepare("SELECT * FROM crates \
                                      WHERE canon_crate_name(name) =
                                            canon_crate_name($1) LIMIT 1"));
        let rows = try!(stmt.query(&[&name]));
        let row = rows.iter().next();
        let row = try!(row.chain_error(|| NotFound));
        Ok(Model::from_row(&row))
    }

    pub fn find_or_insert(conn: &GenericConnection,
                          name: &str,
                          user_id: i32,
                          description: &Option<String>,
                          homepage: &Option<String>,
                          documentation: &Option<String>,
                          readme: &Option<String>,
                          repository: &Option<String>,
                          license: &Option<String>,
                          license_file: &Option<String>,
                          max_upload_size: Option<i32>)
                          -> CargoResult<Crate> {
        let description = description.as_ref().map(|s| &s[..]);
        let homepage = homepage.as_ref().map(|s| &s[..]);
        let documentation = documentation.as_ref().map(|s| &s[..]);
        let readme = readme.as_ref().map(|s| &s[..]);
        let repository = repository.as_ref().map(|s| &s[..]);
        let mut license = license.as_ref().map(|s| &s[..]);
        let license_file = license_file.as_ref().map(|s| &s[..]);
        try!(validate_url(homepage, "homepage"));
        try!(validate_url(documentation, "documentation"));
        try!(validate_url(repository, "repository"));

        match license {
            // If a license is given, validate it to make sure it's actually a
            // valid license
            Some(..) => try!(validate_license(license)),

            // If no license is given, but a license file is given, flag this
            // crate as having a nonstandard license. Note that we don't
            // actually do anything else with license_file currently.
            None if license_file.is_some() => {
                license = Some("non-standard");
            }

            None => {}
        }

        // TODO: like with users, this is sadly racy
        let stmt = try!(conn.prepare("UPDATE crates
                                         SET documentation = $1,
                                             homepage = $2,
                                             description = $3,
                                             readme = $4,
                                             license = $5,
                                             repository = $6
                                       WHERE canon_crate_name(name) =
                                             canon_crate_name($7)
                                   RETURNING *"));
        let rows = try!(stmt.query(&[&documentation, &homepage,
                                     &description, &readme,
                                     &license, &repository,
                                     &name]));
        match rows.iter().next() {
            Some(row) => return Ok(Model::from_row(&row)),
            None => {}
        }

        // Blacklist the current set of crates in the rust distribution
        const RESERVED: &'static str = include_str!("reserved_crates.txt");

        if RESERVED.lines().any(|krate| name == krate) {
            return Err(human("cannot upload a crate with a reserved name"))
        }

        let stmt = try!(conn.prepare("INSERT INTO crates
                                      (name, description, homepage,
                                       documentation, readme,
                                       repository, license, max_upload_size)
                                      VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
                                      RETURNING *"));
        let rows = try!(stmt.query(&[&name, &description, &homepage,
                                     &documentation, &readme,
                                     &repository, &license, &max_upload_size]));
        let ret: Crate = Model::from_row(&try!(rows.iter().next().chain_error(|| {
            internal("no crate returned")
        })));

        try!(conn.execute("INSERT INTO crate_owners
                           (crate_id, owner_id, created_by, owner_kind)
                           VALUES ($1, $2, $2, $3)",
                          &[&ret.id, &user_id, &(OwnerKind::User as i32)]));
        return Ok(ret);

        fn validate_url(url: Option<&str>, field: &str) -> CargoResult<()> {
            let url = match url {
                Some(s) => s,
                None => return Ok(())
            };
            let url = try!(Url::parse(url).map_err(|_| {
                human(format!("`{}` is not a valid url: `{}`", field, url))
            }));
            match &url.scheme()[..] {
                "http" | "https" => {}
                s => return Err(human(format!("`{}` has an invalid url \
                                               scheme: `{}`", field, s)))
            }
            if url.cannot_be_a_base() {
                return Err(human(format!("`{}` must have relative scheme \
                                                        data: {}", field, url)))
            }
            Ok(())
        }

        fn validate_license(license: Option<&str>) -> CargoResult<()> {
            license.iter().flat_map(|s| s.split("/"))
                   .map(license_exprs::validate_license_expr)
                   .collect::<Result<Vec<_>, _>>()
                   .map(|_| ())
                   .map_err(|e| human(format!("{}; see http://opensource.org/licenses \
                                                  for options, and http://spdx.org/licenses/ \
                                                  for their identifiers", e)))
        }

    }

    pub fn valid_name(name: &str) -> bool {
        if name.len() == 0 { return false }
        name.chars().next().unwrap().is_alphabetic() &&
            name.chars().all(|c| c.is_alphanumeric() || c == '_' || c == '-') &&
            name.chars().all(|c| c.is_ascii())
    }

    pub fn valid_feature_name(name: &str) -> bool {
        let mut parts = name.split('/');
        match parts.next() {
            Some(part) if !Crate::valid_name(part) => return false,
            None => return false,
            _ => {}
        }
        match parts.next() {
            Some(part) if !Crate::valid_name(part) => return false,
            _ => {}
        }
        parts.next().is_none()
    }

    pub fn minimal_encodable(self) -> EncodableCrate {
        self.encodable(None, None, None)
    }

    pub fn encodable(self,
                     versions: Option<Vec<i32>>,
                     keywords: Option<&[Keyword]>,
                     categories: Option<&[Category]>)
                     -> EncodableCrate {
        let Crate {
            name, created_at, updated_at, downloads, max_version, description,
            homepage, documentation, license, repository,
            readme: _, id: _, max_upload_size: _,
        } = self;
        let versions_link = match versions {
            Some(..) => None,
            None => Some(format!("/api/v1/crates/{}/versions", name)),
        };
        let keyword_ids = keywords.map(|kws| kws.iter().map(|kw| kw.keyword.clone()).collect());
        let category_ids = categories.map(|cats| cats.iter().map(|cat| cat.category.clone()).collect());
        EncodableCrate {
            id: name.clone(),
            name: name.clone(),
            updated_at: ::encode_time(updated_at),
            created_at: ::encode_time(created_at),
            downloads: downloads,
            versions: versions,
            keywords: keyword_ids,
            categories: category_ids,
            max_version: max_version.to_string(),
            documentation: documentation,
            homepage: homepage,
            description: description,
            license: license,
            repository: repository,
            links: CrateLinks {
                version_downloads: format!("/api/v1/crates/{}/downloads", name),
                versions: versions_link,
                owners: Some(format!("/api/v1/crates/{}/owners", name)),
                reverse_dependencies: format!("/api/v1/crates/{}/reverse_dependencies", name)
            },
        }
    }

    pub fn versions(&self, conn: &GenericConnection) -> CargoResult<Vec<Version>> {
        let stmt = try!(conn.prepare("SELECT * FROM versions \
                                      WHERE crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        let mut ret = rows.iter().map(|r| {
            Model::from_row(&r)
        }).collect::<Vec<Version>>();
        ret.sort_by(|a, b| b.num.cmp(&a.num));
        Ok(ret)
    }

    pub fn owners(&self, conn: &GenericConnection) -> CargoResult<Vec<Owner>> {
        let stmt = try!(conn.prepare("SELECT * FROM users
                                      INNER JOIN crate_owners
                                         ON crate_owners.owner_id = users.id
                                      WHERE crate_owners.crate_id = $1
                                        AND crate_owners.deleted = FALSE
                                        AND crate_owners.owner_kind = $2"));
        let user_rows = try!(stmt.query(&[&self.id, &(OwnerKind::User as i32)]));

        let stmt = try!(conn.prepare("SELECT * FROM teams
                                      INNER JOIN crate_owners
                                         ON crate_owners.owner_id = teams.id
                                      WHERE crate_owners.crate_id = $1
                                        AND crate_owners.deleted = FALSE
                                        AND crate_owners.owner_kind = $2"));
        let team_rows = try!(stmt.query(&[&self.id, &(OwnerKind::Team as i32)]));

        let mut owners = vec![];
        owners.extend(user_rows.iter().map(|r| Owner::User(Model::from_row(&r))));
        owners.extend(team_rows.iter().map(|r| Owner::Team(Model::from_row(&r))));
        Ok(owners)
    }

    pub fn owner_add(&self, app: &App, conn: &GenericConnection, req_user: &User,
                     login: &str) -> CargoResult<()> {
        let owner = match Owner::find_by_login(conn, login) {
            Ok(owner @ Owner::User(_)) => { owner }
            Ok(Owner::Team(team)) => if try!(team.contains_user(app, req_user)) {
                Owner::Team(team)
            } else {
                return Err(human(format!("only members of {} can add it as \
                                          an owner", login)));
            },
            Err(err) => if login.contains(":") {
                Owner::Team(try!(Team::create(app, conn, login, req_user)))
            } else {
                return Err(err);
            },
        };

        // First try to un-delete if they've been soft deleted previously, then
        // do an insert if that didn't actually affect anything.
        let amt = try!(conn.execute("UPDATE crate_owners
                                        SET deleted = FALSE
                                      WHERE crate_id = $1 AND owner_id = $2
                                        AND owner_kind = $3",
                                    &[&self.id, &owner.id(), &owner.kind()]));
        assert!(amt <= 1);
        if amt == 0 {
            try!(conn.execute("INSERT INTO crate_owners
                               (crate_id, owner_id, created_by, owner_kind)
                               VALUES ($1, $2, $3, $4)",
                              &[&self.id, &owner.id(), &req_user.id,
                                &owner.kind()]));
        }

        Ok(())
    }

    pub fn owner_remove(&self,
                        conn: &GenericConnection,
                        _req_user: &User,
                        login: &str) -> CargoResult<()> {
        let owner = try!(Owner::find_by_login(conn, login).map_err(|_| {
            human(format!("could not find owner with login `{}`", login))
        }));
        try!(conn.execute("UPDATE crate_owners
                              SET deleted = TRUE
                            WHERE crate_id = $1 AND owner_id = $2
                              AND owner_kind = $3",
                          &[&self.id, &owner.id(), &owner.kind()]));
        Ok(())
    }

    pub fn s3_path(&self, version: &str) -> String {
        format!("/crates/{}/{}-{}.crate", self.name, self.name, version)
    }

    pub fn add_version(&mut self,
                       conn: &GenericConnection,
                       ver: &semver::Version,
                       features: &HashMap<String, Vec<String>>,
                       authors: &[String])
                       -> CargoResult<Version> {
        match try!(Version::find_by_num(conn, self.id, ver)) {
            Some(..) => {
                return Err(human(format!("crate version `{}` is already uploaded",
                                         ver)))
            }
            None => {}
        }
        let zero = semver::Version::parse("0.0.0").unwrap();
        if *ver > self.max_version || self.max_version == zero {
            self.max_version = ver.clone();
        }
        let stmt = try!(conn.prepare("UPDATE crates SET max_version = $1
                           WHERE id = $2 RETURNING updated_at"));
        let rows = try!(stmt.query(&[&self.max_version.to_string(), &self.id]));
        self.updated_at = rows.get(0).get("updated_at");
        Version::insert(conn, self.id, ver, features, authors)
    }

    pub fn keywords(&self, conn: &GenericConnection) -> CargoResult<Vec<Keyword>> {
        let stmt = try!(conn.prepare("SELECT keywords.* FROM keywords
                                      LEFT JOIN crates_keywords
                                      ON keywords.id = crates_keywords.keyword_id
                                      WHERE crates_keywords.crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        Ok(rows.iter().map(|r| Model::from_row(&r)).collect())
    }

    pub fn categories(&self, conn: &GenericConnection) -> CargoResult<Vec<Category>> {
        let stmt = try!(conn.prepare("SELECT categories.* FROM categories \
                                      LEFT JOIN crates_categories \
                                      ON categories.id = \
                                         crates_categories.category_id \
                                      WHERE crates_categories.crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        Ok(rows.iter().map(|r| Model::from_row(&r)).collect())
    }

    /// Returns (dependency, dependent crate name)
    pub fn reverse_dependencies(&self,
                                conn: &GenericConnection,
                                offset: i64,
                                limit: i64)
                                -> CargoResult<(Vec<(Dependency, String)>, i64)> {
        let select_sql = "
              FROM dependencies
              INNER JOIN versions
                ON versions.id = dependencies.version_id
              INNER JOIN crates
                ON crates.id = versions.crate_id
              WHERE dependencies.crate_id = $1
                AND versions.num = crates.max_version
        ";
        let fetch_sql = format!("SELECT DISTINCT ON (crate_name)
                                        dependencies.*,
                                        crates.name AS crate_name
                                        {}
                               ORDER BY crate_name ASC
                                 OFFSET $2
                                  LIMIT $3", select_sql);
        let count_sql = format!("SELECT COUNT(DISTINCT(crates.id)) {}",
                                select_sql);

        let stmt = try!(conn.prepare(&fetch_sql));
        let vec: Vec<_> = try!(stmt.query(&[&self.id, &offset, &limit]))
                                   .iter().map(|r| {
            (Model::from_row(&r), r.get("crate_name"))
        }).collect();
        let stmt = try!(conn.prepare(&count_sql));
        let cnt: i64 = try!(stmt.query(&[&self.id])).iter().next().unwrap().get(0);

        Ok((vec, cnt))
    }
}

impl Model for Crate {
    fn from_row(row: &Row) -> Crate {
        let max: String = row.get("max_version");
        Crate {
            id: row.get("id"),
            name: row.get("name"),
            updated_at: row.get("updated_at"),
            created_at: row.get("created_at"),
            downloads: row.get("downloads"),
            description: row.get("description"),
            documentation: row.get("documentation"),
            homepage: row.get("homepage"),
            readme: row.get("readme"),
            max_version: semver::Version::parse(&max).unwrap(),
            license: row.get("license"),
            repository: row.get("repository"),
            max_upload_size: row.get("max_upload_size"),
        }
    }
    fn table_name(_: Option<Crate>) -> &'static str { "crates" }
}

/// Handles the `GET /crates` route.
#[allow(trivial_casts)]
pub fn index(req: &mut Request) -> CargoResult<Response> {
    let conn = try!(req.tx());
    let (offset, limit) = try!(req.pagination(10, 100));
    let query = req.query();
    let sort = query.get("sort").map(|s| &s[..]).unwrap_or("alpha");
    let sort_sql = match sort {
        "downloads" => "crates.downloads DESC",
        _ => "crates.name ASC",
    };

    // Different queries for different parameters.
    //
    // Sure wish we had an arel-like thing here...
    let mut pattern = String::new();
    let mut id = -1;
    let (mut needs_id, mut needs_pattern) = (false, false);
    let mut args = vec![&limit as &ToSql, &offset];
    let (q, cnt) = query.get("q").map(|query| {
        args.insert(0, query);
        let rank_sort_sql = match sort {
            "downloads" => format!("{}, rank DESC", sort_sql),
            _ => format!("rank DESC, {}", sort_sql),
        };
        format!("{},", sort_sql); // Append Comma
        (format!("SELECT crates.* FROM crates,
                               plainto_tsquery($1) q,
                               ts_rank_cd(textsearchable_index_col, q) rank
          WHERE q @@ textsearchable_index_col
          ORDER BY name = $1 DESC, {}
          LIMIT $2 OFFSET $3", rank_sort_sql),
         "SELECT COUNT(crates.*) FROM crates,
                                      plainto_tsquery($1) q
          WHERE q @@ textsearchable_index_col".to_string())
    }).or_else(|| {
        query.get("letter").map(|letter| {
            pattern = format!("{}%", letter.chars().next().unwrap()
                                           .to_lowercase().collect::<String>());
            needs_pattern = true;
            (format!("SELECT * FROM crates WHERE canon_crate_name(name) \
                      LIKE $1 ORDER BY {} LIMIT $2 OFFSET $3", sort_sql),
             "SELECT COUNT(*) FROM crates WHERE canon_crate_name(name) \
              LIKE $1".to_string())
        })
    }).or_else(|| {
        query.get("keyword").map(|kw| {
            args.insert(0, kw);
            let base = "FROM crates
                        INNER JOIN crates_keywords
                                ON crates.id = crates_keywords.crate_id
                        INNER JOIN keywords
                                ON crates_keywords.keyword_id = keywords.id
                        WHERE lower(keywords.keyword) = lower($1)";
            (format!("SELECT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql),
             format!("SELECT COUNT(crates.*) {}", base))
        })
    }).or_else(|| {
        query.get("category").map(|cat| {
            args.insert(0, cat);
            let base = "FROM crates \
                        INNER JOIN crates_categories \
                                ON crates.id = crates_categories.crate_id \
                        INNER JOIN categories \
                                ON crates_categories.category_id = \
                                   categories.id \
                        WHERE lower(categories.category) = lower($1)";
            (format!("SELECT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql),
             format!("SELECT COUNT(crates.*) {}", base))
        })
    }).or_else(|| {
        query.get("user_id").and_then(|s| s.parse::<i32>().ok()).map(|user_id| {
            id = user_id;
            needs_id = true;
            (format!("SELECT crates.* FROM crates
                       INNER JOIN crate_owners
                          ON crate_owners.crate_id = crates.id
                       WHERE crate_owners.owner_id = $1
                       AND crate_owners.owner_kind = {}
                       ORDER BY {}
                      LIMIT $2 OFFSET $3",
                     OwnerKind::User as i32, sort_sql),
             format!("SELECT COUNT(crates.*) FROM crates
               INNER JOIN crate_owners
                  ON crate_owners.crate_id = crates.id
               WHERE crate_owners.owner_id = $1 \
                 AND crate_owners.owner_kind = {}",
                 OwnerKind::User as i32))
        })
    }).or_else(|| {
        query.get("following").map(|_| {
            needs_id = true;
            (format!("SELECT crates.* FROM crates
                      INNER JOIN follows
                         ON follows.crate_id = crates.id AND
                            follows.user_id = $1 ORDER BY
                      {} LIMIT $2 OFFSET $3", sort_sql),
             "SELECT COUNT(crates.*) FROM crates
              INNER JOIN follows
                 ON follows.crate_id = crates.id AND
                    follows.user_id = $1".to_string())
        })
    }).unwrap_or_else(|| {
        (format!("SELECT * FROM crates ORDER BY {} LIMIT $1 OFFSET $2",
                 sort_sql),
         "SELECT COUNT(*) FROM crates".to_string())
    });

    if needs_id {
        if id == -1 {
            id = try!(req.user()).id;
        }
        args.insert(0, &id);
    } else if needs_pattern {
        args.insert(0, &pattern);
    }

    // Collect all the crates
    let stmt = try!(conn.prepare(&q));
    let mut crates = Vec::new();
    for row in try!(stmt.query(&args)).iter() {
        let krate: Crate = Model::from_row(&row);
        crates.push(krate.minimal_encodable());
    }

    // Query for the total count of crates
    let stmt = try!(conn.prepare(&cnt));
    let args = if args.len() > 2 {&args[..1]} else {&args[..0]};
    let rows = try!(stmt.query(args));
    let row = rows.iter().next().unwrap();
    let total = row.get(0);

    #[derive(RustcEncodable)]
    struct R { crates: Vec<EncodableCrate>, meta: Meta }
    #[derive(RustcEncodable)]
    struct Meta { total: i64 }

    Ok(req.json(&R {
        crates: crates,
        meta: Meta { total: total },
    }))
}

/// Handles the `GET /summary` route.
pub fn summary(req: &mut Request) -> CargoResult<Response> {
    let tx = try!(req.tx());
    let num_crates = try!(Crate::count(tx));
    let num_downloads = {
        let stmt = try!(tx.prepare("SELECT total_downloads FROM metadata"));
        let rows = try!(stmt.query(&[]));
        rows.iter().next().unwrap().get("total_downloads")
    };

    let to_crates = |stmt: pg::stmt::Statement| -> CargoResult<Vec<_>> {
        let rows = try!(stmt.query(&[]));
        Ok(rows.iter().map(|r| {
            let krate: Crate = Model::from_row(&r);
            krate.minimal_encodable()
        }).collect::<Vec<EncodableCrate>>())
    };
    let new_crates = try!(tx.prepare("SELECT * FROM crates \
                                        ORDER BY created_at DESC LIMIT 10"));
    let just_updated = try!(tx.prepare("SELECT * FROM crates \
                                        WHERE updated_at::timestamp(0) !=
                                              created_at::timestamp(0)
                                        ORDER BY updated_at DESC LIMIT 10"));
    let most_downloaded = try!(tx.prepare("SELECT * FROM crates \
                                           ORDER BY downloads DESC LIMIT 10"));

    #[derive(RustcEncodable)]
    struct R {
        num_downloads: i64,
        num_crates: i64,
        new_crates: Vec<EncodableCrate>,
        most_downloaded: Vec<EncodableCrate>,
        just_updated: Vec<EncodableCrate>,
    }
    Ok(req.json(&R {
        num_downloads: num_downloads,
        num_crates: num_crates,
        new_crates: try!(to_crates(new_crates)),
        most_downloaded: try!(to_crates(most_downloaded)),
        just_updated: try!(to_crates(just_updated)),
    }))
}

/// Handles the `GET /crates/:crate_id` route.
pub fn show(req: &mut Request) -> CargoResult<Response> {
    let name = &req.params()["crate_id"];
    let conn = try!(req.tx());
    let krate = try!(Crate::find_by_name(conn, &name));
    let versions = try!(krate.versions(conn));
    let ids = versions.iter().map(|v| v.id).collect();
    let kws = try!(krate.keywords(conn));
    let cats = try!(krate.categories(conn));

    #[derive(RustcEncodable)]
    struct R {
        krate: EncodableCrate,
        versions: Vec<EncodableVersion>,
        keywords: Vec<EncodableKeyword>,
        categories: Vec<EncodableCategory>,
    }
    Ok(req.json(&R {
        krate: krate.clone().encodable(Some(ids), Some(&kws), Some(&cats)),
        versions: versions.into_iter().map(|v| {
            v.encodable(&krate.name)
        }).collect(),
        keywords: kws.into_iter().map(|k| k.encodable()).collect(),
        categories: cats.into_iter().map(|k| k.encodable()).collect(),
    }))
}

/// Handles the `PUT /crates/new` route.
pub fn new(req: &mut Request) -> CargoResult<Response> {
    let app = req.app().clone();

    let (new_crate, user) = try!(parse_new_headers(req));
    let name = &*new_crate.name;
    let vers = &*new_crate.vers;
    let features = new_crate.features.iter().map(|(k, v)| {
        (k[..].to_string(), v.iter().map(|v| v[..].to_string()).collect())
    }).collect::<HashMap<String, Vec<String>>>();
    let keywords = new_crate.keywords.as_ref().map(|s| &s[..])
                                     .unwrap_or(&[]);
    let keywords = keywords.iter().map(|k| k[..].to_string()).collect::<Vec<_>>();

    let categories = new_crate.categories.as_ref().map(|s| &s[..])
                                     .unwrap_or(&[]);
    let categories = categories.iter().map(|k| k[..].to_string()).collect::<Vec<_>>();

    // Persist the new crate, if it doesn't already exist
    let mut krate = try!(Crate::find_or_insert(try!(req.tx()), name, user.id,
                                               &new_crate.description,
                                               &new_crate.homepage,
                                               &new_crate.documentation,
                                               &new_crate.readme,
                                               &new_crate.repository,
                                               &new_crate.license,
                                               &new_crate.license_file,
                                               None));

    let owners = try!(krate.owners(try!(req.tx())));
    if try!(rights(req.app(), &owners, &user)) < Rights::Publish {
        return Err(human("crate name has already been claimed by \
                          another user"))
    }

    if krate.name != name {
        return Err(human(format!("crate was previously named `{}`", krate.name)))
    }

    let length = try!(req.content_length().chain_error(|| {
        human("missing header: Content-Length")<commit_msg>Prefer type annotations over turbofish<commit_after>use std::ascii::AsciiExt;
use std::cmp;
use std::collections::HashMap;
use std::io::prelude::*;
use std::io;
use std::mem;
use std::sync::Arc;

use conduit::{Request, Response};
use conduit_router::RequestParams;
use curl::easy::Easy;
use license_exprs;
use pg::GenericConnection;
use pg::rows::Row;
use pg::types::ToSql;
use pg;
use rustc_serialize::hex::ToHex;
use rustc_serialize::json;
use semver;
use time::{Timespec, Duration};
use url::Url;

use {Model, User, Keyword, Version, Category};
use app::{App, RequestApp};
use db::RequestTransaction;
use dependency::{Dependency, EncodableDependency};
use download::{VersionDownload, EncodableVersionDownload};
use git;
use keyword::EncodableKeyword;
use category::EncodableCategory;
use upload;
use user::RequestUser;
use owner::{EncodableOwner, Owner, Rights, OwnerKind, Team, rights};
use util::errors::NotFound;
use util::{LimitErrorReader, HashingReader};
use util::{RequestUtils, CargoResult, internal, ChainError, human};
use version::EncodableVersion;

#[derive(Clone)]
pub struct Crate {
    pub id: i32,
    pub name: String,
    pub updated_at: Timespec,
    pub created_at: Timespec,
    pub downloads: i32,
    pub max_version: semver::Version,
    pub description: Option<String>,
    pub homepage: Option<String>,
    pub documentation: Option<String>,
    pub readme: Option<String>,
    pub license: Option<String>,
    pub repository: Option<String>,
    pub max_upload_size: Option<i32>,
}

#[derive(RustcEncodable, RustcDecodable)]
pub struct EncodableCrate {
    pub id: String,
    pub name: String,
    pub updated_at: String,
    pub versions: Option<Vec<i32>>,
    pub keywords: Option<Vec<String>>,
    pub categories: Option<Vec<String>>,
    pub created_at: String,
    pub downloads: i32,
    pub max_version: String,
    pub description: Option<String>,
    pub homepage: Option<String>,
    pub documentation: Option<String>,
    pub license: Option<String>,
    pub repository: Option<String>,
    pub links: CrateLinks,
}

#[derive(RustcEncodable, RustcDecodable)]
pub struct CrateLinks {
    pub version_downloads: String,
    pub versions: Option<String>,
    pub owners: Option<String>,
    pub reverse_dependencies: String,
}

impl Crate {
    pub fn find_by_name(conn: &GenericConnection,
                        name: &str) -> CargoResult<Crate> {
        let stmt = try!(conn.prepare("SELECT * FROM crates \
                                      WHERE canon_crate_name(name) =
                                            canon_crate_name($1) LIMIT 1"));
        let rows = try!(stmt.query(&[&name]));
        let row = rows.iter().next();
        let row = try!(row.chain_error(|| NotFound));
        Ok(Model::from_row(&row))
    }

    pub fn find_or_insert(conn: &GenericConnection,
                          name: &str,
                          user_id: i32,
                          description: &Option<String>,
                          homepage: &Option<String>,
                          documentation: &Option<String>,
                          readme: &Option<String>,
                          repository: &Option<String>,
                          license: &Option<String>,
                          license_file: &Option<String>,
                          max_upload_size: Option<i32>)
                          -> CargoResult<Crate> {
        let description = description.as_ref().map(|s| &s[..]);
        let homepage = homepage.as_ref().map(|s| &s[..]);
        let documentation = documentation.as_ref().map(|s| &s[..]);
        let readme = readme.as_ref().map(|s| &s[..]);
        let repository = repository.as_ref().map(|s| &s[..]);
        let mut license = license.as_ref().map(|s| &s[..]);
        let license_file = license_file.as_ref().map(|s| &s[..]);
        try!(validate_url(homepage, "homepage"));
        try!(validate_url(documentation, "documentation"));
        try!(validate_url(repository, "repository"));

        match license {
            // If a license is given, validate it to make sure it's actually a
            // valid license
            Some(..) => try!(validate_license(license)),

            // If no license is given, but a license file is given, flag this
            // crate as having a nonstandard license. Note that we don't
            // actually do anything else with license_file currently.
            None if license_file.is_some() => {
                license = Some("non-standard");
            }

            None => {}
        }

        // TODO: like with users, this is sadly racy
        let stmt = try!(conn.prepare("UPDATE crates
                                         SET documentation = $1,
                                             homepage = $2,
                                             description = $3,
                                             readme = $4,
                                             license = $5,
                                             repository = $6
                                       WHERE canon_crate_name(name) =
                                             canon_crate_name($7)
                                   RETURNING *"));
        let rows = try!(stmt.query(&[&documentation, &homepage,
                                     &description, &readme,
                                     &license, &repository,
                                     &name]));
        match rows.iter().next() {
            Some(row) => return Ok(Model::from_row(&row)),
            None => {}
        }

        // Blacklist the current set of crates in the rust distribution
        const RESERVED: &'static str = include_str!("reserved_crates.txt");

        if RESERVED.lines().any(|krate| name == krate) {
            return Err(human("cannot upload a crate with a reserved name"))
        }

        let stmt = try!(conn.prepare("INSERT INTO crates
                                      (name, description, homepage,
                                       documentation, readme,
                                       repository, license, max_upload_size)
                                      VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
                                      RETURNING *"));
        let rows = try!(stmt.query(&[&name, &description, &homepage,
                                     &documentation, &readme,
                                     &repository, &license, &max_upload_size]));
        let ret: Crate = Model::from_row(&try!(rows.iter().next().chain_error(|| {
            internal("no crate returned")
        })));

        try!(conn.execute("INSERT INTO crate_owners
                           (crate_id, owner_id, created_by, owner_kind)
                           VALUES ($1, $2, $2, $3)",
                          &[&ret.id, &user_id, &(OwnerKind::User as i32)]));
        return Ok(ret);

        fn validate_url(url: Option<&str>, field: &str) -> CargoResult<()> {
            let url = match url {
                Some(s) => s,
                None => return Ok(())
            };
            let url = try!(Url::parse(url).map_err(|_| {
                human(format!("`{}` is not a valid url: `{}`", field, url))
            }));
            match &url.scheme()[..] {
                "http" | "https" => {}
                s => return Err(human(format!("`{}` has an invalid url \
                                               scheme: `{}`", field, s)))
            }
            if url.cannot_be_a_base() {
                return Err(human(format!("`{}` must have relative scheme \
                                                        data: {}", field, url)))
            }
            Ok(())
        }

        fn validate_license(license: Option<&str>) -> CargoResult<()> {
            license.iter().flat_map(|s| s.split("/"))
                   .map(license_exprs::validate_license_expr)
                   .collect::<Result<Vec<_>, _>>()
                   .map(|_| ())
                   .map_err(|e| human(format!("{}; see http://opensource.org/licenses \
                                                  for options, and http://spdx.org/licenses/ \
                                                  for their identifiers", e)))
        }

    }

    pub fn valid_name(name: &str) -> bool {
        if name.len() == 0 { return false }
        name.chars().next().unwrap().is_alphabetic() &&
            name.chars().all(|c| c.is_alphanumeric() || c == '_' || c == '-') &&
            name.chars().all(|c| c.is_ascii())
    }

    pub fn valid_feature_name(name: &str) -> bool {
        let mut parts = name.split('/');
        match parts.next() {
            Some(part) if !Crate::valid_name(part) => return false,
            None => return false,
            _ => {}
        }
        match parts.next() {
            Some(part) if !Crate::valid_name(part) => return false,
            _ => {}
        }
        parts.next().is_none()
    }

    pub fn minimal_encodable(self) -> EncodableCrate {
        self.encodable(None, None, None)
    }

    pub fn encodable(self,
                     versions: Option<Vec<i32>>,
                     keywords: Option<&[Keyword]>,
                     categories: Option<&[Category]>)
                     -> EncodableCrate {
        let Crate {
            name, created_at, updated_at, downloads, max_version, description,
            homepage, documentation, license, repository,
            readme: _, id: _, max_upload_size: _,
        } = self;
        let versions_link = match versions {
            Some(..) => None,
            None => Some(format!("/api/v1/crates/{}/versions", name)),
        };
        let keyword_ids = keywords.map(|kws| kws.iter().map(|kw| kw.keyword.clone()).collect());
        let category_ids = categories.map(|cats| cats.iter().map(|cat| cat.category.clone()).collect());
        EncodableCrate {
            id: name.clone(),
            name: name.clone(),
            updated_at: ::encode_time(updated_at),
            created_at: ::encode_time(created_at),
            downloads: downloads,
            versions: versions,
            keywords: keyword_ids,
            categories: category_ids,
            max_version: max_version.to_string(),
            documentation: documentation,
            homepage: homepage,
            description: description,
            license: license,
            repository: repository,
            links: CrateLinks {
                version_downloads: format!("/api/v1/crates/{}/downloads", name),
                versions: versions_link,
                owners: Some(format!("/api/v1/crates/{}/owners", name)),
                reverse_dependencies: format!("/api/v1/crates/{}/reverse_dependencies", name)
            },
        }
    }

    pub fn versions(&self, conn: &GenericConnection) -> CargoResult<Vec<Version>> {
        let stmt = try!(conn.prepare("SELECT * FROM versions \
                                      WHERE crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        let mut ret = rows.iter().map(|r| {
            Model::from_row(&r)
        }).collect::<Vec<Version>>();
        ret.sort_by(|a, b| b.num.cmp(&a.num));
        Ok(ret)
    }

    pub fn owners(&self, conn: &GenericConnection) -> CargoResult<Vec<Owner>> {
        let stmt = try!(conn.prepare("SELECT * FROM users
                                      INNER JOIN crate_owners
                                         ON crate_owners.owner_id = users.id
                                      WHERE crate_owners.crate_id = $1
                                        AND crate_owners.deleted = FALSE
                                        AND crate_owners.owner_kind = $2"));
        let user_rows = try!(stmt.query(&[&self.id, &(OwnerKind::User as i32)]));

        let stmt = try!(conn.prepare("SELECT * FROM teams
                                      INNER JOIN crate_owners
                                         ON crate_owners.owner_id = teams.id
                                      WHERE crate_owners.crate_id = $1
                                        AND crate_owners.deleted = FALSE
                                        AND crate_owners.owner_kind = $2"));
        let team_rows = try!(stmt.query(&[&self.id, &(OwnerKind::Team as i32)]));

        let mut owners = vec![];
        owners.extend(user_rows.iter().map(|r| Owner::User(Model::from_row(&r))));
        owners.extend(team_rows.iter().map(|r| Owner::Team(Model::from_row(&r))));
        Ok(owners)
    }

    pub fn owner_add(&self, app: &App, conn: &GenericConnection, req_user: &User,
                     login: &str) -> CargoResult<()> {
        let owner = match Owner::find_by_login(conn, login) {
            Ok(owner @ Owner::User(_)) => { owner }
            Ok(Owner::Team(team)) => if try!(team.contains_user(app, req_user)) {
                Owner::Team(team)
            } else {
                return Err(human(format!("only members of {} can add it as \
                                          an owner", login)));
            },
            Err(err) => if login.contains(":") {
                Owner::Team(try!(Team::create(app, conn, login, req_user)))
            } else {
                return Err(err);
            },
        };

        // First try to un-delete if they've been soft deleted previously, then
        // do an insert if that didn't actually affect anything.
        let amt = try!(conn.execute("UPDATE crate_owners
                                        SET deleted = FALSE
                                      WHERE crate_id = $1 AND owner_id = $2
                                        AND owner_kind = $3",
                                    &[&self.id, &owner.id(), &owner.kind()]));
        assert!(amt <= 1);
        if amt == 0 {
            try!(conn.execute("INSERT INTO crate_owners
                               (crate_id, owner_id, created_by, owner_kind)
                               VALUES ($1, $2, $3, $4)",
                              &[&self.id, &owner.id(), &req_user.id,
                                &owner.kind()]));
        }

        Ok(())
    }

    pub fn owner_remove(&self,
                        conn: &GenericConnection,
                        _req_user: &User,
                        login: &str) -> CargoResult<()> {
        let owner = try!(Owner::find_by_login(conn, login).map_err(|_| {
            human(format!("could not find owner with login `{}`", login))
        }));
        try!(conn.execute("UPDATE crate_owners
                              SET deleted = TRUE
                            WHERE crate_id = $1 AND owner_id = $2
                              AND owner_kind = $3",
                          &[&self.id, &owner.id(), &owner.kind()]));
        Ok(())
    }

    pub fn s3_path(&self, version: &str) -> String {
        format!("/crates/{}/{}-{}.crate", self.name, self.name, version)
    }

    pub fn add_version(&mut self,
                       conn: &GenericConnection,
                       ver: &semver::Version,
                       features: &HashMap<String, Vec<String>>,
                       authors: &[String])
                       -> CargoResult<Version> {
        match try!(Version::find_by_num(conn, self.id, ver)) {
            Some(..) => {
                return Err(human(format!("crate version `{}` is already uploaded",
                                         ver)))
            }
            None => {}
        }
        let zero = semver::Version::parse("0.0.0").unwrap();
        if *ver > self.max_version || self.max_version == zero {
            self.max_version = ver.clone();
        }
        let stmt = try!(conn.prepare("UPDATE crates SET max_version = $1
                           WHERE id = $2 RETURNING updated_at"));
        let rows = try!(stmt.query(&[&self.max_version.to_string(), &self.id]));
        self.updated_at = rows.get(0).get("updated_at");
        Version::insert(conn, self.id, ver, features, authors)
    }

    pub fn keywords(&self, conn: &GenericConnection) -> CargoResult<Vec<Keyword>> {
        let stmt = try!(conn.prepare("SELECT keywords.* FROM keywords
                                      LEFT JOIN crates_keywords
                                      ON keywords.id = crates_keywords.keyword_id
                                      WHERE crates_keywords.crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        Ok(rows.iter().map(|r| Model::from_row(&r)).collect())
    }

    pub fn categories(&self, conn: &GenericConnection) -> CargoResult<Vec<Category>> {
        let stmt = try!(conn.prepare("SELECT categories.* FROM categories \
                                      LEFT JOIN crates_categories \
                                      ON categories.id = \
                                         crates_categories.category_id \
                                      WHERE crates_categories.crate_id = $1"));
        let rows = try!(stmt.query(&[&self.id]));
        Ok(rows.iter().map(|r| Model::from_row(&r)).collect())
    }

    /// Returns (dependency, dependent crate name)
    pub fn reverse_dependencies(&self,
                                conn: &GenericConnection,
                                offset: i64,
                                limit: i64)
                                -> CargoResult<(Vec<(Dependency, String)>, i64)> {
        let select_sql = "
              FROM dependencies
              INNER JOIN versions
                ON versions.id = dependencies.version_id
              INNER JOIN crates
                ON crates.id = versions.crate_id
              WHERE dependencies.crate_id = $1
                AND versions.num = crates.max_version
        ";
        let fetch_sql = format!("SELECT DISTINCT ON (crate_name)
                                        dependencies.*,
                                        crates.name AS crate_name
                                        {}
                               ORDER BY crate_name ASC
                                 OFFSET $2
                                  LIMIT $3", select_sql);
        let count_sql = format!("SELECT COUNT(DISTINCT(crates.id)) {}",
                                select_sql);

        let stmt = try!(conn.prepare(&fetch_sql));
        let vec: Vec<_> = try!(stmt.query(&[&self.id, &offset, &limit]))
                                   .iter().map(|r| {
            (Model::from_row(&r), r.get("crate_name"))
        }).collect();
        let stmt = try!(conn.prepare(&count_sql));
        let cnt: i64 = try!(stmt.query(&[&self.id])).iter().next().unwrap().get(0);

        Ok((vec, cnt))
    }
}

impl Model for Crate {
    fn from_row(row: &Row) -> Crate {
        let max: String = row.get("max_version");
        Crate {
            id: row.get("id"),
            name: row.get("name"),
            updated_at: row.get("updated_at"),
            created_at: row.get("created_at"),
            downloads: row.get("downloads"),
            description: row.get("description"),
            documentation: row.get("documentation"),
            homepage: row.get("homepage"),
            readme: row.get("readme"),
            max_version: semver::Version::parse(&max).unwrap(),
            license: row.get("license"),
            repository: row.get("repository"),
            max_upload_size: row.get("max_upload_size"),
        }
    }
    fn table_name(_: Option<Crate>) -> &'static str { "crates" }
}

/// Handles the `GET /crates` route.
#[allow(trivial_casts)]
pub fn index(req: &mut Request) -> CargoResult<Response> {
    let conn = try!(req.tx());
    let (offset, limit) = try!(req.pagination(10, 100));
    let query = req.query();
    let sort = query.get("sort").map(|s| &s[..]).unwrap_or("alpha");
    let sort_sql = match sort {
        "downloads" => "crates.downloads DESC",
        _ => "crates.name ASC",
    };

    // Different queries for different parameters.
    //
    // Sure wish we had an arel-like thing here...
    let mut pattern = String::new();
    let mut id = -1;
    let (mut needs_id, mut needs_pattern) = (false, false);
    let mut args = vec![&limit as &ToSql, &offset];
    let (q, cnt) = query.get("q").map(|query| {
        args.insert(0, query);
        let rank_sort_sql = match sort {
            "downloads" => format!("{}, rank DESC", sort_sql),
            _ => format!("rank DESC, {}", sort_sql),
        };
        format!("{},", sort_sql); // Append Comma
        (format!("SELECT crates.* FROM crates,
                               plainto_tsquery($1) q,
                               ts_rank_cd(textsearchable_index_col, q) rank
          WHERE q @@ textsearchable_index_col
          ORDER BY name = $1 DESC, {}
          LIMIT $2 OFFSET $3", rank_sort_sql),
         "SELECT COUNT(crates.*) FROM crates,
                                      plainto_tsquery($1) q
          WHERE q @@ textsearchable_index_col".to_string())
    }).or_else(|| {
        query.get("letter").map(|letter| {
            pattern = format!("{}%", letter.chars().next().unwrap()
                                           .to_lowercase().collect::<String>());
            needs_pattern = true;
            (format!("SELECT * FROM crates WHERE canon_crate_name(name) \
                      LIKE $1 ORDER BY {} LIMIT $2 OFFSET $3", sort_sql),
             "SELECT COUNT(*) FROM crates WHERE canon_crate_name(name) \
              LIKE $1".to_string())
        })
    }).or_else(|| {
        query.get("keyword").map(|kw| {
            args.insert(0, kw);
            let base = "FROM crates
                        INNER JOIN crates_keywords
                                ON crates.id = crates_keywords.crate_id
                        INNER JOIN keywords
                                ON crates_keywords.keyword_id = keywords.id
                        WHERE lower(keywords.keyword) = lower($1)";
            (format!("SELECT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql),
             format!("SELECT COUNT(crates.*) {}", base))
        })
    }).or_else(|| {
        query.get("category").map(|cat| {
            args.insert(0, cat);
            let base = "FROM crates \
                        INNER JOIN crates_categories \
                                ON crates.id = crates_categories.crate_id \
                        INNER JOIN categories \
                                ON crates_categories.category_id = \
                                   categories.id \
                        WHERE lower(categories.category) = lower($1)";
            (format!("SELECT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql),
             format!("SELECT COUNT(crates.*) {}", base))
        })
    }).or_else(|| {
        query.get("user_id").and_then(|s| s.parse::<i32>().ok()).map(|user_id| {
            id = user_id;
            needs_id = true;
            (format!("SELECT crates.* FROM crates
                       INNER JOIN crate_owners
                          ON crate_owners.crate_id = crates.id
                       WHERE crate_owners.owner_id = $1
                       AND crate_owners.owner_kind = {}
                       ORDER BY {}
                      LIMIT $2 OFFSET $3",
                     OwnerKind::User as i32, sort_sql),
             format!("SELECT COUNT(crates.*) FROM crates
               INNER JOIN crate_owners
                  ON crate_owners.crate_id = crates.id
               WHERE crate_owners.owner_id = $1 \
                 AND crate_owners.owner_kind = {}",
                 OwnerKind::User as i32))
        })
    }).or_else(|| {
        query.get("following").map(|_| {
            needs_id = true;
            (format!("SELECT crates.* FROM crates
                      INNER JOIN follows
                         ON follows.crate_id = crates.id AND
                            follows.user_id = $1 ORDER BY
                      {} LIMIT $2 OFFSET $3", sort_sql),
             "SELECT COUNT(crates.*) FROM crates
              INNER JOIN follows
                 ON follows.crate_id = crates.id AND
                    follows.user_id = $1".to_string())
        })
    }).unwrap_or_else(|| {
        (format!("SELECT * FROM crates ORDER BY {} LIMIT $1 OFFSET $2",
                 sort_sql),
         "SELECT COUNT(*) FROM crates".to_string())
    });

    if needs_id {
        if id == -1 {
            id = try!(req.user()).id;
        }
        args.insert(0, &id);
    } else if needs_pattern {
        args.insert(0, &pattern);
    }

    // Collect all the crates
    let stmt = try!(conn.prepare(&q));
    let mut crates = Vec::new();
    for row in try!(stmt.query(&args)).iter() {
        let krate: Crate = Model::from_row(&row);
        crates.push(krate.minimal_encodable());
    }

    // Query for the total count of crates
    let stmt = try!(conn.prepare(&cnt));
    let args = if args.len() > 2 {&args[..1]} else {&args[..0]};
    let rows = try!(stmt.query(args));
    let row = rows.iter().next().unwrap();
    let total = row.get(0);

    #[derive(RustcEncodable)]
    struct R { crates: Vec<EncodableCrate>, meta: Meta }
    #[derive(RustcEncodable)]
    struct Meta { total: i64 }

    Ok(req.json(&R {
        crates: crates,
        meta: Meta { total: total },
    }))
}

/// Handles the `GET /summary` route.
pub fn summary(req: &mut Request) -> CargoResult<Response> {
    let tx = try!(req.tx());
    let num_crates = try!(Crate::count(tx));
    let num_downloads = {
        let stmt = try!(tx.prepare("SELECT total_downloads FROM metadata"));
        let rows = try!(stmt.query(&[]));
        rows.iter().next().unwrap().get("total_downloads")
    };

    let to_crates = |stmt: pg::stmt::Statement| -> CargoResult<Vec<_>> {
        let rows = try!(stmt.query(&[]));
        Ok(rows.iter().map(|r| {
            let krate: Crate = Model::from_row(&r);
            krate.minimal_encodable()
        }).collect::<Vec<EncodableCrate>>())
    };
    let new_crates = try!(tx.prepare("SELECT * FROM crates \
                                        ORDER BY created_at DESC LIMIT 10"));
    let just_updated = try!(tx.prepare("SELECT * FROM crates \
                                        WHERE updated_at::timestamp(0) !=
                                              created_at::timestamp(0)
                                        ORDER BY updated_at DESC LIMIT 10"));
    let most_downloaded = try!(tx.prepare("SELECT * FROM crates \
                                           ORDER BY downloads DESC LIMIT 10"));

    #[derive(RustcEncodable)]
    struct R {
        num_downloads: i64,
        num_crates: i64,
        new_crates: Vec<EncodableCrate>,
        most_downloaded: Vec<EncodableCrate>,
        just_updated: Vec<EncodableCrate>,
    }
    Ok(req.json(&R {
        num_downloads: num_downloads,
        num_crates: num_crates,
        new_crates: try!(to_crates(new_crates)),
        most_downloaded: try!(to_crates(most_downloaded)),
        just_updated: try!(to_crates(just_updated)),
    }))
}

/// Handles the `GET /crates/:crate_id` route.
pub fn show(req: &mut Request) -> CargoResult<Response> {
    let name = &req.params()["crate_id"];
    let conn = try!(req.tx());
    let krate = try!(Crate::find_by_name(conn, &name));
    let versions = try!(krate.versions(conn));
    let ids = versions.iter().map(|v| v.id).collect();
    let kws = try!(krate.keywords(conn));
    let cats = try!(krate.categories(conn));

    #[derive(RustcEncodable)]
    struct R {
        krate: EncodableCrate,
        versions: Vec<EncodableVersion>,
        keywords: Vec<EncodableKeyword>,
        categories: Vec<EncodableCategory>,
    }
    Ok(req.json(&R {
        krate: krate.clone().encodable(Some(ids), Some(&kws), Some(&cats)),
        versions: versions.into_iter().map(|v| {
            v.encodable(&krate.name)
        }).collect(),
        keywords: kws.into_iter().map(|k| k.encodable()).collect(),
        categories: cats.into_iter().map(|k| k.encodable()).collect(),
    }))
}

/// Handles the `PUT /crates/new` route.
pub fn new(req: &mut Request) -> CargoResult<Response> {
    let app = req.app().clone();

    let (new_crate, user) = try!(parse_new_headers(req));
    let name = &*new_crate.name;
    let vers = &*new_crate.vers;
    let features = new_crate.features.iter().map(|(k, v)| {
        (k[..].to_string(), v.iter().map(|v| v[..].to_string()).collect())
    }).collect::<HashMap<String, Vec<String>>>();
    let keywords = new_crate.keywords.as_ref().map(|s| &s[..])
                                     .unwrap_or(&[]);
    let keywords = keywords.iter().map(|k| k[..].to_string()).collect::<Vec<_>>();

    let categories = new_crate.categories.as_ref().map(|s| &s[..])
                                     .unwrap_or(&[]);
    let categories: Vec<_> = categories.iter().map(|k| k[..].to_string()).collect();

    // Persist the new crate, if it doesn't already exist
    let mut krate = try!(Crate::find_or_insert(try!(req.tx()), name, user.id,
                                               &new_crate.description,
                                               &new_crate.homepage,
                                               &new_crate.documentation,
                                               &new_crate.readme,
                                               &new_crate.repository,
                                               &new_crate.license,
                                               &new_crate.license_file,
                                               None));

    let owners = try!(krate.owners(try!(req.tx())));
    if try!(rights(req.app(), &owners, &user)) < Rights::Publish {
        return Err(human("crate name has already been claimed by \
                          another user"))
    }

    if krate.name != name {
        return Err(human(format!("crate was previously named `{}`", krate.name)))
    }

    let length = try!(req.content_length().chain_error(|| {
        human("missing header: Content-Length")
{'fraction_non_alphanumeric': 0.1362612800453704, 'fraction_numerical': 0.004904004937365515, 'mean_word_length': 1.58860103626943, 'pattern_counts': {'":': 2, '<': 174, '<?xml version=': 0, '>': 272, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 292, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2428120', 'n_tokens_mistral': 16635, 'n_tokens_neox': 16324, 'n_words': 3720}