<commit_before>// cargo.rs - Rust package manager

use rustc;
use std;

import rustc::syntax::{ast, codemap, visit};
import rustc::syntax::parse::parser;
import rustc::util::filesearch::get_cargo_root;

import std::fs;
import std::generic_os;
import std::io;
import std::json;
import option;
import option::{none, some};
import result;
import std::map;
import std::os;
import std::run;
import str;
import std::tempfile;
import vec;

tag _src {
    /* Break cycles in package <-> source */
    _source(source);
}

type package = {
//    source: _src,
    name: str,
    uuid: str,
    url: str,
    method: str,
    ref: option::t<str>,
    tags: [str]
};

type source = {
    name: str,
    url: str,
    sig: option::t<str>,
    key: option::t<str>,
    keyfp: option::t<str>,
    mutable packages: [package]
};

type cargo = {
    pgp: bool,
    root: str,
    bindir: str,
    libdir: str,
    workdir: str,
    sourcedir: str,
    sources: map::hashmap<str, source>
};

type pkg = {
    name: str,
    vers: str,
    uuid: str,
    desc: option::t<str>,
    sigs: option::t<str>,
    crate_type: option::t<str>
};

fn info(msg: str) {
    io::stdout().write_line(msg);
}

fn warn(msg: str) {
    io::stdout().write_line("warning: " + msg);
}

fn error(msg: str) {
    io::stdout().write_line("error: " + msg);
}

fn load_link(mis: [@ast::meta_item]) -> (option::t<str>,
                                         option::t<str>,
                                         option::t<str>) {
    let name = none;
    let vers = none;
    let uuid = none;
    for a: @ast::meta_item in mis {
        alt a.node {
            ast::meta_name_value(v, {node: ast::lit_str(s), span: _}) {
                alt v {
                    "name" { name = some(s); }
                    "vers" { vers = some(s); }
                    "uuid" { uuid = some(s); }
                    _ { }
                }
            }
        }
    }
    (name, vers, uuid)
}

fn load_pkg(filename: str) -> option::t<pkg> {
    let sess = @{cm: codemap::new_codemap(), mutable next_id: 0};
    let c = parser::parse_crate_from_crate_file(filename, [], sess);

    let name = none;
    let vers = none;
    let uuid = none;
    let desc = none;
    let sigs = none;
    let crate_type = none;

    for a in c.node.attrs {
        alt a.node.value.node {
            ast::meta_name_value(v, {node: ast::lit_str(s), span: _}) {
                alt v {
                    "desc" { desc = some(v); }
                    "sigs" { sigs = some(v); }
                    "crate_type" { crate_type = some(v); }
                    _ { }
                }
            }
            ast::meta_list(v, mis) {
                if v == "link" {
                    let (n, v, u) = load_link(mis);
                    name = n;
                    vers = v;
                    uuid = u;
                }
            }
        }
    }

    alt (name, vers, uuid) {
        (some(name0), some(vers0), some(uuid0)) {
            some({
                name: name0,
                vers: vers0,
                uuid: uuid0,
                desc: desc,
                sigs: sigs,
                crate_type: crate_type})
        }
        _ { ret none; }
    }
}

fn print(s: str) {
    io::stdout().write_line(s);
}

fn rest(s: str, start: uint) -> str {
    if (start >= str::char_len(s)) {
        ""
    } else {
        str::char_slice(s, start, str::char_len(s))
    }
}

fn need_dir(s: str) {
    if fs::path_is_dir(s) { ret; }
    if !fs::make_dir(s, 0x1c0i32) {
        fail #fmt["can't make_dir %s", s];
    }
}

fn parse_source(name: str, j: json::json) -> source {
    alt j {
        json::dict(_j) {
            let url = alt _j.find("url") {
                some(json::string(u)) {
                    u
                }
                _ { fail "Needed 'url' field in source."; }
            };
            let sig = alt _j.find("sig") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            let key = alt _j.find("key") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            let keyfp = alt _j.find("keyfp") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            ret { name: name, url: url, sig: sig, key: key, keyfp: keyfp,
                  mutable packages: [] };
        }
        _ { fail "Needed dict value in source."; }
    };
}

fn try_parse_sources(filename: str, sources: map::hashmap<str, source>) {
    if !fs::path_exists(filename)  { ret; }
    let c = io::read_whole_file_str(filename);
    let j = json::from_str(result::get(c));
    alt j {
        some(json::dict(_j)) {
            _j.items { |k, v|
                sources.insert(k, parse_source(k, v));
                #debug("source: %s", k);
            }
        }
        _ { fail "malformed sources.json"; }
    }
}

fn load_one_source_package(&src: source, p: map::hashmap<str, json::json>) {
    let name = alt p.find("name") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing name)");
            ret;
        }
    };

    let uuid = alt p.find("uuid") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing uuid)");
            ret;
        }
    };

    let url = alt p.find("url") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing url)");
            ret;
        }
    };

    let method = alt p.find("method") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing method)");
            ret;
        }
    };

    let ref = alt p.find("ref") {
        some(json::string(_n)) { some(_n) }
        _ { none }
    };

    let tags = [];
    alt p.find("tags") {
        some(json::list(js)) {
            for j in *js {
                alt j {
                    json::string(_j) { vec::grow(tags, 1u, _j); }
                    _ { }
                }
            }
        }
        _ { }
    }
    vec::grow(src.packages, 1u, {
        // source: _source(src),
        name: name,
        uuid: uuid,
        url: url,
        method: method,
        ref: ref,
        tags: tags
    });
    log(debug, "  Loaded package: " + src.name + "/" + name);
}

fn load_source_packages(&c: cargo, &src: source) {
    log(debug, "Loading source: " + src.name);
    let dir = fs::connect(c.sourcedir, src.name);
    let pkgfile = fs::connect(dir, "packages.json");
    if !fs::path_exists(pkgfile) { ret; }
    let pkgstr = io::read_whole_file_str(pkgfile);
    let j = json::from_str(result::get(pkgstr));
    alt j {
        some(json::list(js)) {
            for _j: json::json in *js {
                alt _j {
                    json::dict(_p) {
                        load_one_source_package(src, _p);
                    }
                    _ {
                        warn("Malformed source json: " + src.name +
                             " (non-dict pkg)");
                    }
                }
            }
        }
        _ {
            warn("Malformed source json: " + src.name);
        }
    };
}

fn configure() -> cargo {
    let p = alt get_cargo_root() {
      result::ok(p) { p }
      result::err(e) { fail e }
    };

    let sources = map::new_str_hash::<source>();
    try_parse_sources(fs::connect(p, "sources.json"), sources);
    try_parse_sources(fs::connect(p, "local-sources.json"), sources);
    let c = {
        pgp: pgp::supported(),
        root: p,
        bindir: fs::connect(p, "bin"),
        libdir: fs::connect(p, "lib"),
        workdir: fs::connect(p, "work"),
        sourcedir: fs::connect(p, "sources"),
        sources: sources
    };

    need_dir(c.root);
    need_dir(c.sourcedir);
    need_dir(c.workdir);
    need_dir(c.libdir);
    need_dir(c.bindir);

    sources.keys { |k|
        let s = sources.get(k);
        load_source_packages(c, s);
        sources.insert(k, s);
    };

    if c.pgp {
        pgp::init(c.root);
    }

    c
}

fn for_each_package(c: cargo, b: block(source, package)) {
    c.sources.values({ |v|
        for p in copy v.packages {
            b(v, p);
        }
    })
}

fn install_one_crate(c: cargo, _path: str, cf: str, _p: pkg) {
    let name = fs::basename(cf);
    let ri = str::index(name, '.' as u8);
    if ri != -1 {
        name = str::slice(name, 0u, ri as uint);
    }
    #debug("Installing: %s", name);
    let old = fs::list_dir(".");
    let p = run::program_output("rustc", [name + ".rc"]);
    if p.status != 0 {
        error(#fmt["rustc failed: %d %s", p.status, p.err]);
        ret;
    }
    let new = fs::list_dir(".");
    let created =
        vec::filter::<str>(new, { |n| !vec::member::<str>(n, old) });
    let exec_suffix = os::exec_suffix();
    for ct: str in created {
        if (exec_suffix != "" && str::ends_with(ct, exec_suffix)) ||
            (exec_suffix == "" && !str::starts_with(ct, "./lib")) {
            #debug("  bin: %s", ct);
            // FIXME: need libstd fs::copy or something
            run::run_program("cp", [ct, c.bindir]);
        } else {
            #debug("  lib: %s", ct);
            run::run_program("cp", [ct, c.libdir]);<commit_msg>cargo: Print rustc's stdout when compilation fails<commit_after>// cargo.rs - Rust package manager

use rustc;
use std;

import rustc::syntax::{ast, codemap, visit};
import rustc::syntax::parse::parser;
import rustc::util::filesearch::get_cargo_root;

import std::fs;
import std::generic_os;
import std::io;
import std::json;
import option;
import option::{none, some};
import result;
import std::map;
import std::os;
import std::run;
import str;
import std::tempfile;
import vec;

tag _src {
    /* Break cycles in package <-> source */
    _source(source);
}

type package = {
//    source: _src,
    name: str,
    uuid: str,
    url: str,
    method: str,
    ref: option::t<str>,
    tags: [str]
};

type source = {
    name: str,
    url: str,
    sig: option::t<str>,
    key: option::t<str>,
    keyfp: option::t<str>,
    mutable packages: [package]
};

type cargo = {
    pgp: bool,
    root: str,
    bindir: str,
    libdir: str,
    workdir: str,
    sourcedir: str,
    sources: map::hashmap<str, source>
};

type pkg = {
    name: str,
    vers: str,
    uuid: str,
    desc: option::t<str>,
    sigs: option::t<str>,
    crate_type: option::t<str>
};

fn info(msg: str) {
    io::stdout().write_line(msg);
}

fn warn(msg: str) {
    io::stdout().write_line("warning: " + msg);
}

fn error(msg: str) {
    io::stdout().write_line("error: " + msg);
}

fn load_link(mis: [@ast::meta_item]) -> (option::t<str>,
                                         option::t<str>,
                                         option::t<str>) {
    let name = none;
    let vers = none;
    let uuid = none;
    for a: @ast::meta_item in mis {
        alt a.node {
            ast::meta_name_value(v, {node: ast::lit_str(s), span: _}) {
                alt v {
                    "name" { name = some(s); }
                    "vers" { vers = some(s); }
                    "uuid" { uuid = some(s); }
                    _ { }
                }
            }
        }
    }
    (name, vers, uuid)
}

fn load_pkg(filename: str) -> option::t<pkg> {
    let sess = @{cm: codemap::new_codemap(), mutable next_id: 0};
    let c = parser::parse_crate_from_crate_file(filename, [], sess);

    let name = none;
    let vers = none;
    let uuid = none;
    let desc = none;
    let sigs = none;
    let crate_type = none;

    for a in c.node.attrs {
        alt a.node.value.node {
            ast::meta_name_value(v, {node: ast::lit_str(s), span: _}) {
                alt v {
                    "desc" { desc = some(v); }
                    "sigs" { sigs = some(v); }
                    "crate_type" { crate_type = some(v); }
                    _ { }
                }
            }
            ast::meta_list(v, mis) {
                if v == "link" {
                    let (n, v, u) = load_link(mis);
                    name = n;
                    vers = v;
                    uuid = u;
                }
            }
        }
    }

    alt (name, vers, uuid) {
        (some(name0), some(vers0), some(uuid0)) {
            some({
                name: name0,
                vers: vers0,
                uuid: uuid0,
                desc: desc,
                sigs: sigs,
                crate_type: crate_type})
        }
        _ { ret none; }
    }
}

fn print(s: str) {
    io::stdout().write_line(s);
}

fn rest(s: str, start: uint) -> str {
    if (start >= str::char_len(s)) {
        ""
    } else {
        str::char_slice(s, start, str::char_len(s))
    }
}

fn need_dir(s: str) {
    if fs::path_is_dir(s) { ret; }
    if !fs::make_dir(s, 0x1c0i32) {
        fail #fmt["can't make_dir %s", s];
    }
}

fn parse_source(name: str, j: json::json) -> source {
    alt j {
        json::dict(_j) {
            let url = alt _j.find("url") {
                some(json::string(u)) {
                    u
                }
                _ { fail "Needed 'url' field in source."; }
            };
            let sig = alt _j.find("sig") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            let key = alt _j.find("key") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            let keyfp = alt _j.find("keyfp") {
                some(json::string(u)) {
                    some(u)
                }
                _ { none }
            };
            ret { name: name, url: url, sig: sig, key: key, keyfp: keyfp,
                  mutable packages: [] };
        }
        _ { fail "Needed dict value in source."; }
    };
}

fn try_parse_sources(filename: str, sources: map::hashmap<str, source>) {
    if !fs::path_exists(filename)  { ret; }
    let c = io::read_whole_file_str(filename);
    let j = json::from_str(result::get(c));
    alt j {
        some(json::dict(_j)) {
            _j.items { |k, v|
                sources.insert(k, parse_source(k, v));
                #debug("source: %s", k);
            }
        }
        _ { fail "malformed sources.json"; }
    }
}

fn load_one_source_package(&src: source, p: map::hashmap<str, json::json>) {
    let name = alt p.find("name") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing name)");
            ret;
        }
    };

    let uuid = alt p.find("uuid") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing uuid)");
            ret;
        }
    };

    let url = alt p.find("url") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing url)");
            ret;
        }
    };

    let method = alt p.find("method") {
        some(json::string(_n)) { _n }
        _ {
            warn("Malformed source json: " + src.name + " (missing method)");
            ret;
        }
    };

    let ref = alt p.find("ref") {
        some(json::string(_n)) { some(_n) }
        _ { none }
    };

    let tags = [];
    alt p.find("tags") {
        some(json::list(js)) {
            for j in *js {
                alt j {
                    json::string(_j) { vec::grow(tags, 1u, _j); }
                    _ { }
                }
            }
        }
        _ { }
    }
    vec::grow(src.packages, 1u, {
        // source: _source(src),
        name: name,
        uuid: uuid,
        url: url,
        method: method,
        ref: ref,
        tags: tags
    });
    log(debug, "  Loaded package: " + src.name + "/" + name);
}

fn load_source_packages(&c: cargo, &src: source) {
    log(debug, "Loading source: " + src.name);
    let dir = fs::connect(c.sourcedir, src.name);
    let pkgfile = fs::connect(dir, "packages.json");
    if !fs::path_exists(pkgfile) { ret; }
    let pkgstr = io::read_whole_file_str(pkgfile);
    let j = json::from_str(result::get(pkgstr));
    alt j {
        some(json::list(js)) {
            for _j: json::json in *js {
                alt _j {
                    json::dict(_p) {
                        load_one_source_package(src, _p);
                    }
                    _ {
                        warn("Malformed source json: " + src.name +
                             " (non-dict pkg)");
                    }
                }
            }
        }
        _ {
            warn("Malformed source json: " + src.name);
        }
    };
}

fn configure() -> cargo {
    let p = alt get_cargo_root() {
      result::ok(p) { p }
      result::err(e) { fail e }
    };

    let sources = map::new_str_hash::<source>();
    try_parse_sources(fs::connect(p, "sources.json"), sources);
    try_parse_sources(fs::connect(p, "local-sources.json"), sources);
    let c = {
        pgp: pgp::supported(),
        root: p,
        bindir: fs::connect(p, "bin"),
        libdir: fs::connect(p, "lib"),
        workdir: fs::connect(p, "work"),
        sourcedir: fs::connect(p, "sources"),
        sources: sources
    };

    need_dir(c.root);
    need_dir(c.sourcedir);
    need_dir(c.workdir);
    need_dir(c.libdir);
    need_dir(c.bindir);

    sources.keys { |k|
        let s = sources.get(k);
        load_source_packages(c, s);
        sources.insert(k, s);
    };

    if c.pgp {
        pgp::init(c.root);
    }

    c
}

fn for_each_package(c: cargo, b: block(source, package)) {
    c.sources.values({ |v|
        for p in copy v.packages {
            b(v, p);
        }
    })
}

fn install_one_crate(c: cargo, _path: str, cf: str, _p: pkg) {
    let name = fs::basename(cf);
    let ri = str::index(name, '.' as u8);
    if ri != -1 {
        name = str::slice(name, 0u, ri as uint);
    }
    #debug("Installing: %s", name);
    let old = fs::list_dir(".");
    let p = run::program_output("rustc", [name + ".rc"]);
    if p.status != 0 {
        error(#fmt["rustc failed: %d\n%s\n%s", p.status, p.err, p.out]);
        ret;
    }
    let new = fs::list_dir(".");
    let created =
        vec::filter::<str>(new, { |n| !vec::member::<str>(n, old) });
    let exec_suffix = os::exec_suffix();
    for ct: str in created {
        if (exec_suffix != "" && str::ends_with(ct, exec_suffix)) ||
            (exec_suffix == "" && !str::starts_with(ct, "./lib")) {
            #debug("  bin: %s", ct);
            // FIXME: need libstd fs::copy or something
            run::run_program("cp", [ct, c.bindir]);
        } else {
            #debug("  lib: %s", ct);
            run::run_program("cp", [ct, c.libdir]);
{'fraction_non_alphanumeric': 0.1610932645215651, 'fraction_numerical': 0.0018958344304597398, 'mean_word_length': 1.548993288590604, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803812', 'n_tokens_mistral': 6475, 'n_tokens_neox': 6319, 'n_words': 1401}