<commit_before>//! Loads "sysroot" crate.
//!
//! One confusing point here is that normally sysroot is a bunch of `.rlib`s,
//! but we can't process `.rlib` and need source code instead. The source code
//! is typically installed with `rustup component add rust-src` command.

use std::{convert::TryFrom, env, ops, path::PathBuf, process::Command};

use anyhow::{format_err, Result};
use arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};

use crate::utf8_stdout;

#[derive(Default, Debug, Clone, Eq, PartialEq)]
pub struct Sysroot {
    crates: Arena<SysrootCrateData>,
}

pub type SysrootCrate = Idx<SysrootCrateData>;

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct SysrootCrateData {
    pub name: String,
    pub root: AbsPathBuf,
    pub deps: Vec<SysrootCrate>,
}

impl ops::Index<SysrootCrate> for Sysroot {
    type Output = SysrootCrateData;
    fn index(&self, index: SysrootCrate) -> &SysrootCrateData {
        &self.crates[index]
    }
}

impl Sysroot {
    pub fn public_deps(&self) -> impl Iterator<Item = (&'static str, SysrootCrate)> + '_ {
        // core is added as a dependency before std in order to
        // mimic rustcs dependency order
        vec!["core", "alloc", "std"].into_iter().filter_map(move |it| Some((it, self.by_name(it)?)))
    }

    pub fn proc_macro(&self) -> Option<SysrootCrate> {
        self.by_name("proc_macro")
    }

    pub fn crates<'a>(&'a self) -> impl Iterator<Item = SysrootCrate> + ExactSizeIterator + 'a {
        self.crates.iter().map(|(id, _data)| id)
    }

    pub fn discover(cargo_toml: &AbsPath) -> Result<Sysroot> {
        log::debug!("Discovering sysroot for {}", cargo_toml.display());
        let current_dir = cargo_toml.parent().unwrap();
        let sysroot_src_dir = discover_sysroot_src_dir(current_dir)?;
        let res = Sysroot::load(&sysroot_src_dir)?;
        Ok(res)
    }

    pub fn load(sysroot_src_dir: &AbsPath) -> Result<Sysroot> {
        let mut sysroot = Sysroot { crates: Arena::default() };

        for name in SYSROOT_CRATES.trim().lines() {
            // FIXME: first path when 1.47 comes out
            // https://github.com/rust-lang/rust/pull/73265
            let root = [format!("lib{}/lib.rs", name), format!("{}/src/lib.rs", name)]
                .iter()
                .map(|it| sysroot_src_dir.join(it))
                .find(|it| it.exists());

            if let Some(root) = root {
                sysroot.crates.alloc(SysrootCrateData {
                    name: name.into(),
                    root,
                    deps: Vec::new(),
                });
            }
        }

        if let Some(std) = sysroot.by_name("std") {
            for dep in STD_DEPS.trim().lines() {
                if let Some(dep) = sysroot.by_name(dep) {
                    sysroot.crates[std].deps.push(dep)
                }
            }
        }

        if let Some(alloc) = sysroot.by_name("alloc") {
            if let Some(core) = sysroot.by_name("core") {
                sysroot.crates[alloc].deps.push(core);
            }
        }

        if sysroot.by_name("core").is_none() {
            let var_note = if env::var_os("RUST_SRC_PATH").is_some() {
                " (`RUST_SRC_PATH` might be incorrect, try unsetting it)"
            } else {
                ""
            };
            anyhow::bail!(
                "could not find libcore in sysroot path `{}`{}",
                sysroot_src_dir.as_ref().display(),
                var_note,
            );
        }

        Ok(sysroot)
    }

    fn by_name(&self, name: &str) -> Option<SysrootCrate> {
        let (id, _data) = self.crates.iter().find(|(_id, data)| data.name == name)?;
        Some(id)
    }
}

fn discover_sysroot_src_dir(current_dir: &AbsPath) -> Result<AbsPathBuf> {
    if let Ok(path) = env::var("RUST_SRC_PATH") {
        let path = AbsPathBuf::try_from(path.as_str())
            .map_err(|path| format_err!("RUST_SRC_PATH must be absolute: {}", path.display()))?;
        log::debug!("Discovered sysroot by RUST_SRC_PATH: {}", path.display());
        return Ok(path);
    }

    let sysroot_path = {
        let mut rustc = Command::new(toolchain::rustc());
        rustc.current_dir(current_dir).args(&["--print", "sysroot"]);
        log::debug!("Discovering sysroot by {:?}", rustc);
        let stdout = utf8_stdout(rustc)?;
        AbsPathBuf::assert(PathBuf::from(stdout))
    };

    get_rust_src(&sysroot_path)
        .or_else(|| {
            let mut rustup = Command::new(toolchain::rustup());
            rustup.current_dir(current_dir).args(&["component", "add", "rust-src"]);
            utf8_stdout(rustup).ok()?;
            get_rust_src(&sysroot_path)
        })
        .ok_or_else(|| {
            format_err!(
                "\
can't load standard library from sysroot
{}
(discovered via `rustc --print sysroot`)
try running `rustup component add rust-src` or set `RUST_SRC_PATH`",
                sysroot_path.display(),
            )
        })
}

fn get_rust_src(sysroot_path: &AbsPath) -> Option<AbsPathBuf> {
    // Try the new path first since the old one still exists.
    //
    // FIXME: remove `src` when 1.47 comes out
    // https://github.com/rust-lang/rust/pull/73265<commit_msg>remove FIXME after rust 1.47 comes out<commit_after>//! Loads "sysroot" crate.
//!
//! One confusing point here is that normally sysroot is a bunch of `.rlib`s,
//! but we can't process `.rlib` and need source code instead. The source code
//! is typically installed with `rustup component add rust-src` command.

use std::{convert::TryFrom, env, ops, path::PathBuf, process::Command};

use anyhow::{format_err, Result};
use arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf};

use crate::utf8_stdout;

#[derive(Default, Debug, Clone, Eq, PartialEq)]
pub struct Sysroot {
    crates: Arena<SysrootCrateData>,
}

pub type SysrootCrate = Idx<SysrootCrateData>;

#[derive(Debug, Clone, Eq, PartialEq)]
pub struct SysrootCrateData {
    pub name: String,
    pub root: AbsPathBuf,
    pub deps: Vec<SysrootCrate>,
}

impl ops::Index<SysrootCrate> for Sysroot {
    type Output = SysrootCrateData;
    fn index(&self, index: SysrootCrate) -> &SysrootCrateData {
        &self.crates[index]
    }
}

impl Sysroot {
    pub fn public_deps(&self) -> impl Iterator<Item = (&'static str, SysrootCrate)> + '_ {
        // core is added as a dependency before std in order to
        // mimic rustcs dependency order
        vec!["core", "alloc", "std"].into_iter().filter_map(move |it| Some((it, self.by_name(it)?)))
    }

    pub fn proc_macro(&self) -> Option<SysrootCrate> {
        self.by_name("proc_macro")
    }

    pub fn crates<'a>(&'a self) -> impl Iterator<Item = SysrootCrate> + ExactSizeIterator + 'a {
        self.crates.iter().map(|(id, _data)| id)
    }

    pub fn discover(cargo_toml: &AbsPath) -> Result<Sysroot> {
        log::debug!("Discovering sysroot for {}", cargo_toml.display());
        let current_dir = cargo_toml.parent().unwrap();
        let sysroot_src_dir = discover_sysroot_src_dir(current_dir)?;
        let res = Sysroot::load(&sysroot_src_dir)?;
        Ok(res)
    }

    pub fn load(sysroot_src_dir: &AbsPath) -> Result<Sysroot> {
        let mut sysroot = Sysroot { crates: Arena::default() };

        for name in SYSROOT_CRATES.trim().lines() {
            let root = [format!("{}/src/lib.rs", name), format!("lib{}/lib.rs", name)]
                .iter()
                .map(|it| sysroot_src_dir.join(it))
                .find(|it| it.exists());

            if let Some(root) = root {
                sysroot.crates.alloc(SysrootCrateData {
                    name: name.into(),
                    root,
                    deps: Vec::new(),
                });
            }
        }

        if let Some(std) = sysroot.by_name("std") {
            for dep in STD_DEPS.trim().lines() {
                if let Some(dep) = sysroot.by_name(dep) {
                    sysroot.crates[std].deps.push(dep)
                }
            }
        }

        if let Some(alloc) = sysroot.by_name("alloc") {
            if let Some(core) = sysroot.by_name("core") {
                sysroot.crates[alloc].deps.push(core);
            }
        }

        if sysroot.by_name("core").is_none() {
            let var_note = if env::var_os("RUST_SRC_PATH").is_some() {
                " (`RUST_SRC_PATH` might be incorrect, try unsetting it)"
            } else {
                ""
            };
            anyhow::bail!(
                "could not find libcore in sysroot path `{}`{}",
                sysroot_src_dir.as_ref().display(),
                var_note,
            );
        }

        Ok(sysroot)
    }

    fn by_name(&self, name: &str) -> Option<SysrootCrate> {
        let (id, _data) = self.crates.iter().find(|(_id, data)| data.name == name)?;
        Some(id)
    }
}

fn discover_sysroot_src_dir(current_dir: &AbsPath) -> Result<AbsPathBuf> {
    if let Ok(path) = env::var("RUST_SRC_PATH") {
        let path = AbsPathBuf::try_from(path.as_str())
            .map_err(|path| format_err!("RUST_SRC_PATH must be absolute: {}", path.display()))?;
        log::debug!("Discovered sysroot by RUST_SRC_PATH: {}", path.display());
        return Ok(path);
    }

    let sysroot_path = {
        let mut rustc = Command::new(toolchain::rustc());
        rustc.current_dir(current_dir).args(&["--print", "sysroot"]);
        log::debug!("Discovering sysroot by {:?}", rustc);
        let stdout = utf8_stdout(rustc)?;
        AbsPathBuf::assert(PathBuf::from(stdout))
    };

    get_rust_src(&sysroot_path)
        .or_else(|| {
            let mut rustup = Command::new(toolchain::rustup());
            rustup.current_dir(current_dir).args(&["component", "add", "rust-src"]);
            utf8_stdout(rustup).ok()?;
            get_rust_src(&sysroot_path)
        })
        .ok_or_else(|| {
            format_err!(
                "\
can't load standard library from sysroot
{}
(discovered via `rustc --print sysroot`)
try running `rustup component add rust-src` or set `RUST_SRC_PATH`",
                sysroot_path.display(),
            )
        })
}

fn get_rust_src(sysroot_path: &AbsPath) -> Option<AbsPathBuf> {
    // Try the new path first since the old one still exists.
{'fraction_non_alphanumeric': 0.14704735918924186, 'fraction_numerical': 0.0024361722861040735, 'mean_word_length': 2.2968197879858656, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 47, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '955354', 'n_tokens_mistral': 3380, 'n_tokens_neox': 3308, 'n_words': 723}