<commit_before>//
// Copyright 2021 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use maplit::hashset;
use std::{
    collections::{HashMap, HashSet},
    io::{BufRead, BufReader},
    path::{Path, PathBuf},
    process::Command,
};

use crate::{
    files::*,
    internal::{CargoManifest, Scope},
};

#[derive(Debug)]
pub struct ModifiedContent {
    /// List of modified files.
    pub files: Option<Vec<String>>,
}

impl ModifiedContent {
    pub fn contains(&self, file_name: &str) -> bool {
        self.files.is_none()
            || self
                .files
                .as_ref()
                .unwrap()
                .contains(&file_name.to_string())
    }
}

/// Get all the files that have been modified in the given `scope`. If the scope is `Scope::All`
/// returns `None` as the list of files in `ModifiedContent`. For `Scope::Commits` and
/// `Scope::DiffToMain` the returned files include tracked files with unstaged changes, but do
/// not include new files that are not tracked (i.e., not added to git yet). For
/// `Scope::Commits(N)`, a positive number of commits must be given. In case of zero or
/// a negative number, only the last commit will be considered for finding the modified files.
pub fn modified_files(scope: &Scope) -> ModifiedContent {
    let args = match scope {
        Scope::All => None,
        Scope::Commits(commits) => Some(format!("HEAD~{}", std::cmp::max(1, *commits))),
        // TODO(#2491): Use `--merge-base` when we have a newer version of Git in the docker image.
        Scope::DiffToMain => Some("main".to_owned()),
    };

    let files: Option<Vec<String>> = args.map(|args| {
        let vec = Command::new("git")
            .args(spread!["diff".to_owned(), "--name-only".to_owned(), args,])
            .output()
            .expect("could not get modified files")
            .stdout;

        // Extract the file names from the git output
        String::from_utf8(vec)
            .expect("could not convert to string")
            .split('\n')
            .map(|s| format!("./{}", s))
            .collect()
    });

    // If something is changed in a Dockerfile, we'd want to run all CI steps, so set `files` to
    // `None` in `ModifiedContent`.
    if let Some(ref paths) = files {
        if paths.iter().any(|path| path.contains("Dockerfile")) {
            return ModifiedContent { files: None };
        }
    }

    ModifiedContent { files }
}

/// Returns the list of paths to `Cargo.toml` files for all crates that include at least one of the
/// given modified files.
pub fn directly_modified_crates(files: &ModifiedContent) -> ModifiedContent {
    let files = files.files.as_ref().map(|modified_files| {
        let mut crates = hashset![];
        for str_path in modified_files {
            if let Some(crate_path) = find_crate_toml_file(str_path.clone()) {
                crates.insert(crate_path);
            }
        }
        crates.iter().cloned().collect()
    });
    ModifiedContent { files }
}

fn find_crate_toml_file(str_path: String) -> Option<String> {
    if str_path.ends_with("Cargo.toml") {
        return Some(str_path);<commit_msg>Run xtask command for the entire repo if diff fails (#2616)<commit_after>//
// Copyright 2021 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use maplit::hashset;
use std::{
    collections::{HashMap, HashSet},
    io::{BufRead, BufReader},
    path::{Path, PathBuf},
    process::Command,
};

use crate::{
    files::*,
    internal::{CargoManifest, Scope},
};

#[derive(Debug)]
pub struct ModifiedContent {
    /// List of modified files.
    pub files: Option<Vec<String>>,
}

impl ModifiedContent {
    pub fn contains(&self, file_name: &str) -> bool {
        self.files.is_none()
            || self
                .files
                .as_ref()
                .unwrap()
                .contains(&file_name.to_string())
    }
}

/// Get all the files that have been modified in the given `scope`. If the scope is `Scope::All`
/// returns `None` as the list of files in `ModifiedContent`. For `Scope::Commits` and
/// `Scope::DiffToMain` the returned files include tracked files with unstaged changes, but do
/// not include new files that are not tracked (i.e., not added to git yet). For
/// `Scope::Commits(N)`, a positive number of commits must be given. In case of zero or
/// a negative number, only the last commit will be considered for finding the modified files.
pub fn modified_files(scope: &Scope) -> ModifiedContent {
    let args = match scope {
        Scope::All => None,
        Scope::Commits(commits) => Some(format!("HEAD~{}", std::cmp::max(1, *commits))),
        // TODO(#2491): Use `--merge-base` when we have a newer version of Git in the docker image.
        Scope::DiffToMain => Some("main".to_owned()),
    };

    let files: Option<Vec<String>> = args
        .and_then(|args| {
            let output = Command::new("git")
                .args(spread!["diff".to_owned(), "--name-only".to_owned(), args,])
                .output()
                .expect("could not get modified files");

            if output.status.success() {
                let vec = output.stdout;

                // Extract the file names from the git output
                Some(
                    String::from_utf8(vec)
                        .expect("could not convert to string")
                        .split('\n')
                        .map(|s| format!("./{}", s))
                        .collect(),
                )
            } else {
                eprintln!(
                    "WARN: git diff failed with error ({}), running the command for the entire code base.",
                    output.status
                );
                None
            }
        });

    // If something is changed in a Dockerfile, we'd want to run all CI steps, so set `files` to
    // `None` in `ModifiedContent`.
    if let Some(ref paths) = files {
        if paths.iter().any(|path| path.contains("Dockerfile")) {
            return ModifiedContent { files: None };
        }
    }

    ModifiedContent { files }
}

/// Returns the list of paths to `Cargo.toml` files for all crates that include at least one of the
/// given modified files.
pub fn directly_modified_crates(files: &ModifiedContent) -> ModifiedContent {
    let files = files.files.as_ref().map(|modified_files| {
        let mut crates = hashset![];
        for str_path in modified_files {
            if let Some(crate_path) = find_crate_toml_file(str_path.clone()) {
                crates.insert(crate_path);
            }
        }
        crates.iter().cloned().collect()
    });
    ModifiedContent { files }
}

fn find_crate_toml_file(str_path: String) -> Option<String> {
    if str_path.ends_with("Cargo.toml") {
        return Some(str_path);
{'fraction_non_alphanumeric': 0.12619138391155166, 'fraction_numerical': 0.004066590418096328, 'mean_word_length': 2.3375742154368107, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '307471', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2181, 'n_words': 767}