<commit_before>use std::io::{self, ErrorKind};
use std::fs::{File, Metadata, read_link};
use std::path::{Path, PathBuf};
use std::os::unix::fs::{PermissionsExt, MetadataExt};
use std::collections::{BTreeMap, BTreeSet, HashSet};

use libc::{uid_t, gid_t};
use quire::ast::{Ast, Tag};
use quire::validate as V;
use regex::{Regex, Error as RegexError};
use scan_dir::{ScanDir, Error as ScanDirError};

use container::root::temporary_change_root;
use file_util::shallow_copy;
use path_util::IterSelfAndParents;
use build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard};
use quick_error::ResultExt;


const DEFAULT_UMASK: u32 = 0o002;

const DIR_MODE: u32 = 0o777;
const FILE_MODE: u32 = 0o666;
const EXE_FILE_MODE: u32 = 0o777;
const EXE_CHECK_MASK: u32 = 0o100;


#[derive(RustcDecodable, Debug)]
pub struct Depends {
    pub path: PathBuf,
    pub ignore_regex: String,
    pub include_regex: Option<String>,
}

fn depends_parser(ast: Ast) -> BTreeMap<String, Ast> {
    match ast {
        Ast::Scalar(pos, _, style, value) => {
            let mut map = BTreeMap::new();
            map.insert("path".to_string(),
                Ast::Scalar(pos.clone(), Tag::NonSpecific, style, value));
            map
        },
        _ => unreachable!(),
    }
}

impl Depends {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
            .member("path", V::Scalar::new())
            .member("ignore_regex", V::Scalar::new().default(
                r#"(^|/)\.(git|hg|svn|vagga)($|/)|~$|\.bak$|\.orig$|^#.*#$"#))
            .member("include_regex", V::Scalar::new().optional())
            .parser(depends_parser)
    }
}

impl BuildStep for Depends {
    fn name(&self) -> &'static str { "Depends" }
    fn hash(&self, _cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let path = Path::new("/work").join(&self.path);
        let filter = Filter::new(
            &self.ignore_regex, &self.include_regex)?;
        hash_path(hash, &path, &filter, |h, p, st| {
            h.field("filename", p);
            // We hash only executable flag for files
            // as mode depends on the host system umask
            if st.is_file() {
                let mode = st.permissions().mode();
                let is_executable = mode & EXE_CHECK_MASK > 0;
                h.field("is_executable", is_executable);
            }
            hash_file_content(h, p, st)
                .map_err(|e| VersionError::Io(e, PathBuf::from(p)))?;
            Ok(())
        })?;
        Ok(())
    }
    fn build(&self, _guard: &mut Guard, _build: bool)
        -> Result<(), StepError>
    {
        Ok(())
    }
    fn is_dependent_on(&self) -> Option<&str> {
        None
    }
}

#[derive(RustcDecodable, Debug)]
pub struct Copy {
    pub source: PathBuf,
    pub path: PathBuf,
    pub owner_uid: Option<uid_t>,
    pub owner_gid: Option<gid_t>,
    pub umask: u32,
    pub preserve_permissions: bool,
    pub ignore_regex: String,
    pub include_regex: Option<String>,
}

impl Copy {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("source", V::Scalar::new())
        .member("path", V::Directory::new().absolute(true))
        .member("ignore_regex", V::Scalar::new().default(
            r#"(^|/)\.(git|hg|svn|vagga)($|/)|~$|\.bak$|\.orig$|^#.*#$"#))
        .member("include_regex", V::Scalar::new().optional())
        .member("owner_uid", V::Numeric::new().min(0).optional())
        .member("owner_gid", V::Numeric::new().min(0).optional())
        .member("umask", V::Numeric::new().min(0).max(0o777).default(
            DEFAULT_UMASK as i64))
        .member("preserve_permissions", V::Scalar::new().default(false))
    }

    fn calc_mode(&self, stat: &Metadata) -> Option<u32> {
        if stat.file_type().is_symlink() {
            // ignore as we do not set permissions for symlinks
            return None;
        }
        if self.preserve_permissions {
            Some(stat.permissions().mode())
        } else {
            let base_mode = if stat.is_dir() {
                DIR_MODE
            } else {
                if stat.permissions().mode() & EXE_CHECK_MASK > 0 {
                    EXE_FILE_MODE
                } else {
                    FILE_MODE
                }
            };
            Some(base_mode & !self.umask)
        }
    }
}

impl BuildStep for Copy {
    fn name(&self) -> &'static str { "Copy" }
    fn hash(&self, _cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let ref src = self.source;
        if src.starts_with("/work") {
            let filter = Filter::new(
                &self.ignore_regex, &self.include_regex)?;
            hash_path(hash, src, &filter, |h, p, st| {
                h.field("filename", p);
                if let Some(mode) = self.calc_mode(st) {
                    h.field("mode", mode);
                }
                h.field("uid", self.owner_uid.unwrap_or(st.uid()));
                h.field("gid", self.owner_gid.unwrap_or(st.gid()));
                hash_file_content(h, p, st)
                    .map_err(|e| VersionError::Io(e, PathBuf::from(p)))?;
                Ok(())
            })?;
            hash.field("path", &self.path);
        } else {
            // We don't version the files outside of the /work because
            // we believe they are result of the commands run above
            //
            // And we need already built container to version the files
            // inside the container which is ugly
            hash.field("source", src);
            hash.field("path", &self.path);
            if !self.preserve_permissions {
                if let Some(uid) = self.owner_uid {
                    hash.field("owner_uid", uid);
                }
                if let Some(gid) = self.owner_gid {
                    hash.field("owner_gid", gid);
                }
                hash.field("umask", self.umask);
            }
            hash.field("preserve_permissions", self.preserve_permissions);
        }
        Ok(())
    }
    fn build(&self, _guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        if !build {
            return Ok(());<commit_msg>Simplified hashing code in Copy<commit_after>use std::io::{self, ErrorKind};
use std::fs::{File, Metadata, read_link};
use std::path::{Path, PathBuf};
use std::os::unix::fs::{PermissionsExt, MetadataExt};
use std::collections::{BTreeMap, BTreeSet, HashSet};

use libc::{uid_t, gid_t};
use quire::ast::{Ast, Tag};
use quire::validate as V;
use regex::{Regex, Error as RegexError};
use scan_dir::{ScanDir, Error as ScanDirError};

use container::root::temporary_change_root;
use file_util::shallow_copy;
use path_util::IterSelfAndParents;
use build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard};
use quick_error::ResultExt;


const DEFAULT_UMASK: u32 = 0o002;

const DIR_MODE: u32 = 0o777;
const FILE_MODE: u32 = 0o666;
const EXE_FILE_MODE: u32 = 0o777;
const EXE_CHECK_MASK: u32 = 0o100;


#[derive(RustcDecodable, Debug)]
pub struct Depends {
    pub path: PathBuf,
    pub ignore_regex: String,
    pub include_regex: Option<String>,
}

fn depends_parser(ast: Ast) -> BTreeMap<String, Ast> {
    match ast {
        Ast::Scalar(pos, _, style, value) => {
            let mut map = BTreeMap::new();
            map.insert("path".to_string(),
                Ast::Scalar(pos.clone(), Tag::NonSpecific, style, value));
            map
        },
        _ => unreachable!(),
    }
}

impl Depends {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
            .member("path", V::Scalar::new())
            .member("ignore_regex", V::Scalar::new().default(
                r#"(^|/)\.(git|hg|svn|vagga)($|/)|~$|\.bak$|\.orig$|^#.*#$"#))
            .member("include_regex", V::Scalar::new().optional())
            .parser(depends_parser)
    }
}

impl BuildStep for Depends {
    fn name(&self) -> &'static str { "Depends" }
    fn hash(&self, _cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let path = Path::new("/work").join(&self.path);
        let filter = Filter::new(
            &self.ignore_regex, &self.include_regex)?;
        hash_path(hash, &path, &filter, |h, p, st| {
            h.field("filename", p);
            // We hash only executable flag for files
            // as mode depends on the host system umask
            if st.is_file() {
                let mode = st.permissions().mode();
                let is_executable = mode & EXE_CHECK_MASK > 0;
                h.field("is_executable", is_executable);
            }
            hash_file_content(h, p, st)
                .map_err(|e| VersionError::Io(e, PathBuf::from(p)))?;
            Ok(())
        })?;
        Ok(())
    }
    fn build(&self, _guard: &mut Guard, _build: bool)
        -> Result<(), StepError>
    {
        Ok(())
    }
    fn is_dependent_on(&self) -> Option<&str> {
        None
    }
}

#[derive(RustcDecodable, Debug)]
pub struct Copy {
    pub source: PathBuf,
    pub path: PathBuf,
    pub owner_uid: Option<uid_t>,
    pub owner_gid: Option<gid_t>,
    pub umask: u32,
    pub preserve_permissions: bool,
    pub ignore_regex: String,
    pub include_regex: Option<String>,
}

impl Copy {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("source", V::Scalar::new())
        .member("path", V::Directory::new().absolute(true))
        .member("ignore_regex", V::Scalar::new().default(
            r#"(^|/)\.(git|hg|svn|vagga)($|/)|~$|\.bak$|\.orig$|^#.*#$"#))
        .member("include_regex", V::Scalar::new().optional())
        .member("owner_uid", V::Numeric::new().min(0).optional())
        .member("owner_gid", V::Numeric::new().min(0).optional())
        .member("umask", V::Numeric::new().min(0).max(0o777).default(
            DEFAULT_UMASK as i64))
        .member("preserve_permissions", V::Scalar::new().default(false))
    }

    fn calc_mode(&self, stat: &Metadata) -> Option<u32> {
        if stat.file_type().is_symlink() {
            // ignore as we do not set permissions for symlinks
            return None;
        }
        if self.preserve_permissions {
            Some(stat.permissions().mode())
        } else {
            let base_mode = if stat.is_dir() {
                DIR_MODE
            } else {
                if stat.permissions().mode() & EXE_CHECK_MASK > 0 {
                    EXE_FILE_MODE
                } else {
                    FILE_MODE
                }
            };
            Some(base_mode & !self.umask)
        }
    }
}

impl BuildStep for Copy {
    fn name(&self) -> &'static str { "Copy" }
    fn hash(&self, _cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let ref src = self.source;
        if src.starts_with("/work") {
            let filter = Filter::new(
                &self.ignore_regex, &self.include_regex)?;
            hash_path(hash, src, &filter, |h, p, st| {
                h.field("filename", p);
                h.opt_field("mode", &self.calc_mode(st));
                h.field("uid", self.owner_uid.unwrap_or(st.uid()));
                h.field("gid", self.owner_gid.unwrap_or(st.gid()));
                hash_file_content(h, p, st)
                    .map_err(|e| VersionError::Io(e, PathBuf::from(p)))?;
                Ok(())
            })?;
            hash.field("path", &self.path);
        } else {
            // We don't version the files outside of the /work because
            // we believe they are result of the commands run above
            //
            // And we need already built container to version the files
            // inside the container which is ugly
            hash.field("source", src);
            hash.field("path", &self.path);
            hash.field("preserve_permissions", self.preserve_permissions);
            if !self.preserve_permissions {
                hash.opt_field("owner_uid", &self.owner_uid);
                hash.opt_field("owner_gid", &self.owner_gid);
                hash.field("umask", self.umask);
            }
        }
        Ok(())
    }
    fn build(&self, _guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        if !build {
            return Ok(());
{'fraction_non_alphanumeric': 0.15980551053484604, 'fraction_numerical': 0.007293354943273906, 'mean_word_length': 2.1038732394366195, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2107376', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3892, 'n_words': 845}