<commit_before>#[allow(unused_imports)]
#[macro_use]
extern crate log;
extern crate libc;
extern crate env_logger;
extern crate fuse;
extern crate time;
extern crate xattr;

use std::env;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::fs::{File, OpenOptions};
use std::io;
use std::io::{Read, Write};
use std::process::Command;
use std::path::{Path, PathBuf};
use std::os::unix::fs::FileExt;

use env_logger::LogBuilder;
use log::LogRecord;

extern crate catfs;

use catfs::CatFS;
use catfs::catfs::error;
use catfs::catfs::flags::DiskSpace;
use catfs::catfs::file;
use catfs::catfs::rlibc;
use catfs::evicter::Evicter;
use catfs::pcatfs::PCatFS;

#[macro_use]
mod test_suite;


trait Fixture {
    fn setup() -> error::Result<Self>
    where
        Self: std::marker::Sized;
    fn init(&mut self) -> error::Result<()>;
    fn teardown(self) -> error::Result<()>;
}

struct CatFSTests<'a> {
    prefix: PathBuf,
    mnt: PathBuf,
    src: PathBuf,
    cache: PathBuf,
    session: Option<fuse::BackgroundSession<'a>>,
    evicter: Option<Evicter>,
    nested: Option<Box<CatFSTests<'a>>>,
}

impl<'a> CatFSTests<'a> {
    fn get_orig_dir() -> PathBuf {
        let manifest = env::var_os("CARGO_MANIFEST_DIR").unwrap();
        return PathBuf::from(manifest).join("tests/resources");
    }

    fn get_from(&self) -> PathBuf {
        return self.src.clone();
    }

    fn get_cache(&self) -> PathBuf {
        return self.cache.clone();
    }

    fn mount(&self) -> error::Result<(fuse::BackgroundSession<'a>, Evicter)> {
        let fs = CatFS::new(&self.src, &self.cache)?;
        let fs = PCatFS::new(fs);

        let cache_dir = fs.get_cache_dir()?;
        // essentially no-op, but ensures that it starts and terminates
        let ev = Evicter::new(cache_dir, &DiskSpace::Bytes(1));
        return Ok((unsafe { fuse::spawn_mount(fs, &self.mnt, &[])? }, ev));
    }

    fn assert_cache_valid(&self, path: &AsRef<Path>) {
        let src_dir = rlibc::open(&self.src, rlibc::O_RDONLY, 0).unwrap();
        let cache_dir = rlibc::open(&self.cache, rlibc::O_RDONLY, 0).unwrap();

        assert!(file::Handle::validate_cache(src_dir, cache_dir, path, false, true).unwrap());
        rlibc::close(src_dir).unwrap();
        rlibc::close(cache_dir).unwrap();
    }
}

impl<'a> Fixture for CatFSTests<'a> {
    fn setup() -> error::Result<CatFSTests<'a>> {
        let format = |record: &LogRecord| {
            let t = time::now();
            format!(
                "{} {:5} - {}",
                time::strftime("%Y-%m-%d %H:%M:%S", &t).unwrap(),
                record.level(),
                record.args()
            )
        };

        let mut builder = LogBuilder::new();
        builder.format(format);

        if env::var("RUST_LOG").is_ok() {
            builder.parse(&env::var("RUST_LOG").unwrap());
        }

        let _ = builder.init();

        let _ = fs::create_dir(CatFSTests::get_orig_dir().join("dir2"));

        let prefix = catfs::catfs::tests::copy_resources();
        let mnt = prefix.join("mnt");
        let resources = prefix.join("resources");
        let cache = prefix.join("cache");

        fs::create_dir_all(&mnt)?;
        fs::create_dir_all(&cache)?;

        let t = CatFSTests {
            prefix: prefix,
            mnt: mnt,
            src: resources,
            cache: cache,
            session: Default::default(),
            evicter: Default::default(),
            nested: Default::default(),
        };

        if let Some(v) = env::var_os("CATFS_SELF_HOST") {
            if v == OsStr::new("1") || v == OsStr::new("true") {
                let mnt = t.mnt.clone();
                let mut mnt2 = mnt.as_os_str().to_os_string();
                mnt2.push("2");
                let cache = t.cache.to_path_buf();
                let mut cache2 = cache.as_os_str().to_os_string();
                cache2.push("2");

                let mnt2 = PathBuf::from(mnt2);
                let cache2 = PathBuf::from(cache2);

                fs::create_dir_all(&mnt2)?;
                fs::create_dir_all(&cache2)?;
                
                let t2 = CatFSTests {
                    prefix: t.prefix.clone(),
                    mnt: mnt2,
                    src: mnt,
                    cache: cache2,
                    session: Default::default(),
                    evicter: Default::default(),
                    nested: Some(Box::new(t)),
                };

                return Ok(t2);
            }
        }

        return Ok(t);
    }

    fn init(&mut self) -> error::Result<()> {
        if let Some(ref mut t) = self.nested {
            t.init()?;
        }

        let (session, ev) = self.mount()?;
        self.session = Some(session);
        self.evicter = Some(ev);
        if let Some(ref mut ev) = self.evicter {
            ev.run();
        }
        return Ok(());
    }

    fn teardown(self) -> error::Result<()> {
        {
            // move out the session to let us umount first
            std::mem::drop(self.session);
        }

        if let Some(t) = self.nested {
            // unmount the inner session
            std::mem::drop(t.session);
        }

        fs::remove_dir_all(&self.prefix)?;
        // TODO do I need to free self.src/cache?
        return Ok(());
    }
}

fn diff(dir1: &AsRef<Path>, dir2: &AsRef<Path>) {
    debug!("diff {:?} {:?}", dir1.as_ref(), dir2.as_ref());
    let status = Command::new("diff")
        .arg("-ru")
        .arg(dir1.as_ref().as_os_str())
        .arg(dir2.as_ref().as_os_str())
        .status()
        .expect("failed to execute `diff'");
    assert!(status.success());
}

unit_tests!{
    fn read_one(f: &CatFSTests) {
        let mut s = String::new();
        File::open(f.mnt.join("dir1/file1")).unwrap().read_to_string(&mut s).unwrap();
        assert_eq!(s, "dir1/file1\n");
    }

    fn read_all(f: &CatFSTests) {
        diff(&CatFSTests::get_orig_dir(), &f.mnt);
    }

    fn create(f: &CatFSTests) {
        {
            let mut fh = OpenOptions::new().write(true).create(true)
                .open(Path::new(&f.mnt).join("foo")).unwrap();
            fh.write_all(b"hello world").unwrap();
        }

        fs::symlink_metadata(&f.get_from().join("foo")).unwrap();
        diff(&f.get_from(), &f.mnt);
    }

    fn large_write(f: &CatFSTests) {
        let mut of=OsString::from("of=");
        of.push(f.mnt.join("foo"));
        let status = Command::new("dd")
            .arg("if=/dev/zero").arg(of)
            .arg("bs=1M").arg("count=10")
            .status().expect("failed to execute `dd'");
        assert!(status.success());
        
        let foo = fs::symlink_metadata(&f.get_from().join("foo")).unwrap();
        assert_eq!(foo.len(), 10 * 1024 * 1024);
        diff(&f.get_from(), &f.mnt);
    }

    fn large_dir(f: &CatFSTests) {
        let dir2 = Path::new(&f.mnt).join("dir2");
        for i in 1..1001 {
            let _fh = OpenOptions::new().write(true).create(true)
                .open(dir2.join(format!("{}", i))).unwrap();
        }

        let mut i = 0;
        let mut total = 0;
        for entry in fs::read_dir(dir2).unwrap() {
            i += 1;
            let num: u32 = entry.unwrap().file_name().to_str().unwrap().parse().unwrap();
            total += num;
        }
        assert_eq!(i, 1000);
        assert_eq!(total, (1000 * (1000 + 1)) / 2);<commit_msg>Redirect dd stderr to null<commit_after>#[allow(unused_imports)]
#[macro_use]
extern crate log;
extern crate libc;
extern crate env_logger;
extern crate fuse;
extern crate time;
extern crate xattr;

use std::env;
use std::ffi::{OsStr, OsString};
use std::fs;
use std::fs::{File, OpenOptions};
use std::io;
use std::io::{Read, Write};
use std::process::{Command, Stdio};
use std::path::{Path, PathBuf};
use std::os::unix::fs::FileExt;

use env_logger::LogBuilder;
use log::LogRecord;

extern crate catfs;

use catfs::CatFS;
use catfs::catfs::error;
use catfs::catfs::flags::DiskSpace;
use catfs::catfs::file;
use catfs::catfs::rlibc;
use catfs::evicter::Evicter;
use catfs::pcatfs::PCatFS;

#[macro_use]
mod test_suite;


trait Fixture {
    fn setup() -> error::Result<Self>
    where
        Self: std::marker::Sized;
    fn init(&mut self) -> error::Result<()>;
    fn teardown(self) -> error::Result<()>;
}

struct CatFSTests<'a> {
    prefix: PathBuf,
    mnt: PathBuf,
    src: PathBuf,
    cache: PathBuf,
    session: Option<fuse::BackgroundSession<'a>>,
    evicter: Option<Evicter>,
    nested: Option<Box<CatFSTests<'a>>>,
}

impl<'a> CatFSTests<'a> {
    fn get_orig_dir() -> PathBuf {
        let manifest = env::var_os("CARGO_MANIFEST_DIR").unwrap();
        return PathBuf::from(manifest).join("tests/resources");
    }

    fn get_from(&self) -> PathBuf {
        return self.src.clone();
    }

    fn get_cache(&self) -> PathBuf {
        return self.cache.clone();
    }

    fn mount(&self) -> error::Result<(fuse::BackgroundSession<'a>, Evicter)> {
        let fs = CatFS::new(&self.src, &self.cache)?;
        let fs = PCatFS::new(fs);

        let cache_dir = fs.get_cache_dir()?;
        // essentially no-op, but ensures that it starts and terminates
        let ev = Evicter::new(cache_dir, &DiskSpace::Bytes(1));
        return Ok((unsafe { fuse::spawn_mount(fs, &self.mnt, &[])? }, ev));
    }

    fn assert_cache_valid(&self, path: &AsRef<Path>) {
        let src_dir = rlibc::open(&self.src, rlibc::O_RDONLY, 0).unwrap();
        let cache_dir = rlibc::open(&self.cache, rlibc::O_RDONLY, 0).unwrap();

        assert!(file::Handle::validate_cache(src_dir, cache_dir, path, false, true).unwrap());
        rlibc::close(src_dir).unwrap();
        rlibc::close(cache_dir).unwrap();
    }
}

impl<'a> Fixture for CatFSTests<'a> {
    fn setup() -> error::Result<CatFSTests<'a>> {
        let format = |record: &LogRecord| {
            let t = time::now();
            format!(
                "{} {:5} - {}",
                time::strftime("%Y-%m-%d %H:%M:%S", &t).unwrap(),
                record.level(),
                record.args()
            )
        };

        let mut builder = LogBuilder::new();
        builder.format(format);

        if env::var("RUST_LOG").is_ok() {
            builder.parse(&env::var("RUST_LOG").unwrap());
        }

        let _ = builder.init();

        let _ = fs::create_dir(CatFSTests::get_orig_dir().join("dir2"));

        let prefix = catfs::catfs::tests::copy_resources();
        let mnt = prefix.join("mnt");
        let resources = prefix.join("resources");
        let cache = prefix.join("cache");

        fs::create_dir_all(&mnt)?;
        fs::create_dir_all(&cache)?;

        let t = CatFSTests {
            prefix: prefix,
            mnt: mnt,
            src: resources,
            cache: cache,
            session: Default::default(),
            evicter: Default::default(),
            nested: Default::default(),
        };

        if let Some(v) = env::var_os("CATFS_SELF_HOST") {
            if v == OsStr::new("1") || v == OsStr::new("true") {
                let mnt = t.mnt.clone();
                let mut mnt2 = mnt.as_os_str().to_os_string();
                mnt2.push("2");
                let cache = t.cache.to_path_buf();
                let mut cache2 = cache.as_os_str().to_os_string();
                cache2.push("2");

                let mnt2 = PathBuf::from(mnt2);
                let cache2 = PathBuf::from(cache2);

                fs::create_dir_all(&mnt2)?;
                fs::create_dir_all(&cache2)?;
                
                let t2 = CatFSTests {
                    prefix: t.prefix.clone(),
                    mnt: mnt2,
                    src: mnt,
                    cache: cache2,
                    session: Default::default(),
                    evicter: Default::default(),
                    nested: Some(Box::new(t)),
                };

                return Ok(t2);
            }
        }

        return Ok(t);
    }

    fn init(&mut self) -> error::Result<()> {
        if let Some(ref mut t) = self.nested {
            t.init()?;
        }

        let (session, ev) = self.mount()?;
        self.session = Some(session);
        self.evicter = Some(ev);
        if let Some(ref mut ev) = self.evicter {
            ev.run();
        }
        return Ok(());
    }

    fn teardown(self) -> error::Result<()> {
        {
            // move out the session to let us umount first
            std::mem::drop(self.session);
        }

        if let Some(t) = self.nested {
            // unmount the inner session
            std::mem::drop(t.session);
        }

        fs::remove_dir_all(&self.prefix)?;
        // TODO do I need to free self.src/cache?
        return Ok(());
    }
}

fn diff(dir1: &AsRef<Path>, dir2: &AsRef<Path>) {
    debug!("diff {:?} {:?}", dir1.as_ref(), dir2.as_ref());
    let status = Command::new("diff")
        .arg("-ru")
        .arg(dir1.as_ref().as_os_str())
        .arg(dir2.as_ref().as_os_str())
        .status()
        .expect("failed to execute `diff'");
    assert!(status.success());
}

unit_tests!{
    fn read_one(f: &CatFSTests) {
        let mut s = String::new();
        File::open(f.mnt.join("dir1/file1")).unwrap().read_to_string(&mut s).unwrap();
        assert_eq!(s, "dir1/file1\n");
    }

    fn read_all(f: &CatFSTests) {
        diff(&CatFSTests::get_orig_dir(), &f.mnt);
    }

    fn create(f: &CatFSTests) {
        {
            let mut fh = OpenOptions::new().write(true).create(true)
                .open(Path::new(&f.mnt).join("foo")).unwrap();
            fh.write_all(b"hello world").unwrap();
        }

        fs::symlink_metadata(&f.get_from().join("foo")).unwrap();
        diff(&f.get_from(), &f.mnt);
    }

    fn large_write(f: &CatFSTests) {
        let mut of=OsString::from("of=");
        of.push(f.mnt.join("foo"));
        let status = Command::new("dd")
            .arg("if=/dev/zero").arg(of)
            .arg("bs=1M").arg("count=10")
            .stderr(Stdio::null())
            .status().expect("failed to execute `dd'");
        assert!(status.success());
        
        let foo = fs::symlink_metadata(&f.get_from().join("foo")).unwrap();
        assert_eq!(foo.len(), 10 * 1024 * 1024);
        diff(&f.get_from(), &f.mnt);
    }

    fn large_dir(f: &CatFSTests) {
        let dir2 = Path::new(&f.mnt).join("dir2");
        for i in 1..1001 {
            let _fh = OpenOptions::new().write(true).create(true)
                .open(dir2.join(format!("{}", i))).unwrap();
        }

        let mut i = 0;
        let mut total = 0;
        for entry in fs::read_dir(dir2).unwrap() {
            i += 1;
            let num: u32 = entry.unwrap().file_name().to_str().unwrap().parse().unwrap();
            total += num;
        }
        assert_eq!(i, 1000);
        assert_eq!(total, (1000 * (1000 + 1)) / 2);
{'fraction_non_alphanumeric': 0.17627666016282043, 'fraction_numerical': 0.009823050528157169, 'mean_word_length': 2.1044277360066834, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972181', 'n_tokens_mistral': 5093, 'n_tokens_neox': 4952, 'n_words': 937}