<commit_before>use failure::{bail, ResultExt};
use flate2::write::GzEncoder;
use std::fs::{read_link, symlink_metadata};
use std::io::{self, empty, BufWriter, Write};
use std::path::Path;
use tar::{Builder, Header};
use walkdir::WalkDir;
use xz2::write::XzEncoder;

use crate::util::*;
use crate::Result;

actor! {
    #[derive(Debug)]
    pub struct Tarballer {
        /// The input folder to be compressed.
        input: String = "package",

        /// The prefix of the tarballs.
        output: String = "./dist",

        /// The folder in which the input is to be found.
        work_dir: String = "./workdir",
    }
}

impl Tarballer {
    /// Generates the actual tarballs
    pub fn run(self) -> Result<()> {
        let tar_gz = self.output.clone() + ".tar.gz";
        let tar_xz = self.output.clone() + ".tar.xz";

        // Remove any existing files.
        for file in &[&tar_gz, &tar_xz] {
            if Path::new(file).exists() {
                remove_file(file)?;
            }
        }

        // Sort files by their suffix, to group files with the same name from
        // different locations (likely identical) and files with the same
        // extension (likely containing similar data).
        let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input)
            .with_context(|_| "failed to collect file paths")?;
        files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev()));

        // Prepare the `.tar.gz` file.
        let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::best());

        // Prepare the `.tar.xz` file.
        let stream = xz2::stream::MtStreamBuilder::new()
            .threads(num_cpus::get() as u32)
            .preset(6)
            .encoder()?;
        let xz = XzEncoder::new_stream(create_new_file(tar_xz)?, stream);

        // Write the tar into both encoded files. We write all directories
        // first, so files may be directly created. (See rust-lang/rustup.rs#1092.)
        let tee = RayonTee(xz, gz);
        let buf = BufWriter::with_capacity(1024 * 1024, tee);
        let mut builder = Builder::new(buf);

        let pool = rayon::ThreadPoolBuilder::new()
            .num_threads(2)
            .build()
            .unwrap();
        pool.install(move || {
            for path in dirs {
                let src = Path::new(&self.work_dir).join(&path);<commit_msg>Use at most 8 threads for the xz stream<commit_after>use failure::{bail, ResultExt};
use flate2::write::GzEncoder;
use std::fs::{read_link, symlink_metadata};
use std::io::{self, empty, BufWriter, Write};
use std::path::Path;
use tar::{Builder, Header};
use walkdir::WalkDir;
use xz2::write::XzEncoder;

use crate::util::*;
use crate::Result;

actor! {
    #[derive(Debug)]
    pub struct Tarballer {
        /// The input folder to be compressed.
        input: String = "package",

        /// The prefix of the tarballs.
        output: String = "./dist",

        /// The folder in which the input is to be found.
        work_dir: String = "./workdir",
    }
}

impl Tarballer {
    /// Generates the actual tarballs
    pub fn run(self) -> Result<()> {
        let tar_gz = self.output.clone() + ".tar.gz";
        let tar_xz = self.output.clone() + ".tar.xz";

        // Remove any existing files.
        for file in &[&tar_gz, &tar_xz] {
            if Path::new(file).exists() {
                remove_file(file)?;
            }
        }

        // Sort files by their suffix, to group files with the same name from
        // different locations (likely identical) and files with the same
        // extension (likely containing similar data).
        let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input)
            .with_context(|_| "failed to collect file paths")?;
        files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev()));

        // Prepare the `.tar.gz` file.
        let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::best());

        // Prepare the `.tar.xz` file. Note that preset 6 takes about 173MB of memory
        // per thread, so we limit the number of threads to not blow out 32-bit hosts.
        // (We could be more precise with `MtStreamBuilder::memusage()` if desired.)
        let stream = xz2::stream::MtStreamBuilder::new()
            .threads(Ord::min(num_cpus::get(), 8) as u32)
            .preset(6)
            .encoder()?;
        let xz = XzEncoder::new_stream(create_new_file(tar_xz)?, stream);

        // Write the tar into both encoded files. We write all directories
        // first, so files may be directly created. (See rust-lang/rustup.rs#1092.)
        let tee = RayonTee(xz, gz);
        let buf = BufWriter::with_capacity(1024 * 1024, tee);
        let mut builder = Builder::new(buf);

        let pool = rayon::ThreadPoolBuilder::new()
            .num_threads(2)
            .build()
            .unwrap();
        pool.install(move || {
            for path in dirs {
                let src = Path::new(&self.work_dir).join(&path);
{'fraction_non_alphanumeric': 0.14152441445017863, 'fraction_numerical': 0.009527590313616514, 'mean_word_length': 2.2977748691099475, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1727920', 'n_tokens_mistral': 1592, 'n_tokens_neox': 1534, 'n_words': 449}