<commit_before>//! Searches, processes and uploads debug information files (DIFs). See
//! `DifUpload` for more information.

use std::collections::{BTreeMap, BTreeSet};
use std::convert::TryInto;
use std::ffi::{OsStr, OsString};
use std::fmt::{self, Display};
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write};
use std::iter::IntoIterator;
use std::mem::transmute;
use std::ops::Deref;
use std::path::{Component, Path, PathBuf};
use std::process::Command;
use std::slice::{Chunks, Iter};
use std::str;
use std::thread;
use std::time::{Duration, Instant};

use console::style;
use failure::{bail, err_msg, Error, SyncFailure};
use indicatif::HumanBytes;
use log::{debug, info, warn};
use sha1::Digest;
use symbolic::common::{AsSelf, ByteView, DebugId, SelfCell, Uuid};
use symbolic::debuginfo::macho::{BcSymbolMap, UuidMapping};
use symbolic::debuginfo::sourcebundle::SourceBundleWriter;
use symbolic::debuginfo::{Archive, FileEntry, FileFormat, Object};
use walkdir::WalkDir;
use which::which;
use zip::{write::FileOptions, ZipArchive, ZipWriter};

use crate::api::{
    Api, ChunkUploadCapability, ChunkUploadOptions, ChunkedDifRequest, ChunkedFileState,
};
use crate::config::Config;
use crate::constants::{DEFAULT_MAX_DIF_SIZE, DEFAULT_MAX_WAIT};
use crate::utils::chunks::{
    upload_chunks, BatchedSliceExt, Chunk, ItemSize, ASSEMBLE_POLL_INTERVAL,
};
use crate::utils::dif::ObjectDifFeatures;
use crate::utils::fs::{get_sha1_checksum, get_sha1_checksums, TempDir, TempFile};
use crate::utils::progress::{ProgressBar, ProgressStyle};
use crate::utils::ui::{copy_with_progress, make_byte_progress_bar};

/// A debug info file on the server.
pub use crate::api::DebugInfoFile;

/// Fallback maximum number of chunks in a batch for the legacy upload.
static MAX_CHUNKS: u64 = 64;

/// An iterator over chunks of data in a `ChunkedDifMatch` object.
///
/// This struct is returned by `ChunkedDifMatch::chunks`.
struct DifChunks<'a> {
    checksums: Iter<'a, Digest>,
    iter: Chunks<'a, u8>,
}

impl<'a> Iterator for DifChunks<'a> {
    type Item = Chunk<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        match (self.checksums.next(), self.iter.next()) {
            (Some(checksum), Some(data)) => Some(Chunk((*checksum, data))),
            (_, _) => None,
        }
    }
}

/// A Debug Information File.
///
/// This is primarily used to store inside the [`DifMatch`] so does not contain any
/// information already present there.  You probably should look whether you can use
/// [`DifMatch`] instead of this instead.
enum ParsedDif<'a> {
    Object(Object<'a>),
    BcSymbolMap(BcSymbolMap<'a>),
    PList(UuidMapping),
}

impl<'slf, 'data: 'slf> AsSelf<'slf> for ParsedDif<'data> {
    type Ref = ParsedDif<'data>;

    fn as_self(&'slf self) -> &Self::Ref {
        self
    }
}

/// Contains backing data for a `DifMatch`.
///
/// This can be used to store the actual data that a `FatObject` might be
/// relying upon, such as temporary files or extracted archives. It will be
/// disposed along with a `DifMatch` once it is dropped.
#[derive(Debug)]
enum DifBacking {
    Temp(TempFile),
}

/// A handle to a debug information file found by `DifUpload`.
///
/// It contains a `FatObject` giving access to the metadata and contents of the
/// debug information file. `DifMatch::attachments` may contain supplemental
/// files used to further process this file, such as dSYM PLists.
struct DifMatch<'data> {
    _backing: Option<DifBacking>,
    dif: SelfCell<ByteView<'data>, ParsedDif<'data>>,
    name: String,
    debug_id: Option<DebugId>,
    attachments: Option<BTreeMap<String, ByteView<'static>>>,
}

impl<'data> DifMatch<'data> {
    fn from_temp_object<S>(temp_file: TempFile, name: S) -> Result<Self, Error>
    where
        S: Into<String>,
    {
        let buffer = ByteView::open(temp_file.path()).map_err(SyncFailure::new)?;
        let dif = SelfCell::try_new(buffer, |b| {
            Object::parse(unsafe { &*b }).map(ParsedDif::Object)
        })?;
        let debug_id = match dif.get() {
            ParsedDif::Object(ref obj) if !obj.debug_id().is_nil() => Some(obj.debug_id()),
            _ => None,
        };

        Ok(DifMatch {
            _backing: Some(DifBacking::Temp(temp_file)),
            dif,
            name: name.into(),
            debug_id,
            attachments: None,
        })
    }

    /// Creates a [`DifMatch`] from a `.bcsymbolmap` file.
    ///
    /// The `uuid` is the DebugID of the symbolmap while `name` is the filename of the file.
    /// Normally the filename should be the `uuid` with `.bcsymbolmap` appended to it.
    fn from_bcsymbolmap(
        uuid: DebugId,
        name: String,
        data: ByteView<'static>,
    ) -> Result<Self, Error> {
        let dif = SelfCell::try_new(data, |buf| {
            BcSymbolMap::parse(unsafe { &*buf }).map(ParsedDif::BcSymbolMap)
        })?;

        Ok(Self {
            _backing: None,
            dif,
            name,
            debug_id: Some(uuid),
            attachments: None,
        })
    }

    fn from_plist(uuid: DebugId, name: String, data: ByteView<'static>) -> Result<Self, Error> {
        let dif = SelfCell::try_new(data, |buf| {
            UuidMapping::parse_plist(uuid, unsafe { &*buf }).map(ParsedDif::PList)
        })?;

        Ok(Self {
            _backing: None,
            dif,
            name,
            debug_id: Some(uuid),
            attachments: None,
        })
    }

    /// Moves the specified temporary debug file to a safe location and assumes
    /// ownership. The file will be deleted in the file system when this
    /// `DifMatch` is dropped.
    ///
    /// The path must point to a `FatObject` containing exactly one `Object`.
    fn take_temp<P, S>(path: P, name: S) -> Result<Self, Error>
    where
        P: AsRef<Path>,
        S: Into<String>,
    {
        let temp_file = TempFile::take(path)?;
        Self::from_temp_object(temp_file, name)
    }

    /// Returns the parsed [`Object`] of this DIF.
    pub fn object(&self) -> Option<&Object<'data>> {
        match self.dif.get() {
            ParsedDif::Object(ref obj) => Some(obj),
            ParsedDif::BcSymbolMap(_) => None,
            ParsedDif::PList(_) => None,
        }
    }

    pub fn format(&self) -> DifFormat {
        match self.dif.get() {
            ParsedDif::Object(ref object) => DifFormat::Object(object.file_format()),
            ParsedDif::BcSymbolMap(_) => DifFormat::BcSymbolMap,
            ParsedDif::PList(_) => DifFormat::PList,
        }
    }

    /// Returns the raw binary data of this DIF.
    pub fn data(&self) -> &[u8] {
        // TODO(flub): Can this not always return self.dif.owner()?
        match self.dif.get() {
            ParsedDif::Object(ref obj) => obj.data(),
            ParsedDif::BcSymbolMap(_) => &self.dif.owner(),
            ParsedDif::PList(_) => &self.dif.owner(),
        }
    }

    /// Returns the size of of this DIF in bytes.
    pub fn size(&self) -> u64 {
        self.data().len() as u64
    }

    /// Returns the path of this DIF relative to the search origin.
    pub fn path(&self) -> &str {
        &self.name
    }

    /// Returns the name of this DIF, including its file extension.
    pub fn file_name(&self) -> &str {<commit_msg>Remove todo, doesn't matter that much<commit_after>//! Searches, processes and uploads debug information files (DIFs). See
//! `DifUpload` for more information.

use std::collections::{BTreeMap, BTreeSet};
use std::convert::TryInto;
use std::ffi::{OsStr, OsString};
use std::fmt::{self, Display};
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write};
use std::iter::IntoIterator;
use std::mem::transmute;
use std::ops::Deref;
use std::path::{Component, Path, PathBuf};
use std::process::Command;
use std::slice::{Chunks, Iter};
use std::str;
use std::thread;
use std::time::{Duration, Instant};

use console::style;
use failure::{bail, err_msg, Error, SyncFailure};
use indicatif::HumanBytes;
use log::{debug, info, warn};
use sha1::Digest;
use symbolic::common::{AsSelf, ByteView, DebugId, SelfCell, Uuid};
use symbolic::debuginfo::macho::{BcSymbolMap, UuidMapping};
use symbolic::debuginfo::sourcebundle::SourceBundleWriter;
use symbolic::debuginfo::{Archive, FileEntry, FileFormat, Object};
use walkdir::WalkDir;
use which::which;
use zip::{write::FileOptions, ZipArchive, ZipWriter};

use crate::api::{
    Api, ChunkUploadCapability, ChunkUploadOptions, ChunkedDifRequest, ChunkedFileState,
};
use crate::config::Config;
use crate::constants::{DEFAULT_MAX_DIF_SIZE, DEFAULT_MAX_WAIT};
use crate::utils::chunks::{
    upload_chunks, BatchedSliceExt, Chunk, ItemSize, ASSEMBLE_POLL_INTERVAL,
};
use crate::utils::dif::ObjectDifFeatures;
use crate::utils::fs::{get_sha1_checksum, get_sha1_checksums, TempDir, TempFile};
use crate::utils::progress::{ProgressBar, ProgressStyle};
use crate::utils::ui::{copy_with_progress, make_byte_progress_bar};

/// A debug info file on the server.
pub use crate::api::DebugInfoFile;

/// Fallback maximum number of chunks in a batch for the legacy upload.
static MAX_CHUNKS: u64 = 64;

/// An iterator over chunks of data in a `ChunkedDifMatch` object.
///
/// This struct is returned by `ChunkedDifMatch::chunks`.
struct DifChunks<'a> {
    checksums: Iter<'a, Digest>,
    iter: Chunks<'a, u8>,
}

impl<'a> Iterator for DifChunks<'a> {
    type Item = Chunk<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        match (self.checksums.next(), self.iter.next()) {
            (Some(checksum), Some(data)) => Some(Chunk((*checksum, data))),
            (_, _) => None,
        }
    }
}

/// A Debug Information File.
///
/// This is primarily used to store inside the [`DifMatch`] so does not contain any
/// information already present there.  You probably should look whether you can use
/// [`DifMatch`] instead of this instead.
enum ParsedDif<'a> {
    Object(Object<'a>),
    BcSymbolMap(BcSymbolMap<'a>),
    PList(UuidMapping),
}

impl<'slf, 'data: 'slf> AsSelf<'slf> for ParsedDif<'data> {
    type Ref = ParsedDif<'data>;

    fn as_self(&'slf self) -> &Self::Ref {
        self
    }
}

/// Contains backing data for a `DifMatch`.
///
/// This can be used to store the actual data that a `FatObject` might be
/// relying upon, such as temporary files or extracted archives. It will be
/// disposed along with a `DifMatch` once it is dropped.
#[derive(Debug)]
enum DifBacking {
    Temp(TempFile),
}

/// A handle to a debug information file found by `DifUpload`.
///
/// It contains a `FatObject` giving access to the metadata and contents of the
/// debug information file. `DifMatch::attachments` may contain supplemental
/// files used to further process this file, such as dSYM PLists.
struct DifMatch<'data> {
    _backing: Option<DifBacking>,
    dif: SelfCell<ByteView<'data>, ParsedDif<'data>>,
    name: String,
    debug_id: Option<DebugId>,
    attachments: Option<BTreeMap<String, ByteView<'static>>>,
}

impl<'data> DifMatch<'data> {
    fn from_temp_object<S>(temp_file: TempFile, name: S) -> Result<Self, Error>
    where
        S: Into<String>,
    {
        let buffer = ByteView::open(temp_file.path()).map_err(SyncFailure::new)?;
        let dif = SelfCell::try_new(buffer, |b| {
            Object::parse(unsafe { &*b }).map(ParsedDif::Object)
        })?;
        let debug_id = match dif.get() {
            ParsedDif::Object(ref obj) if !obj.debug_id().is_nil() => Some(obj.debug_id()),
            _ => None,
        };

        Ok(DifMatch {
            _backing: Some(DifBacking::Temp(temp_file)),
            dif,
            name: name.into(),
            debug_id,
            attachments: None,
        })
    }

    /// Creates a [`DifMatch`] from a `.bcsymbolmap` file.
    ///
    /// The `uuid` is the DebugID of the symbolmap while `name` is the filename of the file.
    /// Normally the filename should be the `uuid` with `.bcsymbolmap` appended to it.
    fn from_bcsymbolmap(
        uuid: DebugId,
        name: String,
        data: ByteView<'static>,
    ) -> Result<Self, Error> {
        let dif = SelfCell::try_new(data, |buf| {
            BcSymbolMap::parse(unsafe { &*buf }).map(ParsedDif::BcSymbolMap)
        })?;

        Ok(Self {
            _backing: None,
            dif,
            name,
            debug_id: Some(uuid),
            attachments: None,
        })
    }

    fn from_plist(uuid: DebugId, name: String, data: ByteView<'static>) -> Result<Self, Error> {
        let dif = SelfCell::try_new(data, |buf| {
            UuidMapping::parse_plist(uuid, unsafe { &*buf }).map(ParsedDif::PList)
        })?;

        Ok(Self {
            _backing: None,
            dif,
            name,
            debug_id: Some(uuid),
            attachments: None,
        })
    }

    /// Moves the specified temporary debug file to a safe location and assumes
    /// ownership. The file will be deleted in the file system when this
    /// `DifMatch` is dropped.
    ///
    /// The path must point to a `FatObject` containing exactly one `Object`.
    fn take_temp<P, S>(path: P, name: S) -> Result<Self, Error>
    where
        P: AsRef<Path>,
        S: Into<String>,
    {
        let temp_file = TempFile::take(path)?;
        Self::from_temp_object(temp_file, name)
    }

    /// Returns the parsed [`Object`] of this DIF.
    pub fn object(&self) -> Option<&Object<'data>> {
        match self.dif.get() {
            ParsedDif::Object(ref obj) => Some(obj),
            ParsedDif::BcSymbolMap(_) => None,
            ParsedDif::PList(_) => None,
        }
    }

    pub fn format(&self) -> DifFormat {
        match self.dif.get() {
            ParsedDif::Object(ref object) => DifFormat::Object(object.file_format()),
            ParsedDif::BcSymbolMap(_) => DifFormat::BcSymbolMap,
            ParsedDif::PList(_) => DifFormat::PList,
        }
    }

    /// Returns the raw binary data of this DIF.
    pub fn data(&self) -> &[u8] {
        match self.dif.get() {
            ParsedDif::Object(ref obj) => obj.data(),
            ParsedDif::BcSymbolMap(_) => &self.dif.owner(),
            ParsedDif::PList(_) => &self.dif.owner(),
        }
    }

    /// Returns the size of of this DIF in bytes.
    pub fn size(&self) -> u64 {
        self.data().len() as u64
    }

    /// Returns the path of this DIF relative to the search origin.
    pub fn path(&self) -> &str {
        &self.name
    }

    /// Returns the name of this DIF, including its file extension.
    pub fn file_name(&self) -> &str {
{'fraction_non_alphanumeric': 0.14726450020567666, 'fraction_numerical': 0.0017825311942959, 'mean_word_length': 2.927571351642434, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 129, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '480079', 'n_tokens_mistral': 4738, 'n_tokens_neox': 4744, 'n_words': 1282}