<commit_before>use std;
use std::fmt;

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FileType {
    GZip,
    Zip,
    Tar,
    BZip2,
    Xz,
    Deb,
    DiskImage,
    Ext4,
    Other,
}

impl fmt::Display for FileType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(self, f)
    }
}

fn read_octal(bytes: &[u8]) -> Option<u32> {
    let mut start = 0;
    while start < bytes.len() && b' ' == bytes[start] {
        start += 1;
    }

    let mut end = bytes.len() - 1;
    while end > start && (b' ' == bytes[end] || 0 == bytes[end]) {
        end -= 1;
    }

    if let Ok(string) = std::str::from_utf8(&bytes[start..(end + 1)]) {
        if let Ok(val) = u32::from_str_radix(string, 8) {
            return Some(val);
        }
    }
    None
}

fn is_probably_tar(header: &[u8]) -> bool {
    if header.len() < 512 {
        return false;
    }

    if let Some(checksum) = read_octal(&header[148..156]) {
        let mut sum: u32 = (b' ' as u32) * 8;
        for i in 0..148 {
            sum += header[i] as u32;
        }
        for i in 156..512 {
            sum += header[i] as u32;
        }

        if checksum == sum {
            return true;
        }
    }

    false
}

const DEB_PREFIX: &[u8] = b"!<arch>\ndebian-binary ";

impl FileType {
    #[cfg_attr(rustfmt, rustfmt_skip)]
    pub fn identify(header: &[u8]) -> FileType {
        if header.len() >= 20
            && 0x1f == header[0] && 0x8b == header[1] {
            FileType::GZip
        } else if header.len() >= 152
            && b'P' == header[0] && b'K' == header[1]
            && 0x03 == header[2] && 0x04 == header[3] {
            FileType::Zip<commit_msg>functional tar header; not sure I like it<commit_after>use std;
use std::fmt;

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FileType {
    GZip,
    Zip,
    Tar,
    BZip2,
    Xz,
    Deb,
    DiskImage,
    Ext4,
    Other,
}

impl fmt::Display for FileType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Debug::fmt(self, f)
    }
}

fn read_octal(bytes: &[u8]) -> Option<u32> {
    let mut start = 0;
    while start < bytes.len() && b' ' == bytes[start] {
        start += 1;
    }

    let mut end = bytes.len() - 1;
    while end > start && (b' ' == bytes[end] || 0 == bytes[end]) {
        end -= 1;
    }

    if let Ok(string) = std::str::from_utf8(&bytes[start..(end + 1)]) {
        if let Ok(val) = u32::from_str_radix(string, 8) {
            return Some(val);
        }
    }
    None
}

fn is_probably_tar(header: &[u8]) -> bool {
    if header.len() < 512 {
        return false;
    }

    if let Some(expected) = read_octal(&header[148..156]) {
        let found: u32 =
            header[0..148].into_iter().map(|x| *x as u32).sum::<u32>() +
            (b' ' as u32) * 8 +
            header[156..512].into_iter().map(|x| *x as u32).sum::<u32>();

        if expected == found {
            return true;
        }
    }

    false
}

const DEB_PREFIX: &[u8] = b"!<arch>\ndebian-binary ";

impl FileType {
    #[cfg_attr(rustfmt, rustfmt_skip)]
    pub fn identify(header: &[u8]) -> FileType {
        if header.len() >= 20
            && 0x1f == header[0] && 0x8b == header[1] {
            FileType::GZip
        } else if header.len() >= 152
            && b'P' == header[0] && b'K' == header[1]
            && 0x03 == header[2] && 0x04 == header[3] {
            FileType::Zip
{'fraction_non_alphanumeric': 0.1756993006993007, 'fraction_numerical': 0.04020979020979021, 'mean_word_length': 1.9167374681393372, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789850', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1258, 'n_words': 301}