<commit_before>use std::str::from_utf8;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: TypeFlag,
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    GlobalExtendedHeaderWithMetadata,
    ExtendedHeaderWithMetadataForNext,
    VendorSpecific,
    Invalid
}

/* TODO: validation */
fn str_to_u64(s: &str, base: u64) -> u64 {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        u += f * ((c as u64) - ('0' as u64));
        f *= base;
    }

    u
}

pub fn octal_to_u64(o: &str) -> u64 {
    str_to_u64(o, 8)
}

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::GlobalExtendedHeaderWithMetadata,
        'x' => TypeFlag::ExtendedHeaderWithMetadataForNext,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::Invalid
    }
}

fn parse_ustar00(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("00")               ~
        uname:    take_str!(32)  ~
        gname:    take_str!(32)  ~
        devmajor: take_str!(8)   ~
        devminor: take_str!(8)   ~
        prefix:   take_str!(155) ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: octal_to_u64(devmajor),
                devminor: octal_to_u64(devminor),
                prefix:   prefix
            })
        }
    )
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: parse_ustar00,
        ||{
            ustar
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str!(100) ~
        mode:     take_str!(8)   ~
        uid:      take_str!(8)   ~
        gid:      take_str!(8)   ~
        size:     take_str!(12)  ~
        mtime:    take_str!(12)  ~
        chksum:   take_str!(8)   ~
        typeflag: take!(1)       ~
        linkname: take_str!(100) ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      octal_to_u64(uid),
                gid:      octal_to_u64(gid),
                size:     octal_to_u64(size),
                mtime:    octal_to_u64(mtime),
                chksum:   chksum,
                typeflag: char_to_type_flag(typeflag[0] as char),<commit_msg>eat garbage after strings<commit_after>use std::str::from_utf8;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: TypeFlag,
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: support vendor specific + sparse */
#[derive(Debug,PartialEq,Eq)]
pub enum TypeFlag {
    NormalFile,
    HardLink,
    SymbolicLink,
    CharacterSpecial,
    BlockSpecial,
    Directory,
    FIFO,
    ContiguousFile,
    GlobalExtendedHeaderWithMetadata,
    ExtendedHeaderWithMetadataForNext,
    VendorSpecific,
    Invalid
}

/* TODO: validation */
fn str_to_u64(s: &str, base: u64) -> u64 {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        u += f * ((c as u64) - ('0' as u64));
        f *= base;
    }

    u
}

pub fn octal_to_u64(o: &str) -> u64 {
    str_to_u64(o, 8)
}

fn char_to_type_flag(c: char) -> TypeFlag {
    match c {
        '0' | '\0' => TypeFlag::NormalFile,
        '1' => TypeFlag::HardLink,
        '2' => TypeFlag::SymbolicLink,
        '3' => TypeFlag::CharacterSpecial,
        '4' => TypeFlag::BlockSpecial,
        '5' => TypeFlag::Directory,
        '6' => TypeFlag::FIFO,
        '7' => TypeFlag::ContiguousFile,
        'g' => TypeFlag::GlobalExtendedHeaderWithMetadata,
        'x' => TypeFlag::ExtendedHeaderWithMetadataForNext,
        'A' ... 'Z' => TypeFlag::VendorSpecific,
        _ => TypeFlag::Invalid
    }
}

macro_rules! take_str_eat_garbage (
 ( $i:expr, $size:expr ) => ( chain!( $i, s: map_res!(take_until!("\0"), from_utf8) ~ take!($size - s.len()), ||{ s } ));
);

fn parse_ustar00(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("00")                           ~
        uname:    take_str_eat_garbage!(32)  ~
        gname:    take_str_eat_garbage!(32)  ~
        devmajor: take_str_eat_garbage!(8)   ~
        devminor: take_str_eat_garbage!(8)   ~
        prefix:   take_str_eat_garbage!(155) ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: octal_to_u64(devmajor),
                devminor: octal_to_u64(devminor),
                prefix:   prefix
            })
        }
    )
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: parse_ustar00,
        ||{
            ustar
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str_eat_garbage!(100) ~
        mode:     take_str_eat_garbage!(8)   ~
        uid:      take_str_eat_garbage!(8)   ~
        gid:      take_str_eat_garbage!(8)   ~
        size:     take_str_eat_garbage!(12)  ~
        mtime:    take_str_eat_garbage!(12)  ~
        chksum:   take_str_eat_garbage!(8)   ~
        typeflag: take!(1)                   ~
        linkname: take_str_eat_garbage!(100) ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      octal_to_u64(uid),
                gid:      octal_to_u64(gid),
                size:     octal_to_u64(size),
                mtime:    octal_to_u64(mtime),
                chksum:   chksum,
                typeflag: char_to_type_flag(typeflag[0] as char),
{'fraction_non_alphanumeric': 0.14526943133846537, 'fraction_numerical': 0.027315619567916564, 'mean_word_length': 1.8442796610169492, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2522393', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2913, 'n_words': 570}