<commit_before>// Module for parsing ISO Base Media Format aka video/mp4 streams.

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

/// Basic ISO box structure.
pub struct BoxHeader {
    /// Four character box type
pub name: u32,
    /// Size of the box in bytes
pub size: u64,
    /// Offset to the start of the contained data (or header size).
pub offset: u64,
}

/// File type box 'ftyp'.
pub struct FileTypeBox {
    name: u32,
    size: u64,
    major_brand: u32,
    minor_version: u32,
    compatible_brands: Vec<u32>,
}

/// Movie header box 'mvhd'.
pub struct MovieHeaderBox {
    pub name: u32,
    pub size: u64,
    pub timescale: u32,
    pub duration: u64,
    // Ignore other fields.
}

pub struct TrackHeaderBox {
    pub name: u32,
    pub size: u64,
    pub track_id: u32,
    pub enabled: bool,
    pub duration: u64,
    pub width: u32,
    pub height: u32,
}

extern crate byteorder;
use byteorder::{BigEndian, ReadBytesExt};
use std::io::{Read, BufRead, Take};

/// Parse a box out of a data buffer.
pub fn read_box_header<T: ReadBytesExt>(src: &mut T) -> byteorder::Result<BoxHeader> {
    let tmp_size = try!(src.read_u32::<BigEndian>());
    let name = try!(src.read_u32::<BigEndian>());
    let size = match tmp_size {
        1 => try!(src.read_u64::<BigEndian>()),
        _ => tmp_size as u64,
    };
    assert!(size >= 8);
    if tmp_size == 1 {
        assert!(size >= 16);
    }
    let offset = match tmp_size {
        1 => 4 + 4 + 8,
        _ => 4 + 4,
    };
    assert!(offset <= size);
    Ok(BoxHeader{
      name: name,
      size: size,
      offset: offset,
    })
}

/// Parse the extra header fields for a full box.
fn read_fullbox_extra<T: ReadBytesExt>(src: &mut T) -> (u8, u32) {
    let version = src.read_u8().unwrap();
    let flags_a = src.read_u8().unwrap();
    let flags_b = src.read_u8().unwrap();
    let flags_c = src.read_u8().unwrap();
    (version, (flags_a as u32) << 16 |
              (flags_b as u32) <<  8 |
              (flags_c as u32))
}

/// Skip over the contents of a box.
pub fn skip_box_content<T: BufRead>
  (src: &mut T, header: &BoxHeader)
  -> std::io::Result<usize>
{
    let bytes_to_skip = (header.size - header.offset) as usize;
    src.consume(bytes_to_skip);
    Ok(bytes_to_skip)
}

/// Helper to construct a Take over the contents of a box.
fn limit<'a, T: BufRead>(f: &'a mut T, h: &BoxHeader) -> Take<&'a mut T> {
    f.take(h.size - h.offset)
}

struct Cursor<T> {
    inner: T,
    pos: u64
}

impl<T> Cursor<T> {
    fn new(inner: T) -> Cursor<T> {
        Cursor { inner: inner, pos: 0 }
    }
}

macro_rules! read {
    () => {
        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
            let n = try!(Read::read(&mut try!(self.fill_buf()), buf));
            self.pos += n as u64;
            Ok(n)
        }
    }
}

impl<'a> Read for Cursor<&'a [u8]> { read!(); }
impl Read for Cursor<Vec<u8>> { read!(); }
impl<T: BufRead> Read for Cursor<Take<T>> { read!(); }

macro_rules! buffer {
    () => {
        fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
            let amt = std::cmp::min(self.pos, self.inner.len() as u64);
            Ok(&self.inner[(amt as usize)..])
        }
        fn consume(&mut self, amt: usize) { self.pos += amt as u64; }
    }
}

macro_rules! take_buffer {
    () => {
        fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
            self.inner.fill_buf()
        }
        fn consume(&mut self, amt: usize) { self.inner.consume(amt); self.pos += amt as u64; }
    }
}

impl<'a> BufRead for Cursor<&'a [u8]> { buffer!(); }
impl BufRead for Cursor<Vec<u8>> { buffer!(); }
impl<T: BufRead> BufRead for Cursor<Take<T>> { take_buffer!(); }

/// Helper to construct a Cursor over the contents of a box.
fn recurse<T: BufRead>(f: &mut T, h: &BoxHeader) -> byteorder::Result<()> {
    use std::error::Error;
    println!("{} -- recursing", h);
    // FIXME: I couldn't figure out how to do this without copying.
    // We use Seek on the Read we return in skip_box_content, but
    // that trait isn't implemented for a Take like our limit()
    // returns. Slurping the buffer and wrapping it in a Cursor
    // functions as a work around.
    let buf = limit(f, &h);
    let mut content = Cursor::new(buf);
    loop {
        match read_box(&mut content) {
            Ok(_) => {},
            Err(byteorder::Error::UnexpectedEOF) => {
                // byteorder returns EOF at the end of the buffer.
                // This isn't an error for us, just an signal to
                // stop recursion.
                println!("Caught byteorder::Error::UnexpectedEOF");
                break;
            },
            Err(byteorder::Error::Io(e)) => {
                println!("I/O Error '{:?}' reading box: {}",
                         e.kind(), e.description());
                return Err(byteorder::Error::Io(e));
            },
        }
    }
    println!("{} -- end", h);
    Ok(())
}

/// Read the contents of a box, including sub boxes.
/// Right now it just prints the box value rather than
/// returning anything.
pub fn read_box<T: BufRead>(f: &mut T) -> byteorder::Result<()> {
    read_box_header(f).and_then(|h| {
        match &(fourcc_to_string(h.name))[..] {
            "ftyp" => {
                let mut content = limit(f, &h);
                let ftyp = try!(read_ftyp(&mut content, &h));
                println!("{}", ftyp);
            },
            "moov" => try!(recurse(f, &h)),
            "mvhd" => {
                let mut content = limit(f, &h);<commit_msg>Revert "Hack: Implement a copy of Cursor and remove the copy in recurse()."<commit_after>// Module for parsing ISO Base Media Format aka video/mp4 streams.

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

/// Basic ISO box structure.
pub struct BoxHeader {
    /// Four character box type
pub name: u32,
    /// Size of the box in bytes
pub size: u64,
    /// Offset to the start of the contained data (or header size).
pub offset: u64,
}

/// File type box 'ftyp'.
pub struct FileTypeBox {
    name: u32,
    size: u64,
    major_brand: u32,
    minor_version: u32,
    compatible_brands: Vec<u32>,
}

/// Movie header box 'mvhd'.
pub struct MovieHeaderBox {
    pub name: u32,
    pub size: u64,
    pub timescale: u32,
    pub duration: u64,
    // Ignore other fields.
}

pub struct TrackHeaderBox {
    pub name: u32,
    pub size: u64,
    pub track_id: u32,
    pub enabled: bool,
    pub duration: u64,
    pub width: u32,
    pub height: u32,
}

extern crate byteorder;
use byteorder::{BigEndian, ReadBytesExt};
use std::io::{Read, BufRead, Take};
use std::io::Cursor;

/// Parse a box out of a data buffer.
pub fn read_box_header<T: ReadBytesExt>(src: &mut T) -> byteorder::Result<BoxHeader> {
    let tmp_size = try!(src.read_u32::<BigEndian>());
    let name = try!(src.read_u32::<BigEndian>());
    let size = match tmp_size {
        1 => try!(src.read_u64::<BigEndian>()),
        _ => tmp_size as u64,
    };
    assert!(size >= 8);
    if tmp_size == 1 {
        assert!(size >= 16);
    }
    let offset = match tmp_size {
        1 => 4 + 4 + 8,
        _ => 4 + 4,
    };
    assert!(offset <= size);
    Ok(BoxHeader{
      name: name,
      size: size,
      offset: offset,
    })
}

/// Parse the extra header fields for a full box.
fn read_fullbox_extra<T: ReadBytesExt>(src: &mut T) -> (u8, u32) {
    let version = src.read_u8().unwrap();
    let flags_a = src.read_u8().unwrap();
    let flags_b = src.read_u8().unwrap();
    let flags_c = src.read_u8().unwrap();
    (version, (flags_a as u32) << 16 |
              (flags_b as u32) <<  8 |
              (flags_c as u32))
}

/// Skip over the contents of a box.
pub fn skip_box_content<T: BufRead>
  (src: &mut T, header: &BoxHeader)
  -> std::io::Result<usize>
{
    let bytes_to_skip = (header.size - header.offset) as usize;
    src.consume(bytes_to_skip);
    Ok(bytes_to_skip)
}

/// Helper to construct a Take over the contents of a box.
fn limit<'a, T: Read>(f: &'a mut T, h: &BoxHeader) -> Take<&'a mut T> {
    f.take(h.size - h.offset)
}

/// Helper to construct a Cursor over the contents of a box.
fn recurse<T: Read>(f: &mut T, h: &BoxHeader) -> byteorder::Result<()> {
    use std::error::Error;
    println!("{} -- recursing", h);
    // FIXME: I couldn't figure out how to do this without copying.
    // We use Seek on the Read we return in skip_box_content, but
    // that trait isn't implemented for a Take like our limit()
    // returns. Slurping the buffer and wrapping it in a Cursor
    // functions as a work around.
    let buf: Vec<u8> = limit(f, &h)
        .bytes()
        .map(|u| u.unwrap())
        .collect();
    let mut content = Cursor::new(buf);
    loop {
        match read_box(&mut content) {
            Ok(_) => {},
            Err(byteorder::Error::UnexpectedEOF) => {
                // byteorder returns EOF at the end of the buffer.
                // This isn't an error for us, just an signal to
                // stop recursion.
                println!("Caught byteorder::Error::UnexpectedEOF");
                break;
            },
            Err(byteorder::Error::Io(e)) => {
                println!("I/O Error '{:?}' reading box: {}",
                         e.kind(), e.description());
                return Err(byteorder::Error::Io(e));
            },
        }
    }
    println!("{} -- end", h);
    Ok(())
}

/// Read the contents of a box, including sub boxes.
/// Right now it just prints the box value rather than
/// returning anything.
pub fn read_box<T: Read + BufRead>(f: &mut T) -> byteorder::Result<()> {
    read_box_header(f).and_then(|h| {
        match &(fourcc_to_string(h.name))[..] {
            "ftyp" => {
                let mut content = limit(f, &h);
                let ftyp = try!(read_ftyp(&mut content, &h));
                println!("{}", ftyp);
            },
            "moov" => try!(recurse(f, &h)),
            "mvhd" => {
                let mut content = limit(f, &h);
{'fraction_non_alphanumeric': 0.15239111456009313, 'fraction_numerical': 0.016587447861092248, 'mean_word_length': 2.2340025094102884, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 98, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1483752', 'n_tokens_mistral': 3585, 'n_tokens_neox': 3464, 'n_words': 1082}