<commit_before>use nom::{
  be_u8, be_u16, be_u32, be_u64,
  le_u32,
  IResult,
  ErrorCode, Err,
  Needed,
};

use std::str::from_utf8;
use std::collections::HashMap;

use metadata::{
  Block, BlockData,
  StreamInfo, Application, VorbisComment, CueSheet, Picture,
  SeekPoint, CueSheetTrack, CueSheetTrackIndex, PictureType,
};

use utility::to_u32;

macro_rules! skip_bytes (
  ($input: expr, $length: expr, $offset: expr) => (
    {
      match take!($input, $length) {
        IResult::Done(i, bytes)   => {
          let head        = bytes[0] << $offset;
          let tail        = &bytes[1..];
          let is_all_zero = tail.iter().all(|byte| *byte == 0);

          if head == 0 && is_all_zero {
            IResult::Done(i, bytes)
          } else {
            IResult::Error(Err::Position(ErrorCode::Digit as u32, $input))
          }
        }
        IResult::Error(error)     => IResult::Error(error),
        IResult::Incomplete(need) => IResult::Incomplete(need),
      }
    }
  );
);

/// Parses all metadata within a file stream
///
/// The first metadata block should always be `StreamInfo` since that is the
/// only required `Block`. At the moment `metadata_parser` parser doesn't
/// check that requirement.
named!(pub metadata_parser <&[u8], (StreamInfo, Vec<Block>)>,
  chain!(
    tag!("fLaC") ~
    blocks: many_blocks,
    || { blocks }
  )
);

named!(pub stream_info <&[u8], BlockData>,
  chain!(
    min_block_size: be_u16 ~
    max_block_size: be_u16 ~
    min_frame_size: map!(take!(3), to_u32) ~
    max_frame_size: map!(take!(3), to_u32) ~
    bytes: take!(8) ~
    md5_sum: count_fixed!(u8, be_u8, 16),
    || {
      let sample_rate     = ((bytes[0] as u32) << 12) +
                            ((bytes[1] as u32) << 4)  +
                            ((bytes[2] as u32) >> 4);
      let channels        = (bytes[2] >> 1) & 0b0111;
      let bits_per_sample = ((bytes[2] & 0b01) << 4) +
                            bytes[3] >> 4;
      let total_samples   = (((bytes[3] as u64) & 0x0f) << 32) +
                            ((bytes[4] as u64) << 24) +
                            ((bytes[5] as u64) << 16) +
                            ((bytes[6] as u64) << 8) +
                            (bytes[7] as u64);

      BlockData::StreamInfo(StreamInfo {
        min_block_size: min_block_size,
        max_block_size: max_block_size,
        min_frame_size: min_frame_size,
        max_frame_size: max_frame_size,
        sample_rate: sample_rate,
        channels: channels + 1,
        bits_per_sample: bits_per_sample + 1,
        total_samples: total_samples,
        md5_sum: md5_sum,
      })
    }
  )
);

pub fn padding(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, skip_bytes!(length, 0), |_| BlockData::Padding(0))
}

pub fn application(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  chain!(input,
    id: take_str!(4) ~
    data: take!(length - 4),
    || {
      BlockData::Application(Application {
        id: id.to_owned(),
        data: data.to_owned(),
      })
    }
  )
}

named!(seek_point <&[u8], SeekPoint>,
  chain!(
    sample_number: be_u64 ~
    stream_offset: be_u64 ~
    frame_samples: be_u16,
    || {
      SeekPoint {
        sample_number: sample_number,
        stream_offset: stream_offset,
        frame_samples: frame_samples,
      }
    }
  )
);

pub fn seek_table(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  let seek_count = (length / 18) as usize;

  map!(input, count!(seek_point, seek_count), BlockData::SeekTable)
}

named!(pub vorbis_comment <&[u8], BlockData>,
  chain!(
    vendor_string_length: le_u32 ~
    vendor_string: take_str!(vendor_string_length)  ~
    number_of_comments: le_u32 ~
    comment_lines: count!(comment_field, number_of_comments as usize),
    || {
      let mut comments = HashMap::with_capacity(comment_lines.len());

      for line in comment_lines {
        let comment: Vec<&str> = line.splitn(2, '=').collect();

        comments.insert(comment[0].to_owned(), comment[1].to_owned());
      }

      BlockData::VorbisComment(VorbisComment {
        vendor_string: vendor_string.to_owned(),
        comments: comments,
      })
    }
  )
);

named!(comment_field <&[u8], String>,
  chain!(
    comment_length: le_u32 ~
    comment: take_str!(comment_length),
    || { comment.to_owned() }
  )
);

named!(pub cue_sheet <&[u8], BlockData>,
  chain!(
    media_catalog_number: take_str!(128) ~
    lead_in: be_u64 ~
    bytes: take!(259) ~ // TODO: last (7 + 258 * 8) bits must be 0
    num_tracks: be_u8 ~
    tracks: count!(cue_sheet_track, num_tracks as usize),
    || {
      let is_cd = ((bytes[0] >> 7) & 0b01) == 1;

      BlockData::CueSheet(CueSheet {
        media_catalog_number: media_catalog_number.to_owned(),
        lead_in: lead_in,
        is_cd: is_cd,
        tracks: tracks,
      })
    }
  )
);

named!(cue_sheet_track <&[u8], CueSheetTrack>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    isrc: take_str!(12) ~
    bytes: take!(14) ~ // TODO: last (6 + 13 * 8) bits must be 0
    num_indices: be_u8 ~
    indices: cond!(
      num_indices != 0,
      count!(cue_sheet_track_index, num_indices as usize)
    ),
    || {
      let is_audio        = ((bytes[0] >> 7) & 0b01) == 0;
      let is_pre_emphasis = ((bytes[0] >> 6) & 0b01) == 1;

      CueSheetTrack {
        offset: offset,
        number: number,
        isrc: isrc.to_owned(),
        is_audio: is_audio,<commit_msg>Use `skip_bytes!` macro for the offset zero bits<commit_after>use nom::{
  be_u8, be_u16, be_u32, be_u64,
  le_u32,
  IResult,
  ErrorCode, Err,
  Needed,
};

use std::str::from_utf8;
use std::collections::HashMap;

use metadata::{
  Block, BlockData,
  StreamInfo, Application, VorbisComment, CueSheet, Picture,
  SeekPoint, CueSheetTrack, CueSheetTrackIndex, PictureType,
};

use utility::to_u32;

macro_rules! skip_bytes (
  ($input: expr, $length: expr, $offset: expr) => (
    {
      match take!($input, $length) {
        IResult::Done(i, bytes)   => {
          let head        = bytes[0] << $offset;
          let tail        = &bytes[1..];
          let is_all_zero = tail.iter().all(|byte| *byte == 0);

          if head == 0 && is_all_zero {
            IResult::Done(i, bytes)
          } else {
            IResult::Error(Err::Position(ErrorCode::Digit as u32, $input))
          }
        }
        IResult::Error(error)     => IResult::Error(error),
        IResult::Incomplete(need) => IResult::Incomplete(need),
      }
    }
  );
);

/// Parses all metadata within a file stream
///
/// The first metadata block should always be `StreamInfo` since that is the
/// only required `Block`. At the moment `metadata_parser` parser doesn't
/// check that requirement.
named!(pub metadata_parser <&[u8], (StreamInfo, Vec<Block>)>,
  chain!(
    tag!("fLaC") ~
    blocks: many_blocks,
    || { blocks }
  )
);

named!(pub stream_info <&[u8], BlockData>,
  chain!(
    min_block_size: be_u16 ~
    max_block_size: be_u16 ~
    min_frame_size: map!(take!(3), to_u32) ~
    max_frame_size: map!(take!(3), to_u32) ~
    bytes: take!(8) ~
    md5_sum: count_fixed!(u8, be_u8, 16),
    || {
      let sample_rate     = ((bytes[0] as u32) << 12) +
                            ((bytes[1] as u32) << 4)  +
                            ((bytes[2] as u32) >> 4);
      let channels        = (bytes[2] >> 1) & 0b0111;
      let bits_per_sample = ((bytes[2] & 0b01) << 4) +
                            bytes[3] >> 4;
      let total_samples   = (((bytes[3] as u64) & 0x0f) << 32) +
                            ((bytes[4] as u64) << 24) +
                            ((bytes[5] as u64) << 16) +
                            ((bytes[6] as u64) << 8) +
                            (bytes[7] as u64);

      BlockData::StreamInfo(StreamInfo {
        min_block_size: min_block_size,
        max_block_size: max_block_size,
        min_frame_size: min_frame_size,
        max_frame_size: max_frame_size,
        sample_rate: sample_rate,
        channels: channels + 1,
        bits_per_sample: bits_per_sample + 1,
        total_samples: total_samples,
        md5_sum: md5_sum,
      })
    }
  )
);

pub fn padding(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, skip_bytes!(length, 0), |_| BlockData::Padding(0))
}

pub fn application(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  chain!(input,
    id: take_str!(4) ~
    data: take!(length - 4),
    || {
      BlockData::Application(Application {
        id: id.to_owned(),
        data: data.to_owned(),
      })
    }
  )
}

named!(seek_point <&[u8], SeekPoint>,
  chain!(
    sample_number: be_u64 ~
    stream_offset: be_u64 ~
    frame_samples: be_u16,
    || {
      SeekPoint {
        sample_number: sample_number,
        stream_offset: stream_offset,
        frame_samples: frame_samples,
      }
    }
  )
);

pub fn seek_table(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  let seek_count = (length / 18) as usize;

  map!(input, count!(seek_point, seek_count), BlockData::SeekTable)
}

named!(pub vorbis_comment <&[u8], BlockData>,
  chain!(
    vendor_string_length: le_u32 ~
    vendor_string: take_str!(vendor_string_length)  ~
    number_of_comments: le_u32 ~
    comment_lines: count!(comment_field, number_of_comments as usize),
    || {
      let mut comments = HashMap::with_capacity(comment_lines.len());

      for line in comment_lines {
        let comment: Vec<&str> = line.splitn(2, '=').collect();

        comments.insert(comment[0].to_owned(), comment[1].to_owned());
      }

      BlockData::VorbisComment(VorbisComment {
        vendor_string: vendor_string.to_owned(),
        comments: comments,
      })
    }
  )
);

named!(comment_field <&[u8], String>,
  chain!(
    comment_length: le_u32 ~
    comment: take_str!(comment_length),
    || { comment.to_owned() }
  )
);

named!(pub cue_sheet <&[u8], BlockData>,
  chain!(
    media_catalog_number: take_str!(128) ~
    lead_in: be_u64 ~
    bytes: skip_bytes!(259, 1) ~
    num_tracks: be_u8 ~
    tracks: count!(cue_sheet_track, num_tracks as usize),
    || {
      let is_cd = ((bytes[0] >> 7) & 0b01) == 1;

      BlockData::CueSheet(CueSheet {
        media_catalog_number: media_catalog_number.to_owned(),
        lead_in: lead_in,
        is_cd: is_cd,
        tracks: tracks,
      })
    }
  )
);

named!(cue_sheet_track <&[u8], CueSheetTrack>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    isrc: take_str!(12) ~
    bytes: skip_bytes!(14, 2) ~
    num_indices: be_u8 ~
    indices: cond!(
      num_indices != 0,
      count!(cue_sheet_track_index, num_indices as usize)
    ),
    || {
      let is_audio        = ((bytes[0] >> 7) & 0b01) == 0;
      let is_pre_emphasis = ((bytes[0] >> 6) & 0b01) == 1;

      CueSheetTrack {
        offset: offset,
        number: number,
        isrc: isrc.to_owned(),
        is_audio: is_audio,
{'fraction_non_alphanumeric': 0.14928519061583578, 'fraction_numerical': 0.031433284457478006, 'mean_word_length': 2.462246192893401, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1167747', 'n_tokens_mistral': 4046, 'n_tokens_neox': 3866, 'n_words': 802}