<commit_before>use std::io;

use tag;
use err;
use serial;
use byte::{read_byte, write_byte};

use std::cmp;

macro_rules! asn1_cereal_int {
  ($rs_type:ty, $size:expr) => (
    impl serial::traits::Asn1Info for $rs_type {
      fn asn1_tag() -> tag::Tag {
        tag::Tag {
          class: tag::Class::Universal,
          tagnum: 2u8.into(),
          constructed: false,
        }
      }

      fn asn1_type() -> tag::Type {
        tag::Type::from("INTEGER")
      }
    }

    impl serial::traits::Asn1Serialize for $rs_type {
      fn serialize_imp<W: io::Write>(&self, writer: &mut W) -> Result<(), err::EncodeError> {
        let mut started = false;
        // Loop through bytes in int backwards, start writing when first non-zero byte is encounted.
        for offset in (0..8).rev() {
          let shifted: $rs_type = self >> (offset * 8);
          let byte: u8 = (shifted & 0xff) as u8;
          if !started {
            if byte == 0 {
              continue;
            }
            started = true;
          }
          try!(write_byte(writer, byte));
        }

        // If we haven't written anything, write a zero byte.
        if !started {
          try!(write_byte(writer, 0 as u8));
        }
        Ok(())
      }
    }

    impl serial::traits::Asn1Deserialize for $rs_type {
      fn deserialize_imp<I: Iterator<Item=io::Result<u8>>>(reader: &mut I, len: tag::Len) -> Result<Self, err::DecodeError> {
        let len_num = try!(len.as_num().ok_or(err::DecodeError::PrimIndef));

        let mut int: $rs_type = 0;
        let size = cmp::min($size, len_num);

        for _ in 0..size {
          let byte = try!(read_byte(reader));
          int = (int << 8) + (byte as $rs_type);
        }

        Ok(int)
      }
    }
  )
}

// Can't bitshift a u8 by 8 bits, need to update deserialize_imp.
// asn1_cereal_int!(u8, 1);
asn1_cereal_int!(u16, 2);
asn1_cereal_int!(u32, 4);
asn1_cereal_int!(u64, 8);<commit_msg>Rename 'int' macro to 'uint'<commit_after>use std::io;

use tag;
use err;
use serial;
use byte::{read_byte, write_byte};

use std::cmp;

macro_rules! asn1_cereal_uint {
  ($rs_type:ty, $size:expr) => (
    impl serial::traits::Asn1Info for $rs_type {
      fn asn1_tag() -> tag::Tag {
        tag::Tag {
          class: tag::Class::Universal,
          tagnum: 2u8.into(),
          constructed: false,
        }
      }

      fn asn1_type() -> tag::Type {
        tag::Type::from("INTEGER")
      }
    }

    impl serial::traits::Asn1Serialize for $rs_type {
      fn serialize_imp<W: io::Write>(&self, writer: &mut W) -> Result<(), err::EncodeError> {
        let mut started = false;
        // Loop through bytes in int backwards, start writing when first non-zero byte is encounted.
        for offset in (0..8).rev() {
          let shifted: $rs_type = self >> (offset * 8);
          let byte: u8 = (shifted & 0xff) as u8;
          if !started {
            if byte == 0 {
              continue;
            }
            started = true;
          }
          try!(write_byte(writer, byte));
        }

        // If we haven't written anything, write a zero byte.
        if !started {
          try!(write_byte(writer, 0 as u8));
        }
        Ok(())
      }
    }

    impl serial::traits::Asn1Deserialize for $rs_type {
      fn deserialize_imp<I: Iterator<Item=io::Result<u8>>>(reader: &mut I, len: tag::Len) -> Result<Self, err::DecodeError> {
        let len_num = try!(len.as_num().ok_or(err::DecodeError::PrimIndef));

        let mut int: $rs_type = 0;
        let size = cmp::min($size, len_num);

        for _ in 0..size {
          let byte = try!(read_byte(reader));
          int = (int << 8) + (byte as $rs_type);
        }

        Ok(int)
      }
    }
  )
}

// Can't bitshift a u8 by 8 bits, need to update deserialize_imp.
// asn1_cereal_int!(u8, 1);
asn1_cereal_uint!(u16, 2);
asn1_cereal_uint!(u32, 4);
asn1_cereal_uint!(u64, 8);
{'fraction_non_alphanumeric': 0.14748660372544017, 'fraction_numerical': 0.01939270221995407, 'mean_word_length': 2.118536197295147, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2242617', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1347, 'n_words': 333}