<commit_before>use byte::{read_byte, write_byte};
use err;

use std::fmt;
use std::cmp::Ordering;
use std::io;

/// A type for ASN.1 type names.
pub type Type = String;

#[derive(PartialEq, Debug, Clone, Copy)]
/// An ASN.1 Class.
pub enum Class {
  /// Universal class.
  Universal,
  /// Application class.
  Application,
  /// Context-specific class.
  ContextSpecific,
  /// Private class.
  Private,
}

impl From<u8> for Class {
  fn from(class: u8) -> Self {
    match class {
      0 => Class::Universal,
      1 => Class::Application,
      2 => Class::ContextSpecific,
      3 => Class::Private,
      _ => unreachable!()
    }
  }
}

impl From<Class> for u8 {
  fn from(class: Class) -> Self {
    match class {
      Class::Universal => 0,
      Class::Application => 1,
      Class::ContextSpecific => 2,
      Class::Private => 3,
    }
  }
}

impl fmt::Display for Class {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    write!(f, "{}", match *self {
      Class::Universal => "Universal",
      Class::Application => "Application",
      Class::ContextSpecific => "Context-specific",
      Class::Private => "Private",
    })
  }
}

/// An ASN.1 tag number.
pub type TagNum = u64;

#[derive(PartialEq, Debug, Clone, Copy)]
/// A struct containing the information from an ASN.1 tag, which represents an ASN.1 element.
pub struct Tag {
  /// The class flag in the ASN.1 tag.
  pub class: Class,
  /// The tag number in the ASN.1 tag.
  pub tagnum: TagNum,
  /// The constructed flag in the ASN.1 tag, indicating whether this element contains another ASN.1 element.
  pub constructed: bool,
}

impl Tag {
  /// Returns true when this element is a SEQUENCE (OF), or SET (OF).
  pub fn is_structured(&self) -> bool {
    match (self.class, self.tagnum) {
      // SEQUENCE (OF)
      (Class::Universal, 16) => true,
      // SET (OF)
      (Class::Universal, 17) => true,
      _ => false,
    }
  }

  /// Given an iterator over a byte stream, read and return a Tag struct.
  pub fn read_tag<I: Iterator<Item=io::Result<u8>>>(bytes: &mut I) -> Result<Self, err::DecodeError> {
    // Decode tag byte, which includes class, constructed flag, and tag number.
    let tag_byte = try!(read_byte(bytes));
    let class_num = (tag_byte & 0xc0) >> 6;
    let constructed = tag_byte & 0x20 == 0x20;
    // If tag is 0x1F, use extended decode format.
    let tagnum = if (tag_byte & 0x1f) == 0x1f {
      let mut tag: TagNum = 0;
      loop {
        // Incrementatlly read bytes, adding base-128 to tag.
        let tag_more = try!(read_byte(bytes));
        tag = (tag << 7) + (tag_more & 0x7f) as TagNum;
        // Stop looping when 0x80 bit is set.
        if tag_more & 0x80 == 0x00 {
          break;
        }
      }
      tag
    // Otherwise it's just bits 5-1.
    } else {
      (tag_byte & 0x1f) as TagNum
    };
    Ok(Tag {
        class: class_num.into(),
        tagnum: tagnum,
        constructed: constructed,
    })
  }

  /// Write this ASN.1 Tag struct to the given writer.
  pub fn write_tag<W: io::Write>(self, writer: &mut W) -> Result<(), err::EncodeError> {
    let (class, tagnum, constructed) =
      (self.class, self.tagnum, self.constructed);

    // Create first tag_byte from class, constructed and tag number.
    let mut tag_byte: u8 = (u8::from(class) << 6) & 0xc0;
    if constructed {
      tag_byte |= 0x20;
    }
    // If tag number is <31, add to single byte.
    if tagnum < 31 {
      tag_byte |= tagnum as u8 & 0x1f;
      try!(write_byte(writer, tag_byte));
    // Otherwise build additional tag bytes.
    } else {
      if tagnum & 0x8000000000000000 != 0 {
        panic!("Bit 63 set on asn1 tag. Not handling, since this is \
                impractically huge, and it messes up my nice little algorithm.");
      }
      tag_byte |= 0x1f;
      try!(write_byte(writer, tag_byte));
      let mut started = false;
      // Take 7 bit slices eg. 62-55, ..., 6-0.
      // The first non-zero slice marks the start of the int.
      for offset in (0..9).rev() {
        // Get 7 bit slice.
        let mut tag_part = ((tagnum >> (offset * 7)) & 0x7f) as u8;

        if !started {
          // Skip if tag_part is zero and we haven't started.
          if tag_part == 0 {
            continue;
          }
          // TODO: Does tagnum have sign issues like length?
          // Emit an initial zero byte if slice starts with a 1 bit.
          // if tag_part & 0x40 != 0 {
          //   try!(write_byte(writer, 0));
          // }
          started = true;
        }

        // For all slices except the last, set 7th bit.
        if offset != 0 {
          tag_part |= 0x80;
        }
        try!(write_byte(writer, tag_part));
      }
    }

    Ok(())
  }
}

pub type LenNum = u64;

#[derive(PartialEq, Debug, Clone, Copy)]
/// An enum representing the length of an ASN.1 element.
pub enum Len {
  /// A Definite length element.
  Def(u64),
  /// An Indefinite length element, not known before decoding.
  Indef,
}

impl Len {
  /// Given an iterator over a byte stream, read and return the ASN.1 element length.
  pub fn read_len<I: Iterator<Item=io::Result<u8>>>(bytes: &mut I) -> Result<Self, err::DecodeError> {
    // Decode len byte.
    let len_byte = try!(read_byte(bytes));
    Ok(match len_byte {
      // When byte is 0x80, this is the start of indefinite length encoding.
      0x80 => Len::Indef,<commit_msg>Add function to write indefinite length ending octets<commit_after>use byte::{read_byte, write_byte};
use err;

use std::fmt;
use std::cmp::Ordering;
use std::io;

/// A type for ASN.1 type names.
pub type Type = String;

#[derive(PartialEq, Debug, Clone, Copy)]
/// An ASN.1 Class.
pub enum Class {
  /// Universal class.
  Universal,
  /// Application class.
  Application,
  /// Context-specific class.
  ContextSpecific,
  /// Private class.
  Private,
}

impl From<u8> for Class {
  fn from(class: u8) -> Self {
    match class {
      0 => Class::Universal,
      1 => Class::Application,
      2 => Class::ContextSpecific,
      3 => Class::Private,
      _ => unreachable!()
    }
  }
}

impl From<Class> for u8 {
  fn from(class: Class) -> Self {
    match class {
      Class::Universal => 0,
      Class::Application => 1,
      Class::ContextSpecific => 2,
      Class::Private => 3,
    }
  }
}

impl fmt::Display for Class {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    write!(f, "{}", match *self {
      Class::Universal => "Universal",
      Class::Application => "Application",
      Class::ContextSpecific => "Context-specific",
      Class::Private => "Private",
    })
  }
}

/// An ASN.1 tag number.
pub type TagNum = u64;

#[derive(PartialEq, Debug, Clone, Copy)]
/// A struct containing the information from an ASN.1 tag, which represents an ASN.1 element.
pub struct Tag {
  /// The class flag in the ASN.1 tag.
  pub class: Class,
  /// The tag number in the ASN.1 tag.
  pub tagnum: TagNum,
  /// The constructed flag in the ASN.1 tag, indicating whether this element contains another ASN.1 element.
  pub constructed: bool,
}

impl Tag {
  /// Returns true when this element is a SEQUENCE (OF), or SET (OF).
  pub fn is_structured(&self) -> bool {
    match (self.class, self.tagnum) {
      // SEQUENCE (OF)
      (Class::Universal, 16) => true,
      // SET (OF)
      (Class::Universal, 17) => true,
      _ => false,
    }
  }

  /// Given an iterator over a byte stream, read and return a Tag struct.
  pub fn read_tag<I: Iterator<Item=io::Result<u8>>>(bytes: &mut I) -> Result<Self, err::DecodeError> {
    // Decode tag byte, which includes class, constructed flag, and tag number.
    let tag_byte = try!(read_byte(bytes));
    let class_num = (tag_byte & 0xc0) >> 6;
    let constructed = tag_byte & 0x20 == 0x20;
    // If tag is 0x1F, use extended decode format.
    let tagnum = if (tag_byte & 0x1f) == 0x1f {
      let mut tag: TagNum = 0;
      loop {
        // Incrementatlly read bytes, adding base-128 to tag.
        let tag_more = try!(read_byte(bytes));
        tag = (tag << 7) + (tag_more & 0x7f) as TagNum;
        // Stop looping when 0x80 bit is set.
        if tag_more & 0x80 == 0x00 {
          break;
        }
      }
      tag
    // Otherwise it's just bits 5-1.
    } else {
      (tag_byte & 0x1f) as TagNum
    };
    Ok(Tag {
        class: class_num.into(),
        tagnum: tagnum,
        constructed: constructed,
    })
  }

  /// Write this ASN.1 Tag struct to the given writer.
  pub fn write_tag<W: io::Write>(self, writer: &mut W) -> Result<(), err::EncodeError> {
    let (class, tagnum, constructed) =
      (self.class, self.tagnum, self.constructed);

    // Create first tag_byte from class, constructed and tag number.
    let mut tag_byte: u8 = (u8::from(class) << 6) & 0xc0;
    if constructed {
      tag_byte |= 0x20;
    }
    // If tag number is <31, add to single byte.
    if tagnum < 31 {
      tag_byte |= tagnum as u8 & 0x1f;
      try!(write_byte(writer, tag_byte));
    // Otherwise build additional tag bytes.
    } else {
      if tagnum & 0x8000000000000000 != 0 {
        panic!("Bit 63 set on asn1 tag. Not handling, since this is \
                impractically huge, and it messes up my nice little algorithm.");
      }
      tag_byte |= 0x1f;
      try!(write_byte(writer, tag_byte));
      let mut started = false;
      // Take 7 bit slices eg. 62-55, ..., 6-0.
      // The first non-zero slice marks the start of the int.
      for offset in (0..9).rev() {
        // Get 7 bit slice.
        let mut tag_part = ((tagnum >> (offset * 7)) & 0x7f) as u8;

        if !started {
          // Skip if tag_part is zero and we haven't started.
          if tag_part == 0 {
            continue;
          }
          // TODO: Does tagnum have sign issues like length?
          // Emit an initial zero byte if slice starts with a 1 bit.
          // if tag_part & 0x40 != 0 {
          //   try!(write_byte(writer, 0));
          // }
          started = true;
        }

        // For all slices except the last, set 7th bit.
        if offset != 0 {
          tag_part |= 0x80;
        }
        try!(write_byte(writer, tag_part));
      }
    }

    Ok(())
  }
}

pub type LenNum = u64;

#[derive(PartialEq, Debug, Clone, Copy)]
/// An enum representing the length of an ASN.1 element.
pub enum Len {
  /// A Definite length element.
  Def(u64),
  /// An Indefinite length element, not known before decoding.
  Indef,
}

impl Len {
  /// Write an indefinite length terminator to the given writer.
  pub fn write_indef_end<W: io::Write>(writer: &mut W) -> Result<(), err::EncodeError> {
    try!(write_byte(writer, 0x0));
    try!(write_byte(writer, 0x0));
    Ok(())
  }

  /// Given an iterator over a byte stream, read and return the ASN.1 element length.
  pub fn read_len<I: Iterator<Item=io::Result<u8>>>(bytes: &mut I) -> Result<Self, err::DecodeError> {
    // Decode len byte.
    let len_byte = try!(read_byte(bytes));
    Ok(match len_byte {
      // When byte is 0x80, this is the start of indefinite length encoding.
      0x80 => Len::Indef,
{'fraction_non_alphanumeric': 0.1321163925537683, 'fraction_numerical': 0.02566419663835171, 'mean_word_length': 2.25117508813161, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6162987', 'n_tokens_mistral': 3801, 'n_tokens_neox': 3607, 'n_words': 1206}