<commit_before>use std::fmt;
use std::str;
use std::fmt::{Result};

static EMPTY : &'static str = "";
static NO_NAME : &'static str = "<none>";

// NOTE: Uncomment when https://github.com/mozilla/rust/issues/13703 is fixed.
// pub struct PreOrderNodes<'a, 'b> {
//   queue: Vec<&'a Node<'b>>
// }
//
// impl<'a, 'b> Iterator<&'a Node<'b>> for PreOrderNodes<'a, 'b> {
//   fn next( &mut self ) -> Option<&'a Node<'b>> {
//     match self.queue.pop() {
//       ex @ Some( node ) => {
//         match node.contents {
//           Children( ref x ) => {
//             for child in x.as_slice().rev_iter() {
//               self.queue.push( child )
//             }
//           }
//           _ => ()
//         };
//         ex
//       }
//       _ => None
//     }
//   }
// }


#[deriving(Show, Eq)]
pub enum NodeContents<'a> {
  /// A `&[u8]` byte slice this node matched in the parse input. Only leaf nodes
  /// have `Data` contents.
  Data( &'a [u8] ),

  /// Children of the node, if any. Only non-leaf nodes have `Children`
  /// contents.
  Children( Vec<Node<'a>> )
}


#[deriving(Eq)]
pub struct Node<'a> {
  /// The name of the node.
  pub name: &'static str,

  /// The (inclusive) start index of the range this node matches. It's the byte
  /// (NOT char) offset of the parse input.
  pub start: uint,

  /// The (exclusive) end index of the range this node matches. It's the byte
  /// (NOT char) offset of the parse input.
  pub end: uint,

  /// The contents of the node; this can be either children nodes or a matched
  /// `&[u8]` slice.
  pub contents: NodeContents<'a>
}

fn indent( formatter: &mut fmt::Formatter, indent_spaces: int )
    -> fmt::Result {
  for _ in range( 0, indent_spaces ) {
    try!( write!( formatter.buf, " " ) )
  }
  Ok(())
}

impl<'a> Node<'a> {
  fn format( &self, formatter: &mut fmt::Formatter, indent_spaces: int )
      -> fmt::Result {
    try!( indent( formatter, indent_spaces ) );
    try!( write!( formatter.buf,
                  "Node \\{name: {0}, start: {1}, end: {2}",
                  self.displayName(), self.start, self.end ) );

    match self.contents {
      Data( data ) => {
        match str::from_utf8( data ) {
          Some( string ) => {
            try!( writeln!( formatter.buf,
                            ", contents: \"{0}\" \\}",
                            string ) );
          }
          _ => {
            try!( writeln!( formatter.buf,
                            ", contents: \"{0}\" \\}",
                            data ) );
          }
        }
      }
      Children( ref children ) => {
        try!( writeln!( formatter.buf, " \\}" ) );<commit_msg>Updated code to latest rustc<commit_after>use std::fmt;
use std::str;
use std::fmt::{Result};

static EMPTY : &'static str = "";
static NO_NAME : &'static str = "<none>";

// NOTE: Uncomment when https://github.com/mozilla/rust/issues/13703 is fixed.
// pub struct PreOrderNodes<'a, 'b> {
//   queue: Vec<&'a Node<'b>>
// }
//
// impl<'a, 'b> Iterator<&'a Node<'b>> for PreOrderNodes<'a, 'b> {
//   fn next( &mut self ) -> Option<&'a Node<'b>> {
//     match self.queue.pop() {
//       ex @ Some( node ) => {
//         match node.contents {
//           Children( ref x ) => {
//             for child in x.as_slice().rev_iter() {
//               self.queue.push( child )
//             }
//           }
//           _ => ()
//         };
//         ex
//       }
//       _ => None
//     }
//   }
// }


#[deriving(Show, Eq)]
pub enum NodeContents<'a> {
  /// A `&[u8]` byte slice this node matched in the parse input. Only leaf nodes
  /// have `Data` contents.
  Data( &'a [u8] ),

  /// Children of the node, if any. Only non-leaf nodes have `Children`
  /// contents.
  Children( Vec<Node<'a>> )
}


#[deriving(Eq)]
pub struct Node<'a> {
  /// The name of the node.
  pub name: &'static str,

  /// The (inclusive) start index of the range this node matches. It's the byte
  /// (NOT char) offset of the parse input.
  pub start: uint,

  /// The (exclusive) end index of the range this node matches. It's the byte
  /// (NOT char) offset of the parse input.
  pub end: uint,

  /// The contents of the node; this can be either children nodes or a matched
  /// `&[u8]` slice.
  pub contents: NodeContents<'a>
}

fn indent( formatter: &mut fmt::Formatter, indent_spaces: int )
    -> fmt::Result {
  for _ in range( 0, indent_spaces ) {
    try!( write!( formatter, " " ) )
  }
  Ok(())
}

impl<'a> Node<'a> {
  fn format( &self, formatter: &mut fmt::Formatter, indent_spaces: int )
      -> fmt::Result {
    try!( indent( formatter, indent_spaces ) );
    try!( write!( formatter,
                  "Node \\{name: {0}, start: {1}, end: {2}",
                  self.displayName(), self.start, self.end ) );

    match self.contents {
      Data( data ) => {
        match str::from_utf8( data ) {
          Some( string ) => {
            try!( writeln!( formatter,
                            ", contents: \"{0}\" \\}",
                            string ) );
          }
          _ => {
            try!( writeln!( formatter,
                            ", contents: \"{0}\" \\}",
                            data ) );
          }
        }
      }
      Children( ref children ) => {
        try!( writeln!( formatter, " \\}" ) );
{'fraction_non_alphanumeric': 0.17967126393349708, 'fraction_numerical': 0.005667863215567731, 'mean_word_length': 1.7572916666666667, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3652603', 'n_tokens_mistral': 1726, 'n_tokens_neox': 1715, 'n_words': 500}