<commit_before>/// Represents a context tree.
pub struct ContextTree {
  root: Node,
}

impl ContextTree {
  pub fn create(depth: usize) -> ContextTree {
    ContextTree {
      root: Node::new(depth, 0),
    }
  }

  /// Returns the total number of nodes in the tree.
  pub fn size(&self) -> usize {
    self.root.size()
  }
}

/// One node in the context tree.
struct Node {
  left: Option<Box<Node>>,
  right: Option<Box<Node>>,
}

impl Node {
  fn new(depth: usize, level: usize) -> Node {
    return if level == depth {
      return Node::new_leaf();
    } else {
      return Node::new_inner(depth, level + 1);
    }
  }

  fn new_leaf() -> Node {
    Node {
      left: None,
      right: None,
    }
  }

  fn new_inner(depth: usize, level: usize) -> Node {
    Node {
      left: Some(Box::new(Node::new(depth, level + 1))),
      right: Some(Box::new(Node::new(depth, level + 1))),
    }
  }

  /// Returns the number of nodes in the subtree rooted at this node.
  fn size(&self) -> usize {
    return if self.is_leaf() { 1 } else {
      let left_size = self.left.as_ref().unwrap().size();
      let right_size = self.right.as_ref().unwrap().size();
      left_size + right_size + 1
    };
  }

  fn is_leaf(&self) -> bool {
    if self.left.is_some() != self.right.is_some() {
      panic!("ContextTree nodes cannot have only one child.");
    }
    return !self.left.is_some();
  }
}<commit_msg>Rename context tree children to make it clear which is which.<commit_after>/// Represents a context tree.
pub struct ContextTree {
  root: Node,
}

impl ContextTree {
  pub fn create(depth: usize) -> ContextTree {
    ContextTree {
      root: Node::new(depth, 0),
    }
  }

  /// Returns the total number of nodes in the tree.
  pub fn size(&self) -> usize {
    self.root.size()
  }
}

/// One node in the context tree.
struct Node {
  zero_child: Option<Box<Node>>,
  one_child: Option<Box<Node>>,
}

impl Node {
  fn new(depth: usize, level: usize) -> Node {
    return if level == depth {
      return Node::new_leaf();
    } else {
      return Node::new_inner(depth, level + 1);
    }
  }

  fn new_leaf() -> Node {
    Node {
      zero_child: None,
      one_child: None,
    }
  }

  fn new_inner(depth: usize, level: usize) -> Node {
    Node {
      zero_child: Some(Box::new(Node::new(depth, level + 1))),
      one_child: Some(Box::new(Node::new(depth, level + 1))),
    }
  }

  /// Returns the number of nodes in the subtree rooted at this node.
  fn size(&self) -> usize {
    return if self.is_leaf() { 1 } else {
      let zero_child_size = self.zero_child.as_ref().unwrap().size();
      let one_child_size = self.one_child.as_ref().unwrap().size();
      zero_child_size + one_child_size + 1
    };
  }

  fn is_leaf(&self) -> bool {
    if self.zero_child.is_some() != self.one_child.is_some() {
      panic!("ContextTree nodes cannot have only one child.");
    }
    return !self.zero_child.is_some();
  }
}
{'fraction_non_alphanumeric': 0.14931972789115647, 'fraction_numerical': 0.004081632653061225, 'mean_word_length': 2.6041666666666665, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '191605', 'n_tokens_mistral': 1018, 'n_tokens_neox': 1012, 'n_words': 278}