<commit_before>from collections import defaultdict
from Heap import *

# Node of Huffman tree.
class HuffmanNode:
  def __init__(self, left = None, right = None):
    self.left = left
    self.right = right

# Returns the frequencies of each characters in the text.
def getFrequencies(text):
  frequencies = defaultdict(int)
  for ch in text:
    frequencies[ch] += 1
  return frequencies

# Returns Huffman tree constructed from the frequency dictionary.
def encode(frequencies):
  p = Heap()
  # Construct priority queue with high priorities for low frequencies.
  for char, freq in frequencies.iteritems():
    p.insert((freq, char))
  # Construct Huffman tree.
  while len(p) > 1:
    left = p.pop()
    right = p.pop()
    node = HuffmanNode(left[1], right[1])
    p.insert((left[0] + right[0], node))
  return p.pop()[1]

# Returns Huffman encoded codebook from the Huffman Tree.
# Recursively calculates the correspoding codewords to each characters.
def getHuffmanCode(root):
  def traverseHuffmanTree(node, codebook, path):
    if node.left:
      path.append('0')
      if isinstance(node.left, HuffmanNode):
        traverseHuffmanTree(node.left, codebook, path)
      else:
        codebook[node.left] = ''.join(path)
      path.pop()
    if node.right:
      path.append('1')
      if isinstance(node.right, HuffmanNode):
        traverseHuffmanTree(node.right, codebook, path)
      else:
        codebook[node.right] = ''.join(path)
      path.pop()<commit_msg>Fix error on encoding special inputs<commit_after>from collections import defaultdict
from Heap import *

# Node of Huffman tree.
class HuffmanNode:
  def __init__(self, left = None, right = None):
    self.left = left
    self.right = right

# Returns the frequencies of each characters in the text.
def getFrequencies(text):
  frequencies = defaultdict(int)
  for ch in text:
    frequencies[ch] += 1
  return frequencies

# Returns Huffman tree constructed from the frequency dictionary.
def encode(frequencies):
  # Construct priority queue with high priorities for low frequencies.
  p = Heap()
  for char, freq in frequencies.iteritems():
    p.insert((freq, char))
  # Special cases
  if len(frequencies) == 0:  # Empty input
    return HuffmanNode()
  if len(frequencies) == 1:  # Repetition of same characters
    return HuffmanNode(p.pop()[1])
  # Construct Huffman tree.
  while len(p) > 1:
    left = p.pop()
    right = p.pop()
    node = HuffmanNode(left[1], right[1])
    p.insert((left[0] + right[0], node))
  return p.pop()[1]

# Returns Huffman encoded codebook from the Huffman Tree.
def getHuffmanCode(root):
  # Recursively calculate the correspoding codewords to each characters
  def traverseHuffmanTree(node, codebook, path):
    if node.left:
      path.append('0')
      if isinstance(node.left, HuffmanNode):
        traverseHuffmanTree(node.left, codebook, path)
      else:
        codebook[node.left] = ''.join(path)
      path.pop()
    if node.right:
      path.append('1')
      if isinstance(node.right, HuffmanNode):
        traverseHuffmanTree(node.right, codebook, path)
      else:
        codebook[node.right] = ''.join(path)
      path.pop()
{'fraction_non_alphanumeric': 0.10928096293949953, 'fraction_numerical': 0.0066518847006651885, 'mean_word_length': 3.416783216783217, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6292193', 'n_tokens_mistral': 1007, 'n_tokens_neox': 983, 'n_words': 298}