<commit_before>package org.cdlib.was.ngIndexer;

import scala.collection.mutable.{HashMap,PriorityQueue};
import scala.collection.Map;
import scala.io.Source;
import org.archive.net.UURIFactory;
import org.apache.commons.httpclient.URIException;

object BuildHuffmanTree {
  class NodeElement (val node : DecoderNode, val prob : Double)
    extends Ordered[NodeElement] {
    
      // override def <  (that : NodeElement) = that.prob <  this.prob;
      // override def <= (that : NodeElement) = that.prob <= this.prob;
      // override def >  (that : NodeElement) = that.prob >  this.prob;
      // override def >= (that : NodeElement) = that.prob >= this.prob;
      def compare (that : NodeElement) =
        -1 * this.prob.compare(that.prob);
      override def toString = "%s (%f)".format(node, prob);
    }
  
  val allowedBytes = Set[Byte](
    /* gen-delims */
    0x3a, /* : */
    0x2f, /* / */
    0x3f, /* ? */
    0x23, /* # */
    0x5b, /* [ */
    0x5d, /* ] */
    0x40, /* @ */
    /* sub-delims */
    0x21, /* ! */
    0x24, /* $ */
    0x26, /* & */
    0x27, /* ' */
    0x28, /* ( */
    0x2a, /* * */
    0x2b, /* + */
    0x2c, /* , */
    /* unreserved */
    0x3b, /* ; */
    0x30, /* 0 */
    0x31, /* 1 */
    0x32, /* 2 */
    0x33, /* 3 */
    0x34, /* 4 */
    0x35, /* 5 */
    0x36, /* 6 */
    0x37, /* 7 */
    0x38, /* 8 */
    0x39, /* 9 */
    0x3d, /* = */
    /* alpha */
    0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
    0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a,
    /* ALPHA */
    0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a,
    0x2d, /* - */
    0x2e, /* . */
    0x5f, /* - */
    0x7e, /* ~ */
    
    0x25, /* % */
    
    /* invalid, but generated by UURI */
    0x7c  /* | */
  );
  
  val longStrings = List[List[Byte]]("http://".getBytes.toList,
                                     ".com/".getBytes.toList,
                                     ".org/".getBytes.toList,
                                     ".net/".getBytes.toList);
  
  def buildFreqTable(path : String) : Pair[Int,Map[List[Byte], Int]] = {
    var freqTable = new HashMap[List[Byte], Int]();
    var occur = 0;

    /* init table */
    for (i <- allowedBytes) {
      /* works as long as i < 128 */
      val b : Byte = i.asInstanceOf[Byte];
      freqTable.update(List(b), 1);
      occur = occur + 1;
    }
<commit_msg>add .html to longStrings, delete some old comments<commit_after>package org.cdlib.was.ngIndexer;

import scala.collection.mutable.{HashMap,PriorityQueue};
import scala.collection.Map;
import scala.io.Source;
import org.archive.net.UURIFactory;
import org.apache.commons.httpclient.URIException;

object BuildHuffmanTree {
  class NodeElement (val node : DecoderNode, val prob : Double)
    extends Ordered[NodeElement] {
      def compare (that : NodeElement) =
        -1 * this.prob.compare(that.prob);
      override def toString = "%s (%f)".format(node, prob);
    }
  
  val allowedBytes = Set[Byte](
    /* gen-delims */
    0x3a, /* : */
    0x2f, /* / */
    0x3f, /* ? */
    0x23, /* # */
    0x5b, /* [ */
    0x5d, /* ] */
    0x40, /* @ */
    /* sub-delims */
    0x21, /* ! */
    0x24, /* $ */
    0x26, /* & */
    0x27, /* ' */
    0x28, /* ( */
    0x2a, /* * */
    0x2b, /* + */
    0x2c, /* , */
    /* unreserved */
    0x3b, /* ; */
    0x30, /* 0 */
    0x31, /* 1 */
    0x32, /* 2 */
    0x33, /* 3 */
    0x34, /* 4 */
    0x35, /* 5 */
    0x36, /* 6 */
    0x37, /* 7 */
    0x38, /* 8 */
    0x39, /* 9 */
    0x3d, /* = */
    /* alpha */
    0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
    0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a,
    /* ALPHA */
    0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a,
    0x2d, /* - */
    0x2e, /* . */
    0x5f, /* - */
    0x7e, /* ~ */
    
    0x25, /* % */
    
    /* invalid, but generated by UURI */
    0x7c  /* | */
  );
  
  val longStrings = List[List[Byte]]("http://".getBytes.toList,
                                     ".com/".getBytes.toList,
                                     ".org/".getBytes.toList,
                                     ".net/".getBytes.toList,
                                     ".html".getBytes.toList);
  
  def buildFreqTable(path : String) : Pair[Int,Map[List[Byte], Int]] = {
    var freqTable = new HashMap[List[Byte], Int]();
    var occur = 0;

    /* init table */
    for (i <- allowedBytes) {
      /* works as long as i < 128 */
      val b : Byte = i.asInstanceOf[Byte];
      freqTable.update(List(b), 1);
      occur = occur + 1;
    }

{'fraction_non_alphanumeric': 0.18662838392124692, 'fraction_numerical': 0.09967186218211649, 'mean_word_length': 1.701939058171745, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5038539', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2028, 'n_words': 398}