<commit_before>package org.cdlib.was.ngIndexer;

import org.apache.commons.codec.digest.DigestUtils;
import org.cdlib.rabinpoly.RabinPoly;
import scala.collection.immutable.SortedMap;
import scala.collection.immutable.TreeMap;

/** A consistent hashing ring.
  * 
  * @author egh
  */
class ConsistentHashRing[T] {
  implicit object UnsignedLongOrdering extends Ordering[Long] {
    /** Compare two longs not as twos complement numbers, but as a collection of raw bits.
      */
    def compare(x : Long, y : Long) : Int = {
      if (x == y) {
        return 0;
      } else {
        val mask = x ^ y; // first set bit will indicate first different bit
        var l = 0; // l will be lg(mask)
        var tmp = mask >>> 1;
        while (tmp != 0) {
          l += 1; tmp = tmp >> 1;
        } 
        return ((x >>> l) - (y >>> l)).asInstanceOf[Int];
      }
    }
  }

  def compare(x : Long, y : Long) : Int = UnsignedLongOrdering.compare(x, y);

  val DEFAULT_LEVEL = 1000;
  
  /* locations of servers on our ring */
  private var locations : SortedMap[Long, T] = SortedMap[Long,T]()(UnsignedLongOrdering);

  private var levels : Map[String, Int] = Map[String, Int]();

  def hash (bytes : Array[Byte]) : Long = 
    UriUtils.bytearray2long(DigestUtils.md5(bytes).slice(8,16));

  def hash (str : String) : Long = hash(str.getBytes("UTF-8"))

  /** Return the server to use for an item with a given place in the ring.
    *
    * @param l The place in the ring.
    */
  def getServerFor(l : Long) : T = {
    val r = locations.from(l);
    if (r.size == 0) {
      /* special case when we are at the end of the ring */
      return locations.head._2;
    } else {
      /* return the next server */
      return r.head._2;
    }
  }

  def getServerFor(id : String) : T = getServerFor(hash(id));

  /** Add a server to the ring.<commit_msg>import functions, not classes<commit_after>package org.cdlib.was.ngIndexer;

import org.apache.commons.codec.digest.DigestUtils.md5;

import org.cdlib.rabinpoly.RabinPoly;

import org.cdlib.was.ngIndexer.UriUtils.bytearray2long;

import scala.collection.immutable.SortedMap;
import scala.collection.immutable.TreeMap;

/** A consistent hashing ring.
  * 
  * @author egh
  */
class ConsistentHashRing[T] {
  implicit object UnsignedLongOrdering extends Ordering[Long] {
    /** Compare two longs not as twos complement numbers, but as a collection of raw bits.
      */
    def compare(x : Long, y : Long) : Int = {
      if (x == y) {
        return 0;
      } else {
        val mask = x ^ y; // first set bit will indicate first different bit
        var l = 0; // l will be lg(mask)
        var tmp = mask >>> 1;
        while (tmp != 0) {
          l += 1; tmp = tmp >> 1;
        } 
        return ((x >>> l) - (y >>> l)).asInstanceOf[Int];
      }
    }
  }

  def compare(x : Long, y : Long) : Int = UnsignedLongOrdering.compare(x, y);

  val DEFAULT_LEVEL = 1000;
  
  /* locations of servers on our ring */
  private var locations : SortedMap[Long, T] = SortedMap[Long,T]()(UnsignedLongOrdering);

  private var levels : Map[String, Int] = Map[String, Int]();

  def hash (bytes : Array[Byte]) : Long = 
    bytearray2long(md5(bytes).slice(8,16));

  def hash (str : String) : Long = hash(str.getBytes("UTF-8"))

  /** Return the server to use for an item with a given place in the ring.
    *
    * @param l The place in the ring.
    */
  def getServerFor(l : Long) : T = {
    val r = locations.from(l);
    if (r.size == 0) {
      /* special case when we are at the end of the ring */
      return locations.head._2;
    } else {
      /* return the next server */
      return r.head._2;
    }
  }

  def getServerFor(id : String) : T = getServerFor(hash(id));

  /** Add a server to the ring.
{'fraction_non_alphanumeric': 0.13138297872340426, 'fraction_numerical': 0.010638297872340425, 'mean_word_length': 2.5248359887535146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7659458', 'n_tokens_mistral': 1287, 'n_tokens_neox': 1233, 'n_words': 386}