<commit_before>package debox

import annotation.tailrec
import scala.math.{min, max}
import scala.{specialized => spec}

object Util {
  def alloc[@spec A:Manifest](src:Array[A], s1:Int, len:Int) = {
    val as = Array.ofDim[A](len)
    System.arraycopy(src, s1, as, 0, len)
    as
  }
}<commit_msg>add 'literal array' macro to debox.Util<commit_after>package debox

import annotation.tailrec
import scala.math.{min, max}
import scala.{specialized => spec}

import language.experimental.macros

import scala.reflect.{ClassTag, TypeTag}
import scala.reflect.makro.Context

object Util {
  def alloc[@spec A:Manifest](src:Array[A], s1:Int, len:Int) = {
    val as = Array.ofDim[A](len)
    System.arraycopy(src, s1, as, 0, len)
    as
  }

  /**
   * Efficient alternative to Array.apply.
   *
   * "As seen on scala-internals!"
   */
  def array[A](as:A*)(implicit ct:ClassTag[A]) = macro arrayMacro[A]

  /**
   * Takes in something like:
   *   ArrayUtil.alloc[Int](11, 22, 33, 44)(ct)
   *
   * and builds a tree like:
   *   {
   *     val arr:Array[Int] = ct.newArray(4)
   *     arr.update(0, 11)
   *     arr.update(1, 22)
   *     arr.update(2, 33)
   *     arr.update(3, 44)
   *     arr
   *   }
   */
  def arrayMacro[A:c.TypeTag](c:Context)(as:c.Expr[A]*)(ct:c.Expr[ClassTag[A]]): c.Expr[Array[A]] = {
    import c.mirror._
    import c.universe._
    def const(x:Int) = Literal(Constant(x))

    val n = as.length
    val arr = newTermName("arr")

    val create = Apply(Select(ct.tree, "newArray"), List(const(n)))
    val valdef = ValDef(Modifiers(), arr, TypeTree(typeOf[Array[A]]), create)

    val updates = (0 until n).map {
      i => Apply(Select(Ident(arr), "update"), List(const(i), as(i).tree))
    }

    val exprs = Seq(valdef) ++ updates ++ Seq(Ident(arr))
    val block = Block(exprs:_*)

    c.Expr[Array[A]](block)
  }

}
{'fraction_non_alphanumeric': 0.16864864864864865, 'fraction_numerical': 0.015135135135135135, 'mean_word_length': 3.2356979405034325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2246468', 'n_tokens_mistral': 738, 'n_tokens_neox': 707, 'n_words': 148}