<commit_before>package pellucid.pframe

import language.experimental.macros

import scala.reflect.ClassTag
import scala.reflect.macros.Context

import scala.collection.immutable.BitSet
import scala.{ specialized => spec }
import scala.annotation.{ unspecialized => unspec }

import spire.algebra._
import spire.syntax.cfor._

/**
 * A `Column` represents an `Int`-indexed set of values. The defining
 * characteristic is that a column is always defined for all `Int` values. In
 * order to interact with a column in an optimal and concrete way, we need an
 * external notion of valid rows to restrict the input by. A [[Series]], for
 * instance, ties a `Column` together with an [[Index]] to restrict the set of
 * rows being used.
 */
trait Column[@spec(Int,Long,Float,Double) +A] extends ColumnLike[Column[A]] {

  /**
   * Returns `true` if the value exists; that is, it is available and
   * meaningful. This method must be called before any calls to `missing` or
   * `value`, since it is essentially the switch that let's you know which
   * one to call.
   */
  def exists(row: Int): Boolean

  /**
   * If `exists(row) == false`, then this will return either `NA` or `NM` to
   * indicate the value is not available or not meaningful (resp.). If
   * `exists(row) == true`, then the result of calling this is undefined.
   */
  def missing(row: Int): Missing

  /**
   * If `exists(row) == true`, then this will return the value stored in row
   * `row`. If `exists(row) == false`, then the result of calling this is
   * undefined.
   */
  def value(row: Int): A

  def foldRow[B](row: Int)(f: A => B, g: Missing => B): B =
    if (exists(row)) f(value(row)) else g(missing(row))

  def apply(row: Int): Cell[A] =
    foldRow(row)(Value(_), m => m)

  /**
   * Map all existing values to a given value.
   */
  def map[B](f: A => B): Column[B] = new MappedColumn(f, this)

  /**
   * Filter a column by a given predicate. All values that have been filtered
   * out are turned into `NA` (Not Available).
   */
  def filter(f: A => Boolean): Column[A] = new FilteredColumn(f, this)

  def zipMap[B, C](rhs: Column[B])(f: (A, B) => C): Column[C] =
    new ZipMapColumn[A, B, C](f, this, rhs)

  /**
   * Masks this column with a given `BitSet`. That is, a value only exists at
   * a row if it exists in both the source `Column` and if `bitset(row)` is
   * `true`. If a value exists in the source `Column`, but `bitset(row)` is
   * `false`, then that value is treated as `NA` (Not Available).
   */
  def mask(bits: Int => Boolean): Column[A] = new MaskedColumn(bits, this)

  /**
   * Shift all rows in this column down by `rows`. If `rows` is negative, then
   * they will be shifted up by `-rows`.
   */
  def shift(rows: Int): Column[A] = new ShiftColumn(rows, this)

  /**
   * Returns a Column whose rows are mapped with the given `index` to rows in
   * this column. If a row doesn't exist in the index (ie. is less than 0 or
   * greater than or equal to `index.length`), then `NA` is returned.
   */
  def reindex(index: Array[Int]): Column[A] = new ReindexColumn(index, this)

  /**
   * Force a specific row to be not available (`NA`).
   */
  def setNA(row: Int): Column[A] = new SetNAColumn(row, this)

  def compact[AA >: A](len: Int)(implicit ct: ClassTag[AA]): Column[AA] = {
    val bldr = new ColumnBuilder[AA]
    cfor(0)(_ < len, _ + 1) { row =>
      if (exists(row)) {
        bldr.addValue(value(row))
      } else {
        bldr.addMissing(missing(row))<commit_msg>Re-organize metric data modules a bit.<commit_after>package pellucid.pframe

import language.experimental.macros

import scala.reflect.ClassTag
import scala.reflect.macros.Context

import scala.collection.immutable.BitSet
import scala.{ specialized => spec }
import scala.annotation.{ unspecialized => unspec }

import spire.algebra._
import spire.syntax.cfor._

/**
 * A `Column` represents an `Int`-indexed set of values. The defining
 * characteristic is that a column is always defined for all `Int` values. In
 * order to interact with a column in an optimal and concrete way, we need an
 * external notion of valid rows to restrict the input by. A [[Series]], for
 * instance, ties a `Column` together with an [[Index]] to restrict the set of
 * rows being used.
 */
trait Column[@spec(Int,Long,Float,Double) +A] extends ColumnLike[Column[A]] {

  /**
   * Returns `true` if the value exists; that is, it is available and
   * meaningful. This method must be called before any calls to `missing` or
   * `value`, since it is essentially the switch that let's you know which
   * one to call.
   */
  def exists(row: Int): Boolean

  /**
   * If `exists(row) == false`, then this will return either `NA` or `NM` to
   * indicate the value is not available or not meaningful (resp.). If
   * `exists(row) == true`, then the result of calling this is undefined.
   */
  def missing(row: Int): Missing

  /**
   * If `exists(row) == true`, then this will return the value stored in row
   * `row`. If `exists(row) == false`, then the result of calling this is
   * undefined.
   */
  def value(row: Int): A

  def foldRow[B](row: Int)(f: A => B, g: Missing => B): B =
    if (exists(row)) f(value(row)) else g(missing(row))

  def apply(row: Int): Cell[A] =
    foldRow(row)(Value(_), m => m)

  /**
   * Map all existing values to a given value.
   */
  def map[B](f: A => B): Column[B] = new MappedColumn(f, this)

  /**
   * Filter a column by a given predicate. All values that have been filtered
   * out are turned into `NA` (Not Available).
   */
  def filter(f: A => Boolean): Column[A] = new FilteredColumn(f, this)

  def zipMap[B, C](rhs: Column[B])(f: (A, B) => C): Column[C] =
    new ZipMapColumn[A, B, C](f, this, rhs)

  /**
   * Masks this column with a given `BitSet`. That is, a value only exists at
   * a row if it exists in both the source `Column` and if `bitset(row)` is
   * `true`. If a value exists in the source `Column`, but `bitset(row)` is
   * `false`, then that value is treated as `NA` (Not Available).
   */
  def mask(bits: Int => Boolean): Column[A] = new MaskedColumn(bits, this)

  /**
   * Shift all rows in this column down by `rows`. If `rows` is negative, then
   * they will be shifted up by `-rows`.
   */
  def shift(rows: Int): Column[A] = new ShiftColumn(rows, this)

  /**
   * Returns a Column whose rows are mapped with the given `index` to rows in
   * this column. If a row doesn't exist in the index (ie. is less than 0 or
   * greater than or equal to `index.length`), then `NA` is returned.
   */
  def reindex(index: Array[Int]): Column[A] = new ReindexColumn(index, this)

  /**
   * Force a specific row to be not available (`NA`).
   */
  def setNA(row: Int): Column[A] = new SetNAColumn(row, this)

  // TODO: This should really just take an Index[_] and compact it for
  //       fast access.
  def compact[AA >: A](len: Int)(implicit ct: ClassTag[AA]): Column[AA] = {
    val bldr = new ColumnBuilder[AA]
    cfor(0)(_ < len, _ + 1) { row =>
      if (exists(row)) {
        bldr.addValue(value(row))
      } else {
        bldr.addMissing(missing(row))
{'fraction_non_alphanumeric': 0.13209668353007306, 'fraction_numerical': 0.0008431703204047217, 'mean_word_length': 3.2642300778909528, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7310210', 'n_tokens_mistral': 2314, 'n_tokens_neox': 2257, 'n_words': 918}