<commit_before>// Copyright 2016 <NAME>
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.dianahep

import scala.language.existentials

import org.dianahep.histogrammar.json._
import org.dianahep.histogrammar.util._

package histogrammar {
  private[histogrammar] trait Compatible[-X, -Y]
  private[histogrammar] object Compatible {
    implicit object BothAreCounting extends Compatible[Counting, Counting]
    implicit object XIsCounting extends Compatible[Counting, AggregationOnData]
    implicit object YIsCounting extends Compatible[AggregationOnData, Counting]
    implicit def dataAreCompatible[X <: AggregationOnData, Y <: AggregationOnData](implicit evidence: X#Datum =:= Y#Datum) = new Compatible[X, Y] {}
  }

  //////////////////////////////////////////////////////////////// Cut/Cutted/Cutting

  /** Accumulate an aggregator for data that satisfy a cut (or more generally, a weighting).
    * 
    * Factory produces mutable [[org.dianahep.histogrammar.Cutting]] and immutable [[org.dianahep.histogrammar.Cutted]] (sic) objects.
    */
  object Cut extends Factory {
    val name = "Cut"
    val help = "Accumulate an aggregator for data that satisfy a cut (or more generally, a weighting)."
    val detailedHelp = """Cut(selection: UserFcn[DATUM, Double], value: V)"""

    /** Create an immutable [[org.dianahep.histogrammar.Cutted]] from arguments (instead of JSON).
      * 
      * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
      * @param value Aggregator that accumulated values that passed the cut.
      */
    def ed[V <: Container[V]](entries: Double, value: V) = new Cutted[V](entries, None, value)

    /** Create an empty, mutable [[org.dianahep.histogrammar.Limiting]].
      * 
      * @param selection Boolean or non-negative function that cuts or weights entries.
      * @param value Aggregator to accumulate for values that pass `selection`.
      */
    def apply[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](selection: UserFcn[DATUM, Double], value: V) = new Cutting[DATUM, V](0.0, selection, value)

    /** Synonym for `apply`. */
    def ing[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](selection: UserFcn[DATUM, Double], value: V) = apply(selection, value)

    /** Use [[org.dianahep.histogrammar.Cutted]] in Scala pattern-matching. */
    def unapply[V <: Container[V]](x: Cutted[V]) = Some(x.value)
    /** Use [[org.dianahep.histogrammar.Cutting]] in Scala pattern-matching. */
    def unapply[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](x: Cutting[DATUM, V]) = Some(x.value)

    import KeySetComparisons._
    def fromJsonFragment(json: Json): Container[_] = json match {
      case JsonObject(pairs @ _*) if (pairs.keySet has Set("entries", "type", "data").maybe("name")) =>
        val get = pairs.toMap

        val entries = get("entries") match {
          case JsonNumber(x) => x
          case x => throw new JsonFormatException(x, name + ".entries")
        }

        val selectionName = get.getOrElse("name", JsonNull) match {
          case JsonString(x) => Some(x)
          case JsonNull => None
          case x => throw new JsonFormatException(x, name + ".name")
        }

        val factory = get("type") match {
          case JsonString(x) => Factory(x)
          case x => throw new JsonFormatException(x, name + ".type")
        }

        val value = factory.fromJsonFragment(get("data"))

        new Cutted[Container[_]](entries, selectionName, value)

      case _ => throw new JsonFormatException(json, name)
    }
  }

  /** An accumulated aggregator of data that passed the cut.
    * 
    * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
    * @param selectionName Optional name given to the selection function, passed for bookkeeping.
    * @param value Aggregator that accumulated values that passed the cut.
    */
  class Cutted[V <: Container[V]] private[histogrammar](val entries: Double, val selectionName: Option[String], val value: V) extends Container[Cutted[V]] {
    type Type = Cutted[V]
    def factory = Cut

    if (entries < 0.0)
      throw new ContainerException(s"entries ($entries) cannot be negative")

    def zero = new Cutted[V](0.0, selectionName, value.zero)
    def +(that: Cutted[V]) =
      if (this.selectionName != that.selectionName)
        throw new ContainerException(s"cannot add ${getClass.getName} because selectionName differs (${this.selectionName} vs ${that.selectionName})")
      else
        new Cutted[V](this.entries + that.entries, this.selectionName, this.value + that.value)

    def toJsonFragment = JsonObject(
      "entries" -> JsonFloat(entries),
      "type" -> JsonString(value.factory.name),
      "data" -> value.toJsonFragment).
      maybe(JsonString("name") -> selectionName.map(JsonString(_)))

    override def toString() = s"""Cutted[$value]"""
    override def equals(that: Any) = that match {
      case that: Cutted[V] => this.entries === that.entries  &&  this.selectionName == that.selectionName  &&  this.value == that.value
      case _ => false
    }
    override def hashCode() = (entries, selectionName, value).hashCode
  }

  /** Accumulating an aggregator of data that passes a cut.
    * 
    * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
    * @param selection Boolean or non-negative function that cuts or weights entries.
    * @param value Aggregator to accumulate values that pass the cut.
    */
  class Cutting[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}] private[histogrammar](var entries: Double, val selection: UserFcn[DATUM, Double], val value: V) extends Container[Cutting[DATUM, V]] with AggregationOnData {
    type Type = Cutting[DATUM, V]
    type Datum = DATUM
    def factory = Cut

    if (entries < 0.0)
      throw new ContainerException(s"entries ($entries) cannot be negative")

    def zero = new Cutting[DATUM, V](0.0, selection, value.zero)
    def +(that: Cutting[DATUM, V]) =
      if (this.selection.name != that.selection.name)
        throw new ContainerException(s"cannot add ${getClass.getName} because selection name differs (${this.selection.name} vs ${that.selection.name})")
      else
        new Cutting[DATUM, V](this.entries + that.entries, this.selection, this.value + that.value)

    def fill[SUB <: Datum](datum: SUB, weight: Double = 1.0) {
      val w = weight * selection(datum)
      if (w > 0.0)
        value.fill(datum, w)
      // no possibility of exception from here on out (for rollback)
      entries += weight
    }

    def toJsonFragment = JsonObject(
      "entries" -> JsonFloat(entries),
      "type" -> JsonString(value.factory.name),
      "data" -> value.toJsonFragment).
      maybe(JsonString("name") -> selection.name.map(JsonString(_)))

    override def toString() = s"""Cutting[$value]"""
    override def equals(that: Any) = that match {
      case that: Cutting[DATUM, V] => this.entries === that.entries  &&  this.selection == that.selection  &&  this.value == that.value
      case _ => false
    }
    override def hashCode() = (entries, selection, value).hashCode
  }

  //////////////////////////////////////////////////////////////// Limit/Limited/Limiting

  /** Accumulate an aggregator until its number of entries reaches a predefined limit.
    * 
    * Factory produces mutable [[org.dianahep.histogrammar.Limiting]] and immutable [[org.dianahep.histogrammar.Limited]] objects.
    */
  object Limit extends Factory {<commit_msg>implemented but have not tested C++ Cut<commit_after>// Copyright 2016 <NAME>
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.dianahep

import scala.language.existentials

import org.dianahep.histogrammar.json._
import org.dianahep.histogrammar.util._

package histogrammar {
  private[histogrammar] trait Compatible[-X, -Y]
  private[histogrammar] object Compatible {
    implicit object BothAreCounting extends Compatible[Counting, Counting]
    implicit object XIsCounting extends Compatible[Counting, AggregationOnData]
    implicit object YIsCounting extends Compatible[AggregationOnData, Counting]
    implicit def dataAreCompatible[X <: AggregationOnData, Y <: AggregationOnData](implicit evidence: X#Datum =:= Y#Datum) = new Compatible[X, Y] {}
  }

  //////////////////////////////////////////////////////////////// Cut/Cutted/Cutting

  /** Accumulate an aggregator for data that satisfy a cut (or more generally, a weighting).
    * 
    * Factory produces mutable [[org.dianahep.histogrammar.Cutting]] and immutable [[org.dianahep.histogrammar.Cutted]] (sic) objects.
    */
  object Cut extends Factory {
    val name = "Cut"
    val help = "Accumulate an aggregator for data that satisfy a cut (or more generally, a weighting)."
    val detailedHelp = """Cut(selection: UserFcn[DATUM, Double], value: V)"""

    /** Create an immutable [[org.dianahep.histogrammar.Cutted]] from arguments (instead of JSON).
      * 
      * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
      * @param value Aggregator that accumulated values that passed the cut.
      */
    def ed[V <: Container[V]](entries: Double, value: V) = new Cutted[V](entries, None, value)

    /** Create an empty, mutable [[org.dianahep.histogrammar.Limiting]].
      * 
      * @param selection Boolean or non-negative function that cuts or weights entries.
      * @param value Aggregator to accumulate for values that pass `selection`.
      */
    def apply[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](selection: UserFcn[DATUM, Double], value: V) = new Cutting[DATUM, V](0.0, selection, value)

    /** Synonym for `apply`. */
    def ing[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](selection: UserFcn[DATUM, Double], value: V) = apply(selection, value)

    /** Use [[org.dianahep.histogrammar.Cutted]] in Scala pattern-matching. */
    def unapply[V <: Container[V]](x: Cutted[V]) = Some(x.value)
    /** Use [[org.dianahep.histogrammar.Cutting]] in Scala pattern-matching. */
    def unapply[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}](x: Cutting[DATUM, V]) = Some(x.value)

    import KeySetComparisons._
    def fromJsonFragment(json: Json): Container[_] = json match {
      case JsonObject(pairs @ _*) if (pairs.keySet has Set("entries", "type", "data").maybe("name")) =>
        val get = pairs.toMap

        val entries = get("entries") match {
          case JsonNumber(x) => x
          case x => throw new JsonFormatException(x, name + ".entries")
        }

        val selectionName = get.getOrElse("name", JsonNull) match {
          case JsonString(x) => Some(x)
          case JsonNull => None
          case x => throw new JsonFormatException(x, name + ".name")
        }

        val factory = get("type") match {
          case JsonString(x) => Factory(x)
          case x => throw new JsonFormatException(x, name + ".type")
        }

        val value = factory.fromJsonFragment(get("data"))

        new Cutted[Container[_]](entries, selectionName, value)

      case _ => throw new JsonFormatException(json, name)
    }
  }

  /** An accumulated aggregator of data that passed the cut.
    * 
    * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
    * @param selectionName Optional name given to the selection function, passed for bookkeeping.
    * @param value Aggregator that accumulated values that passed the cut.
    */
  class Cutted[V <: Container[V]] private[histogrammar](val entries: Double, val selectionName: Option[String], val value: V) extends Container[Cutted[V]] {
    type Type = Cutted[V]
    def factory = Cut

    if (entries < 0.0)
      throw new ContainerException(s"entries ($entries) cannot be negative")

    def zero = new Cutted[V](0.0, selectionName, value.zero)
    def +(that: Cutted[V]) =
      if (this.selectionName != that.selectionName)
        throw new ContainerException(s"cannot add ${getClass.getName} because selectionName differs (${this.selectionName} vs ${that.selectionName})")
      else
        new Cutted[V](this.entries + that.entries, this.selectionName, this.value + that.value)

    def toJsonFragment = JsonObject(
      "entries" -> JsonFloat(entries),
      "type" -> JsonString(value.factory.name),
      "data" -> value.toJsonFragment).
      maybe(JsonString("name") -> selectionName.map(JsonString(_)))

    override def toString() = s"""Cutted[$value]"""
    override def equals(that: Any) = that match {
      case that: Cutted[V] => this.entries === that.entries  &&  this.selectionName == that.selectionName  &&  this.value == that.value
      case _ => false
    }
    override def hashCode() = (entries, selectionName, value).hashCode
  }

  /** Accumulating an aggregator of data that passes a cut.
    * 
    * @param entries Weighted number of entries (sum of all observed weights without the cut applied).
    * @param selection Boolean or non-negative function that cuts or weights entries.
    * @param value Aggregator to accumulate values that pass the cut.
    */
  class Cutting[DATUM, V <: Container[V] with Aggregation{type Datum >: DATUM}] private[histogrammar](var entries: Double, val selection: UserFcn[DATUM, Double], val value: V) extends Container[Cutting[DATUM, V]] with AggregationOnData {
    type Type = Cutting[DATUM, V]
    type Datum = DATUM
    def factory = Cut

    if (entries < 0.0)
      throw new ContainerException(s"entries ($entries) cannot be negative")

    def zero = new Cutting[DATUM, V](0.0, selection, value.zero)
    def +(that: Cutting[DATUM, V]) =
      if (this.selection.name != that.selection.name)
        throw new ContainerException(s"cannot add ${getClass.getName} because selection name differs (${this.selection.name} vs ${that.selection.name})")
      else
        new Cutting[DATUM, V](this.entries + that.entries, this.selection, this.value + that.value)

    def fill[SUB <: Datum](datum: SUB, weight: Double = 1.0) {
      val w = weight * selection(datum)
      if (w > 0.0)
        value.fill(datum, w)

      // no possibility of exception from here on out (for rollback)
      entries += weight
    }

    def toJsonFragment = JsonObject(
      "entries" -> JsonFloat(entries),
      "type" -> JsonString(value.factory.name),
      "data" -> value.toJsonFragment).
      maybe(JsonString("name") -> selection.name.map(JsonString(_)))

    override def toString() = s"""Cutting[$value]"""
    override def equals(that: Any) = that match {
      case that: Cutting[DATUM, V] => this.entries === that.entries  &&  this.selection == that.selection  &&  this.value == that.value
      case _ => false
    }
    override def hashCode() = (entries, selection, value).hashCode
  }

  //////////////////////////////////////////////////////////////// Limit/Limited/Limiting

  /** Accumulate an aggregator until its number of entries reaches a predefined limit.
    * 
    * Factory produces mutable [[org.dianahep.histogrammar.Limiting]] and immutable [[org.dianahep.histogrammar.Limited]] objects.
    */
  object Limit extends Factory {
{'fraction_non_alphanumeric': 0.1325471408390148, 'fraction_numerical': 0.0027025366992199495, 'mean_word_length': 3.174871794871795, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6080725', 'n_tokens_mistral': 4810, 'n_tokens_neox': 4608, 'n_words': 1506}