<commit_before>/*
Copyright 2012 Twitter, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.twitter.scalding.typed

import java.io.Serializable
import java.util.PriorityQueue
import scala.collection.JavaConverters._

import com.twitter.algebird.{ Fold, Semigroup, Ring, Aggregator }
import com.twitter.algebird.mutable.PriorityQueueMonoid

import com.twitter.scalding._

object KeyedListLike {
  /** KeyedListLike items are implicitly convertable to TypedPipe */
  implicit def toTypedPipe[K, V, S[K, +V] <: KeyedListLike[K, V, S]](keyed: KeyedListLike[K, V, S]): TypedPipe[(K, V)] = keyed.toTypedPipe
}

/**
 * This is for the case where you don't want to expose any structure
 * but the ability to operate on an iterator of the values
 */
trait KeyedList[K, +T] extends KeyedListLike[K, T, KeyedList]

/**
 * Represents sharded lists of items of type T
 * There are exactly two the fundamental operations:
 * toTypedPipe: marks the end of the grouped-on-key operations.
 * mapValueStream: further transforms all values, in order, one at a time,
 *  with a function from Iterator to another Iterator
 */
trait KeyedListLike[K, +T, +This[K, +T] <: KeyedListLike[K, T, This]]
  extends java.io.Serializable {

  /**
   * End of the operations on values. From this point on the keyed structure
   * is lost and another shuffle is generally required to reconstruct it
   */
  def toTypedPipe: TypedPipe[(K, T)]

  /**
   * This is like take except that the items are kept in memory
   * and we attempt to partially execute on the mappers if possible
   * For very large values of n, this could create memory pressure.
   * (as you may aggregate n items in a memory heap for each key)
   * If you get OOM issues, try to resolve using the method `take` instead.
   */
  def bufferedTake(n: Int): This[K, T]<commit_msg>Fixes minor KeyedList docs wording<commit_after>/*
Copyright 2012 Twitter, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.twitter.scalding.typed

import java.io.Serializable
import java.util.PriorityQueue
import scala.collection.JavaConverters._

import com.twitter.algebird.{ Fold, Semigroup, Ring, Aggregator }
import com.twitter.algebird.mutable.PriorityQueueMonoid

import com.twitter.scalding._

object KeyedListLike {
  /** KeyedListLike items are implicitly convertable to TypedPipe */
  implicit def toTypedPipe[K, V, S[K, +V] <: KeyedListLike[K, V, S]](keyed: KeyedListLike[K, V, S]): TypedPipe[(K, V)] = keyed.toTypedPipe
}

/**
 * This is for the case where you don't want to expose any structure
 * but the ability to operate on an iterator of the values
 */
trait KeyedList[K, +T] extends KeyedListLike[K, T, KeyedList]

/**
 * Represents sharded lists of items of type T
 * There are exactly two fundamental operations:
 * toTypedPipe: marks the end of the grouped-on-key operations.
 * mapValueStream: further transforms all values, in order, one at a time,
 *  with a function from Iterator to another Iterator
 */
trait KeyedListLike[K, +T, +This[K, +T] <: KeyedListLike[K, T, This]]
  extends java.io.Serializable {

  /**
   * End of the operations on values. From this point on the keyed structure
   * is lost and another shuffle is generally required to reconstruct it
   */
  def toTypedPipe: TypedPipe[(K, T)]

  /**
   * This is like take except that the items are kept in memory
   * and we attempt to partially execute on the mappers if possible
   * For very large values of n, this could create memory pressure.
   * (as you may aggregate n items in a memory heap for each key)
   * If you get OOM issues, try to resolve using the method `take` instead.
   */
  def bufferedTake(n: Int): This[K, T]
{'fraction_non_alphanumeric': 0.08086486486486487, 'fraction_numerical': 0.0034594594594594594, 'mean_word_length': 4.14, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1117443', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1351, 'n_words': 604}