<commit_before>package ch.uzh.ifi.pdeboer.pplib.process.stdlib

import ch.uzh.ifi.pdeboer.pplib.process.entities.{Patch, PassableProcessParam}
import ch.uzh.ifi.pdeboer.pplib.process._

/**
 * Created by pdeboer on 05/12/14.
 */
class CollectDecideProcess(_params: Map[String, Any] = Map.empty) extends ProcessStub[Patch, Patch](_params) {

	import ch.uzh.ifi.pdeboer.pplib.process.stdlib.CollectDecideProcess._

	override def expectedParametersBeforeRun: List[ProcessParameter[_]] =
		List(COLLECT, DECIDE).asInstanceOf[List[ProcessParameter[_]]]


	override protected def processCategoryNames: List[String] = List("create.refine.collectdecide")

	override protected def run(data: Patch): Patch = {
		val memoizer: ProcessMemoizer = getProcessMemoizer(data.hashCode() + "").getOrElse(new NoProcessMemoizer())

		logger.info("Running collect-phase for patch")
		val collection: List[Patch] = memoizer.mem("collectProcess")(COLLECT.get.create(params).process(data))
		val collectionDistinct = collection.distinct
		logger.info(s"got ${collection.length} results. ${collectionDistinct.length} after pruning. Running decide")
		val res = memoizer.mem(getClass.getSimpleName + "decideProcess")(DECIDE.get.create(params).process(collectionDistinct))
		logger.info(s"Collect/decide for $res has finished with Patch $res")
		res
	}
}

object CollectDecideProcess {
	val COLLECT = new ProcessParameter[PassableProcessParam[Patch, List[Patch]]]("collect", WorkflowParam(), None)
	val DECIDE = new ProcessParameter[PassableProcessParam[List[Patch], Patch]]("decide", WorkflowParam(), None)
}<commit_msg>option to not forward params<commit_after>package ch.uzh.ifi.pdeboer.pplib.process.stdlib

import ch.uzh.ifi.pdeboer.pplib.process._
import ch.uzh.ifi.pdeboer.pplib.process.entities.{PassableProcessParam, Patch}

/**
 * Created by pdeboer on 05/12/14.
 */
class CollectDecideProcess(_params: Map[String, Any] = Map.empty) extends ProcessStub[Patch, Patch](_params) {

	import ch.uzh.ifi.pdeboer.pplib.process.stdlib.CollectDecideProcess._

	override def expectedParametersBeforeRun: List[ProcessParameter[_]] =
		List(COLLECT, DECIDE).asInstanceOf[List[ProcessParameter[_]]]


	override protected def processCategoryNames: List[String] = List("create.refine.collectdecide")

	override protected def run(data: Patch): Patch = {
		val memoizer: ProcessMemoizer = getProcessMemoizer(data.hashCode() + "").getOrElse(new NoProcessMemoizer())

		logger.info("Running collect-phase for patch")
		val collection: List[Patch] = memoizer.mem("collectProcess")(COLLECT.get.create(if (FORWARD_PARAMS_TO_COLLECT.get) params else Map.empty).process(data))
		val collectionDistinct = collection.distinct
		logger.info(s"got ${collection.length} results. ${collectionDistinct.length} after pruning. Running decide")
		val res = memoizer.mem(getClass.getSimpleName + "decideProcess")(DECIDE.get.create(if (FORWARD_PARAMS_TO_DECIDE.get) params else Map.empty).process(collectionDistinct))
		logger.info(s"Collect/decide for $res has finished with Patch $res")
		res
	}

	override def optionalParameters: List[ProcessParameter[_]] = List(FORWARD_PARAMS_TO_COLLECT, FORWARD_PARAMS_TO_DECIDE)
}

object CollectDecideProcess {
	val FORWARD_PARAMS_TO_COLLECT = new ProcessParameter[Boolean]("forwardParamsToCollect", OtherParam(), Some(List(true)))
	val FORWARD_PARAMS_TO_DECIDE = new ProcessParameter[Boolean]("forwardParamsToDecide", OtherParam(), Some(List(true)))
	val COLLECT = new ProcessParameter[PassableProcessParam[Patch, List[Patch]]]("collect", WorkflowParam(), None)
	val DECIDE = new ProcessParameter[PassableProcessParam[List[Patch], Patch]]("decide", WorkflowParam(), None)
}
{'fraction_non_alphanumeric': 0.12482928161704453, 'fraction_numerical': 0.0032777929527451517, 'mean_word_length': 7.030701754385965, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1922323', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1142, 'n_words': 218}