<commit_before>package slamdata.engine.physical.mongodb

import com.mongodb.DBObject

import scalaz._
import Scalaz._

case class PipelineOpMergeError(left: PipelineOp, right: PipelineOp, hint: Option[String] = None) {
  def message = "The pipeline op " + left + " cannot be merged with the pipeline op " + right + hint.map(": " + _).getOrElse("")
}

case class PipelineMergeError(merged: List[PipelineOp], lrest: List[PipelineOp], rrest: List[PipelineOp], hint: Option[String] = None) {
  def message = "The pipeline " + lrest + " cannot be merged with the pipeline " + rrest + hint.map(": " + _).getOrElse("")
}

final case class Pipeline(ops: List[PipelineOp]) {
  def repr: java.util.List[DBObject] = ops.foldLeft(new java.util.ArrayList[DBObject](): java.util.List[DBObject]) {
    case (list, op) =>
      list.add(op.bson.repr)

      list
  }

  def merge(that: Pipeline): PipelineMergeError \/ Pipeline = {
    def merge0(merged: List[PipelineOp], left: List[PipelineOp], right: List[PipelineOp]): PipelineMergeError \/ List[PipelineOp] = {
      (left, right) match {
        case (left, right) if left == right => \/- (merged ++ left)

        case (left, Nil) => \/- (merged ++ left)
        case (Nil, right) => \/- (merged ++ right)

        case (lh :: lt, rh :: rt) => 
          for {
            h <- merge1(lh, rh).bimap(_ => PipelineMergeError(merged, left, right), identity)
            m <- merge0(merged ++ h, lt, rt)
          } yield m
      }
    }

    merge0(Nil, this.ops, that.ops).map(Pipeline.apply)
  }

  private def merge1(left: PipelineOp, right: PipelineOp): PipelineMergeError \/ List[PipelineOp] = {
    ???
  }
}

import scalaz.NonEmptyList
import scalaz.\/

import scalaz.syntax.monad._<commit_msg>sketch out structure of pipeline merge<commit_after>package slamdata.engine.physical.mongodb

import com.mongodb.DBObject

import scalaz._
import Scalaz._

case class PipelineOpMergeError(left: PipelineOp, right: PipelineOp, hint: Option[String] = None) {
  def message = "The pipeline op " + left + " cannot be merged with the pipeline op " + right + hint.map(": " + _).getOrElse("")
}

case class PipelineMergeError(merged: List[PipelineOp], lrest: List[PipelineOp], rrest: List[PipelineOp], hint: Option[String] = None) {
  def message = "The pipeline " + lrest + " cannot be merged with the pipeline " + rrest + hint.map(": " + _).getOrElse("")
}

final case class Pipeline(ops: List[PipelineOp]) {
  def repr: java.util.List[DBObject] = ops.foldLeft(new java.util.ArrayList[DBObject](): java.util.List[DBObject]) {
    case (list, op) =>
      list.add(op.bson.repr)

      list
  }

  def merge(that: Pipeline): PipelineMergeError \/ Pipeline = {
    def merge0(merged: List[PipelineOp], left: List[PipelineOp], right: List[PipelineOp]): PipelineMergeError \/ List[PipelineOp] = {
      (left, right) match {
        case (left, right) if left == right => \/- (merged ++ left)

        case (left, Nil) => \/- (merged ++ left)
        case (Nil, right) => \/- (merged ++ right)

        case (lh :: lt, rh :: rt) => 
          for {
            h <- lh.merge(rh).bimap(_ => PipelineMergeError(merged, left, right), identity)
            m <- merge0(merged ++ h, lt, rt)
          } yield m
      }
    }

    merge0(Nil, this.ops, that.ops).map(Pipeline.apply)
  }
}

import scalaz.NonEmptyList
import scalaz.\/

import scalaz.syntax.monad._
{'fraction_non_alphanumeric': 0.15062020082693445, 'fraction_numerical': 0.002362669816893089, 'mean_word_length': 3.212686567164179, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '137151', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1099, 'n_words': 291}