<commit_before>/*
 * Copyright 2014–2018 SlamData Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quasar.physical.mongodb

import slamdata.Predef._
import quasar._
import quasar.common.{Map => _, _}
import quasar.contrib.pathy._
import quasar.contrib.specs2._
import quasar.ejson.{EJson, Fixed}
import quasar.fp.ski._
import quasar.fs._
import quasar.javascript._
import quasar.physical.mongodb.accumulator._
import quasar.physical.mongodb.expression._
import quasar.physical.mongodb.planner._
import quasar.physical.mongodb.workflow._
import quasar.qscript.{OnUndefined, ShiftType}
import quasar.sql._

import java.time.Instant
import scala.Either

import eu.timepit.refined.auto._
import matryoshka.data.Fix
import org.specs2.execute._
import pathy.Path._
import scalaz._, Scalaz._

/**
 * Tests of which the workflow expectation should be fairly stable
 * and/or fairly easy to maintain by hand
 */
class PlannerSql2ExactSpec extends
    PlannerHelpers with
    PendingWithActualTracking {

  //to write the new actuals:
  //override val mode = WriteMode

  import Grouped.grouped
  import Reshape.reshape
  import jscore._
  import CollectionUtil._

  import fixExprOp._
  import PlannerHelpers._, expr3_4Fp._

  val dsl =
    quasar.qscript.construction.mkDefaults[Fix, fs.MongoQScript[Fix, ?]]
  import dsl._

  val json = Fixed[Fix[EJson]]

  def plan(query: Fix[Sql]): Either[FileSystemError, Crystallized[WorkflowF]] =
    PlannerHelpers.plan(query)

  val specs = List(
    PlanSpec(
      "simple join ($lookup)",
      sqlToWf = Ok,
      sqlE"select smallZips.city from zips join smallZips on zips.`_id` = smallZips.`_id`",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.EquiJoin(
          fix.Unreferenced,
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("zips"), qscript.ExcludeId),
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("smallZips"), qscript.ExcludeId),
          List((
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined),
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined))),
          JoinType.Inner,
          func.Guard(func.RightSide, Type.AnyObject, func.ProjectKeyS(func.RightSide, "city"), func.Undefined))).some,
      chain[Workflow](
        $read(collection("db", "zips")),
        $match(Selector.Doc(
          BsonField.Name("_id") -> Selector.Exists(true))),
        $project(reshape(JoinDir.Left.name -> $$ROOT)),
        $lookup(
          CollectionName("smallZips"),
          JoinHandler.LeftName \ BsonField.Name("_id"),
          BsonField.Name("_id"),
          JoinHandler.RightName),
        $unwind(DocField(JoinHandler.RightName), None, None),
        $project(
          reshape(sigil.Quasar -> $field(JoinDir.Right.name, "city")),
          ExcludeId))),

    PlanSpec(
      "simple inner equi-join ($lookup)",
      sqlToWf = Ok,
      sqlE"select cars.name, cars2.year from cars join cars2 on cars.`_id` = cars2.`_id`",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.EquiJoin(
          fix.Unreferenced,
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("cars"), qscript.ExcludeId),
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("cars2"), qscript.ExcludeId),
          List((
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined),
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined))),
          JoinType.Inner,
          func.ConcatMaps(
            func.MakeMap(
              func.Constant(json.str("name")),
              func.Guard(
                func.Guard(func.LeftSide, Type.AnyObject, func.LeftSide, func.Undefined),
                Type.AnyObject,
                func.ProjectKeyS(func.LeftSide, "name"),
                func.Undefined)),
            func.MakeMap(
              func.Constant(json.str("year")),
              func.Guard(
                func.Guard(func.RightSide, Type.AnyObject, func.RightSide, func.Undefined),
                Type.AnyObject,
                func.ProjectKeyS(func.RightSide, "year"),
                func.Undefined))))).some,
      chain[Workflow](
        $read(collection("db", "cars")),
        $match(Selector.Doc(
          BsonField.Name("_id") -> Selector.Exists(true))),
        $project(reshape(JoinDir.Left.name -> $$ROOT)),
        $lookup(
          CollectionName("cars2"),
          JoinHandler.LeftName \ BsonField.Name("_id"),
          BsonField.Name("_id"),
          JoinHandler.RightName),
        $unwind(DocField(JoinHandler.RightName), None, None),
        $project(reshape(
          "name" -> $field(JoinDir.Left.name, "name"),
          "year" -> $field(JoinDir.Right.name, "year")),
          ExcludeId))),

    PlanSpec(
      "$match before $unwind",
      sqlToWf = Ok,
      sqlE"SELECT measureEnrollments[*].measureKey FROM zips WHERE year=2017 and memberNumber=123456",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.LeftShift(
          fix.Filter(fix.ShiftedRead[AFile](rootDir </> dir("db") </> file("zips"), qscript.ExcludeId),
            func.Guard(
              func.Hole,
              Type.Obj(Map(), Some(Type.Top)),
              func.And(
                func.Eq(
                  func.ProjectKey(func.Hole, func.Constant(json.str("year"))),
                  func.Constant(json.int(2017))),
                func.Eq(
                  func.ProjectKey(func.Hole, func.Constant(json.str("memberNumber"))),
                  func.Constant(json.int(123456)))),
              func.Undefined)),
          func.Guard(
            func.Guard(
              func.Hole,
              Type.Obj(Map(), Some(Type.Top)),
              func.ProjectKey(
                func.Hole,
                func.Constant(json.str("measureEnrollments"))),
              func.Undefined),
            Type.FlexArr(0, None, Type.Obj(Map(), Some(Type.Top))),
            func.Guard(
              func.Hole,
              Type.Obj(Map(), Some(Type.Top)),
              func.ProjectKey(func.Hole, func.Constant(json.str("measureEnrollments"))),
              func.Undefined),
            func.Undefined),
          qscript.ExcludeId,
          ShiftType.Array,
          OnUndefined.Omit,
          func.ProjectKey(
            func.RightSide, func.Constant(json.str("measureKey"))))
      ).some,
      chain[Workflow](
        $read(collection("db", "zips")),
        $match(Selector.And(
          Selector.Doc(BsonField.Name("year") -> Selector.Eq(Bson.Int32(2017))),
          Selector.Doc(BsonField.Name("memberNumber") -> Selector.Eq(Bson.Int32(123456))))),
        $project(reshape(
          "0" ->
            $cond(
              $and(
                $lte($literal(Bson.Arr()), $field("measureEnrollments")),
                $lt($field("measureEnrollments"), $literal(Bson.Binary.fromArray(scala.Array[Byte]())))),
              $field("measureEnrollments"),
              $literal(Bson.Undefined))),
          ExcludeId),
        $unwind(DocField(BsonField.Name("0")), None, None),<commit_msg>Fix sql2 mongo spec expectation.<commit_after>/*
 * Copyright 2014–2018 SlamData Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quasar.physical.mongodb

import slamdata.Predef._
import quasar._
import quasar.common.{Map => _, _}
import quasar.contrib.pathy._
import quasar.contrib.specs2._
import quasar.ejson.{EJson, Fixed}
import quasar.fp.ski._
import quasar.fs._
import quasar.javascript._
import quasar.physical.mongodb.accumulator._
import quasar.physical.mongodb.expression._
import quasar.physical.mongodb.planner._
import quasar.physical.mongodb.workflow._
import quasar.qscript.{OnUndefined, ShiftType}
import quasar.sql._

import java.time.Instant
import scala.Either

import eu.timepit.refined.auto._
import matryoshka.data.Fix
import org.specs2.execute._
import pathy.Path._
import scalaz._, Scalaz._

/**
 * Tests of which the workflow expectation should be fairly stable
 * and/or fairly easy to maintain by hand
 */
class PlannerSql2ExactSpec extends
    PlannerHelpers with
    PendingWithActualTracking {

  //to write the new actuals:
  //override val mode = WriteMode

  import Grouped.grouped
  import Reshape.reshape
  import jscore._
  import CollectionUtil._

  import fixExprOp._
  import PlannerHelpers._, expr3_4Fp._

  val dsl =
    quasar.qscript.construction.mkDefaults[Fix, fs.MongoQScript[Fix, ?]]
  import dsl._

  val json = Fixed[Fix[EJson]]

  def plan(query: Fix[Sql]): Either[FileSystemError, Crystallized[WorkflowF]] =
    PlannerHelpers.plan(query)

  val specs = List(
    PlanSpec(
      "simple join ($lookup)",
      sqlToWf = Ok,
      sqlE"select smallZips.city from zips join smallZips on zips.`_id` = smallZips.`_id`",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.EquiJoin(
          fix.Unreferenced,
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("zips"), qscript.ExcludeId),
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("smallZips"), qscript.ExcludeId),
          List((
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined),
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined))),
          JoinType.Inner,
          func.Guard(func.RightSide, Type.AnyObject, func.ProjectKeyS(func.RightSide, "city"), func.Undefined))).some,
      chain[Workflow](
        $read(collection("db", "zips")),
        $match(Selector.Doc(
          BsonField.Name("_id") -> Selector.Exists(true))),
        $project(reshape(JoinDir.Left.name -> $$ROOT)),
        $lookup(
          CollectionName("smallZips"),
          JoinHandler.LeftName \ BsonField.Name("_id"),
          BsonField.Name("_id"),
          JoinHandler.RightName),
        $unwind(DocField(JoinHandler.RightName), None, None),
        $project(
          reshape(sigil.Quasar -> $field(JoinDir.Right.name, "city")),
          ExcludeId))),

    PlanSpec(
      "simple inner equi-join ($lookup)",
      sqlToWf = Ok,
      sqlE"select cars.name, cars2.year from cars join cars2 on cars.`_id` = cars2.`_id`",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.EquiJoin(
          fix.Unreferenced,
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("cars"), qscript.ExcludeId),
          free.ShiftedRead[AFile](rootDir </> dir("db") </> file("cars2"), qscript.ExcludeId),
          List((
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined),
            func.Guard(func.Hole, Type.AnyObject, func.ProjectKeyS(func.Hole, "_id"), func.Undefined))),
          JoinType.Inner,
          func.ConcatMaps(
            func.MakeMap(
              func.Constant(json.str("name")),
              func.Guard(
                func.Guard(func.LeftSide, Type.AnyObject, func.LeftSide, func.Undefined),
                Type.AnyObject,
                func.ProjectKeyS(func.LeftSide, "name"),
                func.Undefined)),
            func.MakeMap(
              func.Constant(json.str("year")),
              func.Guard(
                func.Guard(func.RightSide, Type.AnyObject, func.RightSide, func.Undefined),
                Type.AnyObject,
                func.ProjectKeyS(func.RightSide, "year"),
                func.Undefined))))).some,
      chain[Workflow](
        $read(collection("db", "cars")),
        $match(Selector.Doc(
          BsonField.Name("_id") -> Selector.Exists(true))),
        $project(reshape(JoinDir.Left.name -> $$ROOT)),
        $lookup(
          CollectionName("cars2"),
          JoinHandler.LeftName \ BsonField.Name("_id"),
          BsonField.Name("_id"),
          JoinHandler.RightName),
        $unwind(DocField(JoinHandler.RightName), None, None),
        $project(reshape(
          "name" -> $field(JoinDir.Left.name, "name"),
          "year" -> $field(JoinDir.Right.name, "year")),
          ExcludeId))),

    PlanSpec(
      "$match before $unwind",
      sqlToWf = Ok,
      sqlE"SELECT measureEnrollments[*].measureKey FROM zips WHERE year=2017 and memberNumber=123456",
      QsSpec(
        sqlToQs = Ok,
        qsToWf = Ok,
        fix.LeftShift(
          fix.Filter(fix.ShiftedRead[AFile](rootDir </> dir("db") </> file("zips"), qscript.ExcludeId),
            func.Guard(
              func.Hole,
              Type.Obj(Map(), Some(Type.Top)),
              func.And(
                func.Eq(
                  func.ProjectKey(func.Hole, func.Constant(json.str("year"))),
                  func.Constant(json.int(2017))),
                func.Eq(
                  func.ProjectKey(func.Hole, func.Constant(json.str("memberNumber"))),
                  func.Constant(json.int(123456)))),
              func.Undefined)),
          func.Guard(
            func.Guard(
              func.Hole,
              Type.Obj(Map(), Some(Type.Top)),
              func.ProjectKey(
                func.Hole,
                func.Constant(json.str("measureEnrollments"))),
              func.Undefined),
            Type.FlexArr(0, None, Type.Obj(Map(), Some(Type.Top))),
            func.ProjectKey(func.Hole, func.Constant(json.str("measureEnrollments"))),
            func.Undefined),
          qscript.ExcludeId,
          ShiftType.Array,
          OnUndefined.Omit,
          func.ProjectKey(
            func.RightSide, func.Constant(json.str("measureKey"))))
      ).some,
      chain[Workflow](
        $read(collection("db", "zips")),
        $match(Selector.And(
          Selector.Doc(BsonField.Name("year") -> Selector.Eq(Bson.Int32(2017))),
          Selector.Doc(BsonField.Name("memberNumber") -> Selector.Eq(Bson.Int32(123456))))),
        $project(reshape(
          "0" ->
            $cond(
              $and(
                $lte($literal(Bson.Arr()), $field("measureEnrollments")),
                $lt($field("measureEnrollments"), $literal(Bson.Binary.fromArray(scala.Array[Byte]())))),
              $field("measureEnrollments"),
              $literal(Bson.Undefined))),
          ExcludeId),
        $unwind(DocField(BsonField.Name("0")), None, None),
{'fraction_non_alphanumeric': 0.1384051329055912, 'fraction_numerical': 0.007791017415215398, 'mean_word_length': 2.4080767514502455, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '984185', 'n_tokens_mistral': 4899, 'n_tokens_neox': 4688, 'n_words': 886}