<commit_before>/*
 * Copyright 2014–2018 SlamData Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quasar.physical.mongodb

import slamdata.Predef._
import quasar._
import quasar.contrib.scalaz.catchable._
import quasar.fp._
import quasar.fp.ski._
import quasar.fp.tree._
import quasar.frontend.{logicalplan => lp}, lp.{LogicalPlan => LP}
import quasar.fs.{DataCursor, FileSystemError}
import quasar.physical.mongodb.WorkflowExecutor.WorkflowCursor
import quasar.physical.mongodb.fs._, bsoncursor._
import quasar.physical.mongodb.workflow._
import quasar.qscript._
import quasar.std._
import quasar.time.TemporalPart

import scala.sys

import java.time.{Instant, OffsetDateTime => JOffsetDateTime, ZoneOffset}
import java.time.temporal.ChronoUnit

import matryoshka.{Hole => _, _}
import matryoshka.data.Fix
import matryoshka.implicits._
import org.specs2.execute._
import org.specs2.matcher._
import org.specs2.main.ArgProperty
import scala.concurrent.duration._
import scalaz._, Scalaz._
import scalaz.concurrent.{Strategy, Task}
import shapeless.{Nat}

/** Test the implementation of the standard library for one of MongoDb's
  * evaluators.
  */
abstract class MongoDbStdLibSpec extends StdLibSpec {
  val lpf = new lp.LogicalPlanR[Fix[LP]]

  val runAt = Instant.parse("2015-01-26T00:00:00Z")

  val noTimeZoneSupport = Skipped("Mongo does not support time zones.")

  args.report(showtimes = ArgProperty(true))

  def shortCircuit[N <: Nat](backend: BackendName, func: GenericFunc[N], args: List[Data]): Result \/ Unit

  def skipTemporalTrunc(part: TemporalPart): Boolean

  def compile(queryModel: MongoQueryModel, coll: Collection, lp: FreeMap[Fix])
      : FileSystemError \/ (Crystallized[WorkflowF], BsonField.Name)

  def advertisedVersion(backend: BackendName): Option[MongoQueryModel] =
    if (backend ≟ TestConfig.MONGO_3_2.name) MongoQueryModel.`3.2`.some
    else if ((backend ≟ TestConfig.MONGO_3_4.name) || (backend ≟ TestConfig.MONGO_READ_ONLY.name))
      MongoQueryModel.`3.4`.some
    else if (backend ≟ TestConfig.MONGO_3_4_13.name) MongoQueryModel.`3.4.4`.some
    else if (backend ≟ TestConfig.MONGO_3_6.name) MongoQueryModel.`3.6`.some
    else None

  MongoDbSpec.clientShould(MongoDb.Type) { (backend, prefix, setupClient, testClient) =>
    import MongoDbIO._

    /** Intercept and transform expected values into the form that's actually
      * produced by the MongoDB backend, in cases where MongoDB cannot represent
      * the type natively. */
    def massageExpected(expected: Data): Data = expected match {
      case Data.LocalDate(time) =>
        Data.OffsetDateTime(JOffsetDateTime.of(time.atStartOfDay, ZoneOffset.UTC))
      case Data.LocalTime(time) =>
        Data.Str(time.toString)
      case Data.LocalDateTime(time) =>
        Data.OffsetDateTime(JOffsetDateTime.of(
          // NB withNano(scala.math.round(...)) fails for a corner case:
          // Invalid value for NanoOfSecond (valid values 0 - 999999999): 1000000000
          time.truncatedTo(ChronoUnit.SECONDS)
            .plusNanos(scala.math.round(time.getNano.toDouble / 1000000).toLong * 1000000), // round to millis
          ZoneOffset.UTC))
      case _ => expected
    }

    def containsOffset(args: List[Data]): Boolean =
      args.filter {
        case Data.OffsetLike(_) => true
        case _ => false
      }.nonEmpty

    /** Identify constructs that are expected not to be implemented. */
    def shortCircuitLP(args: List[Data]): AlgebraM[Result \/ ?, LP, Unit] = {
      case lp.Invoke(_, _) if containsOffset(args) => noTimeZoneSupport.left
      case lp.Invoke(func, _) => shortCircuit(backend, func, args)
      case lp.TemporalTrunc(part, _) if skipTemporalTrunc(part) =>
        Pending(s"TemporalTrunc for $part not implemented.").left
      case lp.TemporalTrunc(_, _) if containsOffset(args) => noTimeZoneSupport.left
      case _ => ().right
    }

    def evaluate(wf: Crystallized[WorkflowF], tmp: Collection): MongoDbIO[List[Data]] =<commit_msg>improve test messages a bit for MongoDbStdLibSpec<commit_after>/*
 * Copyright 2014–2018 SlamData Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package quasar.physical.mongodb

import slamdata.Predef._
import quasar._
import quasar.contrib.scalaz.catchable._
import quasar.fp._
import quasar.fp.ski._
import quasar.fp.tree._
import quasar.frontend.{logicalplan => lp}, lp.{LogicalPlan => LP}
import quasar.fs.{DataCursor, FileSystemError}
import quasar.physical.mongodb.WorkflowExecutor.WorkflowCursor
import quasar.physical.mongodb.fs._, bsoncursor._
import quasar.physical.mongodb.workflow._
import quasar.qscript._
import quasar.std._
import quasar.time.TemporalPart

import scala.sys

import java.time.{Instant, OffsetDateTime => JOffsetDateTime, ZoneOffset}
import java.time.temporal.ChronoUnit

import matryoshka.{Hole => _, _}
import matryoshka.data.Fix
import matryoshka.implicits._
import org.specs2.execute._
import org.specs2.matcher._
import org.specs2.main.ArgProperty
import scala.concurrent.duration._
import scalaz._, Scalaz._
import scalaz.concurrent.{Strategy, Task}
import shapeless.{Nat}

/** Test the implementation of the standard library for one of MongoDb's
  * evaluators.
  */
abstract class MongoDbStdLibSpec extends StdLibSpec {
  val lpf = new lp.LogicalPlanR[Fix[LP]]

  val runAt = Instant.parse("2015-01-26T00:00:00Z")

  val noTimeZoneSupport = Pending("Mongo does not support time zones.")

  args.report(showtimes = ArgProperty(true))

  def shortCircuit[N <: Nat](backend: BackendName, func: GenericFunc[N], args: List[Data]): Result \/ Unit

  def skipTemporalTrunc(part: TemporalPart): Boolean

  def compile(queryModel: MongoQueryModel, coll: Collection, lp: FreeMap[Fix])
      : FileSystemError \/ (Crystallized[WorkflowF], BsonField.Name)

  def advertisedVersion(backend: BackendName): Option[MongoQueryModel] =
    if (backend ≟ TestConfig.MONGO_3_2.name) MongoQueryModel.`3.2`.some
    else if ((backend ≟ TestConfig.MONGO_3_4.name) || (backend ≟ TestConfig.MONGO_READ_ONLY.name))
      MongoQueryModel.`3.4`.some
    else if (backend ≟ TestConfig.MONGO_3_4_13.name) MongoQueryModel.`3.4.4`.some
    else if (backend ≟ TestConfig.MONGO_3_6.name) MongoQueryModel.`3.6`.some
    else None

  MongoDbSpec.clientShould(MongoDb.Type) { (backend, prefix, setupClient, testClient) =>
    import MongoDbIO._

    /** Intercept and transform expected values into the form that's actually
      * produced by the MongoDB backend, in cases where MongoDB cannot represent
      * the type natively. */
    def massageExpected(expected: Data): Data = expected match {
      case Data.LocalDate(time) =>
        Data.OffsetDateTime(JOffsetDateTime.of(time.atStartOfDay, ZoneOffset.UTC))
      case Data.LocalTime(time) =>
        Data.Str(time.toString)
      case Data.LocalDateTime(time) =>
        Data.OffsetDateTime(JOffsetDateTime.of(
          // NB withNano(scala.math.round(...)) fails for a corner case:
          // Invalid value for NanoOfSecond (valid values 0 - 999999999): 1000000000
          time.truncatedTo(ChronoUnit.SECONDS)
            .plusNanos(scala.math.round(time.getNano.toDouble / 1000000).toLong * 1000000), // round to millis
          ZoneOffset.UTC))
      case _ => expected
    }

    def containsOffset(args: List[Data]): Boolean =
      args.filter {
        case Data.OffsetLike(_) => true
        case _ => false
      }.nonEmpty

    /** Identify constructs that are expected not to be implemented. */
    def shortCircuitLP(args: List[Data]): AlgebraM[Result \/ ?, LP, Unit] = {
      case lp.Invoke(_, _) if containsOffset(args) => noTimeZoneSupport.left
      case lp.Invoke(func, _) => shortCircuit(backend, func, args)
      case lp.TemporalTrunc(_, _) if containsOffset(args) => noTimeZoneSupport.left
      case lp.TemporalTrunc(part, _) if skipTemporalTrunc(part) =>
        Pending(s"TemporalTrunc for $part not implemented.").left
      case _ => ().right
    }

    def evaluate(wf: Crystallized[WorkflowF], tmp: Collection): MongoDbIO[List[Data]] =
{'fraction_non_alphanumeric': 0.10672546857772877, 'fraction_numerical': 0.018081587651598677, 'mean_word_length': 3.874261149919398, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4334053', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2746, 'n_words': 748}