<commit_before>package pl.touk.nussknacker.ui.integration

import java.io.File
import java.util.UUID

import akka.http.javadsl.model.headers.HttpCredentials
import akka.http.scaladsl.model.{ContentTypeRange, ContentTypes, HttpEntity, StatusCodes}
import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
import com.typesafe.config.Config
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
import io.circe.{Decoder, Json}
import org.apache.commons.io.FileUtils
import org.scalatest._
import pl.touk.nussknacker.engine.api.StreamMetaData
import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.RedundantParameters
import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, FixedValuesValidator, LiteralIntValidator, MandatoryParameterValidator, StringParameterEditor}
import pl.touk.nussknacker.engine.api.process.{ParameterConfig, SingleNodeConfig}
import pl.touk.nussknacker.engine.build.EspProcessBuilder
import pl.touk.nussknacker.engine.canonize.ProcessCanonizer
import pl.touk.nussknacker.engine.graph.EspProcess
import pl.touk.nussknacker.engine.graph.exceptionhandler.ExceptionHandlerRef
import pl.touk.nussknacker.engine.graph.expression.Expression
import pl.touk.nussknacker.engine.graph.node.SubprocessInputDefinition.{SubprocessClazzRef, SubprocessParameter}
import pl.touk.nussknacker.engine.graph.node.{SubprocessInputDefinition, SubprocessOutputDefinition}
import pl.touk.nussknacker.engine.spel
import pl.touk.nussknacker.restmodel.displayedgraph.displayablenode.Edge
import pl.touk.nussknacker.restmodel.displayedgraph.{DisplayableProcess, ProcessProperties}
import pl.touk.nussknacker.restmodel.validation.ValidationResults.{ValidationErrors, ValidationResult}
import pl.touk.nussknacker.test.PatientScalaFutures
import pl.touk.nussknacker.ui.NussknackerApp
import pl.touk.nussknacker.ui.api.helpers.{TestFactory, TestProcessUtil, TestProcessingTypes}
import pl.touk.nussknacker.ui.definition.additionalproperty.UiAdditionalPropertyConfig
import pl.touk.nussknacker.ui.process.marshall.ProcessConverter
import pl.touk.nussknacker.ui.util.{ConfigWithScalaVersion, MultipartUtils}
import pl.touk.nussknacker.ui.validation.PrettyValidationErrors

import scala.concurrent.duration._
import scala.util.Properties

class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSupport
  with Matchers with PatientScalaFutures with BeforeAndAfterEach with BeforeAndAfterAll {

  import io.circe.syntax._

  override def testConfig: Config = ConfigWithScalaVersion.config

  private implicit final val string: FromEntityUnmarshaller[String] = Unmarshaller.stringUnmarshaller.forContentTypes(ContentTypeRange.*)

  private val (mainRoute, _) = NussknackerApp.initializeRoute(ConfigWithScalaVersion.config)

  private val credentials = HttpCredentials.createBasicHttpCredentials("admin", "admin")

  implicit val timeout: RouteTestTimeout = RouteTestTimeout(1.minute)

  test("saves, updates and retrieves sample process") {

    val processId = UUID.randomUUID().toString

    val process = EspProcessBuilder
      .id(processId)
      .exceptionHandler()
      .source("source", "csv-source").processorEnd("end", "monitor")

    saveProcess(process)
  }

  test("initializes custom processes") {
    Get("/api/processes/customProcess1") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.OK
    }
  }

  test("ensure nodes config is properly parsed") {
    Post("/api/processDefinitionData/streaming?isSubprocess=false", HttpEntity(ContentTypes.`application/json`, "{}")) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      val settingsJson = responseAs[Json].hcursor.downField("nodesConfig").focus.get
      val settings = Decoder[Map[String, SingleNodeConfig]].decodeJson(settingsJson).right.get

      val underTest = Map(
        //docs url comes from reference.conf in managementSample
        "filter" -> SingleNodeConfig(None, None, Some("https://touk.github.io/nussknacker/filter"), None),
        "test1" -> SingleNodeConfig(None, Some("Sink.svg"), None, None),
        "enricher" -> SingleNodeConfig(
          Some(Map("param" -> ParameterConfig(Some("'default value'"), Some(StringParameterEditor), None, None))),
          Some("Filter.svg"),
          //docs url comes from reference.conf in managementSample
          Some("https://touk.github.io/nussknacker/enricher"),
          None
        ),
        "multipleParamsService" -> SingleNodeConfig(
          Some(Map(
            "foo" -> ParameterConfig(None, Some(FixedValuesParameterEditor(List(FixedExpressionValue("test", "test")))), None, None),
            "bar" -> ParameterConfig(None, Some(StringParameterEditor), None, None),
            "baz" -> ParameterConfig(None, Some(FixedValuesParameterEditor(List(FixedExpressionValue("1", "1"), FixedExpressionValue("2", "2")))), None, None)
          )),
          None,
          None,
          None
        ),
        "accountService" -> SingleNodeConfig(None, None, Some("accountServiceDocs"), None),
        "sub1" -> SingleNodeConfig(
          Some(Map(
            "param1" -> ParameterConfig(None, Some(StringParameterEditor), None, None)
          )),
          None,
          None,
          None
        ),
        "optionalTypesService" -> SingleNodeConfig(
          Some(Map(
            "overriddenByFileConfigParam" -> ParameterConfig(None, None, Some(List.empty), None),
            "overriddenByDevConfigParam" -> ParameterConfig(None, None, Some(List(MandatoryParameterValidator)), None)
          )),
          None,
          None,
          Some("types")
        )
      )

      val (relevant, other) = settings.partition { case (k, _) => underTest.keySet contains k }
      relevant shouldBe underTest
      other.values.forall(_.docsUrl.isEmpty) shouldBe true
    }
  }

  test("ensure additional properties config is properly applied") {
    Post("/api/processDefinitionData/streaming?isSubprocess=false", HttpEntity(ContentTypes.`application/json`, "{}")) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      val settingsJson = responseAs[Json].hcursor.downField("additionalPropertiesConfig").focus.get
      val fixedPossibleValues = List(FixedExpressionValue("1", "1"), FixedExpressionValue("2", "2"))

      val settings = Decoder[Map[String, UiAdditionalPropertyConfig]].decodeJson(settingsJson).right.get

      val underTest = Map(
        "stringRequiredProperty" -> new UiAdditionalPropertyConfig(
          Some("default"),
          StringParameterEditor,
          List(MandatoryParameterValidator),
          Some("label")
        ),
        "intOptionalProperty" -> new UiAdditionalPropertyConfig(
          None,
          StringParameterEditor,
          List(LiteralIntValidator),
          None
        ),
        "fixedValueOptionalProperty" -> new UiAdditionalPropertyConfig(
          None,
          FixedValuesParameterEditor(fixedPossibleValues),
          List(FixedValuesValidator(fixedPossibleValues)),
          None
        )
      )

      settings shouldBe underTest
    }
  }

  test("validate process additional properties") {
    Post(
      "/api/processValidation",
      HttpEntity(ContentTypes.`application/json`, TestFactory.processWithInvalidAdditionalProperties.asJson.spaces2)
    ) ~> addCredentials(credentials) ~> mainRoute ~> check {
      status shouldEqual StatusCodes.OK
      val entity = responseAs[String]

      entity should include("Configured property stringRequiredProperty (label) is missing")
      entity should include("Property intOptionalProperty has value of invalid type")
      entity should include("Unknown property unknown")
      entity should include("Property fixedValueOptionalProperty has invalid value")
    }
  }

  test("be able to work with subprocess with custom class inputs") {
    val processId = UUID.randomUUID().toString
    val endpoint = s"/api/processes/$processId"

    val process = DisplayableProcess(
      id = processId,
      properties = ProcessProperties(StreamMetaData(), ExceptionHandlerRef(List()), isSubprocess = true, subprocessVersions = Map()),
      nodes = List(SubprocessInputDefinition("input1", List(SubprocessParameter("badParam", SubprocessClazzRef("i.do.not.exist")))),
        SubprocessOutputDefinition("output1", "out1", List.empty)),
      edges = List(Edge("input1", "output1", None)),
      processingType = TestProcessingTypes.Streaming
    )

    Post(s"$endpoint/Category1?isSubprocess=true") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.Created
      Put(endpoint, TestFactory.posting.toEntityAsProcessToSave(process)) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
        status shouldEqual StatusCodes.OK

        val res = responseAs[ValidationResult]
        //TODO: in the future should be more local error
        res.errors.globalErrors.map(_.description) shouldBe List(
          "Fatal error: Failed to load subprocess parameter: i.do.not.exist for input1, please check configuration")

        Get(endpoint) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
          status shouldEqual StatusCodes.OK
        }
      }
    }
  }

  import spel.Implicits._

  test("should test process with complexReturnObjectService") {
    val processId = "complexObjectProcess" + UUID.randomUUID().toString

    val process = EspProcessBuilder
      .id(processId)
      .exceptionHandler()
      .source("source", "csv-source")
      .enricher("enricher", "out", "complexReturnObjectService")
      .sink("end", "#input", "sendSms")

    saveProcess(process)

    val multiPart = MultipartUtils.prepareMultiParts("testData" -> "record1|field2", "processJson" -> TestProcessUtil.toJson(process).noSpaces)()
    Post(s"/api/processManagement/test/${process.id}", multiPart) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.OK
    }
  }

  test("should reload ConfigCreator") {

    //@see DevProcessConfigCreator.DynamicService
    val dynamicServiceFile = new File(Properties.tmpDir, "nk-dynamic-params.lst")
    dynamicServiceFile.delete()
    
    def generationTime: Option[String] = {
      Get("/api/app/buildInfo") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
        status shouldEqual StatusCodes.OK
        responseAs[Json].hcursor
          .downField("processingType")
          .downField("streaming")
          .downField("generation-time")
          .focus.flatMap(_.asString)<commit_msg>[FIX] more robust test for reload (#815)<commit_after>package pl.touk.nussknacker.ui.integration

import java.io.File
import java.util.UUID

import akka.http.javadsl.model.headers.HttpCredentials
import akka.http.scaladsl.model.{ContentTypeRange, ContentTypes, HttpEntity, StatusCodes}
import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller}
import com.typesafe.config.Config
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport
import io.circe.{Decoder, Json}
import org.apache.commons.io.FileUtils
import org.scalatest._
import pl.touk.nussknacker.engine.api.StreamMetaData
import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.RedundantParameters
import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, FixedValuesValidator, LiteralIntValidator, MandatoryParameterValidator, StringParameterEditor}
import pl.touk.nussknacker.engine.api.process.{ParameterConfig, SingleNodeConfig}
import pl.touk.nussknacker.engine.build.EspProcessBuilder
import pl.touk.nussknacker.engine.canonize.ProcessCanonizer
import pl.touk.nussknacker.engine.graph.EspProcess
import pl.touk.nussknacker.engine.graph.exceptionhandler.ExceptionHandlerRef
import pl.touk.nussknacker.engine.graph.expression.Expression
import pl.touk.nussknacker.engine.graph.node.SubprocessInputDefinition.{SubprocessClazzRef, SubprocessParameter}
import pl.touk.nussknacker.engine.graph.node.{SubprocessInputDefinition, SubprocessOutputDefinition}
import pl.touk.nussknacker.engine.spel
import pl.touk.nussknacker.restmodel.displayedgraph.displayablenode.Edge
import pl.touk.nussknacker.restmodel.displayedgraph.{DisplayableProcess, ProcessProperties}
import pl.touk.nussknacker.restmodel.validation.ValidationResults.{ValidationErrors, ValidationResult}
import pl.touk.nussknacker.test.PatientScalaFutures
import pl.touk.nussknacker.ui.NussknackerApp
import pl.touk.nussknacker.ui.api.helpers.{TestFactory, TestProcessUtil, TestProcessingTypes}
import pl.touk.nussknacker.ui.definition.additionalproperty.UiAdditionalPropertyConfig
import pl.touk.nussknacker.ui.process.marshall.ProcessConverter
import pl.touk.nussknacker.ui.util.{ConfigWithScalaVersion, MultipartUtils}
import pl.touk.nussknacker.ui.validation.PrettyValidationErrors

import scala.concurrent.duration._
import scala.util.Properties

class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSupport
  with Matchers with PatientScalaFutures with BeforeAndAfterEach with BeforeAndAfterAll {

  import io.circe.syntax._

  override def testConfig: Config = ConfigWithScalaVersion.config

  private implicit final val string: FromEntityUnmarshaller[String] = Unmarshaller.stringUnmarshaller.forContentTypes(ContentTypeRange.*)

  private val (mainRoute, _) = NussknackerApp.initializeRoute(ConfigWithScalaVersion.config)

  private val credentials = HttpCredentials.createBasicHttpCredentials("admin", "admin")

  implicit val timeout: RouteTestTimeout = RouteTestTimeout(1.minute)

  //@see DevProcessConfigCreator.DynamicService, TODO: figure out how to make reload test more robust...
  //currently we delete file in beforeAll, because it's used *also* in initialization...
  val dynamicServiceFile = new File(Properties.tmpDir, "nk-dynamic-params.lst")

  override def beforeAll(): Unit = {
    super.beforeAll()
    dynamicServiceFile.delete()
  }

  override def afterAll(): Unit = {
    dynamicServiceFile.delete()
    super.afterAll()
  }


  test("saves, updates and retrieves sample process") {

    val processId = UUID.randomUUID().toString

    val process = EspProcessBuilder
      .id(processId)
      .exceptionHandler()
      .source("source", "csv-source").processorEnd("end", "monitor")

    saveProcess(process)
  }

  test("initializes custom processes") {
    Get("/api/processes/customProcess1") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.OK
    }
  }

  test("ensure nodes config is properly parsed") {
    Post("/api/processDefinitionData/streaming?isSubprocess=false", HttpEntity(ContentTypes.`application/json`, "{}")) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      val settingsJson = responseAs[Json].hcursor.downField("nodesConfig").focus.get
      val settings = Decoder[Map[String, SingleNodeConfig]].decodeJson(settingsJson).right.get

      val underTest = Map(
        //docs url comes from reference.conf in managementSample
        "filter" -> SingleNodeConfig(None, None, Some("https://touk.github.io/nussknacker/filter"), None),
        "test1" -> SingleNodeConfig(None, Some("Sink.svg"), None, None),
        "enricher" -> SingleNodeConfig(
          Some(Map("param" -> ParameterConfig(Some("'default value'"), Some(StringParameterEditor), None, None))),
          Some("Filter.svg"),
          //docs url comes from reference.conf in managementSample
          Some("https://touk.github.io/nussknacker/enricher"),
          None
        ),
        "multipleParamsService" -> SingleNodeConfig(
          Some(Map(
            "foo" -> ParameterConfig(None, Some(FixedValuesParameterEditor(List(FixedExpressionValue("test", "test")))), None, None),
            "bar" -> ParameterConfig(None, Some(StringParameterEditor), None, None),
            "baz" -> ParameterConfig(None, Some(FixedValuesParameterEditor(List(FixedExpressionValue("1", "1"), FixedExpressionValue("2", "2")))), None, None)
          )),
          None,
          None,
          None
        ),
        "accountService" -> SingleNodeConfig(None, None, Some("accountServiceDocs"), None),
        "sub1" -> SingleNodeConfig(
          Some(Map(
            "param1" -> ParameterConfig(None, Some(StringParameterEditor), None, None)
          )),
          None,
          None,
          None
        ),
        "optionalTypesService" -> SingleNodeConfig(
          Some(Map(
            "overriddenByFileConfigParam" -> ParameterConfig(None, None, Some(List.empty), None),
            "overriddenByDevConfigParam" -> ParameterConfig(None, None, Some(List(MandatoryParameterValidator)), None)
          )),
          None,
          None,
          Some("types")
        )
      )

      val (relevant, other) = settings.partition { case (k, _) => underTest.keySet contains k }
      relevant shouldBe underTest
      other.values.forall(_.docsUrl.isEmpty) shouldBe true
    }
  }

  test("ensure additional properties config is properly applied") {
    Post("/api/processDefinitionData/streaming?isSubprocess=false", HttpEntity(ContentTypes.`application/json`, "{}")) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      val settingsJson = responseAs[Json].hcursor.downField("additionalPropertiesConfig").focus.get
      val fixedPossibleValues = List(FixedExpressionValue("1", "1"), FixedExpressionValue("2", "2"))

      val settings = Decoder[Map[String, UiAdditionalPropertyConfig]].decodeJson(settingsJson).right.get

      val underTest = Map(
        "stringRequiredProperty" -> new UiAdditionalPropertyConfig(
          Some("default"),
          StringParameterEditor,
          List(MandatoryParameterValidator),
          Some("label")
        ),
        "intOptionalProperty" -> new UiAdditionalPropertyConfig(
          None,
          StringParameterEditor,
          List(LiteralIntValidator),
          None
        ),
        "fixedValueOptionalProperty" -> new UiAdditionalPropertyConfig(
          None,
          FixedValuesParameterEditor(fixedPossibleValues),
          List(FixedValuesValidator(fixedPossibleValues)),
          None
        )
      )

      settings shouldBe underTest
    }
  }

  test("validate process additional properties") {
    Post(
      "/api/processValidation",
      HttpEntity(ContentTypes.`application/json`, TestFactory.processWithInvalidAdditionalProperties.asJson.spaces2)
    ) ~> addCredentials(credentials) ~> mainRoute ~> check {
      status shouldEqual StatusCodes.OK
      val entity = responseAs[String]

      entity should include("Configured property stringRequiredProperty (label) is missing")
      entity should include("Property intOptionalProperty has value of invalid type")
      entity should include("Unknown property unknown")
      entity should include("Property fixedValueOptionalProperty has invalid value")
    }
  }

  test("be able to work with subprocess with custom class inputs") {
    val processId = UUID.randomUUID().toString
    val endpoint = s"/api/processes/$processId"

    val process = DisplayableProcess(
      id = processId,
      properties = ProcessProperties(StreamMetaData(), ExceptionHandlerRef(List()), isSubprocess = true, subprocessVersions = Map()),
      nodes = List(SubprocessInputDefinition("input1", List(SubprocessParameter("badParam", SubprocessClazzRef("i.do.not.exist")))),
        SubprocessOutputDefinition("output1", "out1", List.empty)),
      edges = List(Edge("input1", "output1", None)),
      processingType = TestProcessingTypes.Streaming
    )

    Post(s"$endpoint/Category1?isSubprocess=true") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.Created
      Put(endpoint, TestFactory.posting.toEntityAsProcessToSave(process)) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
        status shouldEqual StatusCodes.OK

        val res = responseAs[ValidationResult]
        //TODO: in the future should be more local error
        res.errors.globalErrors.map(_.description) shouldBe List(
          "Fatal error: Failed to load subprocess parameter: i.do.not.exist for input1, please check configuration")

        Get(endpoint) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
          status shouldEqual StatusCodes.OK
        }
      }
    }
  }

  import spel.Implicits._

  test("should test process with complexReturnObjectService") {
    val processId = "complexObjectProcess" + UUID.randomUUID().toString

    val process = EspProcessBuilder
      .id(processId)
      .exceptionHandler()
      .source("source", "csv-source")
      .enricher("enricher", "out", "complexReturnObjectService")
      .sink("end", "#input", "sendSms")

    saveProcess(process)

    val multiPart = MultipartUtils.prepareMultiParts("testData" -> "record1|field2", "processJson" -> TestProcessUtil.toJson(process).noSpaces)()
    Post(s"/api/processManagement/test/${process.id}", multiPart) ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
      status shouldEqual StatusCodes.OK
    }
  }

  test("should reload ConfigCreator") {

    def generationTime: Option[String] = {
      Get("/api/app/buildInfo") ~> addCredentials(credentials) ~> mainRoute ~> checkWithClue {
        status shouldEqual StatusCodes.OK
        responseAs[Json].hcursor
          .downField("processingType")
          .downField("streaming")
          .downField("generation-time")
          .focus.flatMap(_.asString)
{'fraction_non_alphanumeric': 0.10827497235532621, 'fraction_numerical': 0.002257648359749355, 'mean_word_length': 4.045327754532775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 97, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3231434', 'n_tokens_mistral': 6196, 'n_tokens_neox': 6026, 'n_words': 1128}