<commit_before>package org.orbeon.oxf.fr.persistence.proxy

import cats.syntax.option._
import org.orbeon.dom.saxon.{DocumentWrapper, NodeWrapper}
import org.orbeon.io.CharsetNames
import org.orbeon.oxf.externalcontext.{ExternalContext, URLRewriter}
import org.orbeon.oxf.fr.FormRunnerPersistence.findFormDefinitionFormatFromStringVersions
import org.orbeon.oxf.fr.XMLNames.{XBLBindingTest, XBLXBLTest}
import org.orbeon.oxf.fr._
import org.orbeon.oxf.fr.datamigration.MigrationSupport
import org.orbeon.oxf.fr.datamigration.MigrationSupport.MigrationsFromForm
import org.orbeon.oxf.fr.permission.Operations
import org.orbeon.oxf.util.CoreUtils._
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.util.{IndentedLogger, URLRewriterUtils, XPath}
import org.orbeon.oxf.xforms.submission.SubmissionUtils
import org.orbeon.oxf.xml.TransformerUtils
import org.orbeon.saxon.om.{DocumentInfo, NodeInfo}
import org.orbeon.scaxon.SimplePath._

import java.io.{InputStream, OutputStream}
import java.net.URI
import javax.xml.transform.OutputKeys
import javax.xml.transform.stream.StreamResult


object Transforms {

  // TODO: Move this.
  def loadComponentBindings(
    appForm         : AppForm,
    frDocCtx        : FormRunnerDocContext)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): DocumentInfo = {

      val orbeonLibraryVersion = (frDocCtx.metadataRootElem / "library-versions" / "orbeon").stringValue.trimAllToOpt.getOrElse(1)
      val appLibraryVersion    = (frDocCtx.metadataRootElem / "library-versions" / "app").stringValue.trimAllToOpt.getOrElse(1)

      SubmissionUtils.readTinyTree(
        headersGetter       = _ => None, // Q: Do we need any header forwarding here?
        resolvedAbsoluteUrl = new URI(
          URLRewriterUtils.rewriteServiceURL(
            externalContext.getRequest,
            s"/fr/service/custom/orbeon/builder/toolbox?application=${appForm.app}&form=${appForm.form}&orbeon-library-version=$orbeonLibraryVersion&app-library-version=$appLibraryVersion",
            URLRewriter.REWRITE_MODE_ABSOLUTE
          )
        ),
        handleXInclude = false
      )._1
    }

  def readFormData(
    appForm         : AppForm,
    documentId      : String)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): (DocumentInfo, Map[String, List[String]]) =
    SubmissionUtils.readTinyTree(
      headersGetter       = _ => None, // Q: Do we need any header forwarding here?
      resolvedAbsoluteUrl = new URI(
        URLRewriterUtils.rewriteServiceURL(
          externalContext.getRequest,
          s"/fr/service/persistence/crud/${appForm.app}/${appForm.form}/data/$documentId/data.xml",
          URLRewriter.REWRITE_MODE_ABSOLUTE
        )
      ),
      handleXInclude = false
    )

  def migrateFormDefinition(
    dstVersion      : DataFormatVersion,
    app             : String,
    form            : String)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): (InputStream, OutputStream) => Unit = {

    val appForm = AppForm(app, form)

      // We are explicitly asked to downgrade a form definition format
      // The database may contain a form definition in any format

      def migrate(is: InputStream, os: OutputStream): Unit = {

        // Read document and component bindings
        val formDoc =
          new DocumentWrapper(TransformerUtils.readOrbeonDom(is, null, false, false), null, XPath.GlobalConfiguration)

        val frDocCtx: FormRunnerDocContext = new FormRunnerDocContext {
          val formDefinitionRootElem: NodeInfo = formDoc.rootElement
        }

        val componentBindings = loadComponentBindings(appForm, frDocCtx)

        val migrationsFromForm =
          new MigrationsFromForm(
            outerDocument        = formDoc,
            availableXBLBindings = componentBindings.some,
            legacyGridsOnly      = false
          )<commit_msg>Performance: id index when migrating form definition<commit_after>package org.orbeon.oxf.fr.persistence.proxy

import cats.syntax.option._
import org.orbeon.dom.saxon.{DocumentWrapper, NodeWrapper}
import org.orbeon.io.CharsetNames
import org.orbeon.oxf.externalcontext.{ExternalContext, URLRewriter}
import org.orbeon.oxf.fr.FormRunnerPersistence.findFormDefinitionFormatFromStringVersions
import org.orbeon.oxf.fr.XMLNames.{XBLBindingTest, XBLXBLTest}
import org.orbeon.oxf.fr._
import org.orbeon.oxf.fr.datamigration.MigrationSupport
import org.orbeon.oxf.fr.datamigration.MigrationSupport.MigrationsFromForm
import org.orbeon.oxf.util.CoreUtils._
import org.orbeon.oxf.util.StaticXPath.DocumentNodeInfoType
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.util.{IndentedLogger, URLRewriterUtils, XPath}
import org.orbeon.oxf.xforms.model.BasicIdIndex
import org.orbeon.oxf.xforms.submission.SubmissionUtils
import org.orbeon.oxf.xml.TransformerUtils
import org.orbeon.saxon.om.DocumentInfo
import org.orbeon.scaxon.SimplePath._

import java.io.{InputStream, OutputStream}
import java.net.URI
import javax.xml.transform.OutputKeys
import javax.xml.transform.stream.StreamResult


object Transforms {

  // TODO: Move this.
  def loadComponentBindings(
    appForm         : AppForm,
    frDocCtx        : FormRunnerDocContext)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): DocumentInfo = {

      val orbeonLibraryVersion = (frDocCtx.metadataRootElem / "library-versions" / "orbeon").stringValue.trimAllToOpt.getOrElse(1)
      val appLibraryVersion    = (frDocCtx.metadataRootElem / "library-versions" / "app").stringValue.trimAllToOpt.getOrElse(1)

      SubmissionUtils.readTinyTree(
        headersGetter       = _ => None, // Q: Do we need any header forwarding here?
        resolvedAbsoluteUrl = new URI(
          URLRewriterUtils.rewriteServiceURL(
            externalContext.getRequest,
            s"/fr/service/custom/orbeon/builder/toolbox?application=${appForm.app}&form=${appForm.form}&orbeon-library-version=$orbeonLibraryVersion&app-library-version=$appLibraryVersion",
            URLRewriter.REWRITE_MODE_ABSOLUTE
          )
        ),
        handleXInclude = false
      )._1
    }

  def readFormData(
    appForm         : AppForm,
    documentId      : String)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): (DocumentInfo, Map[String, List[String]]) =
    SubmissionUtils.readTinyTree(
      headersGetter       = _ => None, // Q: Do we need any header forwarding here?
      resolvedAbsoluteUrl = new URI(
        URLRewriterUtils.rewriteServiceURL(
          externalContext.getRequest,
          s"/fr/service/persistence/crud/${appForm.app}/${appForm.form}/data/$documentId/data.xml",
          URLRewriter.REWRITE_MODE_ABSOLUTE
        )
      ),
      handleXInclude = false
    )

  def migrateFormDefinition(
    dstVersion      : DataFormatVersion,
    app             : String,
    form            : String)(implicit
    logger          : IndentedLogger,
    externalContext : ExternalContext
  ): (InputStream, OutputStream) => Unit = {

    val appForm = AppForm(app, form)

      // We are explicitly asked to downgrade a form definition format
      // The database may contain a form definition in any format

      def migrate(is: InputStream, os: OutputStream): Unit = {

        // Read document and component bindings
        val formDoc =
          new DocumentWrapper(TransformerUtils.readOrbeonDom(is, null, false, false), null, XPath.GlobalConfiguration)

        val frDocCtx: FormRunnerDocContext = new FormRunnerDocContext {
          val formDefinitionRootElem: NodeInfo = formDoc.rootElement
        }

        // Set an id index to help with migration performance
        // The index is not updated with `insert` and `delete`, but we convinced ourselves that it's ok because:
        //
        // - We only support migrating "down" right now, from 2019.1.0 to 4.8.0.
        // - Inline data: migration is not affected by id changes.
        // - Binds: elements are moved down one level.
        // - Controls: `bind` attributes are removed and that's it.
        // - Templates: search for controls.
        object Index extends BasicIdIndex {
          def documentInfo: DocumentNodeInfoType = formDoc
        }

        formDoc.setIdGetter(Index.idGetter(formDoc))

        val componentBindings = loadComponentBindings(appForm, frDocCtx)

        val migrationsFromForm =
          new MigrationsFromForm(
            outerDocument        = formDoc,
            availableXBLBindings = componentBindings.some,
            legacyGridsOnly      = false
          )
{'fraction_non_alphanumeric': 0.08785648574057038, 'fraction_numerical': 0.0017249310027598896, 'mean_word_length': 3.0208044382801664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1797034', 'n_tokens_mistral': 2569, 'n_tokens_neox': 2519, 'n_words': 507}