<commit_before>package org.orbeon.oxf.fr

import cats.syntax.option._
import io.circe.generic.auto._
import io.circe.syntax._
import org.orbeon.io.{CharsetNames, IOUtils}
import org.orbeon.oxf.externalcontext.URLRewriter
import org.orbeon.oxf.fr.Names.{FormInstance, FormResources}
import org.orbeon.oxf.http.HttpMethod
import org.orbeon.oxf.http.HttpMethod.GET
import org.orbeon.oxf.pipeline.api.PipelineContext
import org.orbeon.oxf.processor.{ProcessorImpl, ProcessorOutput}
import org.orbeon.oxf.util._
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.xforms.analysis.model.{Instance, Submission}
import org.orbeon.oxf.xforms.model.XFormsInstanceSupport
import org.orbeon.oxf.xforms.processor.XFormsCompiler
import org.orbeon.oxf.xml.XMLReceiver
import org.orbeon.scaxon.SimplePath._
import org.orbeon.xforms.ManifestEntry

import java.net.URI
import java.util.zip.{ZipEntry, ZipOutputStream}


class FormRunnerCompiler extends ProcessorImpl {

  private val Logger = LoggerFactory.createLogger(classOf[FormRunnerCompiler])
  private implicit val indentedLogger: IndentedLogger = new IndentedLogger(Logger)

  override def createOutput(outputName: String): ProcessorOutput =
    addOutput(
      outputName,
      new ProcessorOutputImpl(FormRunnerCompiler.this, outputName) {
        def readImpl(pipelineContext: PipelineContext, xmlReceiver: XMLReceiver): Unit = {

          implicit val rcv = xmlReceiver
          implicit val ec  = CoreCrossPlatformSupport.externalContext

          val params       = readCacheInputAsDOM4J(pipelineContext, "instance")
          val formDocument = readCacheInputAsDOM4J(pipelineContext, "data")

          val appName     = params.getRootElement.element("app").getText
          val formName    = params.getRootElement.element("form").getText
          val formVersion = params.getRootElement.element("form-version").getText

          val (jsonString, staticState) = XFormsCompiler.compile(formDocument)

          val cacheableResourcesToInclude = {

            // Find all the languages used and return a comma-separated list of them
            val usedLangsOpt =
              staticState.topLevelPart.findControlAnalysis(FormResources) collect {
                case instance: Instance =>
                  instance.inlineRootElemOpt.get.elementIterator() map
                    (_.attributeValue(XMLNames.XMLLangQName))      mkString
                    ","
              }

            val urlOptIt =
              staticState.topLevelPart.iterateControls collect {
                case instance: Instance
                  if instance.readonly &&
                     instance.cache    &&
                     instance.dependencyURL.isDefined =>
                  instance.dependencyURL
                case submission: Submission
                  if submission.avtXxfReadonlyOpt.contains("true") &&
                     submission.avtXxfCacheOpt.contains("true")    &&
                     submission.avtMethod.exists(s => HttpMethod.withNameInsensitiveOption(s).contains(HttpMethod.GET)) =>

                  if (submission.avtActionOrResource == "/fr/service/i18n/fr-resources/{$app}/{$form}")
                    PathUtils.recombineQuery("/fr/service/i18n/fr-resources/orbeon/offline", usedLangsOpt.map("langs" ->)).some
                  else
                    submission.avtActionOrResource.some
              }

            val entriesIt =
              urlOptIt.flatten map { uri =>
                ManifestEntry(new URI(uri), ContentTypes.XmlContentType)
              }

            entriesIt.toList
          }

          val formInstanceDataAttachments = {

            val opt =
              staticState.topLevelPart.findControlAnalysis(FormInstance) collect {
                case instance: Instance =>

                  val basePath = FormRunner.createFormDefinitionBasePath(appName, formName)

                  val dataDoc =
                    XFormsInstanceSupport.extractDocument(
                      instance.inlineRootElemOpt.get, // FIXME: `get`
                      instance.excludeResultPrefixes,
                      instance.readonly,
                      instance.exposeXPathTypes,
                      removeInstanceData = false
                    )

                  FormRunner.collectAttachments(
                    data             = dataDoc,
                    fromBasePath     = basePath,
                    toBasePath       = basePath,
                    forceAttachments = true
                  ) map { awh =>
                    val holder    = awh.holder
                    val uri       = holder.getStringValue.trimAllToEmpty
                    val mediatype = awh.holder.attValue("mediatype")
                    ManifestEntry(new URI(uri), mediatype)
                  }
              }

            opt.toList.flatten
          }

          val distinctResources = (formInstanceDataAttachments ::: cacheableResourcesToInclude).distinct

          val useZipFormat =
            ec.getRequest.getFirstParamAsString("format").contains("zip")

          if (useZipFormat) {
            val chos = new ContentHandlerOutputStream(xmlReceiver, true)
            val zos  = new ZipOutputStream(chos)

            chos.setContentType("application/zip")

            val jsonFormPath = s"_forms/$appName/$formName/$formVersion/form/form.json"<commit_msg>Offline: fix static attachments detection<commit_after>package org.orbeon.oxf.fr

import cats.syntax.option._
import io.circe.generic.auto._
import io.circe.syntax._
import org.orbeon.io.{CharsetNames, IOUtils}
import org.orbeon.oxf.externalcontext.URLRewriter
import org.orbeon.oxf.fr.Names.{FormInstance, FormResources}
import org.orbeon.oxf.http.HttpMethod
import org.orbeon.oxf.http.HttpMethod.GET
import org.orbeon.oxf.pipeline.api.PipelineContext
import org.orbeon.oxf.processor.{ProcessorImpl, ProcessorOutput}
import org.orbeon.oxf.util._
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.xforms.analysis.model.{Instance, Submission}
import org.orbeon.oxf.xforms.model.XFormsInstanceSupport
import org.orbeon.oxf.xforms.processor.XFormsCompiler
import org.orbeon.oxf.xml.XMLReceiver
import org.orbeon.scaxon.SimplePath._
import org.orbeon.xforms.ManifestEntry

import java.net.URI
import java.util.zip.{ZipEntry, ZipOutputStream}


class FormRunnerCompiler extends ProcessorImpl {

  private val Logger = LoggerFactory.createLogger(classOf[FormRunnerCompiler])
  private implicit val indentedLogger: IndentedLogger = new IndentedLogger(Logger)

  override def createOutput(outputName: String): ProcessorOutput =
    addOutput(
      outputName,
      new ProcessorOutputImpl(FormRunnerCompiler.this, outputName) {
        def readImpl(pipelineContext: PipelineContext, xmlReceiver: XMLReceiver): Unit = {

          implicit val rcv = xmlReceiver
          implicit val ec  = CoreCrossPlatformSupport.externalContext

          val params       = readCacheInputAsDOM4J(pipelineContext, "instance")
          val formDocument = readCacheInputAsDOM4J(pipelineContext, "data")

          val appName     = params.getRootElement.element("app").getText
          val formName    = params.getRootElement.element("form").getText
          val formVersion = params.getRootElement.element("form-version").getText.trimAllToOpt.getOrElse("1")

          val (jsonString, staticState) = XFormsCompiler.compile(formDocument)

          val cacheableResourcesToInclude = {

            // Find all the languages used and return a comma-separated list of them
            val usedLangsOpt =
              staticState.topLevelPart.findControlAnalysis(FormResources) collect {
                case instance: Instance =>
                  instance.inlineRootElemOpt.get.elementIterator() map
                    (_.attributeValue(XMLNames.XMLLangQName))      mkString
                    ","
              }

            val urlOptIt =
              staticState.topLevelPart.iterateControls collect {
                case instance: Instance
                  if instance.readonly &&
                     instance.cache    &&
                     instance.dependencyURL.isDefined =>
                  instance.dependencyURL
                case submission: Submission
                  if submission.avtXxfReadonlyOpt.contains("true") &&
                     submission.avtXxfCacheOpt.contains("true")    &&
                     submission.avtMethod.exists(s => HttpMethod.withNameInsensitiveOption(s).contains(HttpMethod.GET)) =>

                  if (submission.avtActionOrResource == "/fr/service/i18n/fr-resources/{$app}/{$form}")
                    PathUtils.recombineQuery("/fr/service/i18n/fr-resources/orbeon/offline", usedLangsOpt.map("langs" ->)).some
                  else
                    submission.avtActionOrResource.some
              }

            val entriesIt =
              urlOptIt.flatten map { uri =>
                ManifestEntry(new URI(uri), ContentTypes.XmlContentType)
              }

            entriesIt.toList
          }

          val formInstanceDataAttachments = {

            val opt =
              staticState.topLevelPart.findControlAnalysis(FormInstance) collect {
                case instance: Instance =>

                  val fbBasePath             = "/fr/service/persistence/crud/orbeon/builder/data/"
                  val topLevelBasePath       = FormRunner.createFormDefinitionBasePath(appName, formName)
                  val appLibraryBasePath     = FormRunner.createFormDefinitionBasePath(appName, "library")
                  val orbeonLibraryBasePath  = FormRunner.createFormDefinitionBasePath("orbeon", "library")

                  val dataDoc =
                    XFormsInstanceSupport.extractDocument(
                      instance.inlineRootElemOpt.get, // FIXME: `get`
                      instance.excludeResultPrefixes,
                      instance.readonly,
                      instance.exposeXPathTypes,
                      removeInstanceData = false
                    )

                  def collectAttachmentsFor(basePath: String) =
                    FormRunner.collectAttachments(
                      data             = dataDoc,
                      fromBasePath     = basePath,
                      toBasePath       = basePath,
                      forceAttachments = true
                    )

                  (
                    collectAttachmentsFor(fbBasePath)         :::
                    collectAttachmentsFor(topLevelBasePath)   :::
                    collectAttachmentsFor(appLibraryBasePath) :::
                    collectAttachmentsFor(orbeonLibraryBasePath)
                  ) map { awh =>
                    val holder    = awh.holder
                    val uri       = holder.getStringValue.trimAllToEmpty
                    val mediatype = awh.holder.attValue("mediatype")
                    ManifestEntry(new URI(uri), mediatype)
                  }
              }

            opt.toList.flatten
          }

          val distinctResources = (formInstanceDataAttachments ::: cacheableResourcesToInclude).distinct

          val useZipFormat =
            ec.getRequest.getFirstParamAsString("format").contains("zip")

          if (useZipFormat) {
            val chos = new ContentHandlerOutputStream(xmlReceiver, true)
            val zos  = new ZipOutputStream(chos)

            chos.setContentType("application/zip")

            val jsonFormPath = s"_forms/$appName/$formName/$formVersion/form/form.json"
{'fraction_non_alphanumeric': 0.07993403922930047, 'fraction_numerical': 0.001128276340913036, 'mean_word_length': 2.037163943068002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375128', 'n_tokens_mistral': 3043, 'n_tokens_neox': 2899, 'n_words': 499}