<commit_before>package com.azavea.rf.batch.export.spark

import java.util.UUID

import scala.concurrent.ExecutionContext.Implicits.global

import com.azavea.rf.batch._
import com.azavea.rf.batch.ast._
import com.azavea.rf.batch.dropbox._
import com.azavea.rf.batch.export._
import com.azavea.rf.batch.util.conf._
import com.azavea.rf.datamodel._

import _root_.io.circe.parser._
import cats.data.Validated._
import cats.implicits._
import com.azavea.rf.batch.util._
import com.azavea.rf.database.Database
import com.azavea.rf.datamodel._
import com.azavea.rf.tool.ast.MapAlgebraAST
import com.azavea.rf.tool.params.EvalParams
import com.dropbox.core.v2.files.{CreateFolderErrorException, WriteMode}
import com.typesafe.scalalogging.LazyLogging
import geotrellis.proj4.{CRS, LatLng}
import geotrellis.raster._
import geotrellis.raster.histogram.Histogram
import geotrellis.raster.io._
import geotrellis.raster.io.geotiff.{GeoTiff, MultibandGeoTiff, SinglebandGeoTiff}
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.file._
import geotrellis.spark.io.hadoop._
import geotrellis.spark.io.s3._
import geotrellis.spark.tiling.MapKeyTransform
import geotrellis.spark.tiling._
import geotrellis.vector.MultiPolygon
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.rdd.RDD
import spray.json.DefaultJsonProtocol._

// --- //

object Export extends SparkJob with Config with LazyLogging {

  val jobName = "Export"

  implicit lazy val database = Database.DEFAULT

  def astExport(
    ed: ExportDefinition,
    ast: MapAlgebraAST,
    params: EvalParams,
    conf: HadoopConfiguration
  )(implicit sc: SparkContext): Unit = {
    interpretRDD(ast, params.sources, ed.input.resolution) match {
      case Invalid(errs) => ???
      case Valid(rdd) => {

        val mt: MapKeyTransform = rdd.metadata.layout.mapTransform
        val crs: CRS = rdd.metadata.crs

        if(!ed.output.stitch) {
          /* Create GeoTiffs and output them */
          val singles: RDD[(SpatialKey, SinglebandGeoTiff)] =
            rdd.map({ case (key, tile) => (key, SinglebandGeoTiff(tile, mt(key), crs)) })

          writeGeoTiffs[Tile, SinglebandGeoTiff](singles, ed, conf)
        } else {
          /* Stitch the Layer into a single GeoTiff and output it */
          val single: SinglebandGeoTiff = GeoTiff(rdd.stitch, crs)

          single.write(
            new Path(s"${ed.output.source.toString}/${ed.input.resolution}-${UUID.randomUUID()}.tiff"),
            conf.get
          )
        }
      }
    }
  }

  /** Get a LayerReader and an attribute store for the catalog located at the provided URI
    *
    *  @param exportLayerDef export job's layer definition
    */
  def getRfLayerManagement(<commit_msg>Handle interpreter errors in Export job<commit_after>package com.azavea.rf.batch.export.spark

import java.util.UUID

import scala.concurrent.ExecutionContext.Implicits.global

import com.azavea.rf.batch._
import com.azavea.rf.batch.ast._
import com.azavea.rf.batch.dropbox._
import com.azavea.rf.batch.export._
import com.azavea.rf.batch.util.conf._
import com.azavea.rf.datamodel._

import _root_.io.circe.parser._
import cats.data.Validated._
import cats.implicits._
import com.azavea.rf.batch.util._
import com.azavea.rf.common.InterpreterException
import com.azavea.rf.database.Database
import com.azavea.rf.datamodel._
import com.azavea.rf.tool.ast.MapAlgebraAST
import com.azavea.rf.tool.params.EvalParams
import com.dropbox.core.v2.files.{CreateFolderErrorException, WriteMode}
import com.typesafe.scalalogging.LazyLogging
import geotrellis.proj4.{CRS, LatLng}
import geotrellis.raster._
import geotrellis.raster.histogram.Histogram
import geotrellis.raster.io._
import geotrellis.raster.io.geotiff.{GeoTiff, MultibandGeoTiff, SinglebandGeoTiff}
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.file._
import geotrellis.spark.io.hadoop._
import geotrellis.spark.io.s3._
import geotrellis.spark.tiling.MapKeyTransform
import geotrellis.spark.tiling._
import geotrellis.vector.MultiPolygon
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.rdd.RDD
import spray.json.DefaultJsonProtocol._

// --- //

object Export extends SparkJob with Config with LazyLogging {

  val jobName = "Export"

  implicit lazy val database = Database.DEFAULT

  def astExport(
    ed: ExportDefinition,
    ast: MapAlgebraAST,
    params: EvalParams,
    conf: HadoopConfiguration
  )(implicit sc: SparkContext): Unit = {
    interpretRDD(ast, params.sources, ed.input.resolution) match {
      case Invalid(errs) => throw new InterpreterException(errs)
      case Valid(rdd) => {

        val mt: MapKeyTransform = rdd.metadata.layout.mapTransform
        val crs: CRS = rdd.metadata.crs

        if(!ed.output.stitch) {
          /* Create GeoTiffs and output them */
          val singles: RDD[(SpatialKey, SinglebandGeoTiff)] =
            rdd.map({ case (key, tile) => (key, SinglebandGeoTiff(tile, mt(key), crs)) })

          writeGeoTiffs[Tile, SinglebandGeoTiff](singles, ed, conf)
        } else {
          /* Stitch the Layer into a single GeoTiff and output it */
          val single: SinglebandGeoTiff = GeoTiff(rdd.stitch, crs)

          single.write(
            new Path(s"${ed.output.source.toString}/${ed.input.resolution}-${UUID.randomUUID()}.tiff"),
            conf.get
          )
        }
      }
    }
  }

  /** Get a LayerReader and an attribute store for the catalog located at the provided URI
    *
    *  @param exportLayerDef export job's layer definition
    */
  def getRfLayerManagement(
{'fraction_non_alphanumeric': 0.10698080279232111, 'fraction_numerical': 0.0010471204188481676, 'mean_word_length': 3.5268562401263823, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5109963', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1914, 'n_words': 387}