<commit_before>package com.azavea.rf.batch.landsat8.airflow

import com.azavea.rf.batch.Job
import com.azavea.rf.batch.util._
import com.azavea.rf.database.{Database => DB}
import com.azavea.rf.database.tables._
import com.azavea.rf.datamodel._

import io.circe._
import io.circe.syntax._
import geotrellis.proj4.CRS
import geotrellis.slick.Projected
import geotrellis.vector._
import jp.ne.opt.chronoscala.Imports._
import org.postgresql.util.PSQLException

import java.time.{LocalDate, ZoneOffset}
import java.util.UUID

import scala.collection.mutable.ListBuffer
import scala.concurrent.Future
import scala.util.{Failure, Success}
import scala.util.control.Breaks._

@deprecated(message = "Use com.azavea.rf.batch.landsat8.airflow.ImportLandsat8C1 instead, as USGS changes LC8 storage rules: https://landsat.usgs.gov/landsat-collections", since = "https://github.com/azavea/raster-foundry/pull/1821")
case class ImportLandsat8(startDate: LocalDate = LocalDate.now(ZoneOffset.UTC), threshold: Int = 10)(implicit val database: DB) extends Job {
  val name = ImportLandsat8.name

  /** Get S3 client per each call */
  def s3Client = S3(region = landsat8Config.awsRegion)

  protected def scenesFromCsv(user: User, srcProj: CRS = CRS.fromName("EPSG:4326"), targetProj: CRS = CRS.fromName("EPSG:3857")): Future[ListBuffer[Option[Scene]]] = {
    val reader = CSV.parse(landsat8Config.usgsLandsatUrl)
    val iterator = reader.iterator()

    val endDate = startDate + 1.day
    val buffer = ListBuffer[Future[Option[Scene]]]()
    val (_, indexToId) = CSV.getBiFunctions(iterator.next())

    var counter = 0
    breakable {
      while(iterator.hasNext) {
        val line = reader.readNext()

        val row =
          line
            .zipWithIndex
            .map { case (v, i) => indexToId(i) -> v }
            .toMap

        row.get("acquisitionDate") foreach { dateStr =>
          val date = LocalDate.parse(dateStr)
          if (startDate <= date && endDate > date) buffer += csvRowToScene(row, user, srcProj, targetProj)
          else if (date < startDate) counter += 1
        }

        if (counter > threshold) break
      }
    }

    Future.sequence(buffer)
  }

  protected def getLandsatPath(productId: String): String = {
    val (wPath, wRow) = productId.substring(3, 6) -> productId.substring(6, 9)
    val path = s"L8/$wPath/$wRow/$productId"
    logger.debug(s"Constructed path: $path")
    path
  }

  protected def getLandsatUrl(productId: String): String = {
    val rootUrl = s"https://landsat-pds.s3.amazonaws.com/${getLandsatPath(productId)}"
    logger.debug(s"Constructed Root URL: $rootUrl")
    rootUrl
  }

  protected def sizeFromPath(tifPath: String, productId: String): Int = {
    val path = s"${getLandsatPath(productId)}/$tifPath"
    logger.info(s"Getting object size for path: $path")
    s3Client.getObject(landsat8Config.bucketName, path).getObjectMetadata.getContentLength.toInt<commit_msg>Thumbnail base url typo fix.<commit_after>package com.azavea.rf.batch.landsat8.airflow

import com.azavea.rf.batch.Job
import com.azavea.rf.batch.util._
import com.azavea.rf.database.{Database => DB}
import com.azavea.rf.database.tables._
import com.azavea.rf.datamodel._

import io.circe._
import io.circe.syntax._
import geotrellis.proj4.CRS
import geotrellis.slick.Projected
import geotrellis.vector._
import jp.ne.opt.chronoscala.Imports._
import org.postgresql.util.PSQLException

import java.time.{LocalDate, ZoneOffset}
import java.util.UUID

import scala.collection.mutable.ListBuffer
import scala.concurrent.Future
import scala.util.{Failure, Success}
import scala.util.control.Breaks._

@deprecated(message = "Use com.azavea.rf.batch.landsat8.airflow.ImportLandsat8C1 instead, as USGS changes LC8 storage rules: https://landsat.usgs.gov/landsat-collections", since = "https://github.com/azavea/raster-foundry/pull/1821")
case class ImportLandsat8(startDate: LocalDate = LocalDate.now(ZoneOffset.UTC), threshold: Int = 10)(implicit val database: DB) extends Job {
  val name = ImportLandsat8.name

  /** Get S3 client per each call */
  def s3Client = S3(region = landsat8Config.awsRegion)

  protected def scenesFromCsv(user: User, srcProj: CRS = CRS.fromName("EPSG:4326"), targetProj: CRS = CRS.fromName("EPSG:3857")): Future[ListBuffer[Option[Scene]]] = {
    val reader = CSV.parse(landsat8Config.usgsLandsatUrl)
    val iterator = reader.iterator()

    val endDate = startDate + 1.day
    val buffer = ListBuffer[Future[Option[Scene]]]()
    val (_, indexToId) = CSV.getBiFunctions(iterator.next())

    var counter = 0
    breakable {
      while(iterator.hasNext) {
        val line = reader.readNext()

        val row =
          line
            .zipWithIndex
            .map { case (v, i) => indexToId(i) -> v }
            .toMap

        row.get("acquisitionDate") foreach { dateStr =>
          val date = LocalDate.parse(dateStr)
          if (startDate <= date && endDate > date) buffer += csvRowToScene(row, user, srcProj, targetProj)
          else if (date < startDate) counter += 1
        }

        if (counter > threshold) break
      }
    }

    Future.sequence(buffer)
  }

  protected def getLandsatPath(productId: String): String = {
    val (wPath, wRow) = productId.substring(3, 6) -> productId.substring(6, 9)
    val path = s"L8/$wPath/$wRow/$productId"
    logger.debug(s"Constructed path: $path")
    path
  }

  protected def getLandsatUrl(productId: String): String = {
    val rootUrl = s"${landsat8Config.awsLandsatBase}${getLandsatPath(productId)}"
    logger.debug(s"Constructed Root URL: $rootUrl")
    rootUrl
  }

  protected def sizeFromPath(tifPath: String, productId: String): Int = {
    val path = s"${getLandsatPath(productId)}/$tifPath"
    logger.info(s"Getting object size for path: $path")
    s3Client.getObject(landsat8Config.bucketName, path).getObjectMetadata.getContentLength.toInt
{'fraction_non_alphanumeric': 0.12241086587436333, 'fraction_numerical': 0.012903225806451613, 'mean_word_length': 3.7052715654952078, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '684305', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1971, 'n_words': 409}