<commit_before>package s3.website

import s3.website.model._
import s3.website.Ruby.rubyRegexMatches
import scala.concurrent.{ExecutionContextExecutor, Future}
import scala.util.{Failure, Success, Try}
import java.io.File
import org.apache.commons.io.FileUtils._
import org.apache.commons.codec.digest.DigestUtils._
import scala.io.Source
import s3.website.Diff.LocalFileDatabase.resolveDiffAgainstLocalDb
import s3.website.Diff.UploadBatch

case class Diff(
  unchanged: Future[Either[ErrorReport, Seq[S3Key]]],
  uploads: Seq[UploadBatch],
  persistenceError: Future[Option[ErrorReport]]
)

object Diff {

  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFileFromDisk]]]

  def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
    if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
    else resolveDiffAgainstGetBucketResponse(s3FilesFuture)

  private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
    val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
      errorOrS3Files.right.flatMap { s3Files =>
        Try {
          val s3KeyIndex = s3Files.map(_.s3Key).toSet
          val s3Md5Index = s3Files.map(_.md5).toSet
          val siteFiles = Files.listSiteFiles
          val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
          val isChangedOnS3 = (localFile: LocalFileFromDisk) => !(s3Md5Index contains localFile.md5)
          val newFiles = siteFiles collect {
            case file if !existsOnS3(file) => LocalFileFromDisk(file, NewFile)
          }
          val changedFiles = siteFiles collect {
            case file if existsOnS3(file) => LocalFileFromDisk(file, FileUpdate)
          } filter isChangedOnS3
          val unchangedFiles = {
            val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
            siteFiles.filterNot(f => newOrChangedFiles contains f)
          }
          val allFiles: Seq[Either[DbRecord, LocalFileFromDisk]] = unchangedFiles.map {
            f => Left(DbRecord(f))
          } ++ (changedFiles ++ newFiles).map {
            Right(_)
          }
          LocalFileDatabase persist allFiles
          allFiles
        } match {
          case Success(ok) => Right(ok)
          case Failure(err) => Left(ErrorReport(err))
        }
      }
    }
    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFileFromDisk],B]) =
      diffAgainstS3.map { errorOrDiffSource =>
        errorOrDiffSource.right map (_ collect pf)
      }
    val unchanged = collectResult {
      case Left(dbRecord) => dbRecord.s3Key
    }
    val uploads: UploadBatch = collectResult {
      case Right(localFile) => localFile
    }
    Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
  }

  def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
                    (implicit config: Config, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
    val localKeys = for {
      errorOrUnchanged <- diff.unchanged
      errorsOrChanges  <- Future.sequence(diff.uploads)
    } yield {
      errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
        for {
          mem <- memo.right
          keysToDelete <- errorOrChanges.right
        } yield {
          mem ++ keysToDelete.map(_.s3Key)
        }
      }
    }
    s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
      for {
        localS3Keys <- errorOrLocalKeys.right
        remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
      } yield {
        val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
        remoteS3Keys filterNot { s3Key =>
          val ignoreOnServer = config.ignore_on_server.exists(_.fold(
            (ignoreRegex: String) => rubyRegexMatches(s3Key, ignoreRegex),
            (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3Key, _))
          ))
          if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")
          (keysToRetain contains s3Key) || ignoreOnServer
        }
      }
    }.tupled
  }
<commit_msg>Shorten<commit_after>package s3.website

import s3.website.model._
import s3.website.Ruby.rubyRegexMatches
import scala.concurrent.{ExecutionContextExecutor, Future}
import scala.util.{Failure, Success, Try}
import java.io.File
import org.apache.commons.io.FileUtils._
import org.apache.commons.codec.digest.DigestUtils._
import scala.io.Source
import s3.website.Diff.LocalFileDatabase.resolveDiffAgainstLocalDb
import s3.website.Diff.UploadBatch

case class Diff(
  unchanged: Future[Either[ErrorReport, Seq[S3Key]]],
  uploads: Seq[UploadBatch],
  persistenceError: Future[Option[ErrorReport]]
)

object Diff {

  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFileFromDisk]]]

  def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
    if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
    else resolveDiffAgainstGetBucketResponse(s3FilesFuture)

  private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
    val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
      errorOrS3Files.right.flatMap { s3Files =>
        Try {
          val s3KeyIndex = s3Files.map(_.s3Key).toSet
          val s3Md5Index = s3Files.map(_.md5).toSet
          val siteFiles = Files.listSiteFiles
          val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
          val isChangedOnS3 = (localFile: LocalFileFromDisk) => !(s3Md5Index contains localFile.md5)
          val newFiles = siteFiles collect {
            case file if !existsOnS3(file) => LocalFileFromDisk(file, NewFile)
          }
          val changedFiles = siteFiles collect {
            case file if existsOnS3(file) => LocalFileFromDisk(file, FileUpdate)
          } filter isChangedOnS3
          val unchangedFiles = {
            val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
            siteFiles.filterNot(f => newOrChangedFiles contains f)
          }
          val allFiles: Seq[Either[DbRecord, LocalFileFromDisk]] = unchangedFiles.map {
            f => Left(DbRecord(f))
          } ++ (changedFiles ++ newFiles).map {
            Right(_)
          }
          LocalFileDatabase persist allFiles
          allFiles
        } match {
          case Success(ok) => Right(ok)
          case Failure(err) => Left(ErrorReport(err))
        }
      }
    }
    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFileFromDisk],B]) =
      diffAgainstS3.map { errorOrDiffSource =>
        errorOrDiffSource.right map (_ collect pf)
      }
    val unchanged = collectResult {
      case Left(dbRecord) => dbRecord.s3Key
    }
    val uploads: UploadBatch = collectResult {
      case Right(localFile) => localFile
    }
    Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
  }

  def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
                    (implicit config: Config, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
    val localKeys = for {
      errorOrUnchanged <- diff.unchanged
      errorsOrChanges  <- Future.sequence(diff.uploads)
    } yield
      errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
        for {
          mem <- memo.right
          keysToDelete <- errorOrChanges.right
        } yield mem ++ keysToDelete.map(_.s3Key)
      }

    s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
      for {
        localS3Keys <- errorOrLocalKeys.right
        remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
      } yield {
        val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
        remoteS3Keys filterNot { s3Key =>
          val ignoreOnServer = config.ignore_on_server.exists(_.fold(
            (ignoreRegex: String) => rubyRegexMatches(s3Key, ignoreRegex),
            (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3Key, _))
          ))
          if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")
          (keysToRetain contains s3Key) || ignoreOnServer
        }
      }
    }.tupled
  }

{'fraction_non_alphanumeric': 0.10353507715880486, 'fraction_numerical': 0.012476742913428915, 'mean_word_length': 3.0184696569920844, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5984592', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2725, 'n_words': 542}