<commit_before>package hmda.regulator.scheduler

import java.sql.Timestamp
import java.text.SimpleDateFormat
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.Calendar

import akka.{Done, NotUsed}
import akka.stream.ActorMaterializer
import akka.stream.alpakka.s3.impl.{ListBucketVersion1, ListBucketVersion2}
import akka.stream.alpakka.s3.scaladsl.{MultipartUploadResult, S3Client}
import akka.stream.alpakka.s3.{MemoryBufferType, S3Settings}
import akka.stream.scaladsl.{Sink, Source}
import akka.util.ByteString
import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.regions.AwsRegionProvider
import com.typesafe.akka.extension.quartz.QuartzSchedulerExtension
import com.typesafe.config.ConfigFactory
import hmda.actor.HmdaActor
import hmda.query.DbConfiguration.dbConfig
import hmda.regulator.query.RegulatorComponent
import hmda.regulator.query.lar.LarEntityImpl
import hmda.regulator.scheduler.schedules.Schedules.LarScheduler
import slick.basic.DatabasePublisher

import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success}
import scala.concurrent.duration._

class LarScheduler extends HmdaActor with RegulatorComponent {

  implicit val ec = context.system.dispatcher
  implicit val materializer = ActorMaterializer()
  private val fullDate = DateTimeFormatter.ofPattern("yyyy-MM-dd-")
  def larRepository = new LarRepository(dbConfig)

  override def preStart() = {
    QuartzSchedulerExtension(context.system)
      .schedule("LarScheduler", self, LarScheduler)

  }

  override def postStop() = {
    QuartzSchedulerExtension(context.system).cancelJob("LarScheduler")
  }

  override def receive: Receive = {

    case LarScheduler =>
      val awsConfig = ConfigFactory.load("application.conf").getConfig("aws")
      val accessKeyId = awsConfig.getString("access-key-id")
      val secretAccess = awsConfig.getString("secret-access-key ")
      val region = awsConfig.getString("region")
      val bucket = awsConfig.getString("public-bucket")
      val environment = awsConfig.getString("environment")
      val year = awsConfig.getString("year")
      val awsCredentialsProvider = new AWSStaticCredentialsProvider(
        new BasicAWSCredentials(accessKeyId, secretAccess))

      val awsRegionProvider = new AwsRegionProvider {
        override def getRegion: String = region
      }

      val s3Settings = new S3Settings(
        MemoryBufferType,
        None,
        awsCredentialsProvider,
        awsRegionProvider,
        false,
        None,
        ListBucketVersion2
      )
      val bankFilter =
        ConfigFactory.load("application.conf").getConfig("filter")
      val bankFilterList =
        bankFilter.getString("bank-filter-list").toUpperCase.split(",")

      val s3Client = new S3Client(s3Settings)(context.system, materializer)

      val now = LocalDateTime.now().minusDays(1)

      val formattedDate = fullDate.format(now)

      val fileName = s"$formattedDate" + s"$year" + "_lar" + ".txt"
      val s3Sink =
        s3Client.multipartUpload(bucket, s"$environment/lar/$fileName")

      try {
        val allResultsPublisher: DatabasePublisher[LarEntityImpl] =
          larRepository.getAllLARs(bankFilterList)
        val allResultsSource: Source[LarEntityImpl, NotUsed] =
          Source.fromPublisher(allResultsPublisher)

        var results: Future[MultipartUploadResult] = allResultsSource
          .map(larEntity => larEntity.toPSV + "\n")
          .map(s => ByteString(s))
          .runWith(s3Sink)

        results onComplete {
          case Success(result) => {
            log.info(
              "Pushing to S3: " + s"$bucket/$environment/lar/$fileName" + ".")
          }
          case Failure(t) =>
            println("An error has occurred getting LAR Data in Future: " + t.getMessage)

        }
      } catch {
        case e: Exception =>
          println("An error has occurred getting LAR Data: " + e.getMessage)
      }
  }
}
<commit_msg>unused imports<commit_after>package hmda.regulator.scheduler

import java.time.LocalDateTime
import java.time.format.DateTimeFormatter

import akka.NotUsed
import akka.stream.ActorMaterializer
import akka.stream.alpakka.s3.impl.ListBucketVersion2
import akka.stream.alpakka.s3.scaladsl.{MultipartUploadResult, S3Client}
import akka.stream.alpakka.s3.{MemoryBufferType, S3Settings}
import akka.stream.scaladsl.Source
import akka.util.ByteString
import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.regions.AwsRegionProvider
import com.typesafe.akka.extension.quartz.QuartzSchedulerExtension
import com.typesafe.config.ConfigFactory
import hmda.actor.HmdaActor
import hmda.query.DbConfiguration.dbConfig
import hmda.regulator.query.RegulatorComponent
import hmda.regulator.query.lar.LarEntityImpl
import hmda.regulator.scheduler.schedules.Schedules.LarScheduler
import slick.basic.DatabasePublisher

import scala.concurrent.Future
import scala.util.{Failure, Success}

class LarScheduler extends HmdaActor with RegulatorComponent {

  implicit val ec = context.system.dispatcher
  implicit val materializer = ActorMaterializer()
  private val fullDate = DateTimeFormatter.ofPattern("yyyy-MM-dd-")
  def larRepository = new LarRepository(dbConfig)

  override def preStart() = {
    QuartzSchedulerExtension(context.system)
      .schedule("LarScheduler", self, LarScheduler)

  }

  override def postStop() = {
    QuartzSchedulerExtension(context.system).cancelJob("LarScheduler")
  }

  override def receive: Receive = {

    case LarScheduler =>
      val awsConfig = ConfigFactory.load("application.conf").getConfig("aws")
      val accessKeyId = awsConfig.getString("access-key-id")
      val secretAccess = awsConfig.getString("secret-access-key ")
      val region = awsConfig.getString("region")
      val bucket = awsConfig.getString("public-bucket")
      val environment = awsConfig.getString("environment")
      val year = awsConfig.getString("year")
      val awsCredentialsProvider = new AWSStaticCredentialsProvider(
        new BasicAWSCredentials(accessKeyId, secretAccess))

      val awsRegionProvider = new AwsRegionProvider {
        override def getRegion: String = region
      }

      val s3Settings = new S3Settings(
        MemoryBufferType,
        None,
        awsCredentialsProvider,
        awsRegionProvider,
        false,
        None,
        ListBucketVersion2
      )
      val bankFilter =
        ConfigFactory.load("application.conf").getConfig("filter")
      val bankFilterList =
        bankFilter.getString("bank-filter-list").toUpperCase.split(",")

      val s3Client = new S3Client(s3Settings)(context.system, materializer)

      val now = LocalDateTime.now().minusDays(1)

      val formattedDate = fullDate.format(now)

      val fileName = s"$formattedDate" + s"$year" + "_lar" + ".txt"
      val s3Sink =
        s3Client.multipartUpload(bucket, s"$environment/lar/$fileName")

      try {
        val allResultsPublisher: DatabasePublisher[LarEntityImpl] =
          larRepository.getAllLARs(bankFilterList)
        val allResultsSource: Source[LarEntityImpl, NotUsed] =
          Source.fromPublisher(allResultsPublisher)

        var results: Future[MultipartUploadResult] = allResultsSource
          .map(larEntity => larEntity.toPSV + "\n")
          .map(s => ByteString(s))
          .runWith(s3Sink)

        results onComplete {
          case Success(result) => {
            log.info(
              "Pushing to S3: " + s"$bucket/$environment/lar/$fileName" + ".")
          }
          case Failure(t) =>
            println("An error has occurred getting LAR Data in Future: " + t.getMessage)

        }
      } catch {
        case e: Exception =>
          println("An error has occurred getting LAR Data: " + e.getMessage)
      }
  }
}

{'fraction_non_alphanumeric': 0.09646140284598917, 'fraction_numerical': 0.004407505351970785, 'mean_word_length': 3.5176336746302614, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375145', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2294, 'n_words': 447}