<commit_before>/*
This file is part of Intake24.

Copyright 2015, 2016 Newcastle University.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package uk.ac.ncl.openlab.intake24.services.dataexport.controllers

import java.io.{File, FileWriter}
import java.time.format.DateTimeFormatter
import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime}
import java.util.UUID

import akka.actor.ActorSystem
import com.opencsv.CSVWriter
import io.circe.generic.auto._
import io.circe.syntax._
import javax.inject.Inject
import org.slf4j.LoggerFactory
import play.api.Configuration
import play.api.cache.SyncCacheApi
import play.api.http.ContentTypes
import play.api.mvc.{BaseController, ControllerComponents}
import uk.ac.ncl.openlab.intake24.errors.ErrorUtils
import uk.ac.ncl.openlab.intake24.play.utils.{DatabaseErrorHandler, JsonBodyParser}
import uk.ac.ncl.openlab.intake24.security.authorization.Intake24RestrictedActionBuilder
import uk.ac.ncl.openlab.intake24.services.dataexport._
import uk.ac.ncl.openlab.intake24.services.systemdb.Roles
import uk.ac.ncl.openlab.intake24.services.systemdb.admin._
import uk.ac.ncl.openlab.intake24.shorturls.{ShortUrlsHttpClient, ShortUrlsRequest}

import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}

case class UserDataExportHandle(requestId: UUID)

sealed trait UserDataExportStatus

object UserDataExportStatus {

  case object Pending extends UserDataExportStatus

  case class Successful(downloadUrl: String) extends UserDataExportStatus

  case class Failed(errorMessage: String) extends UserDataExportStatus

}

case class RespondentWithAuthUrl(id: Long, userName: String, authUrl: String)

case class RespondentWithAuthAndShortUrls(id: Long, userName: String, authUrl: String, shortUrl: String)

class UserDataExportController @Inject()(configuration: Configuration,
                                         service: UserAdminService,
                                         secureUrlService: SecureUrlService,
                                         syncCacheApi: SyncCacheApi,
                                         shortUrlsClient: ShortUrlsHttpClient,
                                         rab: Intake24RestrictedActionBuilder,
                                         jsonBodyParser: JsonBodyParser,
                                         actorSystem: ActorSystem,
                                         val controllerComponents: ControllerComponents,
                                         implicit val executionContext: ExecutionContext) extends BaseController
  with DatabaseErrorHandler {

  val logger = LoggerFactory.getLogger(classOf[UserDataExportController])

  val surveyFrontendUrl = configuration.get[String]("intake24.surveyFrontendUrl")


  private def appendShortUrls(respondents: Seq[RespondentWithAuthUrl]): Try[Seq[RespondentWithAuthAndShortUrls]] = {

    shortUrlsClient.getShortUrls(ShortUrlsRequest(respondents.map(_.authUrl))).attempt.unsafeRunSync.map {
      response =>
        respondents.zip(response.shortUrls).map {
          case (r, shortUrl) => RespondentWithAuthAndShortUrls(r.id, r.userName, r.authUrl, shortUrl)
        }
    } match {
      case Left(error) => Failure(error)
      case Right(value) => Success(value)
    }
  }

  private def exportImpl(surveyId: String): Try[File] = {

    def getRespondentRecords(offset: Int, limit: Int): Try[Seq[RespondentWithAuthUrl]] =
      ErrorUtils.asTry(for (users <- service.listUsersByRole(Roles.surveyRespondent(surveyId), offset, limit);
                            aliases <- service.getSurveyUserAliases(users.map(_.id), surveyId)) yield {
        users.filter(u => aliases.contains(u.id)).map {
          user =>
            RespondentWithAuthUrl(user.id, aliases(user.id).userName, surveyFrontendUrl + s"/surveys/$surveyId?auth=${aliases(user.id).urlAuthToken}")
        }
      })


    def writeRows(respondents: Seq[RespondentWithAuthAndShortUrls], writer: CSVWriter): Try[Unit] = Try {
      respondents.foreach {
        row =>
          writer.writeNext(Array(row.id.toString, row.userName, row.authUrl, row.shortUrl))
      }
    }

    def exportRemaining(offset: Int, batchSize: Int, writer: CSVWriter): Try[Unit] =
      getRespondentRecords(offset, batchSize).flatMap {
        respondents =>
          if (respondents.isEmpty)
            Success(())
          else for (withShortUrls <- appendShortUrls(respondents);
                    _ <- writeRows(withShortUrls, writer);
                    _ <- exportRemaining(offset + respondents.length, batchSize, writer)) yield ()
      }

    Try {
      val file = File.createTempFile("intake24", ".csv")
      val writer = new CSVWriter(new FileWriter(file))

      (file, writer)
    }.flatMap {
      case (file, writer) =>
        writer.writeNext(Array("Intake24 user ID", "Survey user ID", "Authentication URL", "Short authentication URL"))
        val result = exportRemaining(0, 1000, writer).map(_ => file)
        Try {
          writer.close()
        }
        result
    }
  }

  def cacheKey(surveyId: String, requestId: String): String = s"intake24.userDataExport.$requestId"
<commit_msg>Make shortening of authentication URLs optional<commit_after>/*
This file is part of Intake24.

Copyright 2015, 2016 Newcastle University.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package uk.ac.ncl.openlab.intake24.services.dataexport.controllers

import java.io.{File, FileWriter}
import java.time.format.DateTimeFormatter
import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime}
import java.util.UUID

import akka.actor.ActorSystem
import com.opencsv.CSVWriter
import io.circe.generic.auto._
import io.circe.syntax._
import javax.inject.Inject
import org.slf4j.LoggerFactory
import play.api.Configuration
import play.api.cache.SyncCacheApi
import play.api.http.ContentTypes
import play.api.mvc.{BaseController, ControllerComponents}
import uk.ac.ncl.openlab.intake24.errors.ErrorUtils
import uk.ac.ncl.openlab.intake24.play.utils.{DatabaseErrorHandler, JsonBodyParser}
import uk.ac.ncl.openlab.intake24.security.authorization.Intake24RestrictedActionBuilder
import uk.ac.ncl.openlab.intake24.services.dataexport._
import uk.ac.ncl.openlab.intake24.services.systemdb.Roles
import uk.ac.ncl.openlab.intake24.services.systemdb.admin._
import uk.ac.ncl.openlab.intake24.shorturls.{ShortUrlsHttpClient, ShortUrlsRequest}

import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}

case class UserDataExportHandle(requestId: UUID)

sealed trait UserDataExportStatus

object UserDataExportStatus {

  case object Pending extends UserDataExportStatus

  case class Successful(downloadUrl: String) extends UserDataExportStatus

  case class Failed(errorMessage: String) extends UserDataExportStatus

}

case class RespondentWithAuthUrl(id: Long, userName: String, authUrl: String)

case class RespondentWithAuthAndShortUrls(id: Long, userName: String, authUrl: String, shortUrl: String)

class UserDataExportController @Inject()(configuration: Configuration,
                                         service: UserAdminService,
                                         secureUrlService: SecureUrlService,
                                         syncCacheApi: SyncCacheApi,
                                         shortUrlsClient: ShortUrlsHttpClient,
                                         rab: Intake24RestrictedActionBuilder,
                                         jsonBodyParser: JsonBodyParser,
                                         actorSystem: ActorSystem,
                                         val controllerComponents: ControllerComponents,
                                         implicit val executionContext: ExecutionContext) extends BaseController
  with DatabaseErrorHandler {

  val logger = LoggerFactory.getLogger(classOf[UserDataExportController])

  val surveyFrontendUrl = configuration.get[String]("intake24.surveyFrontendUrl")
  val shortenAuthUrls = configuration.get[Boolean]("intake24.dataExport.authenticationUrls.shorten")

  private def appendShortUrls(respondents: Seq[RespondentWithAuthUrl]): Try[Seq[RespondentWithAuthAndShortUrls]] = {

    shortUrlsClient.getShortUrls(ShortUrlsRequest(respondents.map(_.authUrl))).attempt.unsafeRunSync.map {
      response =>
        respondents.zip(response.shortUrls).map {
          case (r, shortUrl) => RespondentWithAuthAndShortUrls(r.id, r.userName, r.authUrl, shortUrl)
        }
    } match {
      case Left(error) => Failure(error)
      case Right(value) => Success(value)
    }
  }

  private def exportImpl(surveyId: String): Try[File] = {

    def getRespondentRecords(offset: Int, limit: Int): Try[Seq[RespondentWithAuthUrl]] =
      ErrorUtils.asTry(for (users <- service.listUsersByRole(Roles.surveyRespondent(surveyId), offset, limit);
                            aliases <- service.getSurveyUserAliases(users.map(_.id), surveyId)) yield {
        users.filter(u => aliases.contains(u.id)).map {
          user =>
            RespondentWithAuthUrl(user.id, aliases(user.id).userName, surveyFrontendUrl + s"/surveys/$surveyId?auth=${aliases(user.id).urlAuthToken}")
        }
      })


    def writeRowsWithShortUrls(respondents: Seq[RespondentWithAuthAndShortUrls], writer: CSVWriter): Try[Unit] = Try {
      respondents.foreach {
        row =>
          writer.writeNext(Array(row.id.toString, row.userName, row.authUrl, row.shortUrl))
      }
    }

    def writeRows(respondents: Seq[RespondentWithAuthUrl], writer: CSVWriter): Try[Unit] = Try {
      respondents.foreach {
        row =>
          writer.writeNext(Array(row.id.toString, row.userName, row.authUrl))
      }
    }

    def exportRemaining(offset: Int, batchSize: Int, writer: CSVWriter): Try[Unit] =
      getRespondentRecords(offset, batchSize).flatMap {
        respondents =>
          if (respondents.isEmpty)
            Success(())
          else {
            val action = if (shortenAuthUrls)
              appendShortUrls(respondents).flatMap(rows => writeRowsWithShortUrls(rows, writer))
            else
              writeRows(respondents, writer)

            action.flatMap(_ => exportRemaining(offset + respondents.length, batchSize, writer))
          }
      }

    Try {
      val file = File.createTempFile("intake24", ".csv")
      val writer = new CSVWriter(new FileWriter(file))

      (file, writer)
    }.flatMap {
      case (file, writer) =>
        val header = Array("Intake24 user ID", "Survey user ID", "Authentication URL") ++ (if (shortenAuthUrls) Array("Short authentication URL") else Array[String]())
        writer.writeNext(header)
        val result = exportRemaining(0, 1000, writer).map(_ => file)
        Try {
          writer.close()
        }
        result
    }
  }

  def cacheKey(surveyId: String, requestId: String): String = s"intake24.userDataExport.$requestId"

{'fraction_non_alphanumeric': 0.0924806982208795, 'fraction_numerical': 0.008224236320913058, 'mean_word_length': 3.1450434782608694, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5401871', 'n_tokens_mistral': 3407, 'n_tokens_neox': 3240, 'n_words': 750}