<commit_before>package models

import play.api.Logger
import play.api.libs.json._
import play.api.libs.ws.WS
import play.api._
import play.api.cache.Cache
import play.api.mvc._
import play.api.Play.current

/**
 * Just a combination of everything needed to do a search.  Probably should
 * combine all this in the future somehow
 *
 * This does not include pagination since pagination for a single remote
 * instance is different from the pagination of the overall request
 */
case class AssetSearchParameters(
  params: util.AttributeResolver.ResultTuple, 
  afinder: AssetFinder, 
  operation: Option[String] = None //"and" or "or"

) {
  def toQueryString: Map[String, String] = {
    val q1: Map[String, String] = (
      params._1.map{case (enum, value) => (enum.toString, value)} ++ 
      params._2.map{case (assetMeta,value) => ("attribute" -> "%s;%s".format(assetMeta.name, value))} ++ 
      params._3.map{i => ("ip_address" -> i)}
    ).toMap ++ afinder.toMap + ("details" -> "true")
    operation.map{op => q1 + ("operation" -> op)}.getOrElse(q1)
  }

  def paginationKey = toQueryString.map{case (k,v) => k + "_" + v}.mkString("&")

}

/**
 * Executes the actual search request to a remote collins instance, returning a
 * sequence of result RemoteAssets on success
 */
trait RemoteAssetClient {
  val tag: String //identifier to match up with cached paginations
  def getRemoteAssets(params: AssetSearchParameters, page: PageParams): Seq[AssetView]
  def getTotal: Long
}

class HttpRemoteAssetClient(val host: String, val user: String, val pass: String) extends RemoteAssetClient {
  val tag = host

  val queryUrl = host + app.routes.Api.getAssets().toString
  val authenticationTuple = (user, pass, com.ning.http.client.Realm.AuthScheme.BASIC)

  var total: Option[Long] = None

  def getTotal = total.getOrElse(0)

  def getRemoteAssets(params: AssetSearchParameters, page: PageParams) = {
    Logger.logger.debug("retrieving assets from %s: %s".format(host, page.toString))
    val request = WS.url(queryUrl).copy(
      queryString = params.toQueryString ++ page.toQueryString,
      auth = Some(authenticationTuple)
    )
    val result = request.get.await.get
    val json = Json.parse(result.body)
    total = (json \ "data" \ "Pagination" \ "TotalResults").asOpt[Long]
    (json \ "data" \ "Data") match {
      case JsArray(items) => items.map{
        case obj: JsObject => Some(new RemoteAsset(host, obj))
        case _ => {
          Logger.logger.warn("Invalid asset in response data")
          None
        }
      }.flatten
      case _ => {
        Logger.logger.warn("Invalid response from %s".format(host))
        Nil
      }
    }
  }

}

//class MockRemoteAssetClient

object LocalAssetClient extends RemoteAssetClient {
  val tag = "local"

  var total = 0L

  def getRemoteAssets(params: AssetSearchParameters, page: PageParams) = {
    val localPage = Asset.find(page, params.params, params.afinder, params.operation)
    total = localPage.total
    localPage.items
  }

  def getTotal = total
}


/**
 * A peek-able queue of assets from a remote location.  Assets are read from
 * the queue one at a time, but it will fetch remote assets in pages
 * from the remote collins instance as needed
 *
 */
class RemoteAssetQueue(val client: RemoteAssetClient, val params: AssetSearchParameters) {

  val PAGE_SIZE = 20
  val SORT = "ASC"<commit_msg>upping remote asset fetch page size<commit_after>package models

import play.api.Logger
import play.api.libs.json._
import play.api.libs.ws.WS
import play.api._
import play.api.cache.Cache
import play.api.mvc._
import play.api.Play.current

/**
 * Just a combination of everything needed to do a search.  Probably should
 * combine all this in the future somehow
 *
 * This does not include pagination since pagination for a single remote
 * instance is different from the pagination of the overall request
 */
case class AssetSearchParameters(
  params: util.AttributeResolver.ResultTuple, 
  afinder: AssetFinder, 
  operation: Option[String] = None //"and" or "or"

) {
  def toQueryString: Map[String, String] = {
    val q1: Map[String, String] = (
      params._1.map{case (enum, value) => (enum.toString, value)} ++ 
      params._2.map{case (assetMeta,value) => ("attribute" -> "%s;%s".format(assetMeta.name, value))} ++ 
      params._3.map{i => ("ip_address" -> i)}
    ).toMap ++ afinder.toMap + ("details" -> "true")
    operation.map{op => q1 + ("operation" -> op)}.getOrElse(q1)
  }

  def paginationKey = toQueryString.map{case (k,v) => k + "_" + v}.mkString("&")

}

/**
 * Executes the actual search request to a remote collins instance, returning a
 * sequence of result RemoteAssets on success
 */
trait RemoteAssetClient {
  val tag: String //identifier to match up with cached paginations
  def getRemoteAssets(params: AssetSearchParameters, page: PageParams): Seq[AssetView]
  def getTotal: Long
}

class HttpRemoteAssetClient(val host: String, val user: String, val pass: String) extends RemoteAssetClient {
  val tag = host

  val queryUrl = host + app.routes.Api.getAssets().toString
  val authenticationTuple = (user, pass, com.ning.http.client.Realm.AuthScheme.BASIC)

  var total: Option[Long] = None

  def getTotal = total.getOrElse(0)

  def getRemoteAssets(params: AssetSearchParameters, page: PageParams) = {
    Logger.logger.debug("retrieving assets from %s: %s".format(host, page.toString))
    val request = WS.url(queryUrl).copy(
      queryString = params.toQueryString ++ page.toQueryString,
      auth = Some(authenticationTuple)
    )
    val result = request.get.await.get
    val json = Json.parse(result.body)
    total = (json \ "data" \ "Pagination" \ "TotalResults").asOpt[Long]
    (json \ "data" \ "Data") match {
      case JsArray(items) => items.map{
        case obj: JsObject => Some(new RemoteAsset(host, obj))
        case _ => {
          Logger.logger.warn("Invalid asset in response data")
          None
        }
      }.flatten
      case _ => {
        Logger.logger.warn("Invalid response from %s".format(host))
        Nil
      }
    }
  }

}

//class MockRemoteAssetClient

object LocalAssetClient extends RemoteAssetClient {
  val tag = "local"

  var total = 0L

  def getRemoteAssets(params: AssetSearchParameters, page: PageParams) = {
    val localPage = Asset.find(page, params.params, params.afinder, params.operation)
    total = localPage.total
    localPage.items
  }

  def getTotal = total
}


/**
 * A peek-able queue of assets from a remote location.  Assets are read from
 * the queue one at a time, but it will fetch remote assets in pages
 * from the remote collins instance as needed
 *
 */
class RemoteAssetQueue(val client: RemoteAssetClient, val params: AssetSearchParameters) {

  val PAGE_SIZE = 50
  val SORT = "ASC"
{'fraction_non_alphanumeric': 0.11569031498380924, 'fraction_numerical': 0.0029437739181630853, 'mean_word_length': 3.4703947368421053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2389731', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2056, 'n_words': 618}