<commit_before>package com.azavea.rf.database.tables

import io.circe._
import com.azavea.rf.database.fields._
import com.azavea.rf.database.sort._
import com.azavea.rf.datamodel._
import com.azavea.rf.database.query._
import com.azavea.rf.database.{Database => DB}
import com.azavea.rf.database.ExtendedPostgresDriver.api._
import com.azavea.rf.tool.ast._

import slick.model.ForeignKeyAction
import com.lonelyplanet.akka.http.extensions.{PageRequest, Order}
import com.typesafe.scalalogging.LazyLogging

import java.util.UUID
import java.util.Date
import java.sql.Timestamp
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

/** Table that represents tools in the Raster Foundry lab
  *
  * These are user generated tools that will eventually
  * contain a set of operations to be applied.
  */
class Tools(_tableTag: Tag) extends Table[Tool](_tableTag, "tools")
    with ToolFields
    with LazyLogging
    with OrganizationFkFields
    with UserFkFields
    with VisibilityField
    with TimestampFields
{
  def * = (id, createdAt, modifiedAt, createdBy, modifiedBy, owner, organizationId,
    title, description, requirements, license, visibility, compatibleDataSources,
    stars, definition) <> (Tool.tupled, Tool.unapply)

  val id: Rep[UUID] = column[UUID]("id", O.PrimaryKey)
  val createdAt: Rep[Timestamp] = column[Timestamp]("created_at")
  val modifiedAt: Rep[Timestamp] = column[Timestamp]("modified_at")
  val createdBy: Rep[String] = column[String]("created_by", O.Length(255, varying = true))
  val modifiedBy: Rep[String] = column[String]("modified_by", O.Length(255, varying = true))
  val owner: Rep[String] = column[String]("owner", O.Length(255,varying=true))
  val organizationId: Rep[UUID] = column[UUID]("organization_id")
  val title: Rep[String] = column[String]("title", O.Length(255, varying = true))
  val description: Rep[String] = column[String]("description")
  val requirements: Rep[String] = column[String]("requirements")
  val license: Rep[String] = column[String]("license")
  val visibility: Rep[Visibility] = column[Visibility]("visibility")
  val compatibleDataSources: Rep[List[String]] = column[List[String]]("compatible_data_sources",
    O.Length(Int.MaxValue, varying = false), O.Default(List.empty))
  val stars: Rep[Float] = column[Float]("stars", O.Default(0.0f))
  val definition: Rep[Json] = column[Json]("definition", O.Length(2147483647,varying=false))

  lazy val organizationsFk =
    foreignKey("tools_organization_id_fkey", organizationId, Organizations)(
      r => r.id,
      onUpdate = ForeignKeyAction.NoAction,
      onDelete = ForeignKeyAction.NoAction)
  lazy val createdByUserFK = foreignKey("tools_created_by_fkey", createdBy, Users)(
    r => r.id,
    onUpdate = ForeignKeyAction.NoAction,
    onDelete = ForeignKeyAction.NoAction)
  lazy val modifiedByUserFK = foreignKey("tools_modified_by_fkey", modifiedBy, Users)(
    r => r.id,
    onUpdate = ForeignKeyAction.NoAction,
    onDelete = ForeignKeyAction.NoAction)
  lazy val ownerUserFK = foreignKey("tools_owner_fkey", owner, Users)(r => r.id, onUpdate=ForeignKeyAction.NoAction, onDelete=ForeignKeyAction.NoAction)
}

object Tools extends TableQuery(tag => new Tools(tag)) with LazyLogging {
  type TableQuery = Query[Tools, Tool, Seq]

  implicit val toolsSorter: QuerySorter[Tools] =
    new QuerySorter(
      new ToolFieldsSort(identity[Tools]),
      new OrganizationFkSort(identity[Tools]),
      new VisibilitySort(identity[Tools]),
      new TimestampSort(identity[Tools]))

  def joinRelated(query: Query[Tools, Tool, Seq])(implicit database: DB) = {
    logger.debug(s"Performing Tools related join -- SQL: ${query.result.statements.headOption}")

    val toolRelatedJoin = (query
      joinLeft ToolTagsToTools on { case (m, t) =>  m.id === t.toolId }
      joinLeft ToolTags on { case ((_, t), tt) => t.map(_.toolTagId) === tt.id }
      joinLeft ToolCategoriesToTools on { case (((m, _), _), c) =>  m.id === c.toolId }
      joinLeft ToolCategories on { case ((((_, _), _), c), cc) => c.map(_.toolCategorySlug) === cc.slugLabel }
      joinLeft Organizations on { case (((((m, _), _), _), _), o) => m.organizationId === o.id }
    )

    for {
      (((((tool, toolTagToTool), toolTag), toolCategoryToTool), toolCategory), organization) <- toolRelatedJoin
    } yield (
      tool,
      toolTag,
      toolCategory,
      organization
    )
  }

  def groupByTool(joins: Seq[Tool.ToolRelationshipJoin]): Seq[Tool.WithRelated] =
    joins.groupBy(_.tool).map { case (tool, toolRelationshipJoins) =>
      Tool.WithRelated(
        tool.id,
        tool.createdAt,
        tool.modifiedAt,
        tool.createdBy,
        tool.modifiedBy,
        tool.owner,
        toolRelationshipJoins.flatMap(_.organization).headOption,
        tool.title,
        tool.description,
        tool.requirements,
        tool.license,
        tool.visibility,
        tool.compatibleDataSources,
        tool.stars,
        tool.definition,
        toolRelationshipJoins.flatMap(_.toolTag).distinct,
        toolRelationshipJoins.flatMap(_.toolCategory).distinct
      )
    }.toSeq

  /**
    * Returns a paginated result with Tools
    *
    * TODO: filtering not yet implemented
    *
    * @param page page request that has limit, offset, and sort parameters
    */
  def listTools(page: PageRequest, user: User)(implicit database: DB):
      Future[PaginatedResponse[Tool.WithRelated]] = {

    val accessibleTools = Tools.filterToSharedOrganizationIfNotInRoot(user)

    val pagedTools = accessibleTools
      .sort(page.sort)
      .drop(page.offset * page.limit)
      .take(page.limit)

    val toolsQueryAction = joinRelated(pagedTools).result

    logger.debug(s"Fetching tools -- SQL: ${toolsQueryAction.statements.headOption}")
    val toolsQueryResult = database.db.run {
      toolsQueryAction
    } map {
      joinTuples => joinTuples.map(joinTuple => Tool.ToolRelationshipJoin.tupled(joinTuple))
    } map {
      groupByTool
    }

    val nToolsAction = accessibleTools.length.result
    logger.debug(s"Counting tools -- SQL: ${nToolsAction.statements.headOption}")
    val totalToolsResult = database.db.run {
      nToolsAction
    }

    for {
      totalTools <- totalToolsResult
      tools <- toolsQueryResult
    } yield {
      val hasNext = (page.offset + 1) * page.limit < totalTools // 0 indexed page offset
      val hasPrevious = page.offset > 0<commit_msg>Filter tools by user, visibility, or organization<commit_after>package com.azavea.rf.database.tables

import io.circe._
import com.azavea.rf.database.fields._
import com.azavea.rf.database.sort._
import com.azavea.rf.datamodel._
import com.azavea.rf.database.query._
import com.azavea.rf.database.{Database => DB}
import com.azavea.rf.database.ExtendedPostgresDriver.api._
import com.azavea.rf.tool.ast._

import slick.model.ForeignKeyAction
import com.lonelyplanet.akka.http.extensions.{PageRequest, Order}
import com.typesafe.scalalogging.LazyLogging

import java.util.UUID
import java.util.Date
import java.sql.Timestamp
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

/** Table that represents tools in the Raster Foundry lab
  *
  * These are user generated tools that will eventually
  * contain a set of operations to be applied.
  */
class Tools(_tableTag: Tag) extends Table[Tool](_tableTag, "tools")
    with ToolFields
    with LazyLogging
    with OrganizationFkFields
    with UserFkVisibleFields
    with TimestampFields
{
  def * = (id, createdAt, modifiedAt, createdBy, modifiedBy, owner, organizationId,
    title, description, requirements, license, visibility, compatibleDataSources,
    stars, definition) <> (Tool.tupled, Tool.unapply)

  val id: Rep[UUID] = column[UUID]("id", O.PrimaryKey)
  val createdAt: Rep[Timestamp] = column[Timestamp]("created_at")
  val modifiedAt: Rep[Timestamp] = column[Timestamp]("modified_at")
  val createdBy: Rep[String] = column[String]("created_by", O.Length(255, varying = true))
  val modifiedBy: Rep[String] = column[String]("modified_by", O.Length(255, varying = true))
  val owner: Rep[String] = column[String]("owner", O.Length(255,varying=true))
  val organizationId: Rep[UUID] = column[UUID]("organization_id")
  val title: Rep[String] = column[String]("title", O.Length(255, varying = true))
  val description: Rep[String] = column[String]("description")
  val requirements: Rep[String] = column[String]("requirements")
  val license: Rep[String] = column[String]("license")
  val visibility: Rep[Visibility] = column[Visibility]("visibility")
  val compatibleDataSources: Rep[List[String]] = column[List[String]]("compatible_data_sources",
    O.Length(Int.MaxValue, varying = false), O.Default(List.empty))
  val stars: Rep[Float] = column[Float]("stars", O.Default(0.0f))
  val definition: Rep[Json] = column[Json]("definition", O.Length(2147483647,varying=false))

  lazy val organizationsFk =
    foreignKey("tools_organization_id_fkey", organizationId, Organizations)(
      r => r.id,
      onUpdate = ForeignKeyAction.NoAction,
      onDelete = ForeignKeyAction.NoAction)
  lazy val createdByUserFK = foreignKey("tools_created_by_fkey", createdBy, Users)(
    r => r.id,
    onUpdate = ForeignKeyAction.NoAction,
    onDelete = ForeignKeyAction.NoAction)
  lazy val modifiedByUserFK = foreignKey("tools_modified_by_fkey", modifiedBy, Users)(
    r => r.id,
    onUpdate = ForeignKeyAction.NoAction,
    onDelete = ForeignKeyAction.NoAction)
  lazy val ownerUserFK = foreignKey("tools_owner_fkey", owner, Users)(r => r.id, onUpdate=ForeignKeyAction.NoAction, onDelete=ForeignKeyAction.NoAction)
}

object Tools extends TableQuery(tag => new Tools(tag)) with LazyLogging {
  type TableQuery = Query[Tools, Tool, Seq]

  implicit val toolsSorter: QuerySorter[Tools] =
    new QuerySorter(
      new ToolFieldsSort(identity[Tools]),
      new OrganizationFkSort(identity[Tools]),
      new VisibilitySort(identity[Tools]),
      new TimestampSort(identity[Tools]))

  def joinRelated(query: Query[Tools, Tool, Seq])(implicit database: DB) = {
    logger.debug(s"Performing Tools related join -- SQL: ${query.result.statements.headOption}")

    val toolRelatedJoin = (query
      joinLeft ToolTagsToTools on { case (m, t) =>  m.id === t.toolId }
      joinLeft ToolTags on { case ((_, t), tt) => t.map(_.toolTagId) === tt.id }
      joinLeft ToolCategoriesToTools on { case (((m, _), _), c) =>  m.id === c.toolId }
      joinLeft ToolCategories on { case ((((_, _), _), c), cc) => c.map(_.toolCategorySlug) === cc.slugLabel }
      joinLeft Organizations on { case (((((m, _), _), _), _), o) => m.organizationId === o.id }
    )

    for {
      (((((tool, toolTagToTool), toolTag), toolCategoryToTool), toolCategory), organization) <- toolRelatedJoin
    } yield (
      tool,
      toolTag,
      toolCategory,
      organization
    )
  }

  def groupByTool(joins: Seq[Tool.ToolRelationshipJoin]): Seq[Tool.WithRelated] =
    joins.groupBy(_.tool).map { case (tool, toolRelationshipJoins) =>
      Tool.WithRelated(
        tool.id,
        tool.createdAt,
        tool.modifiedAt,
        tool.createdBy,
        tool.modifiedBy,
        tool.owner,
        toolRelationshipJoins.flatMap(_.organization).headOption,
        tool.title,
        tool.description,
        tool.requirements,
        tool.license,
        tool.visibility,
        tool.compatibleDataSources,
        tool.stars,
        tool.definition,
        toolRelationshipJoins.flatMap(_.toolTag).distinct,
        toolRelationshipJoins.flatMap(_.toolCategory).distinct
      )
    }.toSeq

  /**
    * Returns a paginated result with Tools
    *
    * TODO: filtering not yet implemented
    *
    * @param page page request that has limit, offset, and sort parameters
    */
  def listTools(page: PageRequest, user: User)(implicit database: DB):
      Future[PaginatedResponse[Tool.WithRelated]] = {

    val accessibleTools = Tools.filterUserVisibility(user)

    val pagedTools = accessibleTools
      .sort(page.sort)
      .drop(page.offset * page.limit)
      .take(page.limit)

    val toolsQueryAction = joinRelated(pagedTools).result

    logger.debug(s"Fetching tools -- SQL: ${toolsQueryAction.statements.headOption}")
    val toolsQueryResult = database.db.run {
      toolsQueryAction
    } map {
      joinTuples => joinTuples.map(joinTuple => Tool.ToolRelationshipJoin.tupled(joinTuple))
    } map {
      groupByTool
    }

    val nToolsAction = accessibleTools.length.result
    logger.debug(s"Counting tools -- SQL: ${nToolsAction.statements.headOption}")
    val totalToolsResult = database.db.run {
      nToolsAction
    }

    for {
      totalTools <- totalToolsResult
      tools <- toolsQueryResult
    } yield {
      val hasNext = (page.offset + 1) * page.limit < totalTools // 0 indexed page offset
      val hasPrevious = page.offset > 0
{'fraction_non_alphanumeric': 0.12088337853545138, 'fraction_numerical': 0.004184424641611778, 'mean_word_length': 3.8683515654470013, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '985121', 'n_tokens_mistral': 4036, 'n_tokens_neox': 3933, 'n_words': 863}