<commit_before>package com.karasiq.nanoboard

import java.time.Instant

import slick.driver.H2Driver.api._

import scala.concurrent.ExecutionContext
import scala.language.postfixOps

package object model {
  case class DBPost(hash: String, parent: String, message: String, firstSeen: Long) {
    def asThread(answers: Int): NanoboardMessageData = {
      NanoboardMessageData(Some(parent), hash, message, answers)
    }
  }
  class Post(tag: Tag) extends Table[DBPost](tag, "posts") {
    def hash = column[String]("hash", O.SqlType("char(32)"), O.PrimaryKey)
    def parent = column[String]("parent_hash", O.SqlType("char(32)"))
    def message = column[String]("message")
    def firstSeen = column[Long]("first_seen")

    def threadIdx = index("thread_index", (parent, firstSeen), unique = false)
    def recentIdx = index("recent_index", firstSeen, unique = false)
    def * = (hash, parent, message, firstSeen) <> (DBPost.tupled, DBPost.unapply)
  }

  val posts = TableQuery[Post]

  class DeletedPost(tag: Tag) extends Table[String](tag, "posts_deleted") {
    def hash = column[String]("hash", O.PrimaryKey)
    def * = hash
  }

  val deletedPosts = TableQuery[DeletedPost]

  class PendingPost(tag: Tag) extends Table[String](tag, "pending_posts") {
    def hash = column[String]("hash", O.SqlType("char(32)"), O.PrimaryKey)
    def post = foreignKey("post_fk", hash, posts)(_.hash, onUpdate = ForeignKeyAction.Restrict, onDelete = ForeignKeyAction.Cascade)
    def * = hash
  }

  val pendingPosts = TableQuery[PendingPost]

  class Place(tag: Tag) extends Table[String](tag, "places") {
    def url = column[String]("place_url", O.PrimaryKey)
    def * = url
  }

  val places = TableQuery[Place]

  class Category(tag: Tag) extends Table[NanoboardCategory](tag, "categories") {
    def hash = column[String]("category_hash", O.SqlType("char(32)"), O.PrimaryKey)
    def name = column[String]("category_name")
    def * = (hash, name) <> (NanoboardCategory.tupled, NanoboardCategory.unapply)
  }

  val categories = TableQuery[Category]

  object Post {
    def addReply(m: NanoboardMessage) = DBIO.seq(insertMessage(m), pendingPosts.forceInsertQuery {
      val exists = (for (p <- pendingPosts if p.hash === m.hash) yield ()).exists
      for (message <- Query(m.hash) if !exists) yield message
    }, deletedPosts.filter(_.hash === m.hash).delete)

    def insertMessage(m: NanoboardMessage) = posts.forceInsertQuery {
      val deleted = (for (p <- deletedPosts if p.hash === m.hash) yield ()).exists
      val exists = (for (p <- posts if p.hash === m.hash) yield ()).exists
      val insert = (m.hash, m.parent, m.text, Instant.now().toEpochMilli) <> (DBPost.tupled, DBPost.unapply)
      for (message <- Query(insert) if !deleted && !exists) yield message
    }

    def insertMessages(messages: Seq[NanoboardMessage]) = {
      DBIO.sequence(messages.map(insertMessage))
    }

    private def recentQuery(offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      posts
        .sortBy(_.firstSeen.desc)
        .drop(offset)
        .take(count)
        .map(post ⇒ (post, posts.filter(_.parent === post.hash).length))
    }

    private val recentCompiled = Compiled(recentQuery _)

    def recent(offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      recentCompiled(offset, count)
        .result
        .map(_.map {
          case (post, answers) ⇒
            post.asThread(answers)
        })
    }

    private def pendingQuery(offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      pendingPosts
        .flatMap(_.post)
        .sortBy(_.firstSeen.asc)
        .drop(offset)
        .take(count)
    }

    private val pendingCompiled = Compiled(pendingQuery _)

    def pending(offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      pendingCompiled(offset, count)
        .result
        .map(_.map(_.asThread(0)))
    }

    private def threadQuery(hash: Rep[String], offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      val query = posts.filter(_.parent === hash).drop(offset)

      def withAnswerCount(query: Query[Post, DBPost, Seq]) = query.map { post ⇒
        (post, posts.filter(_.parent === post.hash).length)
      }

      (withAnswerCount(posts.filter(_.hash === hash)) ++ withAnswerCount(query))
        .sortBy(p ⇒ ((p._1.hash === hash).desc, p._1.firstSeen.desc))
        .take(count)
    }

    private val threadCompiled = Compiled(threadQuery _)

    def thread(hash: String, offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      threadCompiled(hash, offset, count + 1)
        .result
        .map(_.map {
          case (post, answersCount) ⇒
            post.asThread(answersCount)
        })
    }

    def get(hash: String)(implicit ec: ExecutionContext) = {
      thread(hash, 0, 0).map(_.headOption)
    }

    def delete(hash: String)(implicit ec: ExecutionContext) = {
      def deleteCascade(hash: String): DBIOAction[Seq[String], NoStream, Effect.Write with Effect.Read] = {
        val query = posts.filter(_.parent === hash)
        for {
          deleted ← query.map(_.hash).result
          descendants ← DBIO.sequence(deleted.map(deleteCascade))
          _ ← DBIO.seq(
            pendingPosts.filter(_.hash === hash).delete,
            Category.delete(hash),
            deletedPosts.insertOrUpdate(hash),
            posts.filter(_.hash === hash).delete
          )
        } yield Seq(hash) ++ deleted ++ descendants.flatten
      }

      deleteCascade(hash)
    }
  }

  object Place {
    def list() = {
      places.result
    }

    def update(newList: Seq[String])(implicit ec: ExecutionContext) = DBIO.sequence(
      newList.map(url ⇒ places.insertOrUpdate(url)) :+
        places.filterNot(_.url inSet newList).delete
    ).map(_ ⇒ ())
  }

  object Category {
    def list()(implicit ec: ExecutionContext) = {
      val query = categories.map { c ⇒
        (c.hash, c.name, posts.filter(_.parent === c.hash).length)
      }
      query.result.map(_.map {
        case (hash, name, answers) ⇒
          NanoboardMessageData(None, hash, name, answers)
      })
    }

    def update(newList: Seq[NanoboardCategory])(implicit ec: ExecutionContext) = DBIO.sequence(
      newList.map(c ⇒ add(c.hash, c.name)):+
        categories.filterNot(_.hash inSet newList.map(_.hash)).delete
    ).map(_ ⇒ ())

    def add(hash: String, name: String) = DBIO.seq(
      deletedPosts.filter(_.hash === hash).delete,
      categories.insertOrUpdate(NanoboardCategory(hash, name))
    )

    def delete(hash: String) = {
      categories.filter(_.hash === hash).delete
    }
  }
}
<commit_msg>thread pagination fix<commit_after>package com.karasiq.nanoboard

import java.time.Instant

import slick.driver.H2Driver.api._

import scala.concurrent.ExecutionContext
import scala.language.postfixOps

package object model {
  case class DBPost(hash: String, parent: String, message: String, firstSeen: Long) {
    def asThread(answers: Int): NanoboardMessageData = {
      NanoboardMessageData(Some(parent), hash, message, answers)
    }
  }
  class Post(tag: Tag) extends Table[DBPost](tag, "posts") {
    def hash = column[String]("hash", O.SqlType("char(32)"), O.PrimaryKey)
    def parent = column[String]("parent_hash", O.SqlType("char(32)"))
    def message = column[String]("message")
    def firstSeen = column[Long]("first_seen")

    def threadIdx = index("thread_index", (parent, firstSeen), unique = false)
    def recentIdx = index("recent_index", firstSeen, unique = false)
    def * = (hash, parent, message, firstSeen) <> (DBPost.tupled, DBPost.unapply)
  }

  val posts = TableQuery[Post]

  class DeletedPost(tag: Tag) extends Table[String](tag, "posts_deleted") {
    def hash = column[String]("hash", O.PrimaryKey)
    def * = hash
  }

  val deletedPosts = TableQuery[DeletedPost]

  class PendingPost(tag: Tag) extends Table[String](tag, "pending_posts") {
    def hash = column[String]("hash", O.SqlType("char(32)"), O.PrimaryKey)
    def post = foreignKey("post_fk", hash, posts)(_.hash, onUpdate = ForeignKeyAction.Restrict, onDelete = ForeignKeyAction.Cascade)
    def * = hash
  }

  val pendingPosts = TableQuery[PendingPost]

  class Place(tag: Tag) extends Table[String](tag, "places") {
    def url = column[String]("place_url", O.PrimaryKey)
    def * = url
  }

  val places = TableQuery[Place]

  class Category(tag: Tag) extends Table[NanoboardCategory](tag, "categories") {
    def hash = column[String]("category_hash", O.SqlType("char(32)"), O.PrimaryKey)
    def name = column[String]("category_name")
    def * = (hash, name) <> (NanoboardCategory.tupled, NanoboardCategory.unapply)
  }

  val categories = TableQuery[Category]

  object Post {
    def addReply(m: NanoboardMessage) = DBIO.seq(insertMessage(m), pendingPosts.forceInsertQuery {
      val exists = (for (p <- pendingPosts if p.hash === m.hash) yield ()).exists
      for (message <- Query(m.hash) if !exists) yield message
    }, deletedPosts.filter(_.hash === m.hash).delete)

    def insertMessage(m: NanoboardMessage) = posts.forceInsertQuery {
      val deleted = (for (p <- deletedPosts if p.hash === m.hash) yield ()).exists
      val exists = (for (p <- posts if p.hash === m.hash) yield ()).exists
      val insert = (m.hash, m.parent, m.text, Instant.now().toEpochMilli) <> (DBPost.tupled, DBPost.unapply)
      for (message <- Query(insert) if !deleted && !exists) yield message
    }

    def insertMessages(messages: Seq[NanoboardMessage]) = {
      DBIO.sequence(messages.map(insertMessage))
    }

    private def recentQuery(offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      posts
        .sortBy(_.firstSeen.desc)
        .drop(offset)
        .take(count)
        .map(post ⇒ (post, posts.filter(_.parent === post.hash).length))
    }

    private val recentCompiled = Compiled(recentQuery _)

    def recent(offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      recentCompiled(offset, count)
        .result
        .map(_.map {
          case (post, answers) ⇒
            post.asThread(answers)
        })
    }

    private def pendingQuery(offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      pendingPosts
        .flatMap(_.post)
        .sortBy(_.firstSeen.asc)
        .drop(offset)
        .take(count)
    }

    private val pendingCompiled = Compiled(pendingQuery _)

    def pending(offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      pendingCompiled(offset, count)
        .result
        .map(_.map(_.asThread(0)))
    }

    private def getQuery(hash: Rep[String]) = {
      posts
        .filter(_.hash === hash)
        .map(post ⇒ (post, posts.filter(_.parent === post.hash).length))
    }

    private val getCompiled = Compiled(getQuery _)

    def get(hash: String)(implicit ec: ExecutionContext) = {
      getCompiled(hash)
        .result
        .headOption
        .map(_.map {
          case (post, answers) ⇒
            post.asThread(answers)
        })
    }

    private def threadQuery(hash: Rep[String], offset: ConstColumn[Long], count: ConstColumn[Long]) = {
      val query = posts.filter(_.parent === hash)
        .sortBy(_.firstSeen.desc)
        .drop(offset)
        .take(count)

      def withAnswerCount(query: Query[Post, DBPost, Seq]) = query.map { post ⇒
        (post, posts.filter(_.parent === post.hash).length)
      }

      withAnswerCount(query)
    }

    private val threadCompiled = Compiled(threadQuery _)

    def thread(hash: String, offset: Long, count: Long)(implicit ec: ExecutionContext) = {
      for {
        opPost ← get(hash)
        answers ← threadCompiled(hash, offset, count).result
      } yield opPost.toVector ++ answers.map {
        case (post, answersCount) ⇒
          post.asThread(answersCount)
      }
    }

    def delete(hash: String)(implicit ec: ExecutionContext) = {
      def deleteCascade(hash: String): DBIOAction[Seq[String], NoStream, Effect.Write with Effect.Read] = {
        val query = posts.filter(_.parent === hash)
        for {
          deleted ← query.map(_.hash).result
          descendants ← DBIO.sequence(deleted.map(deleteCascade))
          _ ← DBIO.seq(
            pendingPosts.filter(_.hash === hash).delete,
            Category.delete(hash),
            deletedPosts.insertOrUpdate(hash),
            posts.filter(_.hash === hash).delete
          )
        } yield Seq(hash) ++ deleted ++ descendants.flatten
      }

      deleteCascade(hash)
    }
  }

  object Place {
    def list() = {
      places.result
    }

    def update(newList: Seq[String])(implicit ec: ExecutionContext) = DBIO.sequence(
      newList.map(url ⇒ places.insertOrUpdate(url)) :+
        places.filterNot(_.url inSet newList).delete
    ).map(_ ⇒ ())
  }

  object Category {
    def list()(implicit ec: ExecutionContext) = {
      val query = categories.map { c ⇒
        (c.hash, c.name, posts.filter(_.parent === c.hash).length)
      }
      query.result.map(_.map {
        case (hash, name, answers) ⇒
          NanoboardMessageData(None, hash, name, answers)
      })
    }

    def update(newList: Seq[NanoboardCategory])(implicit ec: ExecutionContext) = DBIO.sequence(
      newList.map(c ⇒ add(c.hash, c.name)):+
        categories.filterNot(_.hash inSet newList.map(_.hash)).delete
    ).map(_ ⇒ ())

    def add(hash: String, name: String) = DBIO.seq(
      deletedPosts.filter(_.hash === hash).delete,
      categories.insertOrUpdate(NanoboardCategory(hash, name))
    )

    def delete(hash: String) = {
      categories.filter(_.hash === hash).delete
    }
  }
}

{'fraction_non_alphanumeric': 0.13856132075471697, 'fraction_numerical': 0.0018425707547169812, 'mean_word_length': 3.0050177095631643, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2365974', 'n_tokens_mistral': 4402, 'n_tokens_neox': 4296, 'n_words': 941}