<commit_before>package lila.security

import org.joda.time.DateTime
import reactivemongo.bson._
import scala.concurrent.duration._

import lila.common.{ EmailAddress, IpAddress }
import lila.db.dsl._
import lila.user.{ User, UserRepo }

// codename UGC
final class GarbageCollector(
    userSpy: UserSpyApi,
    ipIntel: IpIntel,
    slack: lila.slack.SlackApi,
    configColl: Coll,
    system: akka.actor.ActorSystem
) {

  /* User just signed up and doesn't have security data yet,
   * so wait a bit */
  def delay(user: User, ip: IpAddress, email: EmailAddress): Unit =
    if (checkable(user, email)) system.scheduler.scheduleOnce(5 seconds) {
      apply(user, ip, email)
    }

  private def apply(user: User, ip: IpAddress, email: EmailAddress): Funit =
    userSpy(user) flatMap { spy =>
      badOtherAccounts(spy.otherUsers.map(_.user)) ?? { others =>
        lila.common.Future.exists(spy.ips)(isBadIp).map {
          _ ?? {
            val ipBan = spy.usersSharingIp.forall { u =>
              isBadAccount(u) || !u.seenAt.exists(DateTime.now.minusMonths(2).isBefore)
            }
            collect(user, email, others, ipBan)
          }
        }
      }
    }

  private def badOtherAccounts(accounts: Set[User]): Option[List[User]] = {
    val others = accounts.toList
      .sortBy(-_.createdAt.getSeconds)
      .takeWhile(_.createdAt.isAfter(DateTime.now minusDays 7))
      .take(4)
    (others.size > 1 && others.forall(isBadAccount)) option others
  }

  private def isBadIp(ip: UserSpy.IPData): Fu[Boolean] = if (ip.blocked ||
    ip.location == Location.unknown ||
    ip.location == Location.tor ||
    ip.location.shortCountry == "Iran" // some undetected proxies
    ) fuccess(true)
  else ipIntel(ip.ip).map { 75 < _ }

  private def isBadAccount(user: User) =
    (user.troll || user.engine) && !user.enabled

  private val emailSuffixes = "yandex.ru yandex.com mailfa.com juno.com mail.ru"
    .split(' ').toList.map("@" + _)

  private def checkable(user: User, email: EmailAddress): Boolean =
    user.createdAt.isAfter(DateTime.now minusDays 3) &&
      emailSuffixes.exists(email.value.endsWith)

  private def isEffective =
    configColl.primitiveOne[Boolean]($id("ugc"), "value").map(~_)

  private def collect(user: User, email: EmailAddress, others: List[User], ipBan: Boolean): Funit = isEffective flatMap { effective =>
    val wait = (30 + scala.util.Random.nextInt(300)).seconds
    val othersStr = others.map(o => "@" + o.username).mkString(", ")
    val message = s"Will dispose of @${user.username} in $wait. Email: $email. Prev users: $othersStr${!effective ?? " [SIMULATION]"}"
    logger.branch("GarbageCollector").info(message)
    slack.garbageCollector(message) >>- {
      if (effective) system.scheduler.scheduleOnce(wait) {
        doCollect(user, ipBan)
      }
    }
  }

  private def doCollect(user: User, ipBan: Boolean): Unit = {
    system.lilaBus.publish(
      lila.hub.actorApi.security.GarbageCollect(user.id, ipBan),
      'garbageCollect
    )
  }
}<commit_msg>ignore email domain in UGC<commit_after>package lila.security

import org.joda.time.DateTime
import reactivemongo.bson._
import scala.concurrent.duration._

import lila.common.{ EmailAddress, IpAddress }
import lila.db.dsl._
import lila.user.{ User, UserRepo }

// codename UGC
final class GarbageCollector(
    userSpy: UserSpyApi,
    ipIntel: IpIntel,
    slack: lila.slack.SlackApi,
    configColl: Coll,
    system: akka.actor.ActorSystem
) {

  /* User just signed up and doesn't have security data yet,
   * so wait a bit */
  def delay(user: User, ip: IpAddress, email: EmailAddress): Unit =
    if (checkable(user, email)) system.scheduler.scheduleOnce(5 seconds) {
      apply(user, ip, email)
    }

  private def apply(user: User, ip: IpAddress, email: EmailAddress): Funit =
    userSpy(user) flatMap { spy =>
      badOtherAccounts(spy.otherUsers.map(_.user)) ?? { others =>
        lila.common.Future.exists(spy.ips)(isBadIp).map {
          _ ?? {
            val ipBan = spy.usersSharingIp.forall { u =>
              isBadAccount(u) || !u.seenAt.exists(DateTime.now.minusMonths(2).isBefore)
            }
            collect(user, email, others, ipBan)
          }
        }
      }
    }

  private def badOtherAccounts(accounts: Set[User]): Option[List[User]] = {
    val others = accounts.toList
      .sortBy(-_.createdAt.getSeconds)
      .takeWhile(_.createdAt.isAfter(DateTime.now minusDays 7))
      .take(4)
    (others.size > 1 && others.forall(isBadAccount)) option others
  }

  private def isBadIp(ip: UserSpy.IPData): Fu[Boolean] = if (ip.blocked ||
    ip.location == Location.unknown ||
    ip.location == Location.tor ||
    ip.location.shortCountry == "Iran" // some undetected proxies
    ) fuccess(true)
  else ipIntel(ip.ip).map { 75 < _ }

  private def isBadAccount(user: User) =
    (user.troll || user.engine) && !user.enabled

  private def checkable(user: User, email: EmailAddress): Boolean =
    user.createdAt.isAfter(DateTime.now minusDays 3)

  private def isEffective =
    configColl.primitiveOne[Boolean]($id("ugc"), "value").map(~_)

  private def collect(user: User, email: EmailAddress, others: List[User], ipBan: Boolean): Funit = isEffective flatMap { effective =>
    val wait = (30 + scala.util.Random.nextInt(300)).seconds
    val othersStr = others.map(o => "@" + o.username).mkString(", ")
    val message = s"Will dispose of @${user.username} in $wait. Email: $email. Prev users: $othersStr${!effective ?? " [SIMULATION]"}"
    logger.branch("GarbageCollector").info(message)
    slack.garbageCollector(message) >>- {
      if (effective) system.scheduler.scheduleOnce(wait) {
        doCollect(user, ipBan)
      }
    }
  }

  private def doCollect(user: User, ipBan: Boolean): Unit = {
    system.lilaBus.publish(
      lila.hub.actorApi.security.GarbageCollect(user.id, ipBan),
      'garbageCollect
    )
  }
}
{'fraction_non_alphanumeric': 0.13660185967878274, 'fraction_numerical': 0.004395604395604396, 'mean_word_length': 3.2378223495702008, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '407754', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1947, 'n_words': 419}