<commit_before>package lila.openingexplorer

import ornicar.scalalib.Validation
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scalaz.Validation.FlatMap._

import chess.format.Forsyth
import chess.format.pgn.{ Parser, Reader, ParsedPgn, InitialPosition, Sans }
import chess.variant.Variant
import chess.{ Hash, PositionHash, Replay }

final class Importer(
    masterDb: MasterDatabase,
    lichessDb: LichessDatabase,
    pgnDb: PgnDatabase,
    gameInfoDb: GameInfoDatabase
) extends Validation with scalaz.syntax.ToValidationOps {

  private val lichessSeparator = "\n\n\n"

  private val logger = play.api.Logger("importer")

  private var nbImported = 0

  def lichess(text: String): (Unit, Int) = Time {
    val pgns = text.split(lichessSeparator)
    val processed = pgns flatMap { pgn =>
      processLichess(pgn) match {
        case scalaz.Success(processed) => Some(processed)
        case scalaz.Failure(errors) =>
          logger.warn(errors.list.toList mkString ", ")
          None
      }
    }
    if (processed.exists { p => gameInfoDb.contains(p.gameRef.gameId) })
      logger.info(s"found a dup, skipping batch")
    else {
      processed foreach {
        case Processed(parsed, replay, gameRef) =>
          GameInfo parse parsed.tags match {
            case None => logger.warn(s"Can't produce GameInfo for game ${gameRef.gameId}")
            case Some(info) =>
              val variant = replay.setup.board.variant
              try {
                if (gameInfoDb.store(gameRef.gameId, info)) {
                  replay.chronoMoves.take(Config.explorer.lichess(variant).maxPlies).foreach {
                    move =>
                      lichessDb.merge(variant, gameRef, move)
                  }
                } else {
                  logger.warn(s"Duplicate lichess game ${gameRef.gameId}")
                }
              } catch {
                case e: Exception => logger.error(s"Can't merge game ${gameRef.gameId}: ${e.getMessage}")
              }
          }
      }
      val nb = processed.size
      nbImported = nbImported + nb
      logger.info(s"Imported $nb lichess games; total $nbImported")
    }
  }

  private val masterInitBoard = chess.Board.init(chess.variant.Standard)
  private val masterMinRating = 2200

  def master(pgn: String): (Valid[Unit], Int) = Time {
    processMaster(pgn) flatMap {
      case Processed(parsed, replay, gameRef) =>
        if ((Forsyth >> replay.setup.situation) != Forsyth.initial)<commit_msg>remove duplicate duplicate detection<commit_after>package lila.openingexplorer

import ornicar.scalalib.Validation
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scalaz.Validation.FlatMap._

import chess.format.Forsyth
import chess.format.pgn.{ Parser, Reader, ParsedPgn, InitialPosition, Sans }
import chess.variant.Variant
import chess.{ Hash, PositionHash, Replay }

final class Importer(
    masterDb: MasterDatabase,
    lichessDb: LichessDatabase,
    pgnDb: PgnDatabase,
    gameInfoDb: GameInfoDatabase
) extends Validation with scalaz.syntax.ToValidationOps {

  private val lichessSeparator = "\n\n\n"

  private val logger = play.api.Logger("importer")

  private var nbImported = 0

  def lichess(text: String): (Unit, Int) = Time {
    val pgns = text.split(lichessSeparator)
    val processed = pgns flatMap { pgn =>
      processLichess(pgn) match {
        case scalaz.Success(processed) => Some(processed)
        case scalaz.Failure(errors) =>
          logger.warn(errors.list.toList mkString ", ")
          None
      }
    }
    processed foreach {
      case Processed(parsed, replay, gameRef) =>
        GameInfo parse parsed.tags match {
          case None => logger.warn(s"Can't produce GameInfo for game ${gameRef.gameId}")
          case Some(info) =>
            val variant = replay.setup.board.variant
            try {
              if (gameInfoDb.store(gameRef.gameId, info)) {
                replay.chronoMoves.take(Config.explorer.lichess(variant).maxPlies).foreach {
                  move =>
                    lichessDb.merge(variant, gameRef, move)
                }
              } else {
                logger.warn(s"Duplicate lichess game ${gameRef.gameId}")
              }
            } catch {
              case e: Exception => logger.error(s"Can't merge game ${gameRef.gameId}: ${e.getMessage}")
            }
        }
    }
    val nb = processed.size
    nbImported = nbImported + nb
    logger.info(s"Imported $nb lichess games; total $nbImported")
  }

  private val masterInitBoard = chess.Board.init(chess.variant.Standard)
  private val masterMinRating = 2200

  def master(pgn: String): (Valid[Unit], Int) = Time {
    processMaster(pgn) flatMap {
      case Processed(parsed, replay, gameRef) =>
        if ((Forsyth >> replay.setup.situation) != Forsyth.initial)
{'fraction_non_alphanumeric': 0.1038908127928295, 'fraction_numerical': 0.002037074760643716, 'mean_word_length': 2.5735080058224162, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2531362', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1441, 'n_words': 330}