<commit_before>package lila.openingexplorer

import java.io.File

import fm.last.commons.kyoto.factory.{ KyotoDbBuilder, Mode, Compressor, PageComparator }
import fm.last.commons.kyoto.{ KyotoDb, WritableVisitor }

import chess.format.Forsyth
import chess.format.pgn.{ ParsedPgn, Pgn, Tag, TagType, Dumper, Turn, Move }
import chess.Replay

final class PgnDatabase extends MasterDatabasePacker {

  private val dbFile = new File("data/master-pgn.kct")
  dbFile.createNewFile

  private val db =
    new KyotoDbBuilder(dbFile)
      .modes(Mode.CREATE, Mode.READ_WRITE)
      .buckets(2000000L)
      .compressor(Compressor.LZMA)
      .pageComparator(PageComparator.LEXICAL)
      .buildAndOpen

  private val relevantTags: Set[TagType] = Tag.tagTypes.toSet

  def get(gameId: String): Option[String] = Option(db.get(gameId))

  def store(gameId: String, parsed: ParsedPgn, replay: Replay) = {

    val tags = parsed.tags.filter { tag =>
      relevantTags contains tag.name
    }
    val fenSituation = tags find (_.name == Tag.FEN) flatMap {
      case Tag(_, fen) => Forsyth <<< fen
    }
    val pgnMoves = replay.chronoMoves.foldLeft(replay.setup) {
      case (game, moveOrDrop) => moveOrDrop.fold(game.apply, game.applyDrop)
    }.pgnMoves
    val moves = if (fenSituation.exists(_.situation.color.black)) ".." :: pgnMoves else pgnMoves
    val initialTurn = fenSituation.map(_.fullMoveNumber) getOrElse 1
    val pgn = Pgn(tags, turns(moves, initialTurn))

    db.set(gameId, pgn.toString)
  }

  private def turns(moves: List[String], from: Int): List[Turn] =
    (moves grouped 2).zipWithIndex.toList map {
      case (moves, index) => Turn(
        number = index + from,
        white = moves.headOption filter (".." !=) map { Move(_) },
        black = moves lift 1 map { Move(_) })
    } filterNot (_.isEmpty)
<commit_msg>exclude some PGN tags<commit_after>package lila.openingexplorer

import java.io.File

import fm.last.commons.kyoto.factory.{ KyotoDbBuilder, Mode, Compressor, PageComparator }
import fm.last.commons.kyoto.{ KyotoDb, WritableVisitor }

import chess.format.Forsyth
import chess.format.pgn.{ ParsedPgn, Pgn, Tag, TagType, Dumper, Turn, Move }
import chess.Replay

final class PgnDatabase extends MasterDatabasePacker {

  private val dbFile = new File("data/master-pgn.kct")
  dbFile.createNewFile

  private val db =
    new KyotoDbBuilder(dbFile)
      .modes(Mode.CREATE, Mode.READ_WRITE)
      .buckets(2000000L)
      .compressor(Compressor.LZMA)
      .pageComparator(PageComparator.LEXICAL)
      .buildAndOpen

  private val relevantTags: Set[TagType] = 
    Tag.tagTypes.toSet diff Set(Tag.ECO, Tag.Opening, Tag.Variant)

  def get(gameId: String): Option[String] = Option(db.get(gameId))

  def store(gameId: String, parsed: ParsedPgn, replay: Replay) = {

    val tags = parsed.tags.filter { tag =>
      relevantTags contains tag.name
    }
    val fenSituation = tags find (_.name == Tag.FEN) flatMap {
      case Tag(_, fen) => Forsyth <<< fen
    }
    val pgnMoves = replay.chronoMoves.foldLeft(replay.setup) {
      case (game, moveOrDrop) => moveOrDrop.fold(game.apply, game.applyDrop)
    }.pgnMoves
    val moves = if (fenSituation.exists(_.situation.color.black)) ".." :: pgnMoves else pgnMoves
    val initialTurn = fenSituation.map(_.fullMoveNumber) getOrElse 1
    val pgn = Pgn(tags, turns(moves, initialTurn))

    db.set(gameId, pgn.toString)
  }

  private def turns(moves: List[String], from: Int): List[Turn] =
    (moves grouped 2).zipWithIndex.toList map {
      case (moves, index) => Turn(
        number = index + from,
        white = moves.headOption filter (".." !=) map { Move(_) },
        black = moves lift 1 map { Move(_) })
    } filterNot (_.isEmpty)

{'fraction_non_alphanumeric': 0.12148028962188254, 'fraction_numerical': 0.0053633681952266025, 'mean_word_length': 3.4939759036144578, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '619573', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1245, 'n_words': 295}