<commit_before>package lv.addresses.indexer

import java.io.File
import java.sql.DriverManager
import java.time.LocalDateTime

import akka.NotUsed
import akka.stream.scaladsl.Source
import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory
import org.tresql.{LogTopic, Query, Resources, SimpleCache}

import scala.collection.mutable.{ArrayBuffer => AB}
import scala.util.Using

case class Address(code: Int, address: String, zipCode: String, typ: Int,
                   coordX: BigDecimal, coordY: BigDecimal, history: List[String],
                   editDistance: Option[Int])
case class AddressStruct(
                          pilCode: Option[Int] = None, pilName: Option[String] = None,
                          novCode: Option[Int] = None, novName: Option[String] = None,
                          pagCode: Option[Int] = None, pagName: Option[String] = None,
                          cieCode: Option[Int] = None, cieName: Option[String] = None,
                          ielCode: Option[Int] = None, ielName: Option[String] = None,
                          nltCode: Option[Int] = None, nltName: Option[String] = None,
                          dzvCode: Option[Int] = None, dzvName: Option[String] = None)
case class ResolvedAddress(address: String, resolvedAddress: Option[Address])

trait AddressFinder
  extends AddressIndexer
    with AddressResolver
    with AddressIndexLoader
    with AddressLoader
    with AddressIndexerConfig
    with SpatialIndexer {

  import Constants._

  protected val logger = Logger(LoggerFactory.getLogger("lv.addresses.indexer"))


  private[this] var _addressMap: Map[Int, AddrObj] = null
  private[this] var _addressHistory: Map[Int, List[String]] = Map()

  def addressMap = _addressMap
  def addressHistory = _addressHistory

  def init: Unit = {
    if (ready) return

    if (addressFileName == null && (dbConfig.isEmpty || dbDataVersion == null))
      logger.error("Address file not set nor database connection parameters specified")
    else {
      if (indexFiles.isDefined) loadIndex else {
        val (am, ah) = dbConfig
          .map(loadAddressesFromDb) //load from db
          .getOrElse(loadAddresses() -> Map[Int, List[String]]()) //load from file
        _addressMap = am
        _addressHistory = ah
        index(am, ah)
        saveIndex
      }
      spatialIndex(addressMap)
    }
  }

  def ready = _index != null && addressMap != null

  def checkIndex = if (_index == null || addressMap == null)
    sys.error("""
           Address index not found. Check whether 'VZD.ak-file' property is set and points to existing file.
           If method is called from console make sure that method index(<address register zip file>) or loadIndex is called first""")

  def search(str: String)(limit: Int = 20, types: Set[Int] = null): Array[Address] = {
    checkIndex
    if (str.trim.length == 0) //search string empty, search only big units - PIL, NOV, PAG, CIE
      if (types == null || (types -- big_unit_types) != Set.empty)
        Array[Address]()
      else {
        val result = new AB[Int]()
        var (s, i) = (0, 0)
        while (i < _sortedPilsNovPagCiem.size && s < limit) {
          if(types contains _addressMap(_sortedPilsNovPagCiem(i)).typ) {
            result += _sortedPilsNovPagCiem(i)
            s += 1
          }
          i += 1
        }
        result.map(address(_)).toArray
      }
    else {
      val words = normalize(str)
      def codesToAddr(codes: AB[Int], editDistance: Int) = {
        var (perfectRankCount, i) = (0, 0)
        val size = Math.min(codes.length, limit)
        val result = new AB[Long](size)
        while (perfectRankCount < size && i < codes.length) {
          val code = codes(i)
          val r = rank(words, code)
          if (r == 0) perfectRankCount += 1
          val key = r.toLong << 53 | i.toLong << 32 | code
          insertIntoHeap(result, key)
          i += 1
        }
        val res =
          if (size < result.size / 2) heap_topx(result, size)
          else if (size < result.size) result.sorted.take(size) else result.sorted
        res
          .map(_ & 0x00000000FFFFFFFFL)
          .map(_.toInt)
          .map(address(_, editDistance))
      }
      val addresses = new AB[Address]()
      searchCodes(words)(1024, types)
        .foreach { case (codes, err) => addresses ++= codesToAddr(codes, err) }
      addresses.toArray
    }
  }

  def searchNearest(coordX: BigDecimal, coordY: BigDecimal)(limit: Int = 1) =
    new Search(Math.min(limit, 20))
      .searchNearest(coordX, coordY)
      .map(result => addressFromObj(result._1))
      .toArray

  def addressStruct(code: Int) = {
    def s(st: AddressStruct, typ: Int, code: Int, name: String) = typ match {
      case PIL => st.copy(pilCode = Option(code), pilName = Option(name))
      case NOV => st.copy(novCode = Option(code), novName = Option(name))
      case PAG => st.copy(pagCode = Option(code), pagName = Option(name))
      case CIE => st.copy(cieCode = Option(code), cieName = Option(name))
      case IEL => st.copy(ielCode = Option(code), ielName = Option(name))
      case NLT => st.copy(nltCode = Option(code), nltName = Option(name))
      case DZI => st.copy(dzvCode = Option(code), dzvName = Option(name))
      case _ => st
    }
    addressMap
      .get(code)
      .map(_.foldLeft(AddressStruct())((st, o) => s(st, o.typ, o.code, o.name)))
      .getOrElse(AddressStruct())
  }<commit_msg>limit fix<commit_after>package lv.addresses.indexer

import java.io.File
import java.sql.DriverManager
import java.time.LocalDateTime

import akka.NotUsed
import akka.stream.scaladsl.Source
import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory
import org.tresql.{LogTopic, Query, Resources, SimpleCache}

import scala.collection.mutable.{ArrayBuffer => AB}
import scala.util.Using

case class Address(code: Int, address: String, zipCode: String, typ: Int,
                   coordX: BigDecimal, coordY: BigDecimal, history: List[String],
                   editDistance: Option[Int])
case class AddressStruct(
                          pilCode: Option[Int] = None, pilName: Option[String] = None,
                          novCode: Option[Int] = None, novName: Option[String] = None,
                          pagCode: Option[Int] = None, pagName: Option[String] = None,
                          cieCode: Option[Int] = None, cieName: Option[String] = None,
                          ielCode: Option[Int] = None, ielName: Option[String] = None,
                          nltCode: Option[Int] = None, nltName: Option[String] = None,
                          dzvCode: Option[Int] = None, dzvName: Option[String] = None)
case class ResolvedAddress(address: String, resolvedAddress: Option[Address])

trait AddressFinder
  extends AddressIndexer
    with AddressResolver
    with AddressIndexLoader
    with AddressLoader
    with AddressIndexerConfig
    with SpatialIndexer {

  import Constants._

  protected val logger = Logger(LoggerFactory.getLogger("lv.addresses.indexer"))


  private[this] var _addressMap: Map[Int, AddrObj] = null
  private[this] var _addressHistory: Map[Int, List[String]] = Map()

  def addressMap = _addressMap
  def addressHistory = _addressHistory

  def init: Unit = {
    if (ready) return

    if (addressFileName == null && (dbConfig.isEmpty || dbDataVersion == null))
      logger.error("Address file not set nor database connection parameters specified")
    else {
      if (indexFiles.isDefined) loadIndex else {
        val (am, ah) = dbConfig
          .map(loadAddressesFromDb) //load from db
          .getOrElse(loadAddresses() -> Map[Int, List[String]]()) //load from file
        _addressMap = am
        _addressHistory = ah
        index(am, ah)
        saveIndex
      }
      spatialIndex(addressMap)
    }
  }

  def ready = _index != null && addressMap != null

  def checkIndex = if (_index == null || addressMap == null)
    sys.error("""
           Address index not found. Check whether 'VZD.ak-file' property is set and points to existing file.
           If method is called from console make sure that method index(<address register zip file>) or loadIndex is called first""")

  def search(str: String)(limit: Int = 20, types: Set[Int] = null): Array[Address] = {
    checkIndex
    if (str.trim.length == 0) //search string empty, search only big units - PIL, NOV, PAG, CIE
      if (types == null || (types -- big_unit_types) != Set.empty)
        Array[Address]()
      else {
        val result = new AB[Int]()
        var (s, i) = (0, 0)
        while (i < _sortedPilsNovPagCiem.size && s < limit) {
          if(types contains _addressMap(_sortedPilsNovPagCiem(i)).typ) {
            result += _sortedPilsNovPagCiem(i)
            s += 1
          }
          i += 1
        }
        result.map(address(_)).toArray
      }
    else {
      val words = normalize(str)
      def codesToAddr(codes: AB[Int], editDistance: Int) = {
        var (perfectRankCount, i) = (0, 0)
        val size = Math.min(codes.length, limit)
        val result = new AB[Long](size)
        while (perfectRankCount < size && i < codes.length) {
          val code = codes(i)
          val r = rank(words, code)
          if (r == 0) perfectRankCount += 1
          val key = r.toLong << 53 | i.toLong << 32 | code
          insertIntoHeap(result, key)
          i += 1
        }
        val res =
          if (size < result.size / 2) heap_topx(result, size)
          else if (size < result.size) result.sorted.take(size) else result.sorted
        res
          .map(_ & 0x00000000FFFFFFFFL)
          .map(_.toInt)
          .map(address(_, editDistance))
      }
      val resultCodes = searchCodes(words)(1024, types)
      val length = resultCodes.length
      val addresses = new AB[Address]()
      var i = 0
      while (i < length && addresses.length < limit) {
        val (codes, err) = resultCodes(i)
        addresses ++= codesToAddr(codes, err)
        i += 1
      }
      val size = Math.min(limit, addresses.length)
      val result = new Array[Address](size)
      i = 0
      while (i < size) {
        result(i) = addresses(i)
        i += 1
      }
      result
    }
  }

  def searchNearest(coordX: BigDecimal, coordY: BigDecimal)(limit: Int = 1) =
    new Search(Math.min(limit, 20))
      .searchNearest(coordX, coordY)
      .map(result => addressFromObj(result._1))
      .toArray

  def addressStruct(code: Int) = {
    def s(st: AddressStruct, typ: Int, code: Int, name: String) = typ match {
      case PIL => st.copy(pilCode = Option(code), pilName = Option(name))
      case NOV => st.copy(novCode = Option(code), novName = Option(name))
      case PAG => st.copy(pagCode = Option(code), pagName = Option(name))
      case CIE => st.copy(cieCode = Option(code), cieName = Option(name))
      case IEL => st.copy(ielCode = Option(code), ielName = Option(name))
      case NLT => st.copy(nltCode = Option(code), nltName = Option(name))
      case DZI => st.copy(dzvCode = Option(code), dzvName = Option(name))
      case _ => st
    }
    addressMap
      .get(code)
      .map(_.foldLeft(AddressStruct())((st, o) => s(st, o.typ, o.code, o.name)))
      .getOrElse(AddressStruct())
  }
{'fraction_non_alphanumeric': 0.11888423491667409, 'fraction_numerical': 0.006594777649050886, 'mean_word_length': 2.4905132192846033, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4429430', 'n_tokens_mistral': 3502, 'n_tokens_neox': 3415, 'n_words': 937}