<commit_before>package utils

import com.sanoma.cda.geoip.{IpLocation, MaxMindIpGeo}
import io.flow.common.v0.models.Address
import io.flow.reference.v0.models.Country
import play.api.Logger
import scala.util.{Failure, Success, Try}

private[utils] case class LatLong(latitude: String, longitude: String)

object MaxMind {
  import sys.process._
  import java.net.URL
  import java.io.File
  import scala.language.postfixOps

  new URL("https://s3.amazonaws.com/io.flow.aws-s3-public/location/GeoLite2-City.mmdb") #> new File("GeoLite2-City.mmdb") !!

  val geoIp = MaxMindIpGeo(dbFile = "GeoLite2-City.mmdb", lruCache = 10000, synchronized = true)

  def getByIp(ip: String): Option[Address] = {
    Try {
      geoIp.getLocation(ip)
    } match {
      case Success(ipl) => ipl.map(address)
      case Failure(ex) => {
        Logger.error(s"Encountered an error retrieving IP location from database: [$ex]")
        None
      }
    }
  }

  private[utils] def address(ipl: IpLocation): Address = {
    val latLong = MaxMind.latLong(ipl)
    Address(
      city = ipl.city,
      province = ipl.region,
      postal = ipl.postalCode,
      country = MaxMind.country(ipl).map(_.iso31663),
      latitude = latLong.map(_.latitude),
      longitude = latLong.map(_.longitude)
    )
  }
  <commit_msg>Update MaxMind DB<commit_after>package utils

import com.sanoma.cda.geoip.{IpLocation, MaxMindIpGeo}
import io.flow.common.v0.models.Address
import io.flow.reference.v0.models.Country
import play.api.Logger
import scala.util.{Failure, Success, Try}

private[utils] case class LatLong(latitude: String, longitude: String)

object MaxMind {
  import sys.process._
  import java.net.URL
  import java.io.File
  import scala.language.postfixOps

  new URL("https://s3.amazonaws.com/io.flow.aws-s3-public/location/GeoIP2-City-20170404-N116b6K2.mmdb") #> new File("GeoLite2-City.mmdb") !!

  val geoIp = MaxMindIpGeo(dbFile = "GeoLite2-City.mmdb", lruCache = 10000, synchronized = true)

  def getByIp(ip: String): Option[Address] = {
    Try {
      geoIp.getLocation(ip)
    } match {
      case Success(ipl) => ipl.map(address)
      case Failure(ex) => {
        Logger.error(s"Encountered an error retrieving IP location from database: [$ex]")
        None
      }
    }
  }

  private[utils] def address(ipl: IpLocation): Address = {
    val latLong = MaxMind.latLong(ipl)
    Address(
      city = ipl.city,
      province = ipl.region,
      postal = ipl.postalCode,
      country = MaxMind.country(ipl).map(_.iso31663),
      latitude = latLong.map(_.latitude),
      longitude = latLong.map(_.longitude)
    )
  }
  
{'fraction_non_alphanumeric': 0.12748091603053435, 'fraction_numerical': 0.017938931297709924, 'mean_word_length': 3.303776683087028, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4899796', 'n_tokens_mistral': 977, 'n_tokens_neox': 914, 'n_words': 176}