<commit_before>package se.gigurra.leavu3.externaldata

import com.twitter.util.{Duration, JavaTimer}
import se.gigurra.heisenberg.MapData._
import se.gigurra.heisenberg.Schema
import se.gigurra.leavu3.Configuration
import se.gigurra.leavu3.util.{Eventually, RestClient}
import se.gigurra.serviceutils.json.JSON
import se.gigurra.serviceutils.twitter.logging.Logging

import scala.collection.concurrent
import scala.language.implicitConversions

case class UnitTypeData(fullName: String,
                        shortName: String,
                        isKnown: Boolean,
                        typ: UnitType.TYPE,
                        power2RangeExponent: Double,
                        maxRangeIndication: Double)

object UnitTypeData {
  val DEFAULT_POWER_MAPPING_EXPONENT = 2.0
  val AWACS_POWER_MAPPING_EXPONENT = 7.0
  val DEFAULT_MAX_RANGE_INDICATION = 1.0
  val DEFAULT_ARH_MAX_RANGE_INDICATION = 0.4
  val UKN = UnitTypeData("unknown", "UKN", isKnown = false, (0,0,0,0), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
  def default(fullName: String) =Data(fullName.split(' ').head, DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)

  case class Data(shortName: String, power2RangeExponent: Double, maxRangeIndication: Double)

  def firstNumber(text: String): String = {
    val iFirst = text.indexWhere(_.isDigit)
    if (iFirst >= 0) {
      val iLast = text.drop(iFirst).indexWhere(!_.isDigit)
      if (iLast > 0) {
        text.substring(iFirst, iLast + iFirst)
      } else {
        text.substring(iFirst, iFirst + 1)
      }
    } else {
      "ukn"
    }
  }

  val unitMappings: PartialFunction[String, Data] = {
    case x if x.startsWith("mig-29")  => Data("29",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-27")   => Data("29",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-3")    => Data("30",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-")     => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("mig-")    => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("f-")      => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("e-3")     => Data("E3",           AWACS_POWER_MAPPING_EXPONENT,   DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("aim-120") => Data("M120",         DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
    case x if x.startsWith("aim-154") => Data("M54",          DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
    case x if x.startsWith("r-77")    => Data("M77",          DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
  }

  def apply(fullName: String, isKnown: Boolean, typ: UnitType.TYPE): UnitTypeData = {
    val data = unitMappings.applyOrElse(fullName.toLowerCase, (_: Any) => default(fullName))
    UnitTypeData(fullName, data.shortName, isKnown, typ, data.power2RangeExponent, data.maxRangeIndication)
  }
}

case class UnitType(source: SourceData = Map.empty) extends SafeParsed[UnitType.type] {
  val level1 = parse(schema.level1)
  val level2 = parse(schema.level2)
  val level3 = parse(schema.level3)
  val level4 = parse(schema.level4)<commit_msg>cleanup<commit_after>package se.gigurra.leavu3.externaldata

import com.twitter.util.{Duration, JavaTimer}
import se.gigurra.heisenberg.MapData._
import se.gigurra.heisenberg.Schema
import se.gigurra.leavu3.Configuration
import se.gigurra.leavu3.util.{Eventually, RestClient}
import se.gigurra.serviceutils.json.JSON
import se.gigurra.serviceutils.twitter.logging.Logging

import scala.collection.concurrent
import scala.language.implicitConversions

case class UnitTypeData(fullName: String,
                        shortName: String,
                        isKnown: Boolean,
                        typ: UnitType.TYPE,
                        power2RangeExponent: Double,
                        maxRangeIndication: Double)

object UnitTypeData {
  val DEFAULT_POWER_MAPPING_EXPONENT = 2.0
  val AWACS_POWER_MAPPING_EXPONENT = 7.0
  val DEFAULT_MAX_RANGE_INDICATION = 1.0
  val DEFAULT_ARH_MAX_RANGE_INDICATION = 0.4
  val UKN = UnitTypeData("unknown", "UKN", isKnown = false, (0,0,0,0), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)

  def defaultShortName(fullName: String) =Data(fullName.split(' ').head.toUpperCase, DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)

  case class Data(shortName: String, power2RangeExponent: Double, maxRangeIndication: Double)

  def firstNumber(text: String): String = {
    val iFirst = text.indexWhere(_.isDigit)
    if (iFirst >= 0) {
      val iLast = text.drop(iFirst).indexWhere(!_.isDigit)
      if (iLast > 0) {
        text.substring(iFirst, iLast + iFirst)
      } else {
        text.substring(iFirst, iFirst + 1)
      }
    } else {
      "ukn"
    }
  }

  val unitMappings: PartialFunction[String, Data] = {
    case x if x.startsWith("mig-29")  => Data("29",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-27")   => Data("29",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-3")    => Data("30",           DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("su-")     => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("mig-")    => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("f-")      => Data(firstNumber(x), DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("e-3")     => Data("E3",           AWACS_POWER_MAPPING_EXPONENT,   DEFAULT_MAX_RANGE_INDICATION)
    case x if x.startsWith("aim-120") => Data("M120",         DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
    case x if x.startsWith("aim-154") => Data("M54",          DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
    case x if x.startsWith("r-77")    => Data("M77",          DEFAULT_POWER_MAPPING_EXPONENT, DEFAULT_ARH_MAX_RANGE_INDICATION)
  }

  def apply(fullName: String, isKnown: Boolean, typ: UnitType.TYPE): UnitTypeData = {
    val data = unitMappings.applyOrElse(fullName.toLowerCase, defaultShortName)
    UnitTypeData(fullName, data.shortName, isKnown, typ, data.power2RangeExponent, data.maxRangeIndication)
  }
}

case class UnitType(source: SourceData = Map.empty) extends SafeParsed[UnitType.type] {
  val level1 = parse(schema.level1)
  val level2 = parse(schema.level2)
  val level3 = parse(schema.level3)
  val level4 = parse(schema.level4)
{'fraction_non_alphanumeric': 0.10316428261811877, 'fraction_numerical': 0.0164716081491114, 'mean_word_length': 3.9092198581560282, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4938269', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2375, 'n_words': 420}