<commit_before>package is.hail.io.vcf

import java.util

import htsjdk.variant.variantcontext.VariantContext
import htsjdk.variant.vcf.VCFConstants
import is.hail.annotations._
import is.hail.expr._
import is.hail.expr.types._
import is.hail.utils._
import is.hail.variant._

class BufferedLineIterator(bit: BufferedIterator[String]) extends htsjdk.tribble.readers.LineIterator {
  override def peek(): String = bit.head

  override def hasNext: Boolean = bit.hasNext

  override def next(): String = bit.next()

  override def remove() {
    throw new UnsupportedOperationException
  }
}

class HtsjdkRecordReader(val callFields: Set[String]) extends Serializable {

  import HtsjdkRecordReader._

  def readVariantInfo(vc: VariantContext, rvb: RegionValueBuilder, infoType: TStruct, infoFlagFieldNames: Set[String]) {
    // locus, alleles added via VCFLine

    // rsid
    val vcID = vc.getID
    val rsid = if (vcID == ".")
      rvb.setMissing()
    else
      rvb.addString(vcID)

    rvb.addDouble(vc.getPhredScaledQual)

    // filters
    if (!vc.filtersWereApplied)
      rvb.setMissing()
    else {
      if (vc.isNotFiltered) {
        rvb.startArray(0)
        rvb.endArray()
      } else {
        rvb.startArray(vc.getFilters.size())
        val fi = vc.getFilters.iterator()
        while (fi.hasNext)
          rvb.addString(fi.next())
        rvb.endArray()
      }
    }

    // info
    rvb.startStruct()
    infoType.fields.foreach { f =>
      val a = vc.getAttribute(f.name)
      addAttribute(rvb, a, f.typ, -1, isFlag = infoFlagFieldNames.contains(f.name))
    }
    rvb.endStruct() // info
  }
}

object HtsjdkRecordReader {
  def addAttribute(rvb: RegionValueBuilder, attr: Any, t: Type, nAlleles: Int, isFlag: Boolean = false) {
    ((attr, t): @unchecked) match {
      case (null, _) =>
        if (isFlag)
          rvb.addBoolean(false)
        else
          rvb.setMissing()
      case (".", _) =>
        rvb.setMissing()
      case (s: String, TArray(_: TInt32, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addInt(x.toInt)
        }
        rvb.endArray()

      case (s: String, TArray(_: TFloat64, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addDouble(x.toDouble)
        }
        rvb.endArray()
      case (s: String, TArray(_: TString, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addString(x)
        }
        rvb.endArray()
      case (s: String, _: TBoolean) =>
        rvb.addBoolean(s.toBoolean)
      case (b: Boolean, _: TBoolean) =>
        rvb.addBoolean(b)
      case (s: String, _: TString) =>
        rvb.addString(s)
      case (s: String, _: TInt32) =>
        rvb.addInt(s.toInt)
      case (s: String, _: TFloat64) =>
        val d = s match {
          case "nan" => Double.NaN
          case "inf" => Double.PositiveInfinity
          case "-inf" => Double.NegativeInfinity
          case _ => s.toDouble
        }
        rvb.addDouble(d)
      case (i: Int, _: TInt32) =>
        rvb.addInt(i)
      case (d: Double, _: TInt32) =>
        rvb.addInt(d.toInt)

      case (d: Double, _: TFloat64) =>
        rvb.addDouble(d)
      case (f: Float, _: TFloat64) =>
        rvb.addDouble(f.toDouble)
<commit_msg>Support "-nan" coming from VCFs. (#3917)<commit_after>package is.hail.io.vcf

import java.util

import htsjdk.variant.variantcontext.VariantContext
import htsjdk.variant.vcf.VCFConstants
import is.hail.annotations._
import is.hail.expr._
import is.hail.expr.types._
import is.hail.utils._
import is.hail.variant._

class BufferedLineIterator(bit: BufferedIterator[String]) extends htsjdk.tribble.readers.LineIterator {
  override def peek(): String = bit.head

  override def hasNext: Boolean = bit.hasNext

  override def next(): String = bit.next()

  override def remove() {
    throw new UnsupportedOperationException
  }
}

class HtsjdkRecordReader(val callFields: Set[String]) extends Serializable {

  import HtsjdkRecordReader._

  def readVariantInfo(vc: VariantContext, rvb: RegionValueBuilder, infoType: TStruct, infoFlagFieldNames: Set[String]) {
    // locus, alleles added via VCFLine

    // rsid
    val vcID = vc.getID
    val rsid = if (vcID == ".")
      rvb.setMissing()
    else
      rvb.addString(vcID)

    rvb.addDouble(vc.getPhredScaledQual)

    // filters
    if (!vc.filtersWereApplied)
      rvb.setMissing()
    else {
      if (vc.isNotFiltered) {
        rvb.startArray(0)
        rvb.endArray()
      } else {
        rvb.startArray(vc.getFilters.size())
        val fi = vc.getFilters.iterator()
        while (fi.hasNext)
          rvb.addString(fi.next())
        rvb.endArray()
      }
    }

    // info
    rvb.startStruct()
    infoType.fields.foreach { f =>
      val a = vc.getAttribute(f.name)
      addAttribute(rvb, a, f.typ, -1, isFlag = infoFlagFieldNames.contains(f.name))
    }
    rvb.endStruct() // info
  }
}

object HtsjdkRecordReader {
  def addAttribute(rvb: RegionValueBuilder, attr: Any, t: Type, nAlleles: Int, isFlag: Boolean = false) {
    ((attr, t): @unchecked) match {
      case (null, _) =>
        if (isFlag)
          rvb.addBoolean(false)
        else
          rvb.setMissing()
      case (".", _) =>
        rvb.setMissing()
      case (s: String, TArray(_: TInt32, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addInt(x.toInt)
        }
        rvb.endArray()

      case (s: String, TArray(_: TFloat64, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addDouble(x.toDouble)
        }
        rvb.endArray()
      case (s: String, TArray(_: TString, _)) =>
        val xs = s.split(",")
        rvb.startArray(xs.length)
        xs.foreach { x =>
          if (x == ".")
            rvb.setMissing()
          else
            rvb.addString(x)
        }
        rvb.endArray()
      case (s: String, _: TBoolean) =>
        rvb.addBoolean(s.toBoolean)
      case (b: Boolean, _: TBoolean) =>
        rvb.addBoolean(b)
      case (s: String, _: TString) =>
        rvb.addString(s)
      case (s: String, _: TInt32) =>
        rvb.addInt(s.toInt)
      case (s: String, _: TFloat64) =>
        val d = s match {
          case "nan" => Double.NaN
          case "-nan" => Double.NaN
          case "inf" => Double.PositiveInfinity
          case "-inf" => Double.NegativeInfinity
          case _ => s.toDouble
        }
        rvb.addDouble(d)
      case (i: Int, _: TInt32) =>
        rvb.addInt(i)
      case (d: Double, _: TInt32) =>
        rvb.addInt(d.toInt)

      case (d: Double, _: TFloat64) =>
        rvb.addDouble(d)
      case (f: Float, _: TFloat64) =>
        rvb.addDouble(f.toDouble)

{'fraction_non_alphanumeric': 0.13360099681572754, 'fraction_numerical': 0.005537865152983525, 'mean_word_length': 2.0378469301934397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5008183', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2474, 'n_words': 506}