<commit_before>package is.hail.methods

import java.io.BufferedInputStream

import com.fasterxml.jackson.core.JsonParseException
import is.hail.HailContext
import is.hail.annotations._
import is.hail.expr._
import is.hail.expr.ir.{ExecuteContext, TableValue}
import is.hail.expr.ir.functions.{RelationalFunctions, TableToTableFunction}
import is.hail.types._
import is.hail.types.physical.{PStruct, PType}
import is.hail.types.virtual._
import is.hail.methods.VEP._
import is.hail.rvd.RVD
import is.hail.sparkextras.ContextRDD
import is.hail.utils._
import is.hail.variant.{Locus, RegionValueVariant, VariantMethods}
import is.hail.io.fs.FS
import org.apache.spark.sql.Row
import org.json4s.{DefaultFormats, Extraction, Formats, JValue}
import org.json4s.jackson.JsonMethods

import scala.collection.JavaConverters._
import scala.collection.mutable

case class VEPConfiguration(
  command: Array[String],
  env: Map[String, String],
  vep_json_schema: TStruct)

object VEP {
  def readConfiguration(fs: FS, path: String): VEPConfiguration = {
    val jv = using(fs.open(path)) { in =>
      JsonMethods.parse(in)
    }
    implicit val formats: Formats = defaultJSONFormats + new TStructSerializer
    jv.extract[VEPConfiguration]
  }

  def printContext(w: (String) => Unit) {
    w("##fileformat=VCFv4.1")
    w("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT")
  }

  def printElement(w: (String) => Unit, v: (Locus, IndexedSeq[String])) {
    val (locus, alleles) = v

    val sb = new StringBuilder()
    sb.append(locus.contig)
    sb += '\t'
    sb.append(locus.position)
    sb.append("\t.\t")
    sb.append(alleles(0))
    sb += '\t'
    sb.append(alleles.tail.filter(_ != "*").mkString(","))
    sb.append("\t.\t.\tGT")
    w(sb.result())
  }

  def variantFromInput(input: String): (Locus, IndexedSeq[String]) = {
    try {
      val a = input.split("\t")
      (Locus(a(0), a(1).toInt), a(3) +: a(4).split(","))
    } catch {
      case e: Throwable => fatal(s"VEP returned invalid variant '$input'", e)
    }
  }

  def waitFor(proc: Process, err: StringBuilder, cmd: Array[String]): Unit = {
    val rc = proc.waitFor()

    if (rc != 0) {
      fatal(s"VEP command '${ cmd.mkString(" ") }' failed with non-zero exit status $rc\n" +
        "  VEP Error output:\n" + err.toString)
    }
  }

  def getCSQHeaderDefinition(cmd: Array[String], confEnv: Map[String, String]): Option[String] = {
    val csqHeaderRegex = "ID=CSQ[^>]+Description=\"([^\"]+)".r
    val pb = new ProcessBuilder(cmd.toList.asJava)
    val env = pb.environment()
    confEnv.foreach { case (key, value) => env.put(key, value) }

    val (jt, err, proc) = List((Locus("1", 13372), FastIndexedSeq("G", "C"))).iterator.pipe(pb,
      printContext,
      printElement,
      _ => ())

    val csqHeader = jt.flatMap(s => csqHeaderRegex.findFirstMatchIn(s).map(m => m.group(1)))
    waitFor(proc, err, cmd)

    if (csqHeader.hasNext)
      Some(csqHeader.next())
    else {
      warn("could not get VEP CSQ header")
      None
    }
  }

  def apply(fs: FS, params: VEPParameters): VEP = {
    val conf = VEP.readConfiguration(fs, params.config)
    new VEP(params, conf)
  }

  def apply(fs: FS, config: String, csq: Boolean, blockSize: Int): VEP =
    VEP(fs, VEPParameters(config, csq, blockSize))

  def fromJValue(fs: FS, jv: JValue): VEP = {
    println(jv)
    implicit val formats: Formats = RelationalFunctions.formats
    val params = jv.extract[VEPParameters]<commit_msg>[query] Remove VEP println (#9803)<commit_after>package is.hail.methods

import java.io.BufferedInputStream

import com.fasterxml.jackson.core.JsonParseException
import is.hail.HailContext
import is.hail.annotations._
import is.hail.expr._
import is.hail.expr.ir.{ExecuteContext, TableValue}
import is.hail.expr.ir.functions.{RelationalFunctions, TableToTableFunction}
import is.hail.types._
import is.hail.types.physical.{PStruct, PType}
import is.hail.types.virtual._
import is.hail.methods.VEP._
import is.hail.rvd.RVD
import is.hail.sparkextras.ContextRDD
import is.hail.utils._
import is.hail.variant.{Locus, RegionValueVariant, VariantMethods}
import is.hail.io.fs.FS
import org.apache.spark.sql.Row
import org.json4s.{DefaultFormats, Extraction, Formats, JValue}
import org.json4s.jackson.JsonMethods

import scala.collection.JavaConverters._
import scala.collection.mutable

case class VEPConfiguration(
  command: Array[String],
  env: Map[String, String],
  vep_json_schema: TStruct)

object VEP {
  def readConfiguration(fs: FS, path: String): VEPConfiguration = {
    val jv = using(fs.open(path)) { in =>
      JsonMethods.parse(in)
    }
    implicit val formats: Formats = defaultJSONFormats + new TStructSerializer
    jv.extract[VEPConfiguration]
  }

  def printContext(w: (String) => Unit) {
    w("##fileformat=VCFv4.1")
    w("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT")
  }

  def printElement(w: (String) => Unit, v: (Locus, IndexedSeq[String])) {
    val (locus, alleles) = v

    val sb = new StringBuilder()
    sb.append(locus.contig)
    sb += '\t'
    sb.append(locus.position)
    sb.append("\t.\t")
    sb.append(alleles(0))
    sb += '\t'
    sb.append(alleles.tail.filter(_ != "*").mkString(","))
    sb.append("\t.\t.\tGT")
    w(sb.result())
  }

  def variantFromInput(input: String): (Locus, IndexedSeq[String]) = {
    try {
      val a = input.split("\t")
      (Locus(a(0), a(1).toInt), a(3) +: a(4).split(","))
    } catch {
      case e: Throwable => fatal(s"VEP returned invalid variant '$input'", e)
    }
  }

  def waitFor(proc: Process, err: StringBuilder, cmd: Array[String]): Unit = {
    val rc = proc.waitFor()

    if (rc != 0) {
      fatal(s"VEP command '${ cmd.mkString(" ") }' failed with non-zero exit status $rc\n" +
        "  VEP Error output:\n" + err.toString)
    }
  }

  def getCSQHeaderDefinition(cmd: Array[String], confEnv: Map[String, String]): Option[String] = {
    val csqHeaderRegex = "ID=CSQ[^>]+Description=\"([^\"]+)".r
    val pb = new ProcessBuilder(cmd.toList.asJava)
    val env = pb.environment()
    confEnv.foreach { case (key, value) => env.put(key, value) }

    val (jt, err, proc) = List((Locus("1", 13372), FastIndexedSeq("G", "C"))).iterator.pipe(pb,
      printContext,
      printElement,
      _ => ())

    val csqHeader = jt.flatMap(s => csqHeaderRegex.findFirstMatchIn(s).map(m => m.group(1)))
    waitFor(proc, err, cmd)

    if (csqHeader.hasNext)
      Some(csqHeader.next())
    else {
      warn("could not get VEP CSQ header")
      None
    }
  }

  def apply(fs: FS, params: VEPParameters): VEP = {
    val conf = VEP.readConfiguration(fs, params.config)
    new VEP(params, conf)
  }

  def apply(fs: FS, config: String, csq: Boolean, blockSize: Int): VEP =
    VEP(fs, VEPParameters(config, csq, blockSize))

  def fromJValue(fs: FS, jv: JValue): VEP = {
    log.info(s"vep config json: ${ jv.toString }")
    implicit val formats: Formats = RelationalFunctions.formats
    val params = jv.extract[VEPParameters]
{'fraction_non_alphanumeric': 0.15381319622964867, 'fraction_numerical': 0.005427020851185376, 'mean_word_length': 3.6254953764861293, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7477972', 'n_tokens_mistral': 2562, 'n_tokens_neox': 2497, 'n_words': 526}