<commit_before>package org.broadinstitute.hail.methods

import org.apache.hadoop
import org.broadinstitute.hail.Utils._
import org.broadinstitute.hail.variant.{Sex, Phenotype}
import org.broadinstitute.hail.variant.Phenotype.{Phenotype, Case, Control}
import org.broadinstitute.hail.variant.Sex.{Sex, Male, Female}
import scala.io.Source

object Role extends Enumeration {
  type Role = Value
  val Kid = Value("0")
  val Dad = Value("1")
  val Mom = Value("2")
}

import org.broadinstitute.hail.methods.Role.{Role, Kid, Dad, Mom}

case class Trio(kid: Int, fam: Option[String], dad: Option[Int], mom: Option[Int],
  sex: Option[Sex], pheno: Option[Phenotype]) {

  def toCompleteTrio: Option[CompleteTrio] =
    dad.flatMap(d =>
      mom.map(m => CompleteTrio(kid, fam, d, m, sex, pheno)))

  def isMale: Boolean = sex.contains(Male)

  def isFemale: Boolean = sex.contains(Female)

  def isCase: Boolean = pheno.contains(Case)

  def isControl: Boolean = pheno.contains(Control)

  def isComplete: Boolean = dad.isDefined && mom.isDefined
}

case class CompleteTrio(kid: Int, fam: Option[String], dad: Int, mom: Int, sex: Option[Sex], pheno: Option[Phenotype])

object Pedigree {

  def read(filename: String, hConf: hadoop.conf.Configuration, sampleIds: IndexedSeq[String]): Pedigree = {
    if (!filename.endsWith(".fam"))
      fatal("-f | --fam filename must end in .fam")

    val sampleIndex: Map[String, Int] = sampleIds.zipWithIndex.toMap

    var nSamplesDiscarded = 0

    // .fam samples not in sampleIds are discarded
    readFile(filename, hConf) { s =>
      val sampleSet = collection.mutable.Set[Int]()

      val trios = Source.fromInputStream(s)
        .getLines()
        .filter(line => !line.isEmpty)
        .flatMap{ line => // FIXME: check that pedigree makes sense (e.g., cannot be own parent)
          val Array(fam, kid, dad, mom, sex, pheno) = line.split("\t")
          sampleIndex.get(kid) match {
            case Some(s) =>
              if (sampleSet(s))
                fatal(s".fam sample name is not unique: $kid")
              else
                sampleSet += s
              Some(Trio(
                s,
                if (fam != "0") Some(fam) else None,
                if (dad != "0") sampleIndex.get(dad) else None,
                if (mom != "0") sampleIndex.get(mom) else None,
                Sex.withNameOption(sex),
                Phenotype.withNameOption(pheno)))
            case None =>
              nSamplesDiscarded += 1
              None
          }
        }.toArray

      if (nSamplesDiscarded > 0)
        warn(s"$nSamplesDiscarded ${plural(nSamplesDiscarded, "sample")} discarded from .fam: missing from variant data set.")

      Pedigree(trios)
    }
  }<commit_msg>changed .fam back to whitespace delimited<commit_after>package org.broadinstitute.hail.methods

import org.apache.hadoop
import org.broadinstitute.hail.Utils._
import org.broadinstitute.hail.variant.{Sex, Phenotype}
import org.broadinstitute.hail.variant.Phenotype.{Phenotype, Case, Control}
import org.broadinstitute.hail.variant.Sex.{Sex, Male, Female}
import scala.io.Source

object Role extends Enumeration {
  type Role = Value
  val Kid = Value("0")
  val Dad = Value("1")
  val Mom = Value("2")
}

case class Trio(kid: Int, fam: Option[String], dad: Option[Int], mom: Option[Int],
  sex: Option[Sex], pheno: Option[Phenotype]) {

  def toCompleteTrio: Option[CompleteTrio] =
    dad.flatMap(d =>
      mom.map(m => CompleteTrio(kid, fam, d, m, sex, pheno)))

  def isMale: Boolean = sex.contains(Male)

  def isFemale: Boolean = sex.contains(Female)

  def isCase: Boolean = pheno.contains(Case)

  def isControl: Boolean = pheno.contains(Control)

  def isComplete: Boolean = dad.isDefined && mom.isDefined
}

case class CompleteTrio(kid: Int, fam: Option[String], dad: Int, mom: Int, sex: Option[Sex], pheno: Option[Phenotype])

object Pedigree {

  def read(filename: String, hConf: hadoop.conf.Configuration, sampleIds: IndexedSeq[String]): Pedigree = {
    if (!filename.endsWith(".fam"))
      fatal("-f | --fam filename must end in .fam")

    val sampleIndex: Map[String, Int] = sampleIds.zipWithIndex.toMap

    var nSamplesDiscarded = 0

    // .fam samples not in sampleIds are discarded
    readFile(filename, hConf) { s =>
      val sampleSet = collection.mutable.Set[Int]()

      val trios = Source.fromInputStream(s)
        .getLines()
        .filter(line => !line.isEmpty)
        .flatMap{ line => // FIXME: check that pedigree makes sense (e.g., cannot be own parent)
          val splitLine = line.split("\\s+")  // FIXME: fails on names with spaces, will fix in PR for adding .fam to annotations by giving delimiter option
          fatalIf(splitLine.size != 6, s"Require 6 fields per line in .fam, but this line has ${splitLine.size}: $line")
          val Array(fam, kid, dad, mom, sex, pheno) = splitLine
          sampleIndex.get(kid) match {
            case Some(s) =>
              if (sampleSet(s))
                fatal(s".fam sample name is not unique: $kid")
              else
                sampleSet += s
              Some(Trio(
                s,
                if (fam != "0") Some(fam) else None,
                if (dad != "0") sampleIndex.get(dad) else None,
                if (mom != "0") sampleIndex.get(mom) else None,
                Sex.withNameOption(sex),
                Phenotype.withNameOption(pheno)))
            case None =>
              nSamplesDiscarded += 1
              None
          }
        }.toArray

      if (nSamplesDiscarded > 0)
        warn(s"$nSamplesDiscarded ${plural(nSamplesDiscarded, "sample")} discarded from .fam: missing from variant data set.")

      Pedigree(trios)
    }
  }
{'fraction_non_alphanumeric': 0.12421163279607568, 'fraction_numerical': 0.00350385423966363, 'mean_word_length': 2.683225806451613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5661931', 'n_tokens_mistral': 1896, 'n_tokens_neox': 1771, 'n_words': 491}