<commit_before>package ingest

import java.io.File
import scala.concurrent.Future
import macros._
import macros.async._
import dbrary._
import models._
import store.Stage
import site.Site

object Adolph extends Ingest {
  import Parse._

  private type ObjectParser[A] = Parser[A => A]
  private object ObjectParser {
    def apply[A](f : (A, String) => A) : ObjectParser[A] =
      Parser[A => A](s => f(_, s))
    def map[A,B](f : (A, B) => A, p : Parser[B]) : ObjectParser[A] =
      p.map(b => f(_, b))
    def option[A,B](f : (A, B) => A, p : Parser[B], nullif : String = "") : ObjectParser[A] =
      Parser[A => A] { s =>
	if (nullif != null && (s.isEmpty || s.equals(nullif)))
	  identity[A]
	else
	  f(_, p(s))
      }
  }

  private class CellParser[A](name : String, parse : ObjectParser[A])
    extends ColumnParser[A => A](name, parse)
  
  private def blankCellParser[A] : CellParser[A] =
    new CellParser[A]("<blank>", const[A => A](identity[A] _))

  private def parseCells[A](a : A, ps : Seq[CellParser[A]]) : ListParser[A] =
    ListParser[A] { l =>
      ps.foldLeft((a, l)) { (al, p) =>
	val (a, l) = al
	p.map(_(a))(l)
      }
    }

  private def measureParser[T](metric : Metric[T], p : Parser[T]) : Parser[MeasureV[T]] =
    Parser[MeasureV[T]](s => new MeasureV(metric, p(s)))

  private def dateMeasureParser(metric : Metric[Date]) =
    measureParser[Date](metric, date)

  private def measureMap(m : Measure[_]*) : Map[Int,Measure[_]] =
    Map(m.map(m => (m.metricId.unId, m)) : _*)

  private abstract class Record(val category : RecordCategory) {
    protected val measures : Map[Int,Measure[_]]
    protected def idents : Iterable[Measure[_]]
    type Key
    def key : Key

    override def toString = category.name + ":" + idents.map(m => m.metric.name + ":" + m.datum).mkString(",")

    def getMeasure(m : Metric[_]) : Option[Measure[_]] =
      measures.get(m.id.unId)
    protected final def addMeasure(m : Measure[_]) : Map[Int,Measure[_]] = {
      val i = m.metricId.unId
      if (measures.contains(i))
	fail("duplicate measure for " + m.metric.name + " on " + this)
      measures.updated(i, m)
    }
    def withMeasure(m : Measure[_]) : Record

    def find(volume : Volume) : Future[Option[models.Record]] =
      for {
	l <- Record.findMeasures(volume, Some(category), idents.toSeq : _*)
	_ <- check(l.length <= 1,
	  PopulateException("multiple matching records for " + this, volume))
      }	yield (l.headOption)
    def populate(volume : Volume, current : Option[models.Record] = None) : Future[models.Record] =
      for {
	ro <- current orElseAsync find(volume)
	r <- ro getOrElseAsync Record.create(volume, Some(category))
	_ <- measures.values foreachAsync { m =>
	  r.measures(m.metric).fold {
	    r.setMeasure(m).flatMap(check(_, 
	      PopulateException("failed to set measure for record " + this, r)))
	  } { c =>
	    check(c === m,
	      PopulateException("inconsistent mesaures for record " + this + ": " + m + " <> " + c, r))
	  }
	}
      } yield (r)
  }

  private final case class SingletonRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = Nil
    type Key = Unit
    def key = ()
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class IdentRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = getMeasure(Metric.Ident)
    type Key = String
    def key = getMeasure(Metric.Ident).fold(fail(category.name + " ident missing"))(_.datum)
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class TotalRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = measures.values
    type Key = Map[Int,String]
    def key = measures.mapValues(_.datum)
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class Participant(measures : Map[Int,Measure[_]] = Map.empty)
    extends Record(RecordCategory.Participant) {
    def idents = getMeasure(Metric.Ident) ++ getMeasure(Metric.Info)
    type Key = (String, String)
    def key = (getMeasure(Metric.Ident).fold(fail("participant ID missing"))(_.datum), getMeasure(Metric.Info).fold("")(_.datum))
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private type ParticipantMap = Map[(String, String), Participant]

  private object Participants {
    private final val empty = Participant()
    private def measure[T](m : Parser[MeasureV[T]], nullif : String = "") : Parser[Participant => Participant] =
      ObjectParser.option[Participant,MeasureV[T]](_.withMeasure(_), m, nullif)
    val parseId = measure(measureParser(Metric.Ident, trimmed), null)
    val parseSet = measure(measureParser(Metric.Info, trimmed))
    private[Adolph] def parseHeader(name : String) : Parser[Participant => Participant] =
      name match {
	case "SUBJECT ID" => parseId
	case "DATASET" => parseSet
	case "BIRTH DATE" => measure(dateMeasureParser(Metric.Birthdate), null)
	case "GENDER" => measure(Gender.measureParse)
	case "RACE" => measure(Race.measureParse)
	case "ETHNICITY" => measure(Ethnicity.measureParse)
	case "TYPICAL DEVELOPMENT/DISABILITY" => measure(measureParser(Metric.Disability, trimmed), "typical")
	case "LANGUAGE" => measure(measureParser(Metric.Language, trimmed))
	case _ => fail("unknown header: " + name)
      }
    private def header : Parser[CellParser[Participant]] =
      Parser[CellParser[Participant]] {
	case "" => blankCellParser
	case s => new CellParser(s, parseHeader(s))
      }
    private def parseData(l : List[List[String]]) : ParticipantMap = l.zipWithIndex match {
      case h :: l =>
	val p = listAll(header).run(h)
	val line = parseCells(empty, p)
	l.foldLeft[ParticipantMap](Map.empty) { (m, l) =>
	  val p = line.run(l)
	  val i = p.key
	  if (m.contains(i))
	    throw ParseException("duplicate participant key: " + i, line = l._2)
	  m.updated(i, p)
	}
      case Nil => Map.empty
    }
    final def parseCSV(f : File) =
      parseData(CSV.parseFile(f))
  }

  private final case class Exclusion(measures : Map[Int,Measure[_]] = Map.empty) extends Record(RecordCategory.Exclusion) {
    def idents = getMeasure(Metric.Reason)<commit_msg>Allow as-in race ingestion (no validation)<commit_after>package ingest

import java.io.File
import scala.concurrent.Future
import macros._
import macros.async._
import dbrary._
import models._
import store.Stage
import site.Site

object Adolph extends Ingest {
  import Parse._

  private type ObjectParser[A] = Parser[A => A]
  private object ObjectParser {
    def apply[A](f : (A, String) => A) : ObjectParser[A] =
      Parser[A => A](s => f(_, s))
    def map[A,B](f : (A, B) => A, p : Parser[B]) : ObjectParser[A] =
      p.map(b => f(_, b))
    def option[A,B](f : (A, B) => A, p : Parser[B], nullif : String = "") : ObjectParser[A] =
      Parser[A => A] { s =>
	if (nullif != null && (s.isEmpty || s.equals(nullif)))
	  identity[A]
	else
	  f(_, p(s))
      }
  }

  private class CellParser[A](name : String, parse : ObjectParser[A])
    extends ColumnParser[A => A](name, parse)
  
  private def blankCellParser[A] : CellParser[A] =
    new CellParser[A]("<blank>", const[A => A](identity[A] _))

  private def parseCells[A](a : A, ps : Seq[CellParser[A]]) : ListParser[A] =
    ListParser[A] { l =>
      ps.foldLeft((a, l)) { (al, p) =>
	val (a, l) = al
	p.map(_(a))(l)
      }
    }

  private def measureParser[T](metric : Metric[T], p : Parser[T]) : Parser[MeasureV[T]] =
    Parser[MeasureV[T]](s => new MeasureV(metric, p(s)))

  private def dateMeasureParser(metric : Metric[Date]) =
    measureParser[Date](metric, date)

  private def measureMap(m : Measure[_]*) : Map[Int,Measure[_]] =
    Map(m.map(m => (m.metricId.unId, m)) : _*)

  private abstract class Record(val category : RecordCategory) {
    protected val measures : Map[Int,Measure[_]]
    protected def idents : Iterable[Measure[_]]
    type Key
    def key : Key

    override def toString = category.name + ":" + idents.map(m => m.metric.name + ":" + m.datum).mkString(",")

    def getMeasure(m : Metric[_]) : Option[Measure[_]] =
      measures.get(m.id.unId)
    protected final def addMeasure(m : Measure[_]) : Map[Int,Measure[_]] = {
      val i = m.metricId.unId
      if (measures.contains(i))
	fail("duplicate measure for " + m.metric.name + " on " + this)
      measures.updated(i, m)
    }
    def withMeasure(m : Measure[_]) : Record

    def find(volume : Volume) : Future[Option[models.Record]] =
      for {
	l <- Record.findMeasures(volume, Some(category), idents.toSeq : _*)
	_ <- check(l.length <= 1,
	  PopulateException("multiple matching records for " + this, volume))
      }	yield (l.headOption)
    def populate(volume : Volume, current : Option[models.Record] = None) : Future[models.Record] =
      for {
	ro <- current orElseAsync find(volume)
	r <- ro getOrElseAsync Record.create(volume, Some(category))
	_ <- measures.values foreachAsync { m =>
	  r.measures(m.metric).fold {
	    r.setMeasure(m).flatMap(check(_, 
	      PopulateException("failed to set measure for record " + this, r)))
	  } { c =>
	    check(c === m,
	      PopulateException("inconsistent mesaures for record " + this + ": " + m + " <> " + c, r))
	  }
	}
      } yield (r)
  }

  private final case class SingletonRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = Nil
    type Key = Unit
    def key = ()
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class IdentRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = getMeasure(Metric.Ident)
    type Key = String
    def key = getMeasure(Metric.Ident).fold(fail(category.name + " ident missing"))(_.datum)
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class TotalRecord(override val category : RecordCategory, measures : Map[Int,Measure[_]] = Map.empty) extends Record(category) {
    def idents = measures.values
    type Key = Map[Int,String]
    def key = measures.mapValues(_.datum)
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private final case class Participant(measures : Map[Int,Measure[_]] = Map.empty)
    extends Record(RecordCategory.Participant) {
    def idents = getMeasure(Metric.Ident) ++ getMeasure(Metric.Info)
    type Key = (String, String)
    def key = (getMeasure(Metric.Ident).fold(fail("participant ID missing"))(_.datum), getMeasure(Metric.Info).fold("")(_.datum))
    def withMeasure(m : Measure[_]) = copy(measures = addMeasure(m))
  }

  private type ParticipantMap = Map[(String, String), Participant]

  private object Participants {
    private final val empty = Participant()
    private def measure[T](m : Parser[MeasureV[T]], nullif : String = "") : Parser[Participant => Participant] =
      ObjectParser.option[Participant,MeasureV[T]](_.withMeasure(_), m, nullif)
    val parseId = measure(measureParser(Metric.Ident, trimmed), null)
    val parseSet = measure(measureParser(Metric.Info, trimmed))
    private[Adolph] def parseHeader(name : String) : Parser[Participant => Participant] =
      name match {
	case "SUBJECT ID" => parseId
	case "DATASET" => parseSet
	case "BIRTH DATE" => measure(dateMeasureParser(Metric.Birthdate), null)
	case "GENDER" => measure(Gender.measureParse)
	case "RACE" => measure(Race.measureParse)
	case "RACE (AS-IS)" => measure(measureParser(Metric.Race, trimmed))
	case "ETHNICITY" => measure(Ethnicity.measureParse)
	case "TYPICAL DEVELOPMENT/DISABILITY" => measure(measureParser(Metric.Disability, trimmed), "typical")
	case "LANGUAGE" => measure(measureParser(Metric.Language, trimmed))
	case _ => fail("unknown header: " + name)
      }
    private def header : Parser[CellParser[Participant]] =
      Parser[CellParser[Participant]] {
	case "" => blankCellParser
	case s => new CellParser(s, parseHeader(s))
      }
    private def parseData(l : List[List[String]]) : ParticipantMap = l.zipWithIndex match {
      case h :: l =>
	val p = listAll(header).run(h)
	val line = parseCells(empty, p)
	l.foldLeft[ParticipantMap](Map.empty) { (m, l) =>
	  val p = line.run(l)
	  val i = p.key
	  if (m.contains(i))
	    throw ParseException("duplicate participant key: " + i, line = l._2)
	  m.updated(i, p)
	}
      case Nil => Map.empty
    }
    final def parseCSV(f : File) =
      parseData(CSV.parseFile(f))
  }

  private final case class Exclusion(measures : Map[Int,Measure[_]] = Map.empty) extends Record(RecordCategory.Exclusion) {
    def idents = getMeasure(Metric.Reason)
{'fraction_non_alphanumeric': 0.14973594283939112, 'fraction_numerical': 0.00031065548306927616, 'mean_word_length': 3.7905505952380953, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6682534', 'n_tokens_mistral': 4351, 'n_tokens_neox': 4185, 'n_words': 1025}