<commit_before>package turksem.iqa

import turksem.util.ContiguousSpan

import cats.implicits._

import nlpdata.datasets.wiktionary.InflectedForms
import nlpdata.util.LowerCaseStrings._
import nlpdata.util.Text

import monocle.macros.Lenses
import monocle.macros.GenPrism

@Lenses case class QuestioningState(
  sentence: Vector[InflectionalWord],
  triggerGroups: Vector[TriggerGroup]
) {

  val entityStrings = triggerGroups.zipWithIndex.flatMap {
    case (group, groupIndex) => group.qas.zipWithIndex.flatMap {
      case (qa, questionIndex) => qa.judgment.getAnswer.map(
        a => (groupIndex -> questionIndex) -> a.standin
      )
    }
  }.toMap

  // TODO more general for fancy question stuff. maybe need to make changes in TemplatedQuestion
  def renderQuestion(groupIndex: Int, qaIndex: Int): Option[String] = {
    triggerGroups.lift(groupIndex).flatMap(_.qas.lift(qaIndex)).map(
      _.question.renderQuestion(
        sentence, entityStrings.collect {
          case ((`groupIndex`, qaIndex), s) => qaIndex -> s
        })
    )
  }
}
object QuestioningState {
  def initFromSentence(sentence: Vector[InflectionalWord]) = {
    QuestioningState(
      sentence,
      sentence.filter(_.pos != "IN").flatMap { w => // skipping prepositions for now
        allTemplates.getGroupForPosTag(w.pos).map { triggerTemplates =>
          TriggerGroup(w, triggerTemplates, Vector.empty[TemplatedQA])
        }
      }
    )
  }
  def empty = QuestioningState(Vector(), Vector())
}

@Lenses case class TriggerGroup(
  trigger: InflectionalWord,
  templates: List[(Template, Int)],
  qas: Vector[TemplatedQA]) { // map from original index
                                // TODO calculate alignments
}

@Lenses case class TemplatedQA(
  question: TemplatedQuestion,
  judgment: QuestionJudgment)
<commit_msg>Fix bad uninflectable trigger groups<commit_after>package turksem.iqa

import turksem.util.ContiguousSpan

import cats.implicits._

import nlpdata.datasets.wiktionary.InflectedForms
import nlpdata.util.LowerCaseStrings._
import nlpdata.util.Text

import monocle.macros.Lenses
import monocle.macros.GenPrism

@Lenses case class QuestioningState(
  sentence: Vector[InflectionalWord],
  triggerGroups: Vector[TriggerGroup]
) {

  val entityStrings = triggerGroups.zipWithIndex.flatMap {
    case (group, groupIndex) => group.qas.zipWithIndex.flatMap {
      case (qa, questionIndex) => qa.judgment.getAnswer.map(
        a => (groupIndex -> questionIndex) -> a.standin
      )
    }
  }.toMap

  // TODO more general for fancy question stuff. maybe need to make changes in TemplatedQuestion
  def renderQuestion(groupIndex: Int, qaIndex: Int): Option[String] = {
    triggerGroups.lift(groupIndex).flatMap(_.qas.lift(qaIndex)).map(
      _.question.renderQuestion(
        sentence, entityStrings.collect {
          case ((`groupIndex`, qaIndex), s) => qaIndex -> s
        })
    )
  }
}
object QuestioningState {
  def initFromSentence(sentence: Vector[InflectionalWord]) = {
    QuestioningState(
      sentence,
      sentence.filter(w => w.pos != "IN" && w.inflectedFormsOpt.nonEmpty)
        .flatMap { w => // skipping prepositions for now
        allTemplates.getGroupForPosTag(w.pos).map { triggerTemplates =>
          TriggerGroup(w, triggerTemplates, Vector.empty[TemplatedQA])
        }
      }
    )
  }
  def empty = QuestioningState(Vector(), Vector())
}

@Lenses case class TriggerGroup(
  trigger: InflectionalWord,
  templates: List[(Template, Int)],
  qas: Vector[TemplatedQA]) { // map from original index
                                // TODO calculate alignments
}

@Lenses case class TemplatedQA(
  question: TemplatedQuestion,
  judgment: QuestionJudgment)

{'fraction_non_alphanumeric': 0.10603191777116582, 'fraction_numerical': 0.0, 'mean_word_length': 3.455421686746988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2892034', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1122, 'n_words': 245}