<commit_before>package mimir.lenses

import java.io._
import org.specs2.specification._
import org.specs2.mutable._

import mimir.algebra._
import mimir.test._
import mimir.models.FeedbackSource
import mimir.models.FeedbackSourceIdentifier
import mimir.statistics.FeedbackStats

object FeedbackSpec 
  extends SQLTestSpecification("FeedbackTests")
  with BeforeAll
{

  def beforeAll = 
  {
    update("CREATE TABLE R(A string, B int, C int)")
    loadCSV("R", new File("test/r_test/r.csv"))
    update("CREATE ADAPTIVE SCHEMA MATCH AS SELECT * FROM R WITH SCHEMA_MATCHING('B int', 'CX int')")
    db.adaptiveSchemas.create("R_TI", "TYPE_INFERENCE", db.table("R"), Seq(FloatPrimitive(.5))) 
		update("CREATE LENS MV AS SELECT * FROM R WITH MISSING_VALUE('B', 'C')")
  }

  "The Edit Distance Match Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MATCH:EDITDISTANCE:B")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(), List()) must be equalTo(str("B"))

      model.feedback(0, List(), str("C"))
      model.bestGuess(0, List(), List()) must be equalTo(str("C"))
    }

    "Support SQL Feedback" >> {
      val model = db.models.get("MATCH:EDITDISTANCE:CX")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(), List()) must be equalTo(str("C"))

      // Test Model C
      update("FEEDBACK MATCH:EDITDISTANCE:CX 0 IS 'B'")
      model.bestGuess(0, List(), List()) must be equalTo(str("B"))
    }

  }

  "The Type Inference Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MIMIR_TI_ATTR_R_TI")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(IntPrimitive(0)), List()) must be equalTo(TypePrimitive(TInt()))
      db.typechecker.schemaOf(db.adaptiveSchemas.viewFor("R_TI", "DATA").get).
        find(_._1.equals("A")).get._2 must be equalTo(TInt())

      model.feedback(0, List(IntPrimitive(0)), TypePrimitive(TFloat()))
      db.models.persist(model)
      model.bestGuess(0, List(IntPrimitive(0)), List()) must be equalTo(TypePrimitive(TFloat()))
      db.typechecker.schemaOf(db.adaptiveSchemas.viewFor("R_TI", "DATA").get).
        find(_._1.equals("A")).get._2 must be equalTo(TFloat())
    }
  }

  "The SparkML Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MV:SPARKML:B")
      val nullRow = querySingleton("SELECT ROWID() FROM R WHERE B IS NULL")

      model.bestGuess(0, List(nullRow), List()) must not be equalTo(50)
      model.feedback(0, List(nullRow), IntPrimitive(50))
      model.bestGuess(0, List(nullRow), List()).asLong must be equalTo(50)

    }<commit_msg>Bug fix in feedback spec<commit_after>package mimir.lenses

import java.io._
import org.specs2.specification._
import org.specs2.mutable._

import mimir.algebra._
import mimir.test._
import mimir.models.FeedbackSource
import mimir.models.FeedbackSourceIdentifier
import mimir.statistics.FeedbackStats

object FeedbackSpec 
  extends SQLTestSpecification("FeedbackTests")
  with BeforeAll
{

  def beforeAll = 
  {
    update("CREATE TABLE R(A string, B int, C int)")
    loadCSV("R", new File("test/r_test/r.csv"))
    update("CREATE ADAPTIVE SCHEMA MATCH AS SELECT * FROM R WITH SCHEMA_MATCHING('B int', 'CX int')")
    db.adaptiveSchemas.create("R_TI", "TYPE_INFERENCE", db.table("R"), Seq(FloatPrimitive(.5))) 
		update("CREATE LENS MV AS SELECT * FROM R WITH MISSING_VALUE('B', 'C')")
  }

  "The Edit Distance Match Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MATCH:EDITDISTANCE:B")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(), List()) must be equalTo(str("B"))

      model.feedback(0, List(), str("C"))
      model.bestGuess(0, List(), List()) must be equalTo(str("C"))
    }

    "Support SQL Feedback" >> {
      val model = db.models.get("MATCH:EDITDISTANCE:CX")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(), List()) must be equalTo(str("C"))

      // Test Model C
      update("FEEDBACK MATCH:EDITDISTANCE:CX 0 IS 'B'")
      model.bestGuess(0, List(), List()) must be equalTo(str("B"))
    }

  }

  "The Type Inference Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MIMIR_TI_ATTR_R_TI")

      // Base assumptions.  These may change, but the feedback tests 
      // below should be updated accordingly
      model.bestGuess(0, List(IntPrimitive(0)), List()) must be equalTo(TypePrimitive(TInt()))
      db.typechecker.schemaOf(db.adaptiveSchemas.viewFor("R_TI", "DATA").get).
        find(_._1.equals("A")).get._2 must be equalTo(TInt())

      model.feedback(0, List(IntPrimitive(0)), TypePrimitive(TFloat()))
      model.bestGuess(0, List(IntPrimitive(0)), List()) must be equalTo(TypePrimitive(TFloat()))
      db.typechecker.schemaOf(db.adaptiveSchemas.viewFor("R_TI", "DATA").get).
        find(_._1.equals("A")).get._2 must be equalTo(TFloat())
    }
  }

  "The SparkML Model" should {

    "Support direct feedback" >> {
      val model = db.models.get("MV:SPARKML:B")
      val nullRow = querySingleton("SELECT ROWID() FROM R WHERE B IS NULL")

      model.bestGuess(0, List(nullRow), List()) must not be equalTo(50)
      model.feedback(0, List(nullRow), IntPrimitive(50))
      model.bestGuess(0, List(nullRow), List()).asLong must be equalTo(50)

    }
{'fraction_non_alphanumeric': 0.14226086956521738, 'fraction_numerical': 0.009739130434782608, 'mean_word_length': 3.5788216560509554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '126234', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1852, 'n_words': 471}