<commit_before>package com.socrata.pg.store

import org.scalatest.{Matchers, BeforeAndAfterAll, FunSuite}

import java.sql.Connection
import com.socrata.soql.types._
import com.socrata.datacoordinator.id._
import com.socrata.datacoordinator.truth.metadata._
import com.socrata.soql.environment.TypeName
import org.postgresql.util.PSQLException
import com.socrata.datacoordinator.util.collection.ColumnIdMap
import com.socrata.datacoordinator.truth.universe.sql.SqlTableCleanup

/**
 *
 */
class PGSecondaryUniverseTest extends FunSuite with Matchers with BeforeAndAfterAll
      with PGSecondaryUniverseTestBase with DatabaseTestBase with PGStoreTestBase {

  override def beforeAll() {
    createDatabases()
  }

  test("Universe can create a table") {
    withDb() { conn =>
      val (pgu, copyInfo, _) = createTable(conn:Connection)
      val blankTableSchema = getSchema(pgu, copyInfo)
      assert(blankTableSchema.size == 0, "We expect no columns")
    }
  }

  test("Universe can add columns") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)

      val cols = SoQLType.typesByName filterKeys (!Set(TypeName("json")).contains(_)) map {
        case (n, t) => pgu.datasetMapWriter.addColumn(copyInfo, new UserColumnId(n + "_USERNAME"), t, n + "_PHYSNAME")
      }
      sLoader.addColumns(cols)

      // if you want to examine the tables...
      //pgu.commit
      validateSchema(cols, getSchema(pgu, copyInfo))
    }
  }

  ignore("Universe can del columns") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)
      val types = SoQLType.typesByName filterKeys (!Set(TypeName("json")).contains(_))

      val cols = types map {
        case (n, t) => pgu.datasetMapWriter.addColumn(copyInfo, new UserColumnId(n + "_USERNAME"), t, n + "_PHYSNAME")
      }
      sLoader.addColumns(cols)

      validateSchema(cols, getSchema(pgu, copyInfo))
      // +1 since location maps to lat and lon columns, so this test is pretty fragile in the face of new data types
      assert(jdbcColumnCount(conn, copyInfo.dataTableName) == cols.size+1, s"Expected table to have ${cols.size+1} columns")

      cols.foreach(pgu.datasetMapWriter.dropColumn(_))

      try {
        sLoader.dropColumns(cols)
      } catch {
        case pex:PSQLException => println("Failing Query: " + pex.getServerErrorMessage.getHint + " - " + pex.getSQLState); throw pex
      }

      assert(getSchema(pgu, copyInfo).size == 0, "We expect no columns");
      assert(jdbcColumnCount(conn, copyInfo.dataTableName) == 0, s"Expected table to have no columns")
    }
  }

  test("Universe can insert rows") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)
      val schema = createTableWithSchema(pgu, copyInfo, sLoader)

      // Setup our row data
      val dummyVals = dummyValues()
      insertDummyRow(new RowId(0), dummyVals, pgu, copyInfo, schema)

      val result = getRow(new RowId(0), pgu, copyInfo, schema)
      assert(result.size == 1)
      val row = result.get(SoQLID(0)).get
      val rowValues = row.row.values.toSet
<commit_msg>Re-enable delete column test.<commit_after>package com.socrata.pg.store

import org.scalatest.{Matchers, BeforeAndAfterAll, FunSuite}

import java.sql.Connection
import com.socrata.soql.types._
import com.socrata.datacoordinator.id._
import com.socrata.datacoordinator.truth.metadata._
import com.socrata.soql.environment.TypeName
import org.postgresql.util.PSQLException
import com.socrata.datacoordinator.util.collection.ColumnIdMap
import com.socrata.datacoordinator.truth.universe.sql.SqlTableCleanup

/**
 *
 */
class PGSecondaryUniverseTest extends FunSuite with Matchers with BeforeAndAfterAll
      with PGSecondaryUniverseTestBase with DatabaseTestBase with PGStoreTestBase {

  override def beforeAll() {
    createDatabases()
  }

  test("Universe can create a table") {
    withDb() { conn =>
      val (pgu, copyInfo, _) = createTable(conn:Connection)
      val blankTableSchema = getSchema(pgu, copyInfo)
      assert(blankTableSchema.size == 0, "We expect no columns")
    }
  }

  test("Universe can add columns") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)

      val cols = SoQLType.typesByName filterKeys (!Set(TypeName("json")).contains(_)) map {
        case (n, t) => pgu.datasetMapWriter.addColumn(copyInfo, new UserColumnId(n + "_USERNAME"), t, n + "_PHYSNAME")
      }
      sLoader.addColumns(cols)

      // if you want to examine the tables...
      //pgu.commit
      validateSchema(cols, getSchema(pgu, copyInfo))
    }
  }

  test("Universe can del columns") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)
      val types = SoQLType.typesByName filterKeys (!Set(TypeName("json")).contains(_))

      val cols = types map {
        case (n, t) => pgu.datasetMapWriter.addColumn(copyInfo, new UserColumnId(n + "_USERNAME"), t, n + "_PHYSNAME")
      }
      sLoader.addColumns(cols)

      validateSchema(cols, getSchema(pgu, copyInfo))
      // +1 since location maps to lat and lon columns, so this test is pretty fragile in the face of new data types
      assert(jdbcColumnCount(conn, copyInfo.dataTableName) == cols.size+1, s"Expected table to have ${cols.size+1} columns")

      cols.foreach(pgu.datasetMapWriter.dropColumn(_))

      try {
        sLoader.dropColumns(cols)
      } catch {
        case pex:PSQLException => println("Failing Query: " + pex.getServerErrorMessage.getHint + " - " + pex.getSQLState); throw pex
      }

      assert(getSchema(pgu, copyInfo).size == 0, "We expect no columns")
      assert(jdbcColumnCount(conn, copyInfo.dataTableName) == 0, s"Expected table to have no columns")
    }
  }

  test("Universe can insert rows") {
    withDb() { conn =>
      val (pgu, copyInfo, sLoader) = createTable(conn:Connection)
      val schema = createTableWithSchema(pgu, copyInfo, sLoader)

      // Setup our row data
      val dummyVals = dummyValues()
      insertDummyRow(new RowId(0), dummyVals, pgu, copyInfo, schema)

      val result = getRow(new RowId(0), pgu, copyInfo, schema)
      assert(result.size == 1)
      val row = result.get(SoQLID(0)).get
      val rowValues = row.row.values.toSet

{'fraction_non_alphanumeric': 0.11647862704592404, 'fraction_numerical': 0.0031781344350866043, 'mean_word_length': 3.5280575539568346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6024174', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1979, 'n_words': 477}