<commit_before>package ch.unibas.dmi.dbis.adam.storage.engine

import java.sql.{Connection, DriverManager}
import java.util.Properties

import ch.unibas.dmi.dbis.adam.config.{FieldNames, AdamConfig}
import ch.unibas.dmi.dbis.adam.entity.Entity.EntityName
import ch.unibas.dmi.dbis.adam.main.SparkStartup
import ch.unibas.dmi.dbis.adam.storage.components.MetadataStorage
import org.apache.spark.sql.jdbc.AdamDialectRegistrar
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{DataFrame, Row, SaveMode}

/**
 * adamtwo
 *
 * <NAME>
 * October 2015
 */
object PostgresMetadataStorage extends MetadataStorage {
  val url = AdamConfig.jdbcUrl
  AdamDialectRegistrar.register(url)

  /**
    * Opens a connection to the PostgreSQL database.
    *
    * @return
    */
  private def openConnection(): Connection ={
    Class.forName("org.postgresql.Driver").newInstance
    DriverManager.getConnection(AdamConfig.jdbcUrl, AdamConfig.jdbcUser, AdamConfig.jdbcPassword)
  }

  override def create(tablename: EntityName, fields: Map[String, DataType]): Boolean = {
    val structFields = fields.map{
      case (name, fieldtype) => StructField(name, fieldtype)
    }.toSeq

    val df = SparkStartup.sqlContext.createDataFrame(SparkStartup.sc.emptyRDD[Row], StructType(structFields))

    write(tablename, df, SaveMode.ErrorIfExists)

    true
  }

  override def read(tablename: EntityName): DataFrame = {
    SparkStartup.sqlContext.read.format("jdbc").options(
      Map("url" -> url, "dbtable" -> tablename, "user" -> AdamConfig.jdbcUser, "password" -> AdamConfig.jdbcPassword)
    ).load()
  }

  override def write(tablename: EntityName, df: DataFrame, mode: SaveMode = SaveMode.Append): Boolean = {
    val props = new Properties()
    props.put("user", AdamConfig.jdbcUser)
    props.put("password", AdamConfig.jdbcPassword)
    df.write.mode(mode).jdbc(url, tablename, props)
    true
  }

  override def drop(tablename: EntityName): Boolean = {
    SparkStartup.sqlContext.read.format("jdbc").options(
      Map("url" -> url, "dbtable" -> tablename, "user" -> AdamConfig.jdbcUser, "password" -> AdamConfig.jdbcPassword)
    ).load()
    true
  }
}<commit_msg>make id field unique and add index over it<commit_after>package ch.unibas.dmi.dbis.adam.storage.engine

import java.sql.{Connection, DriverManager}
import java.util.Properties

import ch.unibas.dmi.dbis.adam.config.{FieldNames, AdamConfig}
import ch.unibas.dmi.dbis.adam.entity.Entity.EntityName
import ch.unibas.dmi.dbis.adam.main.SparkStartup
import ch.unibas.dmi.dbis.adam.storage.components.MetadataStorage
import org.apache.spark.sql.jdbc.AdamDialectRegistrar
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{DataFrame, Row, SaveMode}

/**
 * adamtwo
 *
 * <NAME>
 * October 2015
 */
object PostgresMetadataStorage extends MetadataStorage {
  val url = AdamConfig.jdbcUrl
  AdamDialectRegistrar.register(url)

  /**
    * Opens a connection to the PostgreSQL database.
    *
    * @return
    */
  private def openConnection(): Connection ={
    Class.forName("org.postgresql.Driver").newInstance
    DriverManager.getConnection(AdamConfig.jdbcUrl, AdamConfig.jdbcUser, AdamConfig.jdbcPassword)
  }

  override def create(tablename: EntityName, fields: Map[String, DataType]): Boolean = {
    val structFields = fields.map{
      case (name, fieldtype) => StructField(name, fieldtype)
    }.toSeq

    val df = SparkStartup.sqlContext.createDataFrame(SparkStartup.sc.emptyRDD[Row], StructType(structFields))

    write(tablename, df, SaveMode.ErrorIfExists)

    //make id field unique
    val idColumnName = FieldNames.idColumnName
    val alterAddIdSql = s"""
                   |ALTER TABLE $tablename
                   | ADD UNIQUE ($idColumnName)
                   |""".stripMargin

    openConnection().createStatement().executeUpdate(alterAddIdSql)

    //crate index over id field
    val idColumnIndexSql = s"""CREATE INDEX ON $tablename ($idColumnName)""".stripMargin

    openConnection().createStatement().executeUpdate(idColumnIndexSql)

    true
  }

  override def read(tablename: EntityName): DataFrame = {
    SparkStartup.sqlContext.read.format("jdbc").options(
      Map("url" -> url, "dbtable" -> tablename, "user" -> AdamConfig.jdbcUser, "password" -> <PASSWORD>Config.jdbcPassword)
    ).load()
  }

  override def write(tablename: EntityName, df: DataFrame, mode: SaveMode = SaveMode.Append): Boolean = {
    val props = new Properties()
    props.put("user", AdamConfig.jdbcUser)
    props.put("password", <PASSWORD>)
    df.write.mode(mode).jdbc(url, tablename, props)
    true
  }

  override def drop(tablename: EntityName): Boolean = {
    SparkStartup.sqlContext.read.format("jdbc").options(
      Map("url" -> url, "dbtable" -> tablename, "user" -> AdamConfig.jdbcUser, "password" -> <PASSWORD>.jdbcPassword)
    ).load()
    true
  }
}
{'fraction_non_alphanumeric': 0.1261663286004057, 'fraction_numerical': 0.0016227180527383367, 'mean_word_length': 4.036772216547497, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114419', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1562, 'n_words': 303}