<commit_before>package ch.unibas.dmi.dbis.adam.entity

import ch.unibas.dmi.dbis.adam.config.FieldNames
import ch.unibas.dmi.dbis.adam.entity.Entity.EntityName
import ch.unibas.dmi.dbis.adam.entity.FieldTypes.LONGTYPE
import ch.unibas.dmi.dbis.adam.exception.{EntityExistingException, EntityNotExistingException, EntityNotProperlyDefinedException}
import ch.unibas.dmi.dbis.adam.index.IndexHandler
import ch.unibas.dmi.dbis.adam.main.{AdamContext, SparkStartup}
import ch.unibas.dmi.dbis.adam.storage.engine.CatalogOperator
import org.apache.log4j.Logger
import org.apache.spark.sql.DataFrame

import scala.util.{Failure, Success, Try}

/**
  * adampro
  *
  * <NAME>
  * April 2016
  */
object EntityHandler {
  val log = Logger.getLogger(getClass.getName)

  private val featureStorage = SparkStartup.featureStorage
  private val metadataStorage = SparkStartup.metadataStorage

  def exists(entityname: EntityName): Boolean = CatalogOperator.existsEntity(entityname)

  /**
    * Creates an entity.
    *
    * @param entityname
    * @param fields if fields is specified, in the metadata storage a table is created with these names, specify fields
    *               as key = name, value = SQL type, note the reserved names
    * @return
    */
  def create(entityname: EntityName, fields: Option[Seq[FieldDefinition]] = None)(implicit ac: AdamContext): Try[Entity] = {
    if (exists(entityname)) {
      log.error("entity " + entityname + " exists already")
      return Failure(EntityExistingException())
    }

    featureStorage.create(entityname)

    if (fields.isDefined) {
      FieldNames.reservedNames.foreach { reservedName =>
        if (fields.get.contains(reservedName)) {
          log.error("entity defined with field " + FieldNames.idColumnName + ", but name is reserved")
          return Failure(EntityNotProperlyDefinedException())
        }
      }

      if (fields.get.map(_.name).distinct.length != fields.get.length) {
        log.error("entity defined with duplicate fields")
        return Failure(EntityNotProperlyDefinedException())
      }

      if (fields.get.filter(_.pk).length > 1) {
        log.error("entity defined with more than one primary key")
        return Failure(EntityNotProperlyDefinedException())
      }

      val fieldsWithId = fields.get.+:(FieldDefinition(FieldNames.idColumnName, LONGTYPE, false, true, true))
      metadataStorage.create(entityname, fieldsWithId)
      CatalogOperator.createEntity(entityname, true)
      Success(Entity(entityname, featureStorage, Option(metadataStorage)))
    } else {
      CatalogOperator.createEntity(entityname, false)
      Success(Entity(entityname, featureStorage, None))
    }
  }

  /**
    * Drops an entity.
    *
    * @param entityname
    * @param ifExists
    * @return
    */
  def drop(entityname: EntityName, ifExists: Boolean = false)(implicit ac: AdamContext): Try[Null] = {
    if (!exists(entityname)) {
      if (!ifExists) {
        return Failure(EntityNotExistingException())
      } else {
        Success(null)
      }
    }
<commit_msg>separate check and performing in creating entity<commit_after>package ch.unibas.dmi.dbis.adam.entity

import ch.unibas.dmi.dbis.adam.config.FieldNames
import ch.unibas.dmi.dbis.adam.entity.Entity.EntityName
import ch.unibas.dmi.dbis.adam.entity.FieldTypes.LONGTYPE
import ch.unibas.dmi.dbis.adam.exception.{EntityExistingException, EntityNotExistingException, EntityNotProperlyDefinedException}
import ch.unibas.dmi.dbis.adam.index.IndexHandler
import ch.unibas.dmi.dbis.adam.main.{AdamContext, SparkStartup}
import ch.unibas.dmi.dbis.adam.storage.engine.CatalogOperator
import org.apache.log4j.Logger
import org.apache.spark.sql.DataFrame

import scala.util.{Failure, Success, Try}

/**
  * adampro
  *
  * <NAME>
  * April 2016
  */
object EntityHandler {
  val log = Logger.getLogger(getClass.getName)

  private val featureStorage = SparkStartup.featureStorage
  private val metadataStorage = SparkStartup.metadataStorage

  def exists(entityname: EntityName): Boolean = CatalogOperator.existsEntity(entityname)

  /**
    * Creates an entity.
    *
    * @param entityname
    * @param fields if fields is specified, in the metadata storage a table is created with these names, specify fields
    *               as key = name, value = SQL type, note the reserved names
    * @return
    */
  def create(entityname: EntityName, fields: Option[Seq[FieldDefinition]] = None)(implicit ac: AdamContext): Try[Entity] = {
    //checks
    if (exists(entityname)) {
      log.error("entity " + entityname + " exists already")
      return Failure(EntityExistingException())
    }

    if (fields.isDefined) {
      FieldNames.reservedNames.foreach { reservedName =>
        if (fields.get.contains(reservedName)) {
          log.error("entity defined with field " + FieldNames.idColumnName + ", but name is reserved")
          return Failure(EntityNotProperlyDefinedException())
        }
      }

      if (fields.get.map(_.name).distinct.length != fields.get.length) {
        log.error("entity defined with duplicate fields")
        return Failure(EntityNotProperlyDefinedException())
      }

      if (fields.get.filter(_.pk).length > 1) {
        log.error("entity defined with more than one primary key")
        return Failure(EntityNotProperlyDefinedException())
      }
    }

    //perform
    featureStorage.create(entityname)

    if (fields.isDefined) {
      val fieldsWithId = fields.get.+:(FieldDefinition(FieldNames.idColumnName, LONGTYPE, false, true, true))
      metadataStorage.create(entityname, fieldsWithId)
      CatalogOperator.createEntity(entityname, true)
      Success(Entity(entityname, featureStorage, Option(metadataStorage)))
    } else {
      CatalogOperator.createEntity(entityname, false)
      Success(Entity(entityname, featureStorage, None))
    }
  }

  /**
    * Drops an entity.
    *
    * @param entityname
    * @param ifExists
    * @return
    */
  def drop(entityname: EntityName, ifExists: Boolean = false)(implicit ac: AdamContext): Try[Null] = {
    if (!exists(entityname)) {
      if (!ifExists) {
        return Failure(EntityNotExistingException())
      } else {
        Success(null)
      }
    }

{'fraction_non_alphanumeric': 0.10819935691318328, 'fraction_numerical': 0.0019292604501607716, 'mean_word_length': 3.4755395683453236, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3836482', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1839, 'n_words': 384}