<commit_before>/***********************************************************************
* Copyright (c) 2013-2016 Commonwealth Computer Research, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License, Version 2.0
* which accompanies this distribution and is available at
* http://www.opensource.org/licenses/apache2.0.php.
*************************************************************************/

package org.locationtech.geomesa.utils.geotools

import java.util.{Date, Locale, UUID}

import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions, ConfigValueFactory}
import com.vividsolutions.jts.geom._
import org.apache.commons.lang.StringEscapeUtils
import org.geotools.feature.AttributeTypeBuilder
import org.geotools.feature.simple.SimpleFeatureTypeBuilder
import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType._
import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.SpecParser.{ListAttributeType, MapAttributeType, SimpleAttributeType}
import org.locationtech.geomesa.utils.stats.{Cardinality, IndexCoverage}
import org.locationtech.geomesa.utils.text.EnhancedTokenParsers
import org.opengis.feature.`type`.{AttributeDescriptor, GeometryDescriptor}
import org.opengis.feature.simple.SimpleFeatureType

import scala.collection.JavaConversions._
import scala.util.parsing.combinator.JavaTokenParsers
import scala.util.{Failure, Success, Try}
object SimpleFeatureTypes {

  import org.locationtech.geomesa.utils.geotools.RichAttributeDescriptors._

  val TABLE_SPLITTER           = "table.splitter.class"
  val TABLE_SPLITTER_OPTIONS   = "table.splitter.options"
  val ENABLED_INDEXES          = "geomesa.indexes.enabled"
  val MIXED_GEOMETRIES         = "geomesa.mixed.geometries"
  val RESERVED_WORDS           = "override.reserved.words" // note: doesn't start with geomesa so we don't persist it

  @deprecated
  val ENABLED_INDEXES_OLD      = "table.indexes.enabled"

  val OPT_DEFAULT              = "default"
  val OPT_SRID                 = "srid"
  val OPT_INDEX_VALUE          = "index-value"
  val OPT_INDEX                = "index"
  val OPT_STATS                = "keep-stats"
  val OPT_CARDINALITY          = "cardinality"
  val OPT_BIN_TRACK_ID         = "bin-track-id"

  val OPTS = Seq(OPT_DEFAULT, OPT_SRID, OPT_INDEX, OPT_STATS, OPT_INDEX_VALUE, OPT_CARDINALITY, OPT_BIN_TRACK_ID)

  val USER_DATA_LIST_TYPE      = "subtype"
  val USER_DATA_MAP_KEY_TYPE   = "keyclass"
  val USER_DATA_MAP_VALUE_TYPE = "valueclass"

  val TypeNamePath   = "type-name"
  val FieldsPath     = "fields"
  val UserDataPath   = "user-data"
  val AttributesPath = "attributes"
  val TypePath       = "type"

  /**
   * Create a SimpleFeatureType from a typesafe Config
   *
   * @param conf config
   * @param typeName optional typename to use for the SFT...will be overridden if the config contains a type-name key
   * @return
   */
  def createType(conf: Config, typeName: Option[String] = None, path: Option[String] = Some("sft")): SimpleFeatureType = {
    import org.locationtech.geomesa.utils.conf.ConfConversions._
    val toParse = path match {
      case Some(p) => conf.getConfigOpt(p).map(conf.withFallback).getOrElse(conf)
      case None    => conf
    }

    val nameSpec = toParse.getStringOpt(TypeNamePath).orElse(typeName).getOrElse(
      throw new IllegalArgumentException("Unable to parse type name from provided argument or config")
    )
    val (namespace, name) = buildTypeName(nameSpec)
    val specParser = new SpecParser

    val fields = getFieldConfig(toParse).map(buildField(_, specParser))
    val userData = if (toParse.hasPath(UserDataPath)) {
      toParse.getConfig(UserDataPath).entrySet().map(e => e.getKey -> e.getValue.unwrapped()).toMap
    } else {
      Map.empty[String, AnyRef]
    }
    val opts = userData.map {
      // Special case to handle adding keywords
      case (KEYWORDS_KEY, v : java.util.ArrayList[String]) =>
        new GenericOption(KEYWORDS_KEY, v.asInstanceOf[java.util.ArrayList[String]].mkString(KEYWORDS_DELIMITER))
      case (k, v) => new GenericOption(k, v)
    }.toSeq
    createType(namespace, name, fields, opts)
  }

  def getFieldConfig(conf: Config): Seq[Config] =
    if (conf.hasPath(FieldsPath)) { conf.getConfigList(FieldsPath) } else { conf.getConfigList(AttributesPath) }

  def buildField(conf: Config, specParser: SpecParser): AttributeSpec = conf.getString(TypePath) match {
    case t if simpleTypeMap.contains(t)   => SimpleAttributeSpec(conf)
    case t if geometryTypeMap.contains(t) => GeomAttributeSpec(conf)

    case t if specParser.parse(specParser.listType, t).successful => ListAttributeSpec(conf)
    case t if specParser.parse(specParser.mapType, t).successful  => MapAttributeSpec(conf)
  }

  def createType(nameSpec: String, spec: String): SimpleFeatureType = {
    val (namespace, name) = buildTypeName(nameSpec)
    val FeatureSpec(attributeSpecs, opts) = parse(spec)
    createType(namespace, name, attributeSpecs, opts)
  }

  def createType(namespace: String, name: String, spec: String): SimpleFeatureType = {
    createType(namespace + ":" + name, spec)
  }

  def buildTypeName(nameSpec: String): (String, String) = {
    val nsIndex = nameSpec.lastIndexOf(':')
    val (namespace, name) = if (nsIndex == -1 || nsIndex == nameSpec.length - 1) {
      (null, nameSpec)
    } else {
      (nameSpec.substring(0, nsIndex), nameSpec.substring(nsIndex + 1))
    }
    (namespace, name)
  }

  def createType(namespace: String,
                 name: String,
                 attributeSpecs: Seq[AttributeSpec],
                 opts: Seq[FeatureOption]): SimpleFeatureType = {
    import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType

    val geomAttributes = attributeSpecs.collect { case g: GeomAttributeSpec => g }
    val dateAttributes = attributeSpecs.collect {
      case s: SimpleAttributeSpec if s.clazz == classOf[Date] => s
    }
    val defaultGeom = geomAttributes.find(_.options.get(OPT_DEFAULT).exists(_.toBoolean))
        .orElse(geomAttributes.headOption)
    // TODO GEOMESA-594 allow for setting default date field
    val defaultDate = dateAttributes.find(_.options.get(OPT_DEFAULT).exists(_.toBoolean))
        .orElse(dateAttributes.headOption)

    val b = new SimpleFeatureTypeBuilder()
    b.setNamespaceURI(namespace)
    b.setName(name)
    b.addAll(attributeSpecs.map(_.toAttribute))
    defaultGeom.foreach(dg => b.setDefaultGeometry(dg.name))
    val sft = b.buildFeatureType()
    defaultDate.foreach(dt => sft.setDtgField(dt.name))
    opts.foreach(_.decorateSFT(sft))
    sft
  }

  def renameSft(sft: SimpleFeatureType, newName: String) = {
    val builder = new SimpleFeatureTypeBuilder()
    builder.init(sft)
    builder.setName(newName)
    val renamed = builder.buildFeatureType()
    renamed.getUserData.putAll(sft.getUserData)<commit_msg>GEOMESA-1072 Invalid types in SFT parsing are not handled well (#1068)<commit_after>/***********************************************************************
* Copyright (c) 2013-2016 Commonwealth Computer Research, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License, Version 2.0
* which accompanies this distribution and is available at
* http://www.opensource.org/licenses/apache2.0.php.
*************************************************************************/

package org.locationtech.geomesa.utils.geotools

import java.util.{Date, Locale, UUID}

import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions, ConfigValueFactory}
import com.vividsolutions.jts.geom._
import org.apache.commons.lang.StringEscapeUtils
import org.geotools.feature.AttributeTypeBuilder
import org.geotools.feature.simple.SimpleFeatureTypeBuilder
import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType._
import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.SpecParser.{ListAttributeType, MapAttributeType, SimpleAttributeType}
import org.locationtech.geomesa.utils.stats.{Cardinality, IndexCoverage}
import org.locationtech.geomesa.utils.text.EnhancedTokenParsers
import org.opengis.feature.`type`.{AttributeDescriptor, GeometryDescriptor}
import org.opengis.feature.simple.SimpleFeatureType

import scala.collection.JavaConversions._
import scala.util.parsing.combinator.JavaTokenParsers
import scala.util.{Failure, Success, Try}
object SimpleFeatureTypes {

  import org.locationtech.geomesa.utils.geotools.RichAttributeDescriptors._

  val TABLE_SPLITTER           = "table.splitter.class"
  val TABLE_SPLITTER_OPTIONS   = "table.splitter.options"
  val ENABLED_INDEXES          = "geomesa.indexes.enabled"
  val MIXED_GEOMETRIES         = "geomesa.mixed.geometries"
  val RESERVED_WORDS           = "override.reserved.words" // note: doesn't start with geomesa so we don't persist it

  @deprecated
  val ENABLED_INDEXES_OLD      = "table.indexes.enabled"

  val OPT_DEFAULT              = "default"
  val OPT_SRID                 = "srid"
  val OPT_INDEX_VALUE          = "index-value"
  val OPT_INDEX                = "index"
  val OPT_STATS                = "keep-stats"
  val OPT_CARDINALITY          = "cardinality"
  val OPT_BIN_TRACK_ID         = "bin-track-id"

  val OPTS = Seq(OPT_DEFAULT, OPT_SRID, OPT_INDEX, OPT_STATS, OPT_INDEX_VALUE, OPT_CARDINALITY, OPT_BIN_TRACK_ID)

  val USER_DATA_LIST_TYPE      = "subtype"
  val USER_DATA_MAP_KEY_TYPE   = "keyclass"
  val USER_DATA_MAP_VALUE_TYPE = "valueclass"

  val TypeNamePath   = "type-name"
  val FieldsPath     = "fields"
  val UserDataPath   = "user-data"
  val AttributesPath = "attributes"
  val TypePath       = "type"

  /**
   * Create a SimpleFeatureType from a typesafe Config
   *
   * @param conf config
   * @param typeName optional typename to use for the SFT...will be overridden if the config contains a type-name key
   * @return
   */
  def createType(conf: Config, typeName: Option[String] = None, path: Option[String] = Some("sft")): SimpleFeatureType = {
    import org.locationtech.geomesa.utils.conf.ConfConversions._
    val toParse = path match {
      case Some(p) => conf.getConfigOpt(p).map(conf.withFallback).getOrElse(conf)
      case None    => conf
    }

    val nameSpec = toParse.getStringOpt(TypeNamePath).orElse(typeName).getOrElse(
      throw new IllegalArgumentException("Unable to parse type name from provided argument or config")
    )
    val (namespace, name) = buildTypeName(nameSpec)
    val specParser = new SpecParser

    val fields = getFieldConfig(toParse).map(buildField(_, specParser))
    val userData = if (toParse.hasPath(UserDataPath)) {
      toParse.getConfig(UserDataPath).entrySet().map(e => e.getKey -> e.getValue.unwrapped()).toMap
    } else {
      Map.empty[String, AnyRef]
    }
    val opts = userData.map {
      // Special case to handle adding keywords
      case (KEYWORDS_KEY, v : java.util.ArrayList[String]) =>
        new GenericOption(KEYWORDS_KEY, v.asInstanceOf[java.util.ArrayList[String]].mkString(KEYWORDS_DELIMITER))
      case (k, v) => new GenericOption(k, v)
    }.toSeq
    createType(namespace, name, fields, opts)
  }

  def getFieldConfig(conf: Config): Seq[Config] =
    if (conf.hasPath(FieldsPath)) { conf.getConfigList(FieldsPath) } else { conf.getConfigList(AttributesPath) }

  def buildField(conf: Config, specParser: SpecParser): AttributeSpec = conf.getString(TypePath) match {
    case t if simpleTypeMap.contains(t)   => SimpleAttributeSpec(conf)
    case t if geometryTypeMap.contains(t) => GeomAttributeSpec(conf)

    case t if specParser.parse(specParser.listType, t).successful => ListAttributeSpec(conf)
    case t if specParser.parse(specParser.mapType, t).successful  => MapAttributeSpec(conf)

    case default => throw new IllegalArgumentException(s"Type $default is not a supported Simple Feature attribute type, "
      + s"valid types are: ${simpleTypeMap.keys.toList.sorted}, ${geometryTypeMap.keys.toList.sorted}, List[?], Map[?,?]")
  }

  def createType(nameSpec: String, spec: String): SimpleFeatureType = {
    val (namespace, name) = buildTypeName(nameSpec)
    val FeatureSpec(attributeSpecs, opts) = parse(spec)
    createType(namespace, name, attributeSpecs, opts)
  }

  def createType(namespace: String, name: String, spec: String): SimpleFeatureType = {
    createType(namespace + ":" + name, spec)
  }

  def buildTypeName(nameSpec: String): (String, String) = {
    val nsIndex = nameSpec.lastIndexOf(':')
    val (namespace, name) = if (nsIndex == -1 || nsIndex == nameSpec.length - 1) {
      (null, nameSpec)
    } else {
      (nameSpec.substring(0, nsIndex), nameSpec.substring(nsIndex + 1))
    }
    (namespace, name)
  }

  def createType(namespace: String,
                 name: String,
                 attributeSpecs: Seq[AttributeSpec],
                 opts: Seq[FeatureOption]): SimpleFeatureType = {
    import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType

    val geomAttributes = attributeSpecs.collect { case g: GeomAttributeSpec => g }
    val dateAttributes = attributeSpecs.collect {
      case s: SimpleAttributeSpec if s.clazz == classOf[Date] => s
    }
    val defaultGeom = geomAttributes.find(_.options.get(OPT_DEFAULT).exists(_.toBoolean))
        .orElse(geomAttributes.headOption)
    val defaultDate = dateAttributes.find(_.options.get(OPT_DEFAULT).exists(_.toBoolean))
        .orElse(dateAttributes.headOption)

    val b = new SimpleFeatureTypeBuilder()
    b.setNamespaceURI(namespace)
    b.setName(name)
    b.addAll(attributeSpecs.map(_.toAttribute))
    defaultGeom.foreach(dg => b.setDefaultGeometry(dg.name))
    val sft = b.buildFeatureType()
    defaultDate.foreach(dt => sft.setDtgField(dt.name))
    opts.foreach(_.decorateSFT(sft))
    sft
  }

  def renameSft(sft: SimpleFeatureType, newName: String) = {
    val builder = new SimpleFeatureTypeBuilder()
    builder.init(sft)
    builder.setName(newName)
    val renamed = builder.buildFeatureType()
    renamed.getUserData.putAll(sft.getUserData)
{'fraction_non_alphanumeric': 0.12226393709711696, 'fraction_numerical': 0.003045972940426436, 'mean_word_length': 3.97989417989418, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6821283', 'n_tokens_mistral': 4402, 'n_tokens_neox': 4278, 'n_words': 928}