<commit_before>/**
 * Copyright 2015, CodiLime Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.deepsense.deeplang.doperations

import scala.collection.immutable.ListMap
import scala.collection.mutable.ListBuffer
import scala.reflect.runtime.{universe => ru}

import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, UserDefinedFunction}

import io.deepsense.deeplang.DOperation.Id
import io.deepsense.deeplang._
import io.deepsense.deeplang.doperables.dataframe._
import io.deepsense.deeplang.doperables.dataframe.types.categorical.{CategoricalMapper, CategoricalMetadata}
import io.deepsense.deeplang.doperables.dataframe.types.{Conversions, SparkConversions}
import io.deepsense.deeplang.inference.{ConversionMayNotBePossibleWarning, InferContext, InferenceWarning, InferenceWarnings}
import io.deepsense.deeplang.parameters.ColumnType.ColumnType
import io.deepsense.deeplang.parameters._

case class ConvertType() extends DOperation1To1[DataFrame, DataFrame] {

  override val name: String = "Convert Type"
  override val id: Id = "f8b3c5d0-febe-11e4-b939-0800200c9a66"

  @transient
  override lazy val tTagTO_0: ru.TypeTag[DataFrame] = ru.typeTag[DataFrame]
  @transient
  override lazy val tTagTI_0: ru.TypeTag[DataFrame] = ru.typeTag[DataFrame]

  val selectedColumnsParameter = ColumnSelectorParameter(
    "Columns to be converted",
    required = true,
    portIndex = 0
  )

  val targetTypeParameter = ChoiceParameter(
    "Target type of the columns",
    None,
    required = true,
    options = ListMap(ColumnType.values.toList.map(_.toString -> ParametersSchema()): _*)
  )

  override val parameters: ParametersSchema = ParametersSchema(
    ConvertType.SelectedColumns -> selectedColumnsParameter,
    ConvertType.TargetType -> targetTypeParameter
  )

  override protected def _execute(context: ExecutionContext)(dataFrame: DataFrame): DataFrame = {
    val targetType = ColumnType.withName(targetTypeParameter.selection.get.label)
    val columns = dataFrame
      .getColumnNames(selectedColumnsParameter.value.get)

    if (targetType == ColumnType.categorical) {<commit_msg>Fix target type list in ConvertType<commit_after>/**
 * Copyright 2015, CodiLime Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package io.deepsense.deeplang.doperations

import scala.collection.immutable.ListMap
import scala.collection.mutable.ListBuffer
import scala.reflect.runtime.{universe => ru}

import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, UserDefinedFunction}

import io.deepsense.deeplang.DOperation.Id
import io.deepsense.deeplang._
import io.deepsense.deeplang.doperables.dataframe._
import io.deepsense.deeplang.doperables.dataframe.types.categorical.{CategoricalMapper, CategoricalMetadata}
import io.deepsense.deeplang.doperables.dataframe.types.{Conversions, SparkConversions}
import io.deepsense.deeplang.inference.{ConversionMayNotBePossibleWarning, InferContext, InferenceWarning, InferenceWarnings}
import io.deepsense.deeplang.parameters.ColumnType.ColumnType
import io.deepsense.deeplang.parameters._

case class ConvertType() extends DOperation1To1[DataFrame, DataFrame] {

  override val name: String = "Convert Type"
  override val id: Id = "f8b3c5d0-febe-11e4-b939-0800200c9a66"

  @transient
  override lazy val tTagTO_0: ru.TypeTag[DataFrame] = ru.typeTag[DataFrame]
  @transient
  override lazy val tTagTI_0: ru.TypeTag[DataFrame] = ru.typeTag[DataFrame]

  val selectedColumnsParameter = ColumnSelectorParameter(
    "Columns to be converted",
    required = true,
    portIndex = 0
  )

  val availableTargetTypes = List(ColumnType.numeric, ColumnType.string, ColumnType.categorical)

  val targetTypeParameter = ChoiceParameter(
    "Target type of the columns",
    None,
    required = true,
    options = ListMap(availableTargetTypes.map(_.toString -> ParametersSchema()): _*)
  )

  override val parameters: ParametersSchema = ParametersSchema(
    ConvertType.SelectedColumns -> selectedColumnsParameter,
    ConvertType.TargetType -> targetTypeParameter
  )

  override protected def _execute(context: ExecutionContext)(dataFrame: DataFrame): DataFrame = {
    val targetType = ColumnType.withName(targetTypeParameter.selection.get.label)
    val columns = dataFrame
      .getColumnNames(selectedColumnsParameter.value.get)

    if (targetType == ColumnType.categorical) {
{'fraction_non_alphanumeric': 0.08674787744555186, 'fraction_numerical': 0.012181616832779624, 'mean_word_length': 4.941885964912281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4672468', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1580, 'n_words': 410}