<commit_before>package net.sansa_stack.ml.spark.similarity.similarity_measures

import org.apache.spark.ml.linalg.Vector
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions.{col, lit, udf}

class RodriguezEgenhoferModel extends GenericSimilarityEstimatorModel {

  protected val rodriguez_egenhofer = udf( (a: Vector, b: Vector, alpha: Double) => {
    val feature_indices_a = a.toSparse.indices
    val feature_indices_b = b.toSparse.indices
    val f_set_a = feature_indices_a.toSet
    val f_set_b = feature_indices_b.toSet
    val rodriguez_egenhofer = (f_set_a.intersect(f_set_b).size.toDouble) /
      ((f_set_a.intersect(f_set_b).size.toDouble) + (alpha * f_set_a.diff(f_set_b).size.toDouble) + ((1 - alpha) * f_set_b.diff(f_set_a).size.toDouble))
    rodriguez_egenhofer
  })

  private var _alpha: Double = _

  def set_alpha(a: Double): Unit = {
    if (a < 0 || a > 1) {
      println("PROBLEM: alpha has to be between 0 and 1")
    }
    else {
      _alpha = a
    }
  }

  override val estimator_name: String = "RodriguezEgenhoferSimilarityEstimator"
  override val estimator_measure_type: String = "similarity"

  override val similarityEstimation = rodriguez_egenhofer

  override def similarityJoin(df_A: DataFrame, df_B: DataFrame, threshold: Double = -1.0, value_column: String = "rodriguez_egenhofer_similarity"): DataFrame = {

    val cross_join_df = createCrossJoinDF(df_A: DataFrame, df_B: DataFrame)

    set_similarity_estimation_column_name(value_column)

    val join_df: DataFrame = cross_join_df.withColumn(<commit_msg>new set opportunitiy. now over this so it is more alligned with the mllib behavior and different error handling<commit_after>package net.sansa_stack.ml.spark.similarity.similarity_measures

import org.apache.spark.ml.linalg.Vector
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions.{col, lit, udf}

class RodriguezEgenhoferModel extends GenericSimilarityEstimatorModel {

  protected val rodriguez_egenhofer = udf( (a: Vector, b: Vector, alpha: Double) => {
    val feature_indices_a = a.toSparse.indices
    val feature_indices_b = b.toSparse.indices
    val f_set_a = feature_indices_a.toSet
    val f_set_b = feature_indices_b.toSet
    val rodriguez_egenhofer = (f_set_a.intersect(f_set_b).size.toDouble) /
      ((f_set_a.intersect(f_set_b).size.toDouble) + (alpha * f_set_a.diff(f_set_b).size.toDouble) + ((1 - alpha) * f_set_b.diff(f_set_a).size.toDouble))
    rodriguez_egenhofer
  })

  private var _alpha: Double = _

  def set_alpha(a: Double): this.type = {
    if (a < 0 || a > 1) {
      throw new Error("PROBLEM: alpha has to be between 0 and 1")
    }
    else {
      _alpha = a
      this
    }
  }

  override val estimator_name: String = "RodriguezEgenhoferSimilarityEstimator"
  override val estimator_measure_type: String = "similarity"

  override val similarityEstimation = rodriguez_egenhofer

  override def similarityJoin(df_A: DataFrame, df_B: DataFrame, threshold: Double = -1.0, value_column: String = "rodriguez_egenhofer_similarity"): DataFrame = {

    val cross_join_df = createCrossJoinDF(df_A: DataFrame, df_B: DataFrame)

    set_similarity_estimation_column_name(value_column)

    val join_df: DataFrame = cross_join_df.withColumn(
{'fraction_non_alphanumeric': 0.094859241126071, 'fraction_numerical': 0.004283965728274173, 'mean_word_length': 4.412251655629139, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2899279', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1154, 'n_words': 238}