<commit_before>package org.tmoerman.plongeur.tda

import org.apache.commons.lang.StringUtils
import org.apache.spark.mllib.regression.LabeledPoint
import org.scalatest.{Matchers, FlatSpec}
import org.tmoerman.plongeur.tda.Model.{Filter, Lens}
import org.tmoerman.plongeur.test.{TestResources, SparkContextSpec}

/**
  * @author <NAME>
  */
class SkeletonSpec extends FlatSpec with SparkContextSpec with TestResources with Matchers {
  import Inspections._

  behavior of "the skeleton"

  it should "work with specified boundaries" in {

    val boundaries = Array((0.0, 12.0), (0.0, 12.0))

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 1.0, 0.5),
                    Filter((l: LabeledPoint) => l.features(1), 1.0, 0.5))

    val result =
      Skeleton.execute(
        lens = lens,
        data = test2DLabeledPointsRDD,
        boundaries = Some(boundaries))

    println(result.dotGraph("test2D"))
  }

  it should "work with calculated boundaries" in {

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 1.0, 0.5),
                    Filter((l: LabeledPoint) => l.features(1), 1.0, 0.5))

    val result =
      Skeleton.execute(
        lens = lens,
        data = test2DLabeledPointsRDD)

    val intro = result.clusterPoints

    println(result.dotGraph("test2Dc"))
  }

  it should "recover the 100 entries circle topology" in {

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 0.5, 0.5))

    val result =
      Skeleton.execute(
        lens = lens,
        data = circle1kRDD)

    println(result.dotGraph("Circle1K"))
  }


}<commit_msg>emit debugging views on test data set<commit_after>package org.tmoerman.plongeur.tda

import org.apache.spark.mllib.regression.LabeledPoint
import org.scalatest.{Matchers, FlatSpec}
import org.tmoerman.plongeur.tda.Model.{Filter, Lens}
import org.tmoerman.plongeur.test.{TestResources, SparkContextSpec}

/**
  * @author <NAME>
  */
class SkeletonSpec extends FlatSpec with SparkContextSpec with TestResources with Matchers {
  import Inspections._

  behavior of "the skeleton"

  implicit val counters = (new MapToInt, new MapToInt)

  it should "work with specified boundaries" in {

    val boundaries = Array((0.0, 12.0), (0.0, 12.0))

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 1.0, 0.5),
                    Filter((l: LabeledPoint) => l.features(1), 1.0, 0.5))

    val result =
      Skeleton.execute(
        lens = lens,
        data = test2DLabeledPointsRDD,
        boundaries = Some(boundaries))

    println(result.dotGraph("test2D"))
  }

  it should "work with calculated boundaries" in {

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 1.0, 0.5),
                    Filter((l: LabeledPoint) => l.features(1), 1.0, 0.5))

    val result =
      Skeleton.execute(
        lens = lens,
        data = test2DLabeledPointsRDD)

    val intro = result.clusterPoints

    println(result.dotGraph("test2Dc"))
  }

  it should "recover the 100 entries circle topology" in {

    val rdd = circle250RDD

    println(rdd.map(l => l.features(0) + "," + l.features(1)).collect.mkString("\n"))

    val data = rdd.collect

    val lens = Lens(Filter((l: LabeledPoint) => l.features(0), 0.25, 0.5))

    val result = Skeleton.execute(lens = lens, data = rdd)

    val intro = result.clusterPoints

    val toCl = result.pointsToClusters

    val byCoords = result.coordsToClusters

    println(byCoords.mkString("\n\n"))

    println(result.dotGraph("Circle250"))
  }


}
{'fraction_non_alphanumeric': 0.12852664576802508, 'fraction_numerical': 0.02707324023938444, 'mean_word_length': 3.0484429065743943, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4673312', 'n_tokens_mistral': 1280, 'n_tokens_neox': 1157, 'n_words': 273}