<commit_before>/*
 * This file is a part of the "sur la plaque" toolkit for cycling
 * data analytics and visualization.
 *
 * Copyright (c) 2013--2014 <NAME> and Red Hat, Inc.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.freevariable.surlaplaque.app;

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._

import org.apache.spark.mllib.clustering._

import com.freevariable.surlaplaque.importer._
import com.freevariable.surlaplaque.data._
import com.freevariable.surlaplaque.app._

object PowerBestsApp extends Common with ActivitySliding with PointClustering {
  import org.json4s._
  import org.json4s.JsonDSL._
  import org.json4s.jackson.JsonMethods._

  import org.apache.spark.rdd.RDD
  import org.apache.spark.SparkConf
    
  import com.freevariable.surlaplaque.geometry.LineString
  import com.freevariable.surlaplaque.geometry.LineString._  

  import scala.collection.immutable.TreeSet

  class PBOptions(periodColors: Map[Int, Triple[Short,Short,Short]], val clusters: Int, val iterations: Int, val files: List[String], val defaultOpacity: Short, val outputFile: String, val httpEndpoint: Option[String]) {
    def periodMap = {
      if (periodColors.size == 0) 
        Map(getEnvValue("SLP_MMP_PERIOD", "60").toInt -> Triple[Short,Short,Short](255,0,0))
      else 
        periodColors
    }
    
    def withPeriodColoring(period: Int, r: Short, g: Short, b: Short) =
      new PBOptions(this.periodColors + Pair(period, Triple[Short,Short,Short](r,g,b)), this.clusters, this.iterations, this.files, this.defaultOpacity, this.outputFile, httpEndpoint)
    
    def withClusters(clusters: Int) = new PBOptions(this.periodColors, clusters, this.iterations, this.files, this.defaultOpacity, this.outputFile, httpEndpoint)

    def withIterations(iterations: Int) = new PBOptions(this.periodColors, this.clusters, iterations, this.files, this.defaultOpacity, this.outputFile, httpEndpoint)
    
    def withFile(file: String) = new PBOptions(this.periodColors, this.clusters, this.iterations, file::this.files, this.defaultOpacity, this.outputFile, httpEndpoint)

    def withFiles(fs: List[String]) = new PBOptions(this.periodColors, this.clusters, this.iterations, fs ++ this.files, this.defaultOpacity, this.outputFile, httpEndpoint)
    
    def withDefaultOpacity(op: Short) = new PBOptions(this.periodColors, this.clusters, this.iterations, this.files, op, this.outputFile, httpEndpoint)
    
    def withOutputFile(f: String) = new PBOptions(this.periodColors, this.clusters, this.iterations, this.files, this.defaultOpacity, f, httpEndpoint)
    
    def withEndpoint(url: String) = new PBOptions(this.periodColors, this.clusters, this.iterations, this.files, this.defaultOpacity, outputFile, Some(url))
  }
  
  object PBOptions {
    val default = new PBOptions(Map(), getEnvValue("SLP_CLUSTERS", "256").toInt, getEnvValue("SLP_ITERATIONS", "10").toInt, List(), 128, getEnvValue("SLP_OUTPUT_FILE", "slp.json"), None)
  }
  
  def parseArgs(args: Array[String]) = {
    val hexRGB = "^((?:[0-9a-fA-F]){2})((?:[0-9a-fA-F]){2})((?:[0-9a-fA-F]){2})$".r
    val dirPattern = "^-d(.*)$".r
<commit_msg>Cleaned up PB option handling<commit_after>/*
 * This file is a part of the "sur la plaque" toolkit for cycling
 * data analytics and visualization.
 *
 * Copyright (c) 2013--2014 <NAME> and Red Hat, Inc.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.freevariable.surlaplaque.app;

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._

import org.apache.spark.mllib.clustering._

import com.freevariable.surlaplaque.importer._
import com.freevariable.surlaplaque.data._
import com.freevariable.surlaplaque.app._

object PowerBestsApp extends Common with ActivitySliding with PointClustering {
  import org.json4s._
  import org.json4s.JsonDSL._
  import org.json4s.jackson.JsonMethods._

  import org.apache.spark.rdd.RDD
  import org.apache.spark.SparkConf
    
  import com.freevariable.surlaplaque.geometry.LineString
  import com.freevariable.surlaplaque.geometry.LineString._  

  import scala.collection.immutable.TreeSet

  class PBOptions(periodColors: Map[Int, Triple[Short,Short,Short]], val clusters: Int, val iterations: Int, val files: List[String], val defaultOpacity: Short, val outputFile: String, val httpEndpoint: Option[String]) {
    def periodMap = {
      if (periodColors.size == 0) 
        Map(getEnvValue("SLP_MMP_PERIOD", "60").toInt -> Triple[Short,Short,Short](255,0,0))
      else 
        periodColors
    }
    
    def withPeriodColoring(period: Int, r: Short, g: Short, b: Short) = this.copy(periodColors = this.periodColors + Pair(period, Triple[Short,Short,Short](r,g,b)))
    
    def withClusters(clusters: Int) = this.copy(clusters = clusters)

    def withIterations(iterations: Int) = this.copy(iterations = iterations)
    
    def withFile(file: String) = this.copy(files = file::this.files)

    def withFiles(fs: List[String]) = this.copy(files = fs ++ this.files) 
    
    def withDefaultOpacity(op: Short) = this.copy(defaultOpacity = op)
    
    def withOutputFile(f: String) = this.copy(outputFile = f)
    
    def withEndpoint(url: String) = this.copy(httpEndpoint = Some(url))
  }
  
  object PBOptions {
    val default = new PBOptions(Map(), getEnvValue("SLP_CLUSTERS", "256").toInt, getEnvValue("SLP_ITERATIONS", "10").toInt, List(), 128, getEnvValue("SLP_OUTPUT_FILE", "slp.json"), None)
  }
  
  def parseArgs(args: Array[String]) = {
    val hexRGB = "^((?:[0-9a-fA-F]){2})((?:[0-9a-fA-F]){2})((?:[0-9a-fA-F]){2})$".r
    val dirPattern = "^-d(.*)$".r

{'fraction_non_alphanumeric': 0.12099322799097066, 'fraction_numerical': 0.012039127163280662, 'mean_word_length': 4.29140127388535, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1118108', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2115, 'n_words': 546}