<commit_before>// spark-shell --master yarn-client --executor-memory 35840m  --driver-memory 64g --num-executors 62 --executor-cores 12 --conf spark.shuffle.io.preferDirectBufs=false --conf shuffle.service.enabled=true --conf spark.driver.maxResultSize="12g" 
// spark-shell --executor-memory 35840m  --driver-memory 64g --num-executors 62 --executor-cores 12 --conf spark.shuffle.io.preferDirectBufs=false --conf shuffle.service.enabled=true --conf spark.driver.maxResultSize="12g" 

val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.mllib.linalg.Vectors
import breeze.stats.DescriptiveStats._
//import breeze.linalg._
import org.apache.log4j.Logger
import org.apache.log4j.Level
import scala.math._
import scala.io.Source

Logger.getLogger("org").setLevel(Level.OFF)
Logger.getLogger("akka").setLevel(Level.OFF)


def removeHeader(input: org.apache.spark.rdd.RDD[String]) = {
    val header = input.first
    val output = input.filter(line => !(line == header))
    output
}

def get_column_names(input: org.apache.spark.rdd.RDD[String], sep : Char = ',') : scala.collection.mutable.Map[String, Int] = {
    val columns = scala.collection.mutable.Map[String, Int]()
    val header = input.first.split(sep).zipWithIndex
    header.foreach(tuple => columns(tuple._1) = tuple._2)
    columns
}

def print_columns(columns : scala.collection.mutable.Map[String, Int]) = {
    val arr = columns.toArray
    arr.sortBy(_._2).foreach(println)
}

def print_lines(input: org.apache.spark.rdd.RDD[String]) = input.take(10).foreach(println)

def print_rdd(input: org.apache.spark.rdd.RDD[Array[String]]) = input.take(20).foreach(m => println(m.mkString(",") ))

/**
def bin_column(row: Array[String], column: String, cuts: Array[Double]) = {
    var bin = 0
    for (cut <- cuts){
        if (row(col(column)).toDouble > cut) { bin = bin+1 }
    }
    col(column + "_bin") = row.length + 1   //update the column index automatically
    row :+ bin.toString
}
*/


def isNumeric(input: String): Boolean = {
    if (input == "") {false}
    else {input.forall(x => (x.isDigit || x == '.' || x =='E' || x == 'e')) && input.count(_ == '.')<2 && (input.count(_ == 'E') + input.count(_ == 'e')) <2}
}
def toDouble(s: String, default:Double=Double.NaN) = {if (isNumeric(s)) s.toDouble else default}


val file_list = System.getenv("DPATH")
val feedback_file = "None"
val duplication_factor = 100
val outputfile = System.getenv("HPATH")  + /word_counts"
val quant = Array(0, 0.1,0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)
val quint = Array(0, 0.2, 0.4, 0.6, 0.8)
var time_cuts = new Array[Double](5)
var frame_length_cuts = new Array[Double](5)
var subdomain_length_cuts = new Array[Double](5)
var numperiods_cuts = new Array[Double](5)
var entropy_cuts = new Array[Double](5)
val compute_quantiles = true

val l_top_domains = Source.fromFile("top-1m.csv").getLines.map(line => {
                      val parts = line.split(",")
                     val l = parts.length
                    parts(1).split("[.]")(0)<commit_msg>timestamp changes<commit_after>// spark-shell --master yarn-client --executor-memory 35840m  --driver-memory 64g --num-executors 62 --executor-cores 12 --conf spark.shuffle.io.preferDirectBufs=false --conf shuffle.service.enabled=true --conf spark.driver.maxResultSize="12g" 
// spark-shell --executor-memory 35840m  --driver-memory 64g --num-executors 62 --executor-cores 12 --conf spark.shuffle.io.preferDirectBufs=false --conf shuffle.service.enabled=true --conf spark.driver.maxResultSize="12g" 

val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import org.apache.spark.sql.types._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.mllib.linalg.Vectors
import breeze.stats.DescriptiveStats._
//import breeze.linalg._
import org.apache.log4j.Logger
import org.apache.log4j.Level
import scala.math._
import scala.io.Source

Logger.getLogger("org").setLevel(Level.OFF)
Logger.getLogger("akka").setLevel(Level.OFF)


def removeHeader(input: org.apache.spark.rdd.RDD[String]) = {
    val header = input.first
    val output = input.filter(line => !(line == header))
    output
}

def get_column_names(input: org.apache.spark.rdd.RDD[String], sep : Char = ',') : scala.collection.mutable.Map[String, Int] = {
    val columns = scala.collection.mutable.Map[String, Int]()
    val header = input.first.split(sep).zipWithIndex
    header.foreach(tuple => columns(tuple._1) = tuple._2)
    columns
}

def print_columns(columns : scala.collection.mutable.Map[String, Int]) = {
    val arr = columns.toArray
    arr.sortBy(_._2).foreach(println)
}

def print_lines(input: org.apache.spark.rdd.RDD[String]) = input.take(10).foreach(println)

def print_rdd(input: org.apache.spark.rdd.RDD[Array[String]]) = input.take(20).foreach(m => println(m.mkString(",") ))

/**
def bin_column(row: Array[String], column: String, cuts: Array[Double]) = {
    var bin = 0
    for (cut <- cuts){
        if (row(col(column)).toDouble > cut) { bin = bin+1 }
    }
    col(column + "_bin") = row.length + 1   //update the column index automatically
    row :+ bin.toString
}
*/


def isNumeric(input: String): Boolean = {
    if (input == "") {false}
    else {input.forall(x => (x.isDigit || x == '.' || x =='E' || x == 'e')) && input.count(_ == '.')<2 && (input.count(_ == 'E') + input.count(_ == 'e')) <2}
}
def toDouble(s: String, default:Double=Double.NaN) = {if (isNumeric(s)) s.toDouble else default}


val file_list = System.getenv("DPATH")
val feedback_file = "None"
val duplication_factor = 100
val outputfile = System.getenv("HPATH")  + "/word_counts"
val quant = Array(0, 0.1,0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)
val quint = Array(0, 0.2, 0.4, 0.6, 0.8)
var time_cuts = new Array[Double](5)
var frame_length_cuts = new Array[Double](5)
var subdomain_length_cuts = new Array[Double](5)
var numperiods_cuts = new Array[Double](5)
var entropy_cuts = new Array[Double](5)
val compute_quantiles = true

val l_top_domains = Source.fromFile("top-1m.csv").getLines.map(line => {
                      val parts = line.split(",")
                     val l = parts.length
                    parts(1).split("[.]")(0)
{'fraction_non_alphanumeric': 0.1671206537413876, 'fraction_numerical': 0.02531645569620253, 'mean_word_length': 3.8015384615384615, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3170628', 'n_tokens_mistral': 2300, 'n_tokens_neox': 2172, 'n_words': 456}