<commit_before>package com.soteradefense.dga.graphx

import com.soteradefense.dga.graphx.harness.Harness
import com.soteradefense.dga.graphx.hbse.HDFSHBSERunner
import com.soteradefense.dga.graphx.io.formats.EdgeInputFormat
import com.soteradefense.dga.graphx.lc.HDFSLCRunner
import com.soteradefense.dga.graphx.louvain.HDFSLouvainRunner
import com.soteradefense.dga.graphx.parser.CommandLineParser
import com.soteradefense.dga.graphx.pr.HDFSPRRunner
import com.soteradefense.dga.graphx.wcc.HDFSWCCRunner
import org.apache.spark.graphx.Graph
import org.apache.spark.{SparkConf, SparkContext}

object DGARunner {
  def main(args: Array[String]) {
    val analytic = args(0)
    println(s"Analytic: $analytic")
    val newArgs = args.slice(1, args.length)
    val cmdLine = new CommandLineParser().parseCommandLine(newArgs)
    cmdLine.properties.foreach({ case (k, v) => System.setProperty(k, v)})
    cmdLine.jars.split(",").foreach(println(_))
    val conf = new SparkConf().setMaster(cmdLine.master)
      .setAppName(cmdLine.appName)
      .setSparkHome(cmdLine.sparkHome)
      .setJars(cmdLine.jars.split(","))
    //.set("spark.serializer", classOf[KryoSerializer].getCanonicalName)
    //.set("spark.kryo.registrator", classOf[DGAKryoRegistrator].getCanonicalName)
    val sc = new SparkContext(conf)
    val inputFormat = new EdgeInputFormat(cmdLine.input, cmdLine.edgeDelimiter)
    var edgeRDD = inputFormat.getEdgeRDD(sc)
    val parallelism = Integer.parseInt(cmdLine.customArguments.getOrElse("parallelism", "-1"))
    if (parallelism != -1)<commit_msg>Added the Log4j.prop file to the runner and removed a println that didnt need to be there<commit_after>package com.soteradefense.dga.graphx

import com.soteradefense.dga.graphx.harness.Harness
import com.soteradefense.dga.graphx.hbse.HDFSHBSERunner
import com.soteradefense.dga.graphx.io.formats.EdgeInputFormat
import com.soteradefense.dga.graphx.lc.HDFSLCRunner
import com.soteradefense.dga.graphx.louvain.HDFSLouvainRunner
import com.soteradefense.dga.graphx.parser.CommandLineParser
import com.soteradefense.dga.graphx.pr.HDFSPRRunner
import com.soteradefense.dga.graphx.wcc.HDFSWCCRunner
import org.apache.log4j.PropertyConfigurator
import org.apache.spark.graphx.Graph
import org.apache.spark.{SparkConf, SparkContext}

object DGARunner {
  def main(args: Array[String]) {
    val analytic = args(0)
    println(s"Analytic: $analytic")
    val newArgs = args.slice(1, args.length)
    PropertyConfigurator.configure(getClass.getResource("/log4j.properties"))
    val cmdLine = new CommandLineParser().parseCommandLine(newArgs)
    cmdLine.properties.foreach({ case (k, v) => System.setProperty(k, v)})
    val conf = new SparkConf().setMaster(cmdLine.master)
      .setAppName(cmdLine.appName)
      .setSparkHome(cmdLine.sparkHome)
      .setJars(cmdLine.jars.split(","))
    //.set("spark.serializer", classOf[KryoSerializer].getCanonicalName)
    //.set("spark.kryo.registrator", classOf[DGAKryoRegistrator].getCanonicalName)
    val sc = new SparkContext(conf)
    val inputFormat = new EdgeInputFormat(cmdLine.input, cmdLine.edgeDelimiter)
    var edgeRDD = inputFormat.getEdgeRDD(sc)
    val parallelism = Integer.parseInt(cmdLine.customArguments.getOrElse("parallelism", "-1"))
    if (parallelism != -1)
{'fraction_non_alphanumeric': 0.1187937861711849, 'fraction_numerical': 0.003350593968930856, 'mean_word_length': 5.149812734082397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3524009', 'n_tokens_mistral': 1197, 'n_tokens_neox': 1170, 'n_words': 173}