<commit_before>package edu.oregonstate.mutation.statementHistory

import java.io.{BufferedInputStream, File, FileInputStream}

import com.brindescu.gumtree.facade.JavaASTDiff
import org.eclipse.jgit.api.Git
import play.api.libs.json._

object NewMain extends App {

	private case class Config(inputJSON: File = null,
														repoLocation: File = null) {	}

	private def parseCmdOptions(args: Array[String]): Option[Config] = {
		val parser = new scopt.OptionParser[Config]("java -jar <jarname>") {
			opt[String]('j', "json") action { (x, c) =>
				c.copy(inputJSON = new File(x))
			} validate( x => {
				val f = new File(x)
				if (f.exists && f.isFile)
					success
				else
					failure("The json file is not a file or doesn't exist")
			} ) text ("The json file with the lines to track") required()
			opt[String]('r', "repo") action { (x, c) =>
				c.copy(repoLocation = new File(x))
			} validate ( x => {
				val f = new File(x)
				if (f.exists)
					if (f.isDirectory)
						success
					else
						failure("The repository must be a directory")
				else
					failure("The given repo location does not exit")
			}) text("Repository location") required()
 		}

		parser.parse(args, Config())
	}

	case class FileToTrack(name: String, lines: List[Int])
	case class MergeCommit(sha: String, files: List[FileToTrack])

	override def main(args: Array[String]) = {
		val config = parseCmdOptions(args) match {
			case Some(c) => c
			case None => sys.exit(1)
		}

		val repo = Git.open(config.repoLocation)

		val detector = new NodeChangeDetector(repo, StatementFinder, JavaASTDiff)

		implicit val filesReader = Json.reads[FileToTrack]
		implicit val commitReader = Json.reads[MergeCommit]
		val parsed = Json.parse(new BufferedInputStream(new FileInputStream(config.inputJSON)))
		val commits = (parsed.validate[List[MergeCommit]] match {
			case s: JsSuccess[List[MergeCommit]] => s.get
			case _ =>
				println("Error parsing " + config.inputJSON)
				List[MergeCommit]()
		}).filterNot { _.files.isEmpty }
			.map (m => {
				MergeCommit(m.sha, m.files.filterNot { _.name == "dev/null" }
					.filterNot { _.lines.isEmpty }
				  .filter { _.name.endsWith("java")})
			})

		commits.flatMap(c => c.files.map{(c.sha, _)})
			.flatMap{ case (sha, f) => f.lines.map((sha, f.name, _))}.toParArray
			.foreach { case (sha, f, l) =>
					println(sha + "," + f + "," + l + "," + detector.findCommits(f, l, sha, Order.FORWARD).mkString(","))
			}
	}
}<commit_msg>Using a ForkJoinPool<commit_after>package edu.oregonstate.mutation.statementHistory

import java.io.{BufferedInputStream, File, FileInputStream}

import com.brindescu.gumtree.facade.JavaASTDiff
import org.eclipse.jgit.api.Git
import play.api.libs.json._

import scala.collection.parallel.ForkJoinTaskSupport

object NewMain extends App {

	private case class Config(inputJSON: File = null,
														repoLocation: File = null) {	}

	private def parseCmdOptions(args: Array[String]): Option[Config] = {
		val parser = new scopt.OptionParser[Config]("java -jar <jarname>") {
			opt[String]('j', "json") action { (x, c) =>
				c.copy(inputJSON = new File(x))
			} validate( x => {
				val f = new File(x)
				if (f.exists && f.isFile)
					success
				else
					failure("The json file is not a file or doesn't exist")
			} ) text ("The json file with the lines to track") required()
			opt[String]('r', "repo") action { (x, c) =>
				c.copy(repoLocation = new File(x))
			} validate ( x => {
				val f = new File(x)
				if (f.exists)
					if (f.isDirectory)
						success
					else
						failure("The repository must be a directory")
				else
					failure("The given repo location does not exit")
			}) text("Repository location") required()
 		}

		parser.parse(args, Config())
	}

	case class FileToTrack(name: String, lines: List[Int])
	case class MergeCommit(sha: String, files: List[FileToTrack])

	override def main(args: Array[String]) = {
		val config = parseCmdOptions(args) match {
			case Some(c) => c
			case None => sys.exit(1)
		}

		val repo = Git.open(config.repoLocation)

		val detector = new NodeChangeDetector(repo, StatementFinder, JavaASTDiff)

		implicit val filesReader = Json.reads[FileToTrack]
		implicit val commitReader = Json.reads[MergeCommit]
		val parsed = Json.parse(new BufferedInputStream(new FileInputStream(config.inputJSON)))
		val commits = (parsed.validate[List[MergeCommit]] match {
			case s: JsSuccess[List[MergeCommit]] => s.get
			case _ =>
				println("Error parsing " + config.inputJSON)
				List[MergeCommit]()
		}).filterNot { _.files.isEmpty }
			.map (m => {
				MergeCommit(m.sha, m.files.filterNot { _.name == "dev/null" }
					.filterNot { _.lines.isEmpty }
				  .filter { _.name.endsWith("java")})
			})

		val parSeq = commits.flatMap(c => c.files.map {(c.sha, _)})
			.flatMap { case (sha, f) => f.lines.map((sha, f.name, _)) }.par
		parSeq.tasksupport = new ForkJoinTaskSupport()
		parSeq.foreach { case (sha, f, l) =>
					println(sha + "," + f + "," + l + "," + detector.findCommits(f, l, sha, Order.FORWARD).mkString(","))
			}
	}
}
{'fraction_non_alphanumeric': 0.14646664034741413, 'fraction_numerical': 0.00039478878799842083, 'mean_word_length': 3.52007136485281, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3021680', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1681, 'n_words': 426}