<commit_before>val annotatorLibrary = Map[String, (ModuleID, String)]()

organization := "com.github.mynlp"

name := "jigg"

scalaVersion := "2.11.8"

version := "0.7.1"

fork in run := true

parallelExecution in Test := false

crossPaths := false

// Allow shell mode from sbt runMain
connectInput in run := true

mainClass in assembly := Some("jp.jigg.nlp.pipeline.Pipeline")

javacOptions ++= Seq("-Xlint:all", "-source", "1.6", "-target", "1.6")

scalacOptions ++= Seq("-deprecation", "-feature")

// The unmanaged jars can be put on both lib/ and jar/ directories.
// The intended use of jar/ is to put the third-party NLP software (e.g., easyccg.jar),
// which we do not want to include in an assembled jar.
val externalJars = Seq("lib", "jar")
  .map(file).flatMap(_.listFiles()).filter(_.getName.endsWith(".jar"))
unmanagedJars in Compile ++= externalJars
unmanagedJars in Runtime ++= externalJars

assemblyExcludedJars in assembly := {
  val cp = (fullClasspath in assembly).value
  val unmanaged = file("jar").listFiles().map(_.getName).filter(_.endsWith(".jar"))
  cp filter { x => unmanaged contains x.data.getName }
}

libraryDependencies ++= Seq(
  "com.novocode" % "junit-interface" % "0.10-M4" % "test->default",
  "org.scalactic" %% "scalactic" % "2.2.6",
  "org.scalatest" %% "scalatest" % "2.2.6" % "test",
  "org.scala-lang.modules" %% "scala-xml" % "1.0.5",
  "org.scala-lang" % "scala-reflect" % "2.11.7",
  "com.ibm.icu" % "icu4j" % "56.1",
  "org.scalanlp" %% "breeze" % "0.12",
  "org.scalanlp" % "breeze-config_2.10" % "0.9.1",
  "org.json4s" %% "json4s-jackson" % "3.3.0",
  "com.typesafe.akka" %% "akka-http" % "10.0.1",
  // "com.typesafe.akka" %% "akka-http-xml" % "10.0.1",
  "edu.ucar" % "cdm" % "4.6.8"
)

libraryDependencies ++= (
  stanfordCoreNLPDependencies
    ++ kuromojiDependencies
    ++ berkeleyParserDependencies)

val stanfordCoreNLPDependencies = Seq(
  "org.slf4j" % "slf4j-api" % "1.7.20", // logger
  "org.slf4j" % "slf4j-simple" % "1.7.6",
  "edu.stanford.nlp" % "stanford-corenlp" % "3.7.0"
)

val kuromojiDependencies = Seq(
  "com.atilika.kuromoji" % "kuromoji-ipadic" % "0.9.0",
  "com.atilika.kuromoji" % "kuromoji-jumandic" % "0.9.0",
  "com.atilika.kuromoji" % "kuromoji-unidic" % "0.9.0"
)

val berkeleyParserDependencies = Seq(
  "edu.berkeley.nlp" % "berkeleyparser" % "r32"
)

libraryDependencies ++= annotatorLibrary.map { case (k, v) => v._1 }.toSeq

lazy val root = (project in file(".")).
  enablePlugins(BuildInfoPlugin).
  settings(
    buildInfoKeys := annotatorLibrary.map { case (k, v) =>
      BuildInfoKey.action("ann_" + k)(v._2)
    }.toSeq,
      // Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
    buildInfoPackage := "jigg.pipeline.annotator"
  )

resolvers ++= Seq(
  "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/",
  "Unidata maven repository" at "https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases"
)<commit_msg>Update CoreNLP version to 3.8<commit_after>val annotatorLibrary = Map[String, (ModuleID, String)]()

organization := "com.github.mynlp"

name := "jigg"

scalaVersion := "2.11.8"

version := "0.7.1"

fork in run := true

parallelExecution in Test := false

crossPaths := false

// Allow shell mode from sbt runMain
connectInput in run := true

mainClass in assembly := Some("jp.jigg.nlp.pipeline.Pipeline")

javacOptions ++= Seq("-Xlint:all", "-source", "1.6", "-target", "1.6")

scalacOptions ++= Seq("-deprecation", "-feature")

// The unmanaged jars can be put on both lib/ and jar/ directories.
// The intended use of jar/ is to put the third-party NLP software (e.g., easyccg.jar),
// which we do not want to include in an assembled jar.
val externalJars = Seq("lib", "jar")
  .map(file).flatMap(_.listFiles()).filter(_.getName.endsWith(".jar"))
unmanagedJars in Compile ++= externalJars
unmanagedJars in Runtime ++= externalJars

assemblyExcludedJars in assembly := {
  val cp = (fullClasspath in assembly).value
  val unmanaged = file("jar").listFiles().map(_.getName).filter(_.endsWith(".jar"))
  cp filter { x => unmanaged contains x.data.getName }
}

libraryDependencies ++= Seq(
  "com.novocode" % "junit-interface" % "0.10-M4" % "test->default",
  "org.scalactic" %% "scalactic" % "2.2.6",
  "org.scalatest" %% "scalatest" % "2.2.6" % "test",
  "org.scala-lang.modules" %% "scala-xml" % "1.0.5",
  "org.scala-lang" % "scala-reflect" % "2.11.7",
  "com.ibm.icu" % "icu4j" % "56.1",
  "org.scalanlp" %% "breeze" % "0.12",
  "org.scalanlp" % "breeze-config_2.10" % "0.9.1",
  "org.json4s" %% "json4s-jackson" % "3.3.0",
  "com.typesafe.akka" %% "akka-http" % "10.0.1",
  // "com.typesafe.akka" %% "akka-http-xml" % "10.0.1",
  "edu.ucar" % "cdm" % "4.6.8"
)

libraryDependencies ++= (
  stanfordCoreNLPDependencies
    ++ kuromojiDependencies
    ++ berkeleyParserDependencies)

val stanfordCoreNLPDependencies = Seq(
  "org.slf4j" % "slf4j-api" % "1.7.20", // logger
  "org.slf4j" % "slf4j-simple" % "1.7.6",
  "edu.stanford.nlp" % "stanford-corenlp" % "3.8.0"
)

val kuromojiDependencies = Seq(
  "com.atilika.kuromoji" % "kuromoji-ipadic" % "0.9.0",
  "com.atilika.kuromoji" % "kuromoji-jumandic" % "0.9.0",
  "com.atilika.kuromoji" % "kuromoji-unidic" % "0.9.0"
)

val berkeleyParserDependencies = Seq(
  "edu.berkeley.nlp" % "berkeleyparser" % "r32"
)

libraryDependencies ++= annotatorLibrary.map { case (k, v) => v._1 }.toSeq

lazy val root = (project in file(".")).
  enablePlugins(BuildInfoPlugin).
  settings(
    buildInfoKeys := annotatorLibrary.map { case (k, v) =>
      BuildInfoKey.action("ann_" + k)(v._2)
    }.toSeq,
      // Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
    buildInfoPackage := "jigg.pipeline.annotator"
  )

resolvers ++= Seq(
  "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/",
  "Unidata maven repository" at "https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases"
)
{'fraction_non_alphanumeric': 0.18953918600739994, 'fraction_numerical': 0.028590649175916583, 'mean_word_length': 3.9558333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2668480', 'n_tokens_mistral': 2452, 'n_tokens_neox': 2363, 'n_words': 460}