<commit_before>import sbt._
import sbt.Keys._
import xerial.sbt.Pack._

object NLP4LBuild extends Build {
  lazy val root = Project(
      id = "nlp4l",
      base = file("."),
      settings = Defaults.defaultSettings ++ packSettings ++
        Seq(
          version := "0.1-dev",
          scalaVersion := "2.11.6",
          libraryDependencies ++= Seq(
            "org.apache.lucene" % "lucene-core" % "5.0.0",
            "org.apache.lucene" % "lucene-analyzers-common" % "5.0.0",
            "org.apache.lucene" % "lucene-analyzers-kuromoji" % "5.0.0",
            "org.apache.lucene" % "lucene-analyzers-icu" % "5.0.0",
            "org.apache.lucene" % "lucene-queries" % "5.0.0",
            "org.apache.lucene" % "lucene-queryparser" % "5.0.0",
            "org.apache.lucene" % "lucene-codecs" % "5.0.0",
            "org.apache.lucene" % "lucene-backward-codecs" % "5.0.0",
            "org.apache.lucene" % "lucene-misc" % "5.0.0",
            "org.apache.httpcomponents" % "httpclient" % "4.4",
            "com.github.scala-incubator.io" % "scala-io-core_2.11" % "0.4.3",
            "com.github.scala-incubator.io" % "scala-io-file_2.11" % "0.4.3",
            "org.scalatest" % "scalatest_2.11" % "2.2.1" % "test",
            "junit" % "junit" % "4.12" % "test",
            "com.novocode" % "junit-interface" % "0.11" % "test",
            "org.scala-lang" % "scala-compiler" % "2.11.6",
            "jline" % "jline" % "2.12",
            "com.typesafe" % "config" % "1.2.1",
            "com.jsuereth" % "scala-arm_2.10" % "1.3"
          ),
          testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v"),
          packMain := Map("nlp4l" -> "org.nlp4l.repl.NLP4LMainGenericRunner"),
          packJvmOpts := Map("nlp4l" -> Seq("-Xmx512m", "-Dfile.encoding=UTF-8", "-Dnlp4l.conf=${PROG_HOME}/bin/repl.init")),
          packResourceDir ++= Map(baseDirectory.value / "docs" -> "docs",
            baseDirectory.value / "examples" -> "examples",
            baseDirectory.value / "repl.init" -> "bin/repl.init",
            baseDirectory.value / "target/scala-2.11/api" -> "api"),
          pack <<= pack.dependsOn(doc in Compile)
      )
    )


}<commit_msg>Upgraded Lucene (from 5.0 to 5.1)<commit_after>import sbt._
import sbt.Keys._
import xerial.sbt.Pack._

object NLP4LBuild extends Build {
  lazy val root = Project(
      id = "nlp4l",
      base = file("."),
      settings = Defaults.defaultSettings ++ packSettings ++
        Seq(
          version := "0.1-dev",
          scalaVersion := "2.11.6",
          libraryDependencies ++= Seq(
            "org.apache.lucene" % "lucene-core" % "5.1.0",
            "org.apache.lucene" % "lucene-analyzers-common" % "5.1.0",
            "org.apache.lucene" % "lucene-analyzers-kuromoji" % "5.1.0",
            "org.apache.lucene" % "lucene-analyzers-icu" % "5.1.0",
            "org.apache.lucene" % "lucene-queries" % "5.1.0",
            "org.apache.lucene" % "lucene-queryparser" % "5.1.0",
            "org.apache.lucene" % "lucene-codecs" % "5.1.0",
            "org.apache.lucene" % "lucene-backward-codecs" % "5.1.0",
            "org.apache.lucene" % "lucene-misc" % "5.1.0",
            "org.apache.httpcomponents" % "httpclient" % "4.4",
            "com.github.scala-incubator.io" % "scala-io-core_2.11" % "0.4.3",
            "com.github.scala-incubator.io" % "scala-io-file_2.11" % "0.4.3",
            "org.scalatest" % "scalatest_2.11" % "2.2.1" % "test",
            "junit" % "junit" % "4.12" % "test",
            "com.novocode" % "junit-interface" % "0.11" % "test",
            "org.scala-lang" % "scala-compiler" % "2.11.6",
            "jline" % "jline" % "2.12",
            "com.typesafe" % "config" % "1.2.1",
            "com.jsuereth" % "scala-arm_2.10" % "1.3"
          ),
          testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v"),
          packMain := Map("nlp4l" -> "org.nlp4l.repl.NLP4LMainGenericRunner"),
          packJvmOpts := Map("nlp4l" -> Seq("-Xmx512m", "-Dfile.encoding=UTF-8", "-Dnlp4l.conf=${PROG_HOME}/bin/repl.init")),
          packResourceDir ++= Map(baseDirectory.value / "docs" -> "docs",
            baseDirectory.value / "examples" -> "examples",
            baseDirectory.value / "repl.init" -> "bin/repl.init",
            baseDirectory.value / "target/scala-2.11/api" -> "api"),
          pack <<= pack.dependsOn(doc in Compile)
      )
    )


}
{'fraction_non_alphanumeric': 0.20018157058556513, 'fraction_numerical': 0.04085338175215615, 'mean_word_length': 2.0351239669421486, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699381', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1677, 'n_words': 239}