<commit_before>import AssemblyKeys._
import sbt.Keys._
import Version._
import com.typesafe.sbt.osgi.SbtOsgi._

lazy val extra = <url>http://scalanlp.org/</url>
  <licenses>
    <license>
      <name>Apache 2</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  <scm>
    <url><EMAIL>:dlwh/epic.git</url>
    <connection>scm:git:<EMAIL>:dlwh/epic.git</connection>
  </scm>
  <developers>
    <developer>
      <id>dlwh</id>
      <name><NAME></name>
      <url>http://cs.berkeley.edu/~dlwh/</url>
    </developer>
  </developers>;

lazy val commonSettings = Seq(
  organization := "org.scalanlp",
  version := "0.4-SNAPSHOT",
  scalaVersion := Version.scala,
  crossScalaVersions := Seq("2.11.6", "2.10.4"),
  libraryDependencies ++= Seq(
    Library.breeze,
    Library.breezeConfig,
    Library.mapdb,
    Library.scalaLoggingSlf4j,
    Library.tikaParsers % "compile,optional",
    Library.boilerpipe,
    Library.nekohtml,
    Library.slf4jSimple,
    Library.commonsLang3,
    Library.jflex,
    Library.scalatest % "test",
    Library.scalacheck % "test",
    Library.junit % "test"
  ),
  scalacOptions ++= Seq("-deprecation", "-language:_", "-optimize"),
  javaOptions += "-Xmx4g",
  javaOptions += "-Xrunhprof:cpu=samples,depth=12",
  fork := true,
  publishMavenStyle := true,
  pomExtra := extra,
  publishTo <<= version { (v: String) =>
    val nexus = "https://oss.sonatype.org/"
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at nexus + "content/repositories/snapshots")
    else
      Some("releases" at nexus + "service/local/staging/deploy/maven2")
  },
  publishArtifact in Test := false,
  pomIncludeRepository := { _ => false },
  assemblyOption in assembly ~= {
    _.copy(cacheOutput = false)
  },
  resolvers ++= Seq(
    "ScalaNLP Maven2" at "http://repo.scalanlp.org/repo",
    "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/",
    "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/"
  ),
  libraryDependencies ++= (CrossVersion.partialVersion(scalaVersion.value) match {
      // if scala 2.11+ is used, add dependency on scala-xml module
      case Some((2, scalaMajor)) if scalaMajor >= 11 =>
      Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1")
      case _ =>
      Seq.empty
      })

  ) ++ assemblySettings ++ osgiSettings ++ sbtjflex.SbtJFlexPlugin.jflexSettings ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++
  IndexedSeq (
      mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
        {
          case PathList("org", "w3c", "dom", _) => MergeStrategy.first
          case PathList("javax", "xml", "stream", _ *) => MergeStrategy.first
          case PathList("scala", "xml", _ *) => MergeStrategy.first
          case PathList("org", "cyberneko", "html", _ *) => MergeStrategy.first
          case x => old(x)
         }
      }
)




lazy val core = project
  .in(file("."))
  .settings(commonSettings: _*)
  .settings(
    name := "epic",
    OsgiKeys.privatePackage := Seq(),
    OsgiKeys.importPackage := Seq("org.apache.tika.*; resolution:=optional", "*"),
    OsgiKeys.exportPackage := Seq("epic.*")
  )
<commit_msg>update version<commit_after>import AssemblyKeys._
import sbt.Keys._
import Version._
import com.typesafe.sbt.osgi.SbtOsgi._

lazy val extra = <url>http://scalanlp.org/</url>
  <licenses>
    <license>
      <name>Apache 2</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
      <distribution>repo</distribution>
    </license>
  </licenses>
  <scm>
    <url><EMAIL>:dlwh/epic.git</url>
    <connection>scm:git:git@github.com:dlwh/epic.git</connection>
  </scm>
  <developers>
    <developer>
      <id>dlwh</id>
      <name><NAME></name>
      <url>http://cs.berkeley.edu/~dlwh/</url>
    </developer>
  </developers>;

lazy val commonSettings = Seq(
  organization := "org.scalanlp",
  version := "0.4-SNAPSHOT",
  scalaVersion := Version.scala,
  crossScalaVersions := Seq("2.11.7", "2.10.4"),
  libraryDependencies ++= Seq(
    Library.breeze,
    Library.breezeConfig,
    Library.mapdb,
    Library.scalaLoggingSlf4j,
    Library.tikaParsers % "compile,optional",
    Library.boilerpipe,
    Library.nekohtml,
    Library.slf4jSimple,
    Library.commonsLang3,
    Library.jflex,
    Library.scalatest % "test",
    Library.scalacheck % "test",
    Library.junit % "test"
  ),
  scalacOptions ++= Seq("-deprecation", "-language:_", "-optimize"),
  javaOptions += "-Xmx4g",
  javaOptions += "-Xrunhprof:cpu=samples,depth=12",
  fork := true,
  publishMavenStyle := true,
  pomExtra := extra,
  publishTo <<= version { (v: String) =>
    val nexus = "https://oss.sonatype.org/"
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at nexus + "content/repositories/snapshots")
    else
      Some("releases" at nexus + "service/local/staging/deploy/maven2")
  },
  publishArtifact in Test := false,
  pomIncludeRepository := { _ => false },
  assemblyOption in assembly ~= {
    _.copy(cacheOutput = false)
  },
  resolvers ++= Seq(
    "ScalaNLP Maven2" at "http://repo.scalanlp.org/repo",
    "Scala Tools Snapshots" at "http://scala-tools.org/repo-snapshots/",
    "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/"
  ),
  libraryDependencies ++= (CrossVersion.partialVersion(scalaVersion.value) match {
      // if scala 2.11+ is used, add dependency on scala-xml module
      case Some((2, scalaMajor)) if scalaMajor >= 11 =>
      Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1")
      case _ =>
      Seq.empty
      })

  ) ++ assemblySettings ++ osgiSettings ++ sbtjflex.SbtJFlexPlugin.jflexSettings ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++
  IndexedSeq (
      mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
        {
          case PathList("org", "w3c", "dom", _) => MergeStrategy.first
          case PathList("javax", "xml", "stream", _ *) => MergeStrategy.first
          case PathList("scala", "xml", _ *) => MergeStrategy.first
          case PathList("org", "cyberneko", "html", _ *) => MergeStrategy.first
          case x => old(x)
         }
      }
)




lazy val core = project
  .in(file("."))
  .settings(commonSettings: _*)
  .settings(
    name := "epic",
    OsgiKeys.privatePackage := Seq(),
    OsgiKeys.importPackage := Seq("org.apache.tika.*; resolution:=optional", "*"),
    OsgiKeys.exportPackage := Seq("epic.*")
  )

{'fraction_non_alphanumeric': 0.1679483235927407, 'fraction_numerical': 0.009535527529990772, 'mean_word_length': 3.198192382182053, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 86, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6011061', 'n_tokens_mistral': 2342, 'n_tokens_neox': 2251, 'n_words': 391}