<commit_before>
import sbt._

/**
  * to set a sparkversion run:
  * {{{buld/sbt compile -DsparkVersion=1.6.2}}}
  */
object SparkShim {

  val sparkVersion = sys.props.getOrElse("sparkVersion", default = "1.6.1")
  val sparkVersion_161 = "1.6.1"
  val sparkVersion_162 = "1.6.2"

  val spark161Dependencies = Seq(
    "org.apache.spark" %% "spark-core" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-sql" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-hive" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-hive-thriftserver" % sparkVersion_161 % "provided"
  )

  val spark162Dependencies = Seq(
    "org.apache.spark" %% "spark-core" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-sql" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-hive" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-hive-thriftserver" % sparkVersion_162 % "provided"
  )

  val sparkDependencies =
    if (sparkVersion == sparkVersion_161 ) spark161Dependencies else spark162Dependencies<commit_msg>set default sparkVersion to 1.6.2<commit_after>
import sbt._

/**
  * to set a sparkversion run:
  * {{{buld/sbt compile -DsparkVersion=1.6.2}}}
  */
object SparkShim {

  val sparkVersion = sys.props.getOrElse("sparkVersion", default = "1.6.2")
  val sparkVersion_161 = "1.6.1"
  val sparkVersion_162 = "1.6.2"

  val spark161Dependencies = Seq(
    "org.apache.spark" %% "spark-core" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-sql" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-hive" % sparkVersion_161 % "provided",
    "org.apache.spark" %% "spark-hive-thriftserver" % sparkVersion_161 % "provided"
  )

  val spark162Dependencies = Seq(
    "org.apache.spark" %% "spark-core" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-sql" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-hive" % sparkVersion_162 % "provided",
    "org.apache.spark" %% "spark-hive-thriftserver" % sparkVersion_162 % "provided"
  )

  val sparkDependencies =
    if (sparkVersion == sparkVersion_161 ) spark161Dependencies else spark162Dependencies
{'fraction_non_alphanumeric': 0.1563071297989031, 'fraction_numerical': 0.05347349177330896, 'mean_word_length': 4.055427251732102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5967524', 'n_tokens_mistral': 895, 'n_tokens_neox': 749, 'n_words': 143}