<commit_before>/*
 * Copyright 2018 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
lazy val scala211Version = "2.11.12"
lazy val scala212Version = "2.12.10"
lazy val sparkVersion = "2.4.0"

lazy val commonSettings = Seq(
  organization := "com.google.cloud.spark",
  version := "0.16.2-SNAPSHOT",
  scalaVersion := scala211Version,
  crossScalaVersions := Seq(scala211Version, scala212Version)
)

// scalastyle:off
// For https://github.com/GoogleCloudPlatform/spark-bigquery-connector/issues/72
// Based on
// https://github.com/sbt/sbt-assembly/#q-despite-the-concerned-friends-i-still-want-publish-fat-jars-what-advice-do-you-have
// scalastyle:on
lazy val root = (project in file("."))
  .disablePlugins(AssemblyPlugin)
  .settings(commonSettings, skip in publish := true)
  .aggregate(connector, fatJar, published)

lazy val commonTestDependencies = Seq(
  "io.grpc" % "grpc-alts" % "1.30.0",
  "io.grpc" % "grpc-netty-shaded" % "1.30.0",
  "com.google.api" % "gax-grpc" % "1.57.0",
  "com.google.guava" % "guava" % "29.0-jre",

  "org.scalatest" %% "scalatest" % "3.1.0" % "test",
  "org.mockito" %% "mockito-scala-scalatest" % "1.10.0" % "test",
  "junit" % "junit" % "4.13" % "test",
  "com.novocode" % "junit-interface" % "0.11" % "test",
  "com.google.truth" % "truth" % "1.0.1" % "test"
)

lazy val connector = (project in file("connector"))
  .enablePlugins(BuildInfoPlugin)
  .configs(ITest)
  .settings(
    commonSettings,
    publishSettings,
    name := "spark-bigquery",
    unmanagedSourceDirectories in Compile += baseDirectory.value /
      "third_party/apache-spark/src/main/java",
    inConfig(ITest)(Defaults.testTasks),
    testOptions in Test := Seq(Tests.Filter(unitFilter)),
    testOptions in ITest := Seq(Tests.Filter(itFilter)),
    buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
    buildInfoPackage := "com.google.cloud.spark.bigquery",
    resourceGenerators in Compile += Def.task {
      val file = (resourceManaged in Compile).value / "spark-bigquery-connector.properties"
      IO.write(file, s"scala.version=${scalaVersion.value}\n")
      Seq(file)
    }.taskValue,
    libraryDependencies ++= (commonTestDependencies ++ Seq(
      "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
      "org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
      "org.slf4j" % "slf4j-api" % "1.7.16" % "provided",
      "aopalliance" % "aopalliance" % "1.0" % "provided",
      "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided",
      "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided",
      "org.apache.arrow" % "arrow-vector" % "0.16.0",
      "com.google.inject" % "guice" % "4.2.3",

      // Keep com.google.cloud dependencies in sync
      "com.google.cloud" % "google-cloud-bigquery" % "1.116.1",
      "com.google.cloud" % "google-cloud-bigquerystorage" % "0.133.2-beta",
      // Keep in sync with com.google.cloud
      "com.fasterxml.jackson.core" % "jackson-databind" % "2.11.0",
      "com.fasterxml.jackson.module" % "jackson-module-paranamer" % "2.11.0",
      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.11.0",
      "io.netty" % "netty-all" % "4.1.27.Final" % "provided",

      // runtime
      // scalastyle:off
      "com.google.cloud.bigdataoss" % "gcs-connector" % "hadoop2-2.0.0" % "runtime" classifier("shaded")
        exclude("com.google.cloud.bigdataoss", "util-hadoop"),
      // scalastyle:on
      // test

      "org.apache.spark" %% "spark-avro" % sparkVersion % "test"
      ))
      .map(_.excludeAll(excludedOrgs.map(ExclusionRule(_)): _*))
  )

lazy val fatJar = project
  .enablePlugins(AssemblyPlugin)
  .settings(
    commonSettings,
    skip in publish := true,
    assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
    assemblyShadeRules in assembly := (
      notRenamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$prefix@1"))
        ++: renamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$relocationPrefix.$prefix@1"))
      ).map(_.inAll),

    assemblyMergeStrategy in assembly := {
      case "module-info.class" => MergeStrategy.discard
      case PathList(ps@_*) if ps.last.endsWith(".properties") => MergeStrategy.filterDistinctLines
      case PathList(ps@_*) if ps.last.endsWith(".proto") => MergeStrategy.discard
      // Relocate netty-tcnative.so. This is necessary even though gRPC shades it, because we shade
      // gRPC.
      case PathList("META-INF", "native", f) if f.contains("netty_tcnative") =>
        RelocationMergeStrategy(path =>
          path.replace("native/lib", s"native/lib${relocationPrefix.replace('.', '_')}_"))

      // Relocate GRPC service registries
      case PathList("META-INF", "services", _) => ServiceResourceMergeStrategy(renamed,
        relocationPrefix)
      case x => (assemblyMergeStrategy in assembly).value(x)
    }
  )
  .dependsOn(connector)

val publishVerified = taskKey[Seq[String]]("Published signed artifact after acceptance test")
lazy val published = project
  .configs(AcceptanceTest)
  .settings(
    commonSettings,
    publishSettings,
    name := "spark-bigquery-with-dependencies",
    packageBin in Compile := (assembly in(fatJar, Compile)).value,
    test in AcceptanceTest := (test in AcceptanceTest).dependsOn(packageBin in Compile).value,
    // publishSigned in SbtPgp := publishSigned.dependsOn(test in AcceptanceTest).value,<commit_msg>Issue #200: Upgrade Netty to a version supporting Java 9+ (#204)<commit_after>/*
 * Copyright 2018 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
lazy val scala211Version = "2.11.12"
lazy val scala212Version = "2.12.10"
lazy val sparkVersion = "2.4.0"
lazy val grpcVersion = "1.30.2"
// should match the dependency from grpc-netty
lazy val nettyVersion = "4.1.48.Final"
// should match the dependency in grpc-netty
lazy val nettyTcnativeVersion = "2.0.29.Final"

lazy val commonSettings = Seq(
  organization := "com.google.cloud.spark",
  version := "0.16.2-SNAPSHOT",
  scalaVersion := scala211Version,
  crossScalaVersions := Seq(scala211Version, scala212Version)
)

// scalastyle:off
// For https://github.com/GoogleCloudPlatform/spark-bigquery-connector/issues/72
// Based on
// https://github.com/sbt/sbt-assembly/#q-despite-the-concerned-friends-i-still-want-publish-fat-jars-what-advice-do-you-have
// scalastyle:on
lazy val root = (project in file("."))
  .disablePlugins(AssemblyPlugin)
  .settings(commonSettings, skip in publish := true)
  .aggregate(connector, fatJar, published)

lazy val commonTestDependencies = Seq(
  "io.grpc" % "grpc-alts" % grpcVersion exclude("io.grpc", "grpc-netty-shaded"),
  "io.grpc" % "grpc-netty" % grpcVersion,
  "com.google.api" % "gax-grpc" % "1.57.0" exclude("io.grpc", "grpc-netty-shaded"),
  "com.google.guava" % "guava" % "29.0-jre",

  "org.scalatest" %% "scalatest" % "3.1.0" % "test",
  "org.mockito" %% "mockito-scala-scalatest" % "1.10.0" % "test",
  "junit" % "junit" % "4.13" % "test",
  "com.novocode" % "junit-interface" % "0.11" % "test",
  "com.google.truth" % "truth" % "1.0.1" % "test"
)

lazy val connector = (project in file("connector"))
  .enablePlugins(BuildInfoPlugin)
  .configs(ITest)
  .settings(
    commonSettings,
    publishSettings,
    name := "spark-bigquery",
    unmanagedSourceDirectories in Compile += baseDirectory.value /
      "third_party/apache-spark/src/main/java",
    inConfig(ITest)(Defaults.testTasks),
    testOptions in Test := Seq(Tests.Filter(unitFilter)),
    testOptions in ITest := Seq(Tests.Filter(itFilter)),
    buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
    buildInfoPackage := "com.google.cloud.spark.bigquery",
    resourceGenerators in Compile += Def.task {
      val file = (resourceManaged in Compile).value / "spark-bigquery-connector.properties"
      IO.write(file, s"scala.version=${scalaVersion.value}\n")
      Seq(file)
    }.taskValue,
    libraryDependencies ++= (commonTestDependencies ++ Seq(
      "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
      "org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
      "org.slf4j" % "slf4j-api" % "1.7.16" % "provided",
      "aopalliance" % "aopalliance" % "1.0" % "provided",
      "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13" % "provided",
      "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13" % "provided",
      "org.apache.arrow" % "arrow-vector" % "0.16.0",
      "com.google.inject" % "guice" % "4.2.3",

      // Keep com.google.cloud dependencies in sync
      "com.google.cloud" % "google-cloud-bigquery" % "1.116.1",
      "com.google.cloud" % "google-cloud-bigquerystorage" % "0.133.2-beta"
        exclude("io.grpc", "grpc-netty-shaded"),
      // Keep in sync with com.google.cloud
      "com.fasterxml.jackson.core" % "jackson-databind" % "2.11.0",
      "com.fasterxml.jackson.module" % "jackson-module-paranamer" % "2.11.0",
      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.11.0",

      // Netty, with a version supporting Java 11
      "io.netty" % "netty-all" % nettyVersion % "provided",
      "io.netty" % "netty-buffer" % nettyVersion,
      "io.netty" % "netty-common" % nettyVersion,
      "org.eclipse.jetty.alpn" % "alpn-api" % "1.1.3.v20160715",
      // scalastyle:off
      // See https://github.com/grpc/grpc-java/blob/master/SECURITY.md#tls-with-netty-tcnative-on-boringssl
      // scalastyle:on
      "io.netty" % "netty-tcnative-boringssl-static" % nettyTcnativeVersion,

      // runtime
      // scalastyle:off
      "com.google.cloud.bigdataoss" % "gcs-connector" % "hadoop2-2.0.0" % "runtime" classifier("shaded")
        exclude("com.google.cloud.bigdataoss", "util-hadoop"),
      // scalastyle:on
      // test

      "org.apache.spark" %% "spark-avro" % sparkVersion % "test"
      ))
      .map(_.excludeAll(excludedOrgs.map(ExclusionRule(_)): _*))
  )

lazy val fatJar = project
  .enablePlugins(AssemblyPlugin)
  .settings(
    commonSettings,
    skip in publish := true,
    assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
    assemblyShadeRules in assembly := (
      notRenamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$prefix@1"))
        ++: renamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$relocationPrefix.$prefix@1"))
      ).map(_.inAll),

    assemblyMergeStrategy in assembly := {
      case x if x.endsWith("/public-suffix-list.txt") => MergeStrategy.filterDistinctLines
      case "module-info.class" => MergeStrategy.discard
      case PathList(ps@_*) if ps.last.endsWith(".properties") => MergeStrategy.filterDistinctLines
      case PathList(ps@_*) if ps.last.endsWith(".proto") => MergeStrategy.discard
      // Relocate netty-tcnative.so. This is necessary even though gRPC shades it, because we shade
      // gRPC.
      case PathList("META-INF", "native", f) if f.contains("netty_tcnative") =>
        RelocationMergeStrategy(path =>
          path.replace("native/lib", s"native/lib${relocationPrefix.replace('.', '_')}_"))

      // Relocate GRPC service registries
      case PathList("META-INF", "services", _) => ServiceResourceMergeStrategy(renamed,
        relocationPrefix)
      case x => (assemblyMergeStrategy in assembly).value(x)
    }
  )
  .dependsOn(connector)

val publishVerified = taskKey[Seq[String]]("Published signed artifact after acceptance test")
lazy val published = project
  .configs(AcceptanceTest)
  .settings(
    commonSettings,
    publishSettings,
    name := "spark-bigquery-with-dependencies",
    packageBin in Compile := (assembly in(fatJar, Compile)).value,
    test in AcceptanceTest := (test in AcceptanceTest).dependsOn(packageBin in Compile).value,
    // publishSigned in SbtPgp := publishSigned.dependsOn(test in AcceptanceTest).value,
{'fraction_non_alphanumeric': 0.15602507545855585, 'fraction_numerical': 0.0213605758068261, 'mean_word_length': 3.5420035149384885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 5, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1106500', 'n_tokens_mistral': 4562, 'n_tokens_neox': 4332, 'n_words': 897}