<commit_before>/*
 * Copyright 2017 Spotify AB.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.spotify

import java.util.UUID

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
import com.google.api.client.json.jackson2.JacksonFactory
import com.google.api.services.dataflow.{Dataflow, DataflowScopes}
import com.google.common.reflect.ClassPath
import com.google.datastore.v1._
import com.google.datastore.v1.client.DatastoreHelper
import com.spotify.scio._
import com.spotify.scio.runners.dataflow.DataflowResult
import com.spotify.scio.values.SCollection
import com.twitter.algebird.Aggregator
import org.apache.beam.runners.dataflow.DataflowPipelineJob
import org.apache.beam.sdk.transforms.DoFn.ProcessElement
import org.apache.beam.sdk.transforms.{DoFn, ParDo}
import org.joda.time.format.{DateTimeFormat, ISODateTimeFormat, PeriodFormat}
import org.joda.time.{DateTimeZone, Instant, Seconds}
import shapeless.datatype.datastore._

import scala.collection.JavaConverters._
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
import scala.util.Random

// This file is symlinked to scio-bench/src/main/scala/com/spotify/ScioBenchmark.scala so that it
// can run with past Scio releases.

object ScioBenchmarkSettings {
  val defaultProjectId: String = "data-integration-test"
  val numOfWorkers = 4
  val commonArgs = Array(
    "--runner=DataflowRunner",
    s"--numWorkers=$numOfWorkers",
    "--workerMachineType=n1-standard-4",
    "--autoscalingAlgorithm=NONE")

  val shuffleConf = Map("ShuffleService" -> Array("--experiments=shuffle_mode=service"))
}

// scalastyle:off number.of.types
// scalastyle:off number.of.methods
object ScioBenchmark {

  import ScioBenchmarkSettings._

  case class CircleCIEnv(buildNum: Long, gitHash: String)
  case class ScioBenchmarkRun(timestamp: Instant,
                              gitHash: String, buildNum: Long, operation: String)
  case class OperationBenchmark(opName: String, metrics: Map[String, String])

  private val dataflow = {
    val transport = GoogleNetHttpTransport.newTrustedTransport()
    val jackson = JacksonFactory.getDefaultInstance
    val credential = GoogleCredential
      .getApplicationDefault.createScoped(Seq(DataflowScopes.CLOUD_PLATFORM).asJava)
    new Dataflow.Builder(transport, jackson, credential).build()
  }

  private val datastore = DatastoreHelper.getDatastoreFromEnv

  private val datastoreKind = "Benchmarks"

  private lazy val datastoreMetricKeys = Set("Elapsed", "TotalMemoryUsage", "TotalPdUsage",
    "TotalShuffleDataProcessed", "TotalSsdUsage", "TotalStreamingDataProcessed", "TotalVcpuTime")

  private val circleCIEnv: Option[CircleCIEnv] = {
    val isCircleCIRun = sys.env.get("CIRCLECI").contains("true")

    if (isCircleCIRun) {
      (sys.env.get("CIRCLE_BUILD_NUM"), sys.env.get("CIRCLE_SHA1")) match {
        case (Some(buildNumber), Some(gitHash)) =>
          Some(CircleCIEnv(buildNumber.toLong, gitHash))
        case _ => throw new IllegalStateException("CIRCLECI env variable is set but not " +
          "CIRCLE_BUILD_NUM and CIRCLE_SHA1")
      }
    } else {
      prettyPrint("CircleCI", "CIRCLECI env variable not found. Will not publish " +
        "benchmark results to Datastore.")
      None
    }
  }

  def main(args: Array[String]): Unit = {
    val argz = Args(args)<commit_msg>Fix: make Datastore env lazy. Local runs dont need Datastore<commit_after>/*
 * Copyright 2017 Spotify AB.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.spotify

import java.util.UUID

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
import com.google.api.client.json.jackson2.JacksonFactory
import com.google.api.services.dataflow.{Dataflow, DataflowScopes}
import com.google.common.reflect.ClassPath
import com.google.datastore.v1._
import com.google.datastore.v1.client.DatastoreHelper
import com.spotify.scio._
import com.spotify.scio.runners.dataflow.DataflowResult
import com.spotify.scio.values.SCollection
import com.twitter.algebird.Aggregator
import org.apache.beam.runners.dataflow.DataflowPipelineJob
import org.apache.beam.sdk.transforms.DoFn.ProcessElement
import org.apache.beam.sdk.transforms.{DoFn, ParDo}
import org.joda.time.format.{DateTimeFormat, ISODateTimeFormat, PeriodFormat}
import org.joda.time.{DateTimeZone, Instant, Seconds}
import shapeless.datatype.datastore._

import scala.collection.JavaConverters._
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
import scala.util.Random

// This file is symlinked to scio-bench/src/main/scala/com/spotify/ScioBenchmark.scala so that it
// can run with past Scio releases.

object ScioBenchmarkSettings {
  val defaultProjectId: String = "data-integration-test"
  val numOfWorkers = 4
  val commonArgs = Array(
    "--runner=DataflowRunner",
    s"--numWorkers=$numOfWorkers",
    "--workerMachineType=n1-standard-4",
    "--autoscalingAlgorithm=NONE")

  val shuffleConf = Map("ShuffleService" -> Array("--experiments=shuffle_mode=service"))
}

// scalastyle:off number.of.types
// scalastyle:off number.of.methods
object ScioBenchmark {

  import ScioBenchmarkSettings._

  case class CircleCIEnv(buildNum: Long, gitHash: String)
  case class ScioBenchmarkRun(timestamp: Instant,
                              gitHash: String, buildNum: Long, operation: String)
  case class OperationBenchmark(opName: String, metrics: Map[String, String])

  private val dataflow = {
    val transport = GoogleNetHttpTransport.newTrustedTransport()
    val jackson = JacksonFactory.getDefaultInstance
    val credential = GoogleCredential
      .getApplicationDefault.createScoped(Seq(DataflowScopes.CLOUD_PLATFORM).asJava)
    new Dataflow.Builder(transport, jackson, credential).build()
  }

  private lazy val datastore = DatastoreHelper.getDatastoreFromEnv

  private val datastoreKind = "Benchmarks"

  private val datastoreMetricKeys = Set("Elapsed", "TotalMemoryUsage", "TotalPdUsage",
    "TotalShuffleDataProcessed", "TotalSsdUsage", "TotalStreamingDataProcessed", "TotalVcpuTime")

  private val circleCIEnv: Option[CircleCIEnv] = {
    val isCircleCIRun = sys.env.get("CIRCLECI").contains("true")

    if (isCircleCIRun) {
      (sys.env.get("CIRCLE_BUILD_NUM"), sys.env.get("CIRCLE_SHA1")) match {
        case (Some(buildNumber), Some(gitHash)) =>
          Some(CircleCIEnv(buildNumber.toLong, gitHash))
        case _ => throw new IllegalStateException("CIRCLECI env variable is set but not " +
          "CIRCLE_BUILD_NUM and CIRCLE_SHA1")
      }
    } else {
      prettyPrint("CircleCI", "CIRCLECI env variable not found. Will not publish " +
        "benchmark results to Datastore.")
      None
    }
  }

  def main(args: Array[String]): Unit = {
    val argz = Args(args)
{'fraction_non_alphanumeric': 0.09963608984816162, 'fraction_numerical': 0.004266532814656795, 'mean_word_length': 4.374241402562373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2491685', 'n_tokens_mistral': 2472, 'n_tokens_neox': 2422, 'n_words': 588}