<commit_before>// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.intellij.build.images.sync

import java.io.File
import java.io.IOException
import java.util.stream.Collectors
import java.util.stream.Stream

internal val GIT = (System.getenv("TEAMCITY_GIT_PATH") ?: System.getenv("GIT") ?: "git").also {
  val noGitFound = "Git is not found, please specify path to git executable in TEAMCITY_GIT_PATH or GIT or add it to PATH"
  try {
    val gitVersion = execute(null, it, "--version")
    if (gitVersion.isBlank()) error(noGitFound)
    log(gitVersion)
  }
  catch (e: IOException) {
    throw IllegalStateException(noGitFound, e)
  }
}

internal fun gitPull(repo: File) = try {
  execute(repo, GIT, "pull", "--rebase")
}
catch (e: Exception) {
  callSafely(printStackTrace = false) {
    execute(repo, GIT, "rebase", "--abort")
  }
  log("Unable to pull changes for $repo: ${e.message}")
}

/**
 * @param dirToList optional dir in [repo] from which to list files
 * @return map of file paths (relative to [dirToList]) to [GitObject]
 */
internal fun listGitObjects(
  repo: File, dirToList: File?,
  fileFilter: (File) -> Boolean = { true }
): Map<String, GitObject> = listGitTree(repo, dirToList, fileFilter)
  .collect(Collectors.toMap({ it.first }, { it.second }))

private fun listGitTree(
  repo: File, dirToList: File?,
  fileFilter: (File) -> Boolean
): Stream<Pair<String, GitObject>> {
  val relativeDirToList = dirToList?.relativeTo(repo)?.path ?: ""
  log("Inspecting $repo")
  if (!isUnderTeamCity()) gitPull(repo)
  return execute(repo, GIT, "ls-tree", "HEAD", "-r", relativeDirToList)
    .trim().lines().stream()
    .filter(String::isNotBlank).map { line ->
      // format: <mode> SP <type> SP <object> TAB <file>
      line.splitWithTab()
        .also { if (it.size != 2) error(line) }
        .let { it[0].splitWithSpace() + it[1] }
        .also { if (it.size != 4) error(line) }
    }
    .filter { fileFilter(repo.resolve(it[3].removeSuffix("\"").removePrefix("\""))) }
    // <file>, <object>, repo
    .map { GitObject(it[3], it[2], repo) }
    .map { it.path.removePrefix("$relativeDirToList/") to it }
}

/**
 * @param repos multiple git repos from which to list files
 * @param root root repo
 * @return map of file paths (relative to [root]) to [GitObject]
 */
internal fun listGitObjects(
  root: File, repos: List<File>,
  fileFilter: (File) -> Boolean = { true }
): Map<String, GitObject> = repos.parallelStream().flatMap { repo ->
  listGitTree(repo, null, fileFilter).map {
    // root relative <file> path to git object
    val rootRelativePath = repo.relativeTo(root).path
    if (rootRelativePath.isEmpty()) {
      it.first
    }
    else {
      "$rootRelativePath/${it.first}"
    } to it.second
  }
}.collect(Collectors.toMap({ it.first }, { it.second }))

/**
 * @param path path relative to [repo]
 */
internal data class GitObject(val path: String, val hash: String, val repo: File) {
  val file = File(repo, path)
}

/**
 * @param dir path in repo
 * @return root of repo
 */
internal fun findGitRepoRoot(dir: File, silent: Boolean = false): File = when {
  dir.isDirectory && dir.listFiles().find { file ->
    file.isDirectory && file.name == ".git"
  } != null -> {
    if (!silent) log("Git repo found in $dir")
    dir
  }
  dir.parentFile != null -> {
    if (!silent) log("No git repo found in $dir")
    findGitRepoRoot(dir.parentFile, silent)
  }
  else -> error("No git repo found in $dir")
}

internal fun unStageFiles(files: List<String>, repo: File) {
  // OS has argument length limit
  splitAndTry(1000, files, repo) {
    execute(repo, GIT, "reset", "HEAD", *it.toTypedArray())
  }
}

internal fun stageFiles(files: List<String>, repo: File) {
  // OS has argument length limit
  splitAndTry(1000, files, repo) {
    execute(repo, GIT, "add", "--ignore-errors", *it.toTypedArray())
  }
}

private fun splitAndTry(factor: Int, files: List<String>, repo: File, block: (files: List<String>) -> Unit) {
  files.split(factor).forEach {
    try {
      block(it)
    }
    catch (e: Exception) {
      if (e.message?.contains("did not match any files") == true) return
      val finerFactor: Int = factor / 2
      if (finerFactor < 1) throw e
      log("Git add command failed with ${e.message}")
      splitAndTry(finerFactor, files, repo, block)
    }
  }
}

internal fun commitAndPush(repo: File, branch: String, message: String, user: String, email: String): CommitInfo {
  execute(
    repo, GIT,
    "-c", "user.name=$user",
    "-c", "user.email=$email",
    "commit", "-m", message,
    "--author=$user <$email>"
  )
  push(repo, branch)
  return commitInfo(repo) ?: error("Unable to read last commit")
}

internal fun checkout(repo: File, branch: String) = execute(repo, GIT, "checkout", branch)

internal fun deleteBranch(repo: File, branch: String) {
  try {
    push(repo, ":$branch")
  }
  catch (e: Exception) {
    if (e.message?.contains("remote ref does not exist") == false) throw e
  }
}

private fun push(repo: File, spec: String) =
  retry(doRetry = { beforePushRetry(it, repo, spec) }) {
    execute(repo, GIT, "push", "origin", spec, withTimer = true)
  }

private fun beforePushRetry(e: Throwable, repo: File, spec: String): Boolean {
  if (!isGitServerUnavailable(e)) {
    val specParts = spec.split(':')
    if (specParts.count() == 2) {
      val flippedSpec = "${specParts[1]}:${specParts[0]}"
      execute(repo, GIT, "pull", "--rebase=true", "origin", flippedSpec)
    }
  }
  return true
}

private fun isGitServerUnavailable(e: Throwable) = with(e.message ?: "") {
  contains("remote end hung up unexpectedly")
  || contains("Service is in readonly mode")
  || contains("failed to lock")
  || contains("Connection timed out")
}

@Volatile
private var origins = emptyMap<File, String>()<commit_msg>[icons sync] skipped pull fix<commit_after>// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.intellij.build.images.sync

import java.io.File
import java.io.IOException
import java.util.stream.Collectors
import java.util.stream.Stream

internal val GIT = (System.getenv("TEAMCITY_GIT_PATH") ?: System.getenv("GIT") ?: "git").also {
  val noGitFound = "Git is not found, please specify path to git executable in TEAMCITY_GIT_PATH or GIT or add it to PATH"
  try {
    val gitVersion = execute(null, it, "--version")
    if (gitVersion.isBlank()) error(noGitFound)
    log(gitVersion)
  }
  catch (e: IOException) {
    throw IllegalStateException(noGitFound, e)
  }
}

internal fun gitPull(repo: File) = try {
  execute(repo, GIT, "pull", "--rebase")
}
catch (e: Exception) {
  callSafely(printStackTrace = false) {
    execute(repo, GIT, "rebase", "--abort")
  }
  log("Unable to pull changes for $repo: ${e.message}")
}

/**
 * @param dirToList optional dir in [repo] from which to list files
 * @return map of file paths (relative to [dirToList]) to [GitObject]
 */
internal fun listGitObjects(
  repo: File, dirToList: File?,
  fileFilter: (File) -> Boolean = { true }
): Map<String, GitObject> = listGitTree(repo, dirToList, fileFilter)
  .collect(Collectors.toMap({ it.first }, { it.second }))

private fun listGitTree(
  repo: File, dirToList: File?,
  fileFilter: (File) -> Boolean
): Stream<Pair<String, GitObject>> {
  val relativeDirToList = dirToList?.relativeTo(repo)?.path ?: ""
  log("Inspecting $repo")
  if (!isUnderTeamCity()) gitPull(repo)
  return execute(repo, GIT, "ls-tree", "HEAD", "-r", relativeDirToList)
    .trim().lines().stream()
    .filter(String::isNotBlank).map { line ->
      // format: <mode> SP <type> SP <object> TAB <file>
      line.splitWithTab()
        .also { if (it.size != 2) error(line) }
        .let { it[0].splitWithSpace() + it[1] }
        .also { if (it.size != 4) error(line) }
    }
    .filter { fileFilter(repo.resolve(it[3].removeSuffix("\"").removePrefix("\""))) }
    // <file>, <object>, repo
    .map { GitObject(it[3], it[2], repo) }
    .map { it.path.removePrefix("$relativeDirToList/") to it }
}

/**
 * @param repos multiple git repos from which to list files
 * @param root root repo
 * @return map of file paths (relative to [root]) to [GitObject]
 */
internal fun listGitObjects(
  root: File, repos: List<File>,
  fileFilter: (File) -> Boolean = { true }
): Map<String, GitObject> = repos.parallelStream().flatMap { repo ->
  listGitTree(repo, null, fileFilter).map {
    // root relative <file> path to git object
    val rootRelativePath = repo.relativeTo(root).path
    if (rootRelativePath.isEmpty()) {
      it.first
    }
    else {
      "$rootRelativePath/${it.first}"
    } to it.second
  }
}.collect(Collectors.toMap({ it.first }, { it.second }))

/**
 * @param path path relative to [repo]
 */
internal data class GitObject(val path: String, val hash: String, val repo: File) {
  val file = File(repo, path)
}

/**
 * @param dir path in repo
 * @return root of repo
 */
internal fun findGitRepoRoot(dir: File, silent: Boolean = false): File = when {
  dir.isDirectory && dir.listFiles().find { file ->
    file.isDirectory && file.name == ".git"
  } != null -> {
    if (!silent) log("Git repo found in $dir")
    dir
  }
  dir.parentFile != null -> {
    if (!silent) log("No git repo found in $dir")
    findGitRepoRoot(dir.parentFile, silent)
  }
  else -> error("No git repo found in $dir")
}

internal fun unStageFiles(files: List<String>, repo: File) {
  // OS has argument length limit
  splitAndTry(1000, files, repo) {
    execute(repo, GIT, "reset", "HEAD", *it.toTypedArray())
  }
}

internal fun stageFiles(files: List<String>, repo: File) {
  // OS has argument length limit
  splitAndTry(1000, files, repo) {
    execute(repo, GIT, "add", "--ignore-errors", *it.toTypedArray())
  }
}

private fun splitAndTry(factor: Int, files: List<String>, repo: File, block: (files: List<String>) -> Unit) {
  files.split(factor).forEach {
    try {
      block(it)
    }
    catch (e: Exception) {
      if (e.message?.contains("did not match any files") == true) return
      val finerFactor: Int = factor / 2
      if (finerFactor < 1) throw e
      log("Git add command failed with ${e.message}")
      splitAndTry(finerFactor, files, repo, block)
    }
  }
}

internal fun commitAndPush(repo: File, branch: String, message: String, user: String, email: String): CommitInfo {
  execute(
    repo, GIT,
    "-c", "user.name=$user",
    "-c", "user.email=$email",
    "commit", "-m", message,
    "--author=$user <$email>"
  )
  push(repo, branch)
  return commitInfo(repo) ?: error("Unable to read last commit")
}

internal fun checkout(repo: File, branch: String) = execute(repo, GIT, "checkout", branch)

internal fun deleteBranch(repo: File, branch: String) {
  try {
    push(repo, ":$branch")
  }
  catch (e: Exception) {
    if (e.message?.contains("remote ref does not exist") == false) throw e
  }
}

private fun push(repo: File, spec: String) =
  retry(doRetry = { beforePushRetry(it, repo, spec) }) {
    execute(repo, GIT, "push", "origin", spec, withTimer = true)
  }

private fun beforePushRetry(e: Throwable, repo: File, spec: String): Boolean {
  if (!isGitServerUnavailable(e)) {
    val specParts = spec.split(':')
    execute(repo, GIT, "pull", "--rebase=true", "origin", if (specParts.count() == 2) {
      "${specParts[1]}:${specParts[0]}"
    } else spec)
  }
  return true
}

private fun isGitServerUnavailable(e: Throwable) = with(e.message ?: "") {
  contains("remote end hung up unexpectedly")
  || contains("Service is in readonly mode")
  || contains("failed to lock")
  || contains("Connection timed out")
}

@Volatile
private var origins = emptyMap<File, String>()
{'fraction_non_alphanumeric': 0.1574796472184532, 'fraction_numerical': 0.00508819538670285, 'mean_word_length': 3.6484036263303112, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '143941', 'n_tokens_mistral': 3974, 'n_tokens_neox': 3898, 'n_words': 1132}