<commit_before>/*
 * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
 * that can be found in the LICENSE file.
 */

import org.w3c.xhr.*
import kotlin.js.json
import kotlin.js.Date
import kotlin.js.Promise
import org.jetbrains.database.*
import org.jetbrains.report.json.*
import org.jetbrains.elastic.*
import org.jetbrains.network.*
import org.jetbrains.buildInfo.Build
import org.jetbrains.analyzer.*
import org.jetbrains.report.*
import org.jetbrains.utils.*

// TODO - create DSL for ES requests?

const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest"
const val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks"

operator fun <K, V> Map<K, V>?.get(key: K) = this?.get(key)

fun getArtifactoryHeader(artifactoryApiKey: String) = Pair("X-JFrog-Art-Api", artifactoryApiKey)

external fun decodeURIComponent(url: String): String

// Convert saved old report to expected new format.
internal fun convertToNewFormat(data: JsonObject): List<Any> {
    val env = Environment.create(data.getRequiredField("env"))
    val benchmarksObj = data.getRequiredField("benchmarks")
    val compilerDescription = data.getRequiredField("kotlin")
    val compiler = Compiler.create(compilerDescription)
    val backend = (compilerDescription as JsonObject).getRequiredField("backend")
    val flagsArray = (backend as JsonObject).getOptionalField("flags")
    var flags: List<String> = emptyList()
    if (flagsArray != null && flagsArray is JsonArray) {
        flags = flagsArray.jsonArray.map { (it as JsonLiteral).unquoted() }
    }
    val benchmarksList = parseBenchmarksArray(benchmarksObj)

    return listOf(env, compiler, benchmarksList, flags)
}

// Convert data results to expected format.
internal fun convert(json: String, buildNumber: String, target: String): List<BenchmarksReport> {
    val data = JsonTreeParser.parse(json)
    val reports = if (data is JsonArray) {
        data.map { convertToNewFormat(it as JsonObject) }
    } else listOf(convertToNewFormat(data as JsonObject))

    // Restored flags for old reports.
    val knownFlags = mapOf(
            "Cinterop" to listOf("-opt"),
            "FrameworkBenchmarksAnalyzer" to listOf("-g"),
            "HelloWorld" to if (target == "Mac OS X")
                listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g")
            else listOf("-g"),
            "Numerical" to listOf("-opt"),
            "ObjCInterop" to listOf("-opt"),
            "Ring" to listOf("-opt"),
            "Startup" to listOf("-opt"),
            "swiftInterop" to listOf("-opt"),
            "Videoplayer" to if (target == "Mac OS X")
                listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g")
            else listOf("-g")
    )

    return reports.map { elements ->
        val benchmarks = (elements[2] as List<BenchmarkResult>).groupBy { it.name.substringBefore('.').substringBefore(':') }
        val parsedFlags = elements[3] as List<String>
        benchmarks.map { (setName, results) ->
            val flags = if (parsedFlags.isNotEmpty() && parsedFlags[0] == "-opt") knownFlags[setName]!! else parsedFlags
            val savedCompiler = elements[1] as Compiler
            val compiler = Compiler(Compiler.Backend(savedCompiler.backend.type, savedCompiler.backend.version, flags),
                    savedCompiler.kotlinVersion)
            val newReport = BenchmarksReport(elements[0] as Environment, results, compiler)
            newReport.buildNumber = buildNumber
            newReport
        }
    }.flatten()
}

// Golden result value used to get normalized results.
data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double)
data class GoldenResultsInfo(val goldenResults: Array<GoldenResult>)

// Convert information about golden results to benchmarks report format.
fun GoldenResultsInfo.toBenchmarksReport(): BenchmarksReport {
    val benchmarksSamples = goldenResults.map {
        BenchmarkResult(it.benchmarkName, BenchmarkResult.Status.PASSED,
                it.value, BenchmarkResult.metricFromString(it.metric)!!, it.value, 1, 0)
    }
    val compiler = Compiler(Compiler.Backend(Compiler.BackendType.NATIVE, "golden", emptyList()), "golden")
    val environment = Environment(Environment.Machine("golden", "golden"), Environment.JDKInstance("golden", "golden"))
    return BenchmarksReport(environment,
            benchmarksSamples, compiler)
}

// Build information provided from request.
data class TCBuildInfo(val buildNumber: String, val branch: String, val startTime: String,
                       val finishTime: String)

data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String,
                         val bundleSize: String?, val fileWithResult: String) {
    companion object {
        fun create(json: String): BuildRegister {
            val requestDetails = JSON.parse<BuildRegister>(json)
            // Parse method doesn't create real instance with all methods. So create it by hands.
            return BuildRegister(requestDetails.buildId, requestDetails.teamCityUser, requestDetails.teamCityPassword,
                    requestDetails.bundleSize, requestDetails.fileWithResult)
        }
    }

    private val teamCityBuildUrl: String by lazy { "builds/id:$buildId" }

    val changesListUrl: String by lazy {
        "changes/?locator=build:id:$buildId"
    }

    val teamCityArtifactsUrl: String by lazy { "builds/id:$buildId/artifacts/content/$fileWithResult" }

    fun sendTeamCityRequest(url: String, json: Boolean = false) =
            UrlNetworkConnector(teamCityUrl).sendRequest(RequestMethod.GET, url, teamCityUser, teamCityPassword, json)

    private fun format(timeValue: Int): String =
            if (timeValue < 10) "0$timeValue" else "$timeValue"

    fun getBuildInformation(): Promise<TCBuildInfo> {
        return Promise.all(arrayOf(sendTeamCityRequest("$teamCityBuildUrl/number"),
                sendTeamCityRequest("$teamCityBuildUrl/branchName"),
                sendTeamCityRequest("$teamCityBuildUrl/startDate"))).then { results ->
            val (buildNumber, branch, startTime) = results
            val currentTime = Date()
            val timeZone = currentTime.getTimezoneOffset() / -60    // Convert to hours.
            // Get finish time as current time, because buid on TeamCity isn't finished.
            val finishTime = "${format(currentTime.getUTCFullYear())}" +
                    "${format(currentTime.getUTCMonth() + 1)}" +
                    "${format(currentTime.getUTCDate())}" +
                    "T${format(currentTime.getUTCHours())}" +
                    "${format(currentTime.getUTCMinutes())}" +
                    "${format(currentTime.getUTCSeconds())}" +
                    "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}"
            TCBuildInfo(buildNumber, branch, startTime, finishTime)
        }
    }
}<commit_msg>Replace request to get branch name from TC in order to get right info in builds with several projects<commit_after>/*
 * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
 * that can be found in the LICENSE file.
 */

import org.w3c.xhr.*
import kotlin.js.json
import kotlin.js.Date
import kotlin.js.Promise
import org.jetbrains.database.*
import org.jetbrains.report.json.*
import org.jetbrains.elastic.*
import org.jetbrains.network.*
import org.jetbrains.buildInfo.Build
import org.jetbrains.analyzer.*
import org.jetbrains.report.*
import org.jetbrains.utils.*

// TODO - create DSL for ES requests?

const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest"
const val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks"

operator fun <K, V> Map<K, V>?.get(key: K) = this?.get(key)

fun getArtifactoryHeader(artifactoryApiKey: String) = Pair("X-JFrog-Art-Api", artifactoryApiKey)

external fun decodeURIComponent(url: String): String

// Convert saved old report to expected new format.
internal fun convertToNewFormat(data: JsonObject): List<Any> {
    val env = Environment.create(data.getRequiredField("env"))
    val benchmarksObj = data.getRequiredField("benchmarks")
    val compilerDescription = data.getRequiredField("kotlin")
    val compiler = Compiler.create(compilerDescription)
    val backend = (compilerDescription as JsonObject).getRequiredField("backend")
    val flagsArray = (backend as JsonObject).getOptionalField("flags")
    var flags: List<String> = emptyList()
    if (flagsArray != null && flagsArray is JsonArray) {
        flags = flagsArray.jsonArray.map { (it as JsonLiteral).unquoted() }
    }
    val benchmarksList = parseBenchmarksArray(benchmarksObj)

    return listOf(env, compiler, benchmarksList, flags)
}

// Convert data results to expected format.
internal fun convert(json: String, buildNumber: String, target: String): List<BenchmarksReport> {
    val data = JsonTreeParser.parse(json)
    val reports = if (data is JsonArray) {
        data.map { convertToNewFormat(it as JsonObject) }
    } else listOf(convertToNewFormat(data as JsonObject))

    // Restored flags for old reports.
    val knownFlags = mapOf(
            "Cinterop" to listOf("-opt"),
            "FrameworkBenchmarksAnalyzer" to listOf("-g"),
            "HelloWorld" to if (target == "Mac OS X")
                listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g")
            else listOf("-g"),
            "Numerical" to listOf("-opt"),
            "ObjCInterop" to listOf("-opt"),
            "Ring" to listOf("-opt"),
            "Startup" to listOf("-opt"),
            "swiftInterop" to listOf("-opt"),
            "Videoplayer" to if (target == "Mac OS X")
                listOf("-Xcache-directory=/Users/teamcity/buildAgent/work/c104dee5223a31c5/test_dist/klib/cache/macos_x64-gSTATIC", "-g")
            else listOf("-g")
    )

    return reports.map { elements ->
        val benchmarks = (elements[2] as List<BenchmarkResult>).groupBy { it.name.substringBefore('.').substringBefore(':') }
        val parsedFlags = elements[3] as List<String>
        benchmarks.map { (setName, results) ->
            val flags = if (parsedFlags.isNotEmpty() && parsedFlags[0] == "-opt") knownFlags[setName]!! else parsedFlags
            val savedCompiler = elements[1] as Compiler
            val compiler = Compiler(Compiler.Backend(savedCompiler.backend.type, savedCompiler.backend.version, flags),
                    savedCompiler.kotlinVersion)
            val newReport = BenchmarksReport(elements[0] as Environment, results, compiler)
            newReport.buildNumber = buildNumber
            newReport
        }
    }.flatten()
}

// Golden result value used to get normalized results.
data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double)
data class GoldenResultsInfo(val goldenResults: Array<GoldenResult>)

// Convert information about golden results to benchmarks report format.
fun GoldenResultsInfo.toBenchmarksReport(): BenchmarksReport {
    val benchmarksSamples = goldenResults.map {
        BenchmarkResult(it.benchmarkName, BenchmarkResult.Status.PASSED,
                it.value, BenchmarkResult.metricFromString(it.metric)!!, it.value, 1, 0)
    }
    val compiler = Compiler(Compiler.Backend(Compiler.BackendType.NATIVE, "golden", emptyList()), "golden")
    val environment = Environment(Environment.Machine("golden", "golden"), Environment.JDKInstance("golden", "golden"))
    return BenchmarksReport(environment,
            benchmarksSamples, compiler)
}

// Build information provided from request.
data class TCBuildInfo(val buildNumber: String, val branch: String, val startTime: String,
                       val finishTime: String)

data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String,
                         val bundleSize: String?, val fileWithResult: String) {
    companion object {
        fun create(json: String): BuildRegister {
            val requestDetails = JSON.parse<BuildRegister>(json)
            // Parse method doesn't create real instance with all methods. So create it by hands.
            return BuildRegister(requestDetails.buildId, requestDetails.teamCityUser, requestDetails.teamCityPassword,
                    requestDetails.bundleSize, requestDetails.fileWithResult)
        }
    }

    private val teamCityBuildUrl: String by lazy { "builds/id:$buildId" }

    val changesListUrl: String by lazy {
        "changes/?locator=build:id:$buildId"
    }

    val teamCityArtifactsUrl: String by lazy { "builds/id:$buildId/artifacts/content/$fileWithResult" }

    fun sendTeamCityRequest(url: String, json: Boolean = false) =
            UrlNetworkConnector(teamCityUrl).sendRequest(RequestMethod.GET, url, teamCityUser, teamCityPassword, json)

    fun getBranchName(project: String): Promise<String> {
        val url = "builds?locator=id:$buildId&fields=build(revisions(revision(vcsBranchName,vcs-root-instance)))"
        var branch: String? = null
        return sendTeamCityRequest(url, true).then { response ->
            val data = JsonTreeParser.parse(response).jsonObject
            data.getArray("build").forEach {
                (it as JsonObject).getObject("revisions").getArray("revision").forEach {
                    val currentBranch = (it as JsonObject).getPrimitive("vcsBranchName").content.removePrefix("refs/heads/")
                    val currentProject = (it as JsonObject).getObject("vcs-root-instance").getPrimitive("name").content
                    if (project == currentProject) {
                        branch = currentBranch
                    }
                    return@forEach
                }
            }
            branch ?: error("No project $project can be found in build $buildId")
        }
    }


    private fun format(timeValue: Int): String =
            if (timeValue < 10) "0$timeValue" else "$timeValue"

    fun getBuildInformation(): Promise<TCBuildInfo> {
        return Promise.all(arrayOf(sendTeamCityRequest("$teamCityBuildUrl/number"),
                getBranchName("Kotlin Native"),
                sendTeamCityRequest("$teamCityBuildUrl/startDate"))).then { results ->
            val (buildNumber, branch, startTime) = results
            val currentTime = Date()
            val timeZone = currentTime.getTimezoneOffset() / -60    // Convert to hours.
            // Get finish time as current time, because buid on TeamCity isn't finished.
            val finishTime = "${format(currentTime.getUTCFullYear())}" +
                    "${format(currentTime.getUTCMonth() + 1)}" +
                    "${format(currentTime.getUTCDate())}" +
                    "T${format(currentTime.getUTCHours())}" +
                    "${format(currentTime.getUTCMinutes())}" +
                    "${format(currentTime.getUTCSeconds())}" +
                    "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}"
            TCBuildInfo(buildNumber, branch, startTime, finishTime)
        }
    }
}
{'fraction_non_alphanumeric': 0.11367197701749804, 'fraction_numerical': 0.006529119874640899, 'mean_word_length': 3.094359796845763, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 33, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4226864', 'n_tokens_mistral': 4386, 'n_tokens_neox': 4237, 'n_words': 1049}