<commit_before>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.gradle.performance.fixture

import org.gradle.integtests.fixtures.executer.GradleDistribution
import org.gradle.integtests.fixtures.executer.IntegrationTestBuildContext
import org.gradle.integtests.fixtures.versions.ReleasedVersionDistributions
import org.gradle.performance.ResultSpecification
import org.gradle.performance.measure.DataAmount
import org.gradle.performance.measure.Duration
import org.gradle.performance.results.DataReporter
import org.gradle.performance.results.MeasuredOperationList
import org.gradle.performance.results.ResultsStore
import org.gradle.test.fixtures.file.TestNameTestDirectoryProvider
import org.gradle.util.GradleVersion
import org.gradle.util.SetSystemProperties
import org.junit.Rule

class CrossVersionPerformanceTestRunnerTest extends ResultSpecification {
    private static interface ReporterAndStore extends DataReporter, ResultsStore {}

    private static final String MOST_RECENT_SNAPSHOT = "2.11-20160101120000+0000"
    private static final String MOST_RECENT_RELEASE = "2.10"

    @Rule
    TestNameTestDirectoryProvider tmpDir = new TestNameTestDirectoryProvider()

    @Rule
    SetSystemProperties systemProperties = new SetSystemProperties(
        'org.gradle.performance.db.url': "jdbc:h2:${tmpDir.testDirectory}"
    )

    final buildContext = new IntegrationTestBuildContext();
    final experimentRunner = Mock(BuildExperimentRunner)
    final reporter = Mock(ReporterAndStore)
    final testProjectLocator = Stub(TestProjectLocator)
    final currentGradle = Stub(GradleDistribution)
    final releases = Stub(ReleasedVersionDistributions)
    final currentVersionBase = GradleVersion.current().baseVersion.version

    def setup() {
        releases.all >> [
            buildContext.distribution("1.0"),
            buildContext.distribution("1.1"),
            buildContext.distribution("2.10"),
            // "2.11" is missing intentionally to test that RC gets used when baseline version has not been released
            buildContext.distribution("2.11-rc-4"),
            buildContext.distribution("2.11-rc-2"),
            buildContext.distribution("2.12"),
            buildContext.distribution("2.13"),
            buildContext.distribution(MOST_RECENT_RELEASE)]
        releases.mostRecentFinalRelease >> buildContext.distribution(MOST_RECENT_RELEASE)
    }

    def "runs tests against version under test plus requested baseline versions and most recent released version and builds results"() {
        given:
        def runner = runner()
        runner.testId = 'some-test'
        runner.previousTestIds = ['prev 1']
        runner.testProject = 'test1'
        runner.targetVersions = ['1.0', '1.1']
        runner.tasksToRun = ['clean', 'build']
        runner.args = ['--arg1', '--arg2']
        runner.gradleOpts = ['-Xmx24']
        runner.useDaemon = true
        runner.warmUpRuns = 1
        runner.runs = 4

        when:
        def results = runner.run()

        then:
        results.testId == 'some-test'
        results.previousTestIds == ['prev 1']
        results.testProject == 'test1'
        results.tasks == ['clean', 'build']
        results.args == ['--arg1', '--arg2']
        runner.gradleOpts.findAll { !it.startsWith('-XX:MaxPermSize=') } == ['-Xmx24']
        results.daemon
        results.versionUnderTest
        results.jvm
        results.operatingSystem
        results.current.size() == 4
        results.current.totalTime.average == Duration.seconds(10)
        results.current.totalMemoryUsed.average == DataAmount.kbytes(10)
        results.baselineVersions*.version == ['1.0', '1.1', MOST_RECENT_RELEASE]
        results.baseline('1.0').results.size() == 4
        results.baseline('1.1').results.size() == 4
        results.baseline(MOST_RECENT_RELEASE).results.size() == 4

        and:
        4 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        0 * reporter._
    }

    def "can use 'last' baseline version to refer to most recently released version"() {
        given:
        def runner = runner()
        runner.targetVersions = ['1.0', 'last']

        when:
        def results = runner.run()

        then:
        results.baselineVersions*.version == ['1.0', MOST_RECENT_RELEASE]

        and:
        3 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
    }

    def "can use 'none' in target versions to leave the baseline unspecified"() {
        given:
        def runner = runner()
        runner.targetVersions = ['none']

        when:
        def results = runner.run()

        then:
        !results.baselineVersions
        1 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            assert result.name == 'Current Gradle'
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        1 * experimentRunner.getHonestProfiler()
        0 * _._
    }

    def "can override baselines with no baselines by using 'none' in the system property"() {
        given:
        def runner = runner()
        runner.targetVersions = ['2.11', '2.12']

        when:
        System.setProperty('org.gradle.performance.baselines', 'none')
        def results = runner.run()

        then:
        !results.baselineVersions
        1 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            assert result.name == 'Current Gradle'
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        1 * experimentRunner.getHonestProfiler()
        0 * _._
    }

    def "shouldn't endlessly resolve 'defaults'"() {
        given:
        def runner = runner()
        runner.targetVersions = ['2.11', 'defaults']

        when:
        System.setProperty('org.gradle.performance.baselines', 'defaults')
        def results = runner.run()

        then:
        thrown(IllegalArgumentException)<commit_msg>Ignored a couple of tests.<commit_after>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.gradle.performance.fixture

import org.gradle.integtests.fixtures.executer.GradleDistribution
import org.gradle.integtests.fixtures.executer.IntegrationTestBuildContext
import org.gradle.integtests.fixtures.versions.ReleasedVersionDistributions
import org.gradle.performance.ResultSpecification
import org.gradle.performance.measure.DataAmount
import org.gradle.performance.measure.Duration
import org.gradle.performance.results.DataReporter
import org.gradle.performance.results.MeasuredOperationList
import org.gradle.performance.results.ResultsStore
import org.gradle.test.fixtures.file.TestNameTestDirectoryProvider
import org.gradle.util.GradleVersion
import org.gradle.util.SetSystemProperties
import org.junit.Rule
import spock.lang.Ignore

class CrossVersionPerformanceTestRunnerTest extends ResultSpecification {
    private static interface ReporterAndStore extends DataReporter, ResultsStore {}

    private static final String MOST_RECENT_SNAPSHOT = "2.11-20160101120000+0000"
    private static final String MOST_RECENT_RELEASE = "2.10"

    @Rule
    TestNameTestDirectoryProvider tmpDir = new TestNameTestDirectoryProvider()

    @Rule
    SetSystemProperties systemProperties = new SetSystemProperties(
        'org.gradle.performance.db.url': "jdbc:h2:${tmpDir.testDirectory}"
    )

    final buildContext = new IntegrationTestBuildContext();
    final experimentRunner = Mock(BuildExperimentRunner)
    final reporter = Mock(ReporterAndStore)
    final testProjectLocator = Stub(TestProjectLocator)
    final currentGradle = Stub(GradleDistribution)
    final releases = Stub(ReleasedVersionDistributions)
    final currentVersionBase = GradleVersion.current().baseVersion.version

    def setup() {
        releases.all >> [
            buildContext.distribution("1.0"),
            buildContext.distribution("1.1"),
            buildContext.distribution("2.10"),
            // "2.11" is missing intentionally to test that RC gets used when baseline version has not been released
            buildContext.distribution("2.11-rc-4"),
            buildContext.distribution("2.11-rc-2"),
            buildContext.distribution("2.12"),
            buildContext.distribution("2.13"),
            buildContext.distribution(MOST_RECENT_RELEASE)]
        releases.mostRecentFinalRelease >> buildContext.distribution(MOST_RECENT_RELEASE)
    }

    def "runs tests against version under test plus requested baseline versions and most recent released version and builds results"() {
        given:
        def runner = runner()
        runner.testId = 'some-test'
        runner.previousTestIds = ['prev 1']
        runner.testProject = 'test1'
        runner.targetVersions = ['1.0', '1.1']
        runner.tasksToRun = ['clean', 'build']
        runner.args = ['--arg1', '--arg2']
        runner.gradleOpts = ['-Xmx24']
        runner.useDaemon = true
        runner.warmUpRuns = 1
        runner.runs = 4

        when:
        def results = runner.run()

        then:
        results.testId == 'some-test'
        results.previousTestIds == ['prev 1']
        results.testProject == 'test1'
        results.tasks == ['clean', 'build']
        results.args == ['--arg1', '--arg2']
        runner.gradleOpts.findAll { !it.startsWith('-XX:MaxPermSize=') } == ['-Xmx24']
        results.daemon
        results.versionUnderTest
        results.jvm
        results.operatingSystem
        results.current.size() == 4
        results.current.totalTime.average == Duration.seconds(10)
        results.current.totalMemoryUsed.average == DataAmount.kbytes(10)
        results.baselineVersions*.version == ['1.0', '1.1', MOST_RECENT_RELEASE]
        results.baseline('1.0').results.size() == 4
        results.baseline('1.1').results.size() == 4
        results.baseline(MOST_RECENT_RELEASE).results.size() == 4

        and:
        4 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        0 * reporter._
    }

    def "can use 'last' baseline version to refer to most recently released version"() {
        given:
        def runner = runner()
        runner.targetVersions = ['1.0', 'last']

        when:
        def results = runner.run()

        then:
        results.baselineVersions*.version == ['1.0', MOST_RECENT_RELEASE]

        and:
        3 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
    }

    @Ignore
    def "can use 'none' in target versions to leave the baseline unspecified"() {
        given:
        def runner = runner()
        runner.targetVersions = ['none']

        when:
        def results = runner.run()

        then:
        !results.baselineVersions
        1 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            assert result.name == 'Current Gradle'
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        1 * experimentRunner.getHonestProfiler()
        0 * _._
    }

    @Ignore
    def "can override baselines with no baselines by using 'none' in the system property"() {
        given:
        def runner = runner()
        runner.targetVersions = ['2.11', '2.12']

        when:
        System.setProperty('org.gradle.performance.baselines', 'none')
        def results = runner.run()

        then:
        !results.baselineVersions
        1 * experimentRunner.run(_, _) >> { BuildExperimentSpec spec, MeasuredOperationList result ->
            assert result.name == 'Current Gradle'
            result.add(operation(totalTime: Duration.seconds(10), totalMemoryUsed: DataAmount.kbytes(10)))
        }
        1 * reporter.report(_)
        1 * experimentRunner.getHonestProfiler()
        0 * _._
    }

    def "shouldn't endlessly resolve 'defaults'"() {
        given:
        def runner = runner()
        runner.targetVersions = ['2.11', 'defaults']

        when:
        System.setProperty('org.gradle.performance.baselines', 'defaults')
        def results = runner.run()

        then:
        thrown(IllegalArgumentException)
{'fraction_non_alphanumeric': 0.10666213460231136, 'fraction_numerical': 0.019578518014955814, 'mean_word_length': 2.9673678532901833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6446377', 'n_tokens_mistral': 4228, 'n_tokens_neox': 3926, 'n_words': 939}