<commit_before>#!/usr/bin/groovy

def call(params, body) {
    def config = [:]
    body.resolveStrategy = Closure.DELEGATE_FIRST
    body.delegate = config
    body(params)

    def customEnv = [
            "SPARK=spark-${config.sparkVersion}-bin-hadoop${config.hadoopVersion}",
            "SPARK_HOME=${env.WORKSPACE}/spark",
            "HADOOP_CONF_DIR=/etc/hadoop/conf",
            "MASTER=yarn-client",
            "H2O_PYTHON_WHEEL=${env.WORKSPACE}/private/h2o.whl",
            "H2O_EXTENDED_JAR=${env.WORKSPACE}/assembly-h2o/private/extended/h2odriver-extended.jar",
            "JAVA_HOME=${JAVA_HOME_8}",
            "PATH=${JAVA_HOME_8}/bin:${PATH}",

            // Properties used in case we are building against specific H2O version
            "BUILD_HADOOP=true",
            "H2O_TARGET=${config.driverHadoopVersion}",
            "H2O_ORIGINAL_JAR=${env.WORKSPACE}/h2o-3/h2o-hadoop/h2o-${config.driverHadoopVersion}-assembly/build/libs/h2odriver.jar"
    ]

    ansiColor('xterm') {
        timestamps {
            withEnv(customEnv) {
                timeout(time: 120, unit: 'MINUTES') {
                    dir("${env.WORKSPACE}") {
                        prepareSparkEnvironment()(config)
                        prepareSparklingWaterEnvironment()(config)
                        buildAndLint()(config)
                        unitTests()(config)
                        localIntegTest()(config)
                        scriptsTest()(config)
                        integTest()(config)
                        pysparklingIntegTest()(config)
                    }
                }
            }
        }
    }
}

def prepareSparkEnvironment() {
    return { config ->
        stage('Prepare Spark Environment') {
            sh  """
                # Download Spark
                wget -q "http://d3kbcqa49mib13.cloudfront.net/${env.SPARK}.tgz"
                mkdir -p "${env.SPARK_HOME}"
                tar zxvf ${env.SPARK}.tgz -C "${env.SPARK_HOME}" --strip-components 1
                rm -rf ${env.SPARK}.tgz
                """

            sh  """
                # Setup Spark
                echo "spark.driver.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                echo "spark.yarn.am.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                echo "spark.executor.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                
                echo "-Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/java-opts
                """
        }
    }
}


def prepareSparklingWaterEnvironment() {
    return { config ->
        stage('QA: Prepare Sparkling Water Environment') {

            sh  """
                # Check if we are bulding against specific H2O branch
                if [ ${config.buildAgainstH2OBranch} = true ]; then
                    # Clone H2O
                    rm -rf h2o-3
                    git clone https://github.com/h2oai/h2o-3.git
                    cd h2o-3
                    git checkout ${config.h2oBranch}
                    ${env.WORKSPACE}/gradlew build -x check
                    cd ..
                    if [ ${config.backendMode} = external ]; then
                        # In this case, PySparkling build is driven by H2O_HOME property
                        # When extending from specific jar the jar has already the desired name
                        ${env.WORKSPACE}/gradlew -q extendJar
                    fi
                fi
    
                # Check if we are building against included H2O 
                if [ ${config.buildAgainstH2OBranch} = false ]; then
                    # Download h2o-python client, save it in private directory
                    # and export variable H2O_PYTHON_WHEEL driving building of pysparkling package
                    mkdir -p ${env.WORKSPACE}/private/
                    curl -s `${env.WORKSPACE}/gradlew -q printH2OWheelPackage` > ${env.WORKSPACE}/private/h2o.whl
                    if [ ${config.backendMode} = external ]; then
                        cp `${env.WORKSPACE}/gradlew -q extendJar -PdownloadH2O=${config.driverHadoopVersion}` ${env.H2O_EXTENDED_JAR}
                    fi
                fi
    
                """
        }
    }
}

def buildAndLint() {
    return { config ->
        stage('QA: Build and Lint') {
            sh  """
                # Build
                if [ ${config.buildAgainstH2OBranch} = true ]; then
                    H2O_HOME=${env.WORKSPACE}/h2o-3 ${env.WORKSPACE}/gradlew clean --include-build ${env.WORKSPACE}/h2o-3 build -x check scalaStyle
                else
                    ${env.WORKSPACE}/gradlew clean build -x check scalaStyle<commit_msg>[SW-565] Fix creation of extended jar in jenkins<commit_after>#!/usr/bin/groovy

def call(params, body) {
    def config = [:]
    body.resolveStrategy = Closure.DELEGATE_FIRST
    body.delegate = config
    body(params)

    def customEnv = [
            "SPARK=spark-${config.sparkVersion}-bin-hadoop${config.hadoopVersion}",
            "SPARK_HOME=${env.WORKSPACE}/spark",
            "HADOOP_CONF_DIR=/etc/hadoop/conf",
            "MASTER=yarn-client",
            "H2O_PYTHON_WHEEL=${env.WORKSPACE}/private/h2o.whl",
            "H2O_EXTENDED_JAR=${env.WORKSPACE}/assembly-h2o/private/extended/h2odriver-extended.jar",
            "JAVA_HOME=${JAVA_HOME_8}",
            "PATH=${JAVA_HOME_8}/bin:${PATH}",

            // Properties used in case we are building against specific H2O version
            "BUILD_HADOOP=true",
            "H2O_TARGET=${config.driverHadoopVersion}",
            "H2O_ORIGINAL_JAR=${env.WORKSPACE}/h2o-3/h2o-hadoop/h2o-${config.driverHadoopVersion}-assembly/build/libs/h2odriver.jar"
    ]

    ansiColor('xterm') {
        timestamps {
            withEnv(customEnv) {
                timeout(time: 120, unit: 'MINUTES') {
                    dir("${env.WORKSPACE}") {
                        prepareSparkEnvironment()(config)
                        prepareSparklingWaterEnvironment()(config)
                        buildAndLint()(config)
                        unitTests()(config)
                        localIntegTest()(config)
                        scriptsTest()(config)
                        integTest()(config)
                        pysparklingIntegTest()(config)
                    }
                }
            }
        }
    }
}

def prepareSparkEnvironment() {
    return { config ->
        stage('Prepare Spark Environment') {
            sh  """
                # Download Spark
                wget -q "http://d3kbcqa49mib13.cloudfront.net/${env.SPARK}.tgz"
                mkdir -p "${env.SPARK_HOME}"
                tar zxvf ${env.SPARK}.tgz -C "${env.SPARK_HOME}" --strip-components 1
                rm -rf ${env.SPARK}.tgz
                """

            sh  """
                # Setup Spark
                echo "spark.driver.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                echo "spark.yarn.am.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                echo "spark.executor.extraJavaOptions -Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/spark-defaults.conf
                
                echo "-Dhdp.version="${config.hdpVersion}"" >> ${env.SPARK_HOME}/conf/java-opts
                """
        }
    }
}


def prepareSparklingWaterEnvironment() {
    return { config ->
        stage('QA: Prepare Sparkling Water Environment') {

            sh  """
                # Check if we are bulding against specific H2O branch
                if [ ${config.buildAgainstH2OBranch} = true ]; then
                    # Clone H2O
                    rm -rf h2o-3
                    git clone https://github.com/h2oai/h2o-3.git
                    cd h2o-3
                    git checkout ${config.h2oBranch}
                    ${env.WORKSPACE}/gradlew build -x check
                    cd ..
                    if [ ${config.backendMode} = external ]; then
                        # In this case, PySparkling build is driven by H2O_HOME property
                        # When extending from specific jar the jar has already the desired name
                        ${env.WORKSPACE}/gradlew -q extendJar
                    fi
                fi
    
                # Check if we are building against included H2O 
                if [ ${config.buildAgainstH2OBranch} = false ]; then
                    # Download h2o-python client, save it in private directory
                    # and export variable H2O_PYTHON_WHEEL driving building of pysparkling package
                    mkdir -p ${env.WORKSPACE}/private/
                    curl -s `${env.WORKSPACE}/gradlew -q printH2OWheelPackage` > ${env.WORKSPACE}/private/h2o.whl
                    if [ ${config.backendMode} = external ]; then
                        cp `${env.WORKSPACE}/gradlew -q :sparkling-water-examples:build -x check extendJar -PdownloadH2O=${config.driverHadoopVersion}` ${env.H2O_EXTENDED_JAR}
                    fi
                fi
    
                """
        }
    }
}

def buildAndLint() {
    return { config ->
        stage('QA: Build and Lint') {
            sh  """
                # Build
                if [ ${config.buildAgainstH2OBranch} = true ]; then
                    H2O_HOME=${env.WORKSPACE}/h2o-3 ${env.WORKSPACE}/gradlew clean --include-build ${env.WORKSPACE}/h2o-3 build -x check scalaStyle
                else
                    ${env.WORKSPACE}/gradlew clean build -x check scalaStyle
{'fraction_non_alphanumeric': 0.11418614186141861, 'fraction_numerical': 0.01055760557605576, 'mean_word_length': 1.549516592631304, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '914816', 'n_tokens_mistral': 2936, 'n_tokens_neox': 2748, 'n_words': 503}