<commit_before>#!/bin/bash
# Copyright (c) 2017 Linaro Limited
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# Author: <NAME>
#
# This script is run in CI and support both pull request and commit modes. So
# it can be used also on commits to the tree or on tags.
# The following options are supports:
# -p  start the script for pull requests
# -m  matrix node number, for example 3 on a 5 node matrix
# -M  total number of nodes in the matrix
# -b  base branch
# -r  the remote to rebase on
#
# The script can be run locally using for example:
# ./scripts/ci/run_ci.sh -b master -r origin  -l -R <commit range>

set -xe

twister_options=" --inline-logs -N -v --integration"
export BSIM_OUT_PATH="${BSIM_OUT_PATH:-/opt/bsim/}"
if [ ! -d "${BSIM_OUT_PATH}" ]; then
        unset BSIM_OUT_PATH
fi
export BSIM_COMPONENTS_PATH="${BSIM_OUT_PATH}/components/"
export EDTT_PATH="${EDTT_PATH:-../tools/edtt}"

bsim_bt_test_results_file="./bsim_bt_out/bsim_results.xml"
west_commands_results_file="./pytest_out/west_commands.xml"

matrix_builds=1
matrix=1

function handle_coverage() {
	# this is for shippable coverage reports
	echo "Calling gcovr"
	gcovr -r ${ZEPHYR_BASE} -x > shippable/codecoverage/coverage.xml


	# Upload to codecov.io only on merged builds or if CODECOV_IO variable
	# is set.
	if [ -n "${CODECOV_IO}" -o -z "${pull_request_nr}" ]; then
		# Capture data
		echo "Running lcov --capture ..."
		lcov --capture \
			--directory twister-out/native_posix/ \
			--directory twister-out/nrf52_bsim/ \
			--directory twister-out/unit_testing/ \
			--directory bsim_bt_out/ \
			--output-file lcov.pre.info -q --rc lcov_branch_coverage=1

		# Remove noise
		echo "Exclude data from coverage report..."
		lcov -q \
			--remove lcov.pre.info mylib.c \
			--remove lcov.pre.info tests/\* \
			--remove lcov.pre.info samples/\* \
			--remove lcov.pre.info ext/\* \
			--remove lcov.pre.info *generated* \
			-o lcov.info --rc lcov_branch_coverage=1

		# Cleanup
		rm lcov.pre.info
		rm -rf twister-out out-2nd-pass

		# Upload to codecov.io
		echo "Upload coverage reports to codecov.io"
		bash <(curl -s https://codecov.io/bash) -f "lcov.info" -X coveragepy -X fixes
		rm -f lcov.info
	fi

	rm -rf twister-out out-2nd-pass

}

function handle_compiler_cache() {
	# Give more details in case we fail because of compiler cache
	if [ -f "$HOME/.cache/zephyr/ToolchainCapabilityDatabase.cmake" ]; then
		echo "Dumping the capability database in case we are affected by #9992"
		cat $HOME/.cache/zephyr/ToolchainCapabilityDatabase.cmake
	fi
}

function on_complete() {
	source zephyr-env.sh
	if [ "$1" == "failure" ]; then
		handle_compiler_cache
	fi

	rm -rf ccache $HOME/.cache/zephyr
	mkdir -p shippable/testresults
	mkdir -p shippable/codecoverage

	if [ -e ./twister-out/twister.xml ]; then
		echo "Copy ./twister-out/twister.xml"
		cp ./twister-out/twister.xml shippable/testresults/
	fi

	if [ -e ./module_tests/twister.xml ]; then
		echo "Copy ./module_tests/twister.xml"
		cp ./module_tests/twister.xml \
			shippable/testresults/module_tests.xml
	fi

	if [ -e ${bsim_bt_test_results_file} ]; then
		echo "Copy ${bsim_bt_test_results_file}"
		cp ${bsim_bt_test_results_file} shippable/testresults/
	fi

	if [ "$matrix" = "1" ]; then
		echo "Skip handling coverage data..."
		#handle_coverage
	else
		rm -rf twister-out out-2nd-pass
	fi
}

function run_bsim_bt_tests() {
	WORK_DIR=${ZEPHYR_BASE}/bsim_bt_out tests/bluetooth/bsim_bt/compile.sh
	RESULTS_FILE=${ZEPHYR_BASE}/${bsim_bt_test_results_file} \
	SEARCH_PATH=tests/bluetooth/bsim_bt/ \
	tests/bluetooth/bsim_bt/run_parallel.sh
}

function get_tests_to_run() {
	./scripts/zephyr_module.py --twister-out module_tests.args
	./scripts/ci/get_modified_tests.py --commits ${commit_range} > modified_tests.args
	./scripts/ci/get_modified_boards.py --commits ${commit_range} > modified_boards.args

	if [ -s modified_boards.args ]; then
		${twister} ${twister_options} +modified_boards.args --save-tests test_file_1.txt || exit 1
	fi
	if [ -s modified_tests.args ]; then
		${twister} ${twister_options} +modified_tests.args --save-tests test_file_2.txt || exit 1
	fi
	rm -f modified_tests.args modified_boards.args
}


function west_setup() {
	# West handling
	git_dir=$(basename $PWD)
	pushd ..
	if [ ! -d .west ]; then
		west init -l ${git_dir}
		west update 1> west.update.log || west update 1> west.update-2.log
		west forall -c 'git reset --hard HEAD'
		# fetch sof submodules
		west sof update
	fi
	popd
}


while getopts ":p:m:b:r:M:cfslR:" opt; do
	case $opt in
		c)
			echo "Execute CI" >&2
			main_ci=1
			;;
		l)
			echo "Executing script locally" >&2
			local_run=1
			main_ci=1
			;;
		s)
			echo "Success" >&2
			success=1
			;;
		f)<commit_msg>ci: remove west extension call<commit_after>#!/bin/bash
# Copyright (c) 2017 Linaro Limited
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# Author: <NAME>
#
# This script is run in CI and support both pull request and commit modes. So
# it can be used also on commits to the tree or on tags.
# The following options are supports:
# -p  start the script for pull requests
# -m  matrix node number, for example 3 on a 5 node matrix
# -M  total number of nodes in the matrix
# -b  base branch
# -r  the remote to rebase on
#
# The script can be run locally using for example:
# ./scripts/ci/run_ci.sh -b master -r origin  -l -R <commit range>

set -xe

twister_options=" --inline-logs -N -v --integration"
export BSIM_OUT_PATH="${BSIM_OUT_PATH:-/opt/bsim/}"
if [ ! -d "${BSIM_OUT_PATH}" ]; then
        unset BSIM_OUT_PATH
fi
export BSIM_COMPONENTS_PATH="${BSIM_OUT_PATH}/components/"
export EDTT_PATH="${EDTT_PATH:-../tools/edtt}"

bsim_bt_test_results_file="./bsim_bt_out/bsim_results.xml"
west_commands_results_file="./pytest_out/west_commands.xml"

matrix_builds=1
matrix=1

function handle_coverage() {
	# this is for shippable coverage reports
	echo "Calling gcovr"
	gcovr -r ${ZEPHYR_BASE} -x > shippable/codecoverage/coverage.xml


	# Upload to codecov.io only on merged builds or if CODECOV_IO variable
	# is set.
	if [ -n "${CODECOV_IO}" -o -z "${pull_request_nr}" ]; then
		# Capture data
		echo "Running lcov --capture ..."
		lcov --capture \
			--directory twister-out/native_posix/ \
			--directory twister-out/nrf52_bsim/ \
			--directory twister-out/unit_testing/ \
			--directory bsim_bt_out/ \
			--output-file lcov.pre.info -q --rc lcov_branch_coverage=1

		# Remove noise
		echo "Exclude data from coverage report..."
		lcov -q \
			--remove lcov.pre.info mylib.c \
			--remove lcov.pre.info tests/\* \
			--remove lcov.pre.info samples/\* \
			--remove lcov.pre.info ext/\* \
			--remove lcov.pre.info *generated* \
			-o lcov.info --rc lcov_branch_coverage=1

		# Cleanup
		rm lcov.pre.info
		rm -rf twister-out out-2nd-pass

		# Upload to codecov.io
		echo "Upload coverage reports to codecov.io"
		bash <(curl -s https://codecov.io/bash) -f "lcov.info" -X coveragepy -X fixes
		rm -f lcov.info
	fi

	rm -rf twister-out out-2nd-pass

}

function handle_compiler_cache() {
	# Give more details in case we fail because of compiler cache
	if [ -f "$HOME/.cache/zephyr/ToolchainCapabilityDatabase.cmake" ]; then
		echo "Dumping the capability database in case we are affected by #9992"
		cat $HOME/.cache/zephyr/ToolchainCapabilityDatabase.cmake
	fi
}

function on_complete() {
	source zephyr-env.sh
	if [ "$1" == "failure" ]; then
		handle_compiler_cache
	fi

	rm -rf ccache $HOME/.cache/zephyr
	mkdir -p shippable/testresults
	mkdir -p shippable/codecoverage

	if [ -e ./twister-out/twister.xml ]; then
		echo "Copy ./twister-out/twister.xml"
		cp ./twister-out/twister.xml shippable/testresults/
	fi

	if [ -e ./module_tests/twister.xml ]; then
		echo "Copy ./module_tests/twister.xml"
		cp ./module_tests/twister.xml \
			shippable/testresults/module_tests.xml
	fi

	if [ -e ${bsim_bt_test_results_file} ]; then
		echo "Copy ${bsim_bt_test_results_file}"
		cp ${bsim_bt_test_results_file} shippable/testresults/
	fi

	if [ "$matrix" = "1" ]; then
		echo "Skip handling coverage data..."
		#handle_coverage
	else
		rm -rf twister-out out-2nd-pass
	fi
}

function run_bsim_bt_tests() {
	WORK_DIR=${ZEPHYR_BASE}/bsim_bt_out tests/bluetooth/bsim_bt/compile.sh
	RESULTS_FILE=${ZEPHYR_BASE}/${bsim_bt_test_results_file} \
	SEARCH_PATH=tests/bluetooth/bsim_bt/ \
	tests/bluetooth/bsim_bt/run_parallel.sh
}

function get_tests_to_run() {
	./scripts/zephyr_module.py --twister-out module_tests.args
	./scripts/ci/get_modified_tests.py --commits ${commit_range} > modified_tests.args
	./scripts/ci/get_modified_boards.py --commits ${commit_range} > modified_boards.args

	if [ -s modified_boards.args ]; then
		${twister} ${twister_options} +modified_boards.args --save-tests test_file_1.txt || exit 1
	fi
	if [ -s modified_tests.args ]; then
		${twister} ${twister_options} +modified_tests.args --save-tests test_file_2.txt || exit 1
	fi
	rm -f modified_tests.args modified_boards.args
}


function west_setup() {
	# West handling
	git_dir=$(basename $PWD)
	pushd ..
	if [ ! -d .west ]; then
		west init -l ${git_dir}
		west update 1> west.update.log || west update 1> west.update-2.log
		west forall -c 'git reset --hard HEAD'
	fi
	popd
}


while getopts ":p:m:b:r:M:cfslR:" opt; do
	case $opt in
		c)
			echo "Execute CI" >&2
			main_ci=1
			;;
		l)
			echo "Executing script locally" >&2
			local_run=1
			main_ci=1
			;;
		s)
			echo "Success" >&2
			success=1
			;;
		f)
{'fraction_non_alphanumeric': 0.1269539501478665, 'fraction_numerical': 0.008660752006759611, 'mean_word_length': 3.4330524344569286, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6470921', 'n_tokens_mistral': 3774, 'n_tokens_neox': 3506, 'n_words': 919}