<commit_before>ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
SHELL := /bin/bash
TOX_DIR := .tox
OS := $(shell uname)

# We separate the OSX X and Linux virtualenvs so we can run in a Docker
# container (st2devbox) while doing things on our host Mac machine
ifeq ($(OS),Darwin)
	VIRTUALENV_DIR ?= virtualenv-osx
	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client-osx
	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components-osx
else
	VIRTUALENV_DIR ?= virtualenv
	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client
	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components
endif

PYTHON_VERSION ?= python2.7

BINARIES := bin

# All components are prefixed by st2 and not .egg-info.
COMPONENTS := $(shell ls -a | grep ^st2 | grep -v .egg-info)
COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter)

COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS)
COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out contrib/runners/mistral_v2,$(COMPONENTS_WITH_RUNNERS)),$(component))

COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/tests)

# Components that implement a component-controlled test-runner. These components provide an
# in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar)
# Note: We also want to ignore egg-info dir created during build
COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info

# nasty hack to get a space into a variable
colon := :
comma := ,
dot := .
slash := /
space_char :=
space_char +=
COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)),$(component))
COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST)))
COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS))
COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES))

COVERAGE_GLOBS := .coverage.unit.* .coverage.integration.* .coverage.mistral.*
COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)')

REQUIREMENTS := test-requirements.txt requirements.txt
PIP_OPTIONS := $(ST2_PIP_OPTIONS)

ifndef PYLINT_CONCURRENCY
	PYLINT_CONCURRENCY := 1
endif

NOSE_OPTS := --rednose --immediate --with-parallel

ifndef NOSE_TIME
	NOSE_TIME := yes
endif

ifeq ($(NOSE_TIME),yes)
	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer
	NOSE_WITH_TIMER := 1
endif

ifndef PIP_OPTIONS
	PIP_OPTIONS :=
endif

# NOTE: We only run coverage on master and version branches and not on pull requests since
# it has a big performance overhead and is very slow.
ifeq ($(ENABLE_COVERAGE),yes)
	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
else
	INCLUDE_TESTS_IN_COVERAGE :=
endif

# If we aren't running test coverage, don't try to include tests in coverage
# results
ifdef INCLUDE_TESTS_IN_COVERAGE
	NOSE_COVERAGE_FLAGS += --cover-tests
	NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
endif

.PHONY: all
all: requirements configgen check tests

.PHONY: .coverage_globs
.coverage_globs:
	@for coverage_result in $$( \
		for coverage_glob in $(COVERAGE_GLOBS_QUOTED); do \
			compgen -G $${coverage_glob}; \
		done; \
	); do \
		echo $${coverage_result}; \
	done

# Target for debugging Makefile variable assembly
.PHONY: play
play:
	@echo COVERAGE_GLOBS=$(COVERAGE_GLOBS_QUOTED)
	@echo
	@echo COMPONENTS=$(COMPONENTS)
	@echo
	@echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS)
	@echo
	@echo COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)
	@echo
	@echo COMPONENTS_TEST=$(COMPONENTS_TEST)
	@echo
	@echo COMPONENTS_TEST_COMMA=$(COMPONENTS_TEST_COMMA)
	@echo
	@echo COMPONENTS_TEST_DIRS=$(COMPONENTS_TEST_DIRS)
	@echo
	@echo COMPONENTS_TEST_MODULES=$(COMPONENTS_TEST_MODULES)
	@echo
	@echo COMPONENTS_TEST_MODULES_COMMA=$(COMPONENTS_TEST_MODULES_COMMA)
	@echo
	@echo COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER)
	@echo
	@echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH)
	@echo
	@echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST)
	@echo
	@echo TRAVIS_EVENT_TYPE=$(TRAVIS_EVENT_TYPE)
	@echo
	@echo NOSE_OPTS=$(NOSE_OPTS)
	@echo
	@echo ENABLE_COVERAGE=$(ENABLE_COVERAGE)
	@echo
	@echo NOSE_COVERAGE_FLAGS=$(NOSE_COVERAGE_FLAGS)
	@echo
	@echo NOSE_COVERAGE_PACKAGES=$(NOSE_COVERAGE_PACKAGES)
	@echo
	@echo INCLUDE_TESTS_IN_COVERAGE=$(INCLUDE_TESTS_IN_COVERAGE)
	@echo

.PHONY: check
check: check-requirements flake8 checklogs

# NOTE: We pass --no-deps to the script so we don't install all the
# package dependencies which are already installed as part of "requirements"
# make targets. This speeds up the build
.PHONY: install-runners
install-runners:
	@echo ""
	@echo "================== INSTALL RUNNERS ===================="
	@echo ""
	@for component in $(COMPONENTS_RUNNERS); do \
		echo "==========================================================="; \
		echo "Installing runner:" $$component; \
		echo "==========================================================="; \
		(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \
	done

.PHONY: check-requirements
check-requirements: requirements
	@echo
	@echo "============== CHECKING REQUIREMENTS =============="
	@echo
	# Update requirements and then make sure no files were changed
	git status -- *requirements.txt */*requirements.txt | grep -q "nothing to commit"
	@echo "All requirements files up-to-date!"

.PHONY: check-python-packages
check-python-packages:
	# Make target which verifies all the components Python packages are valid
	@echo ""
	@echo "================== CHECK PYTHON PACKAGES ===================="
	@echo ""

	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download
	@for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \<commit_msg>Split out part of check-requirements into .check-requirements<commit_after>ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
SHELL := /bin/bash
TOX_DIR := .tox
OS := $(shell uname)

# We separate the OSX X and Linux virtualenvs so we can run in a Docker
# container (st2devbox) while doing things on our host Mac machine
ifeq ($(OS),Darwin)
	VIRTUALENV_DIR ?= virtualenv-osx
	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client-osx
	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components-osx
else
	VIRTUALENV_DIR ?= virtualenv
	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client
	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components
endif

PYTHON_VERSION ?= python2.7

BINARIES := bin

# All components are prefixed by st2 and not .egg-info.
COMPONENTS := $(shell ls -a | grep ^st2 | grep -v .egg-info)
COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter)

COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS)
COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out contrib/runners/mistral_v2,$(COMPONENTS_WITH_RUNNERS)),$(component))

COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/tests)

# Components that implement a component-controlled test-runner. These components provide an
# in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar)
# Note: We also want to ignore egg-info dir created during build
COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info

# nasty hack to get a space into a variable
colon := :
comma := ,
dot := .
slash := /
space_char :=
space_char +=
COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)),$(component))
COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST)))
COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS))
COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES))

COVERAGE_GLOBS := .coverage.unit.* .coverage.integration.* .coverage.mistral.*
COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)')

REQUIREMENTS := test-requirements.txt requirements.txt
PIP_OPTIONS := $(ST2_PIP_OPTIONS)

ifndef PYLINT_CONCURRENCY
	PYLINT_CONCURRENCY := 1
endif

NOSE_OPTS := --rednose --immediate --with-parallel

ifndef NOSE_TIME
	NOSE_TIME := yes
endif

ifeq ($(NOSE_TIME),yes)
	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer
	NOSE_WITH_TIMER := 1
endif

ifndef PIP_OPTIONS
	PIP_OPTIONS :=
endif

# NOTE: We only run coverage on master and version branches and not on pull requests since
# it has a big performance overhead and is very slow.
ifeq ($(ENABLE_COVERAGE),yes)
	NOSE_COVERAGE_FLAGS := --with-coverage --cover-branches --cover-erase
	NOSE_COVERAGE_PACKAGES := --cover-package=$(COMPONENTS_TEST_COMMA)
else
	INCLUDE_TESTS_IN_COVERAGE :=
endif

# If we aren't running test coverage, don't try to include tests in coverage
# results
ifdef INCLUDE_TESTS_IN_COVERAGE
	NOSE_COVERAGE_FLAGS += --cover-tests
	NOSE_COVERAGE_PACKAGES := $(NOSE_COVERAGE_PACKAGES),$(COMPONENTS_TEST_MODULES_COMMA)
endif

.PHONY: all
all: requirements configgen check tests

.PHONY: .coverage_globs
.coverage_globs:
	@for coverage_result in $$( \
		for coverage_glob in $(COVERAGE_GLOBS_QUOTED); do \
			compgen -G $${coverage_glob}; \
		done; \
	); do \
		echo $${coverage_result}; \
	done

# Target for debugging Makefile variable assembly
.PHONY: play
play:
	@echo COVERAGE_GLOBS=$(COVERAGE_GLOBS_QUOTED)
	@echo
	@echo COMPONENTS=$(COMPONENTS)
	@echo
	@echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS)
	@echo
	@echo COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)
	@echo
	@echo COMPONENTS_TEST=$(COMPONENTS_TEST)
	@echo
	@echo COMPONENTS_TEST_COMMA=$(COMPONENTS_TEST_COMMA)
	@echo
	@echo COMPONENTS_TEST_DIRS=$(COMPONENTS_TEST_DIRS)
	@echo
	@echo COMPONENTS_TEST_MODULES=$(COMPONENTS_TEST_MODULES)
	@echo
	@echo COMPONENTS_TEST_MODULES_COMMA=$(COMPONENTS_TEST_MODULES_COMMA)
	@echo
	@echo COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER)
	@echo
	@echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH)
	@echo
	@echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST)
	@echo
	@echo TRAVIS_EVENT_TYPE=$(TRAVIS_EVENT_TYPE)
	@echo
	@echo NOSE_OPTS=$(NOSE_OPTS)
	@echo
	@echo ENABLE_COVERAGE=$(ENABLE_COVERAGE)
	@echo
	@echo NOSE_COVERAGE_FLAGS=$(NOSE_COVERAGE_FLAGS)
	@echo
	@echo NOSE_COVERAGE_PACKAGES=$(NOSE_COVERAGE_PACKAGES)
	@echo
	@echo INCLUDE_TESTS_IN_COVERAGE=$(INCLUDE_TESTS_IN_COVERAGE)
	@echo

.PHONY: check
check: check-requirements flake8 checklogs

# NOTE: We pass --no-deps to the script so we don't install all the
# package dependencies which are already installed as part of "requirements"
# make targets. This speeds up the build
.PHONY: install-runners
install-runners:
	@echo ""
	@echo "================== INSTALL RUNNERS ===================="
	@echo ""
	@for component in $(COMPONENTS_RUNNERS); do \
		echo "==========================================================="; \
		echo "Installing runner:" $$component; \
		echo "==========================================================="; \
		(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \
	done

.PHONY: check-requirements
.check-requirements:
	@echo
	@echo "============== CHECKING REQUIREMENTS =============="
	@echo
	# Update requirements and then make sure no files were changed
	git status -- *requirements.txt */*requirements.txt | grep -q "nothing to commit"
	@echo "All requirements files up-to-date!"

.PHONY: check-requirements
check-requirements: .requirements .check-requirements

.PHONY: check-python-packages
check-python-packages:
	# Make target which verifies all the components Python packages are valid
	@echo ""
	@echo "================== CHECK PYTHON PACKAGES ===================="
	@echo ""

	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download
	@for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \
{'fraction_non_alphanumeric': 0.14610591900311526, 'fraction_numerical': 0.003426791277258567, 'mean_word_length': 6.3629587155963305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751077', 'n_tokens_mistral': 4857, 'n_tokens_neox': 4573, 'n_words': 1000}