<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by <NAME>, <EMAIL>, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
This module contains all routines related to setting up the package
build environment.  All of this is set up by package.py just before
install() is called.

There are two parts to the build environment:

1. Python build environment (i.e. install() method)

   This is how things are set up when install() is called.  Spack
   takes advantage of each package being in its own module by adding a
   bunch of command-like functions (like configure(), make(), etc.) in
   the package's module scope.  Ths allows package writers to call
   them all directly in Package.install() without writing 'self.'
   everywhere.  No, this isn't Pythonic.  Yes, it makes the code more
   readable and more like the shell script from which someone is
   likely porting.

2. Build execution environment

   This is the set of environment variables, like PATH, CC, CXX,
   etc. that control the build.  There are also a number of
   environment variables used to pass information (like RPATHs and
   other information about dependencies) to Spack's compiler wrappers.
   All of these env vars are also set up here.

Skimming this module is a nice way to get acquainted with the types of
calls you can make from within the install() function.
"""
import inspect
import multiprocessing
import os
import shutil
import sys
import traceback
from six import iteritems
from six import StringIO

import llnl.util.tty as tty
from llnl.util.tty.color import colorize
from llnl.util.filesystem import *

import spack
import spack.store
from spack.environment import EnvironmentModifications, validate
from spack.util.environment import *
from spack.util.executable import Executable
from spack.util.module_cmd import load_module, get_path_from_module
from spack.util.log_parse import *


#
# This can be set by the user to globally disable parallel builds.
#
SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'

#
# These environment variables are set by
# set_build_environment_variables and used to pass parameters to
# Spack's compiler wrappers.
#
SPACK_ENV_PATH = 'SPACK_ENV_PATH'
SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
SPACK_RPATH_DEPS = 'SPACK_RPATH_DEPS'
SPACK_LINK_DEPS = 'SPACK_LINK_DEPS'
SPACK_PREFIX = 'SPACK_PREFIX'
SPACK_INSTALL = 'SPACK_INSTALL'
SPACK_DEBUG = 'SPACK_DEBUG'
SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'


# Platform-specific library suffix.
dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so'


class MakeExecutable(Executable):
    """Special callable executable object for make so the user can
       specify parallel or not on a per-invocation basis.  Using
       'parallel' as a kwarg will override whatever the package's
       global setting is, so you can either default to true or false
       and override particular calls.

       Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides
       everything.
    """

    def __init__(self, name, jobs):
        super(MakeExecutable, self).__init__(name)
        self.jobs = jobs

    def __call__(self, *args, **kwargs):
        disable = env_flag(SPACK_NO_PARALLEL_MAKE)
        parallel = not disable and kwargs.get('parallel', self.jobs > 1)

        if parallel:
            jobs = "-j%d" % self.jobs
            args = (jobs,) + args

        return super(MakeExecutable, self).__call__(*args, **kwargs)


def set_compiler_environment_variables(pkg, env):
    assert(pkg.spec.concrete)
    compiler = pkg.compiler
    flags = pkg.spec.compiler_flags

    # Set compiler variables used by CMake and autotools
    assert all(key in compiler.link_paths for key in (
        'cc', 'cxx', 'f77', 'fc'))

    # Populate an object with the list of environment modifications
    # and return it
    # TODO : add additional kwargs for better diagnostics, like requestor,
    # ttyout, ttyerr, etc.
    link_dir = spack.build_env_path

    # Set SPACK compiler variables so that our wrapper knows what to call
    if compiler.cc:
        env.set('SPACK_CC', compiler.cc)
        env.set('CC', join_path(link_dir, compiler.link_paths['cc']))
    if compiler.cxx:
        env.set('SPACK_CXX', compiler.cxx)
        env.set('CXX', join_path(link_dir, compiler.link_paths['cxx']))
    if compiler.f77:
        env.set('SPACK_F77', compiler.f77)
        env.set('F77', join_path(link_dir, compiler.link_paths['f77']))
    if compiler.fc:
        env.set('SPACK_FC',  compiler.fc)
        env.set('FC', join_path(link_dir, compiler.link_paths['fc']))

    # Set SPACK compiler rpath flags so that our wrapper knows what to use
    env.set('SPACK_CC_RPATH_ARG',  compiler.cc_rpath_arg)
    env.set('SPACK_CXX_RPATH_ARG', compiler.cxx_rpath_arg)
    env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg)
    env.set('SPACK_FC_RPATH_ARG',  compiler.fc_rpath_arg)

    # Add every valid compiler flag to the environment, prefixed with "SPACK_"
    for flag in spack.spec.FlagMap.valid_compiler_flags():
        # Concreteness guarantees key safety here
        if flags[flag] != []:
            env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag]))

    env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))

    for mod in compiler.modules:
        # Fixes issue https://github.com/LLNL/spack/issues/3153
        if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
            load_module("cce")
        load_module(mod)

    compiler.setup_custom_environment(pkg, env)

    return env


def set_build_environment_variables(pkg, env, dirty):
    """Ensure a clean install environment when we build packages.

    This involves unsetting pesky environment variables that may
    affect the build. It also involves setting environment variables
    used by Spack's compiler wrappers.

    Args:
        pkg: The package we are building
        env: The build environment
        dirty (bool): Skip unsetting the user's environment settings
    """
    # Gather information about various types of dependencies
    build_deps      = pkg.spec.dependencies(deptype=('build', 'test'))
    link_deps       = pkg.spec.traverse(root=False, deptype=('link'))
    build_link_deps = list(build_deps) + list(link_deps)
    rpath_deps      = get_rpath_deps(pkg)

    build_prefixes      = [dep.prefix for dep in build_deps]
    link_prefixes       = [dep.prefix for dep in link_deps]
    build_link_prefixes = [dep.prefix for dep in build_link_deps]
    rpath_prefixes      = [dep.prefix for dep in rpath_deps]

    # add run-time dependencies of direct build-time dependencies:
    for build_dep in build_deps:
        for run_dep in build_dep.traverse(deptype='run'):
            build_prefixes.append(run_dep.prefix)

    # Filter out system paths: ['/', '/usr', '/usr/local']
    # These paths can be introduced into the build when an external package
    # is added as a dependency. The problem with these paths is that they often<commit_msg>wrap generator with collection to avoid exhausting it with single iteration (#5586)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by <NAME>, <EMAIL>, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""
This module contains all routines related to setting up the package
build environment.  All of this is set up by package.py just before
install() is called.

There are two parts to the build environment:

1. Python build environment (i.e. install() method)

   This is how things are set up when install() is called.  Spack
   takes advantage of each package being in its own module by adding a
   bunch of command-like functions (like configure(), make(), etc.) in
   the package's module scope.  Ths allows package writers to call
   them all directly in Package.install() without writing 'self.'
   everywhere.  No, this isn't Pythonic.  Yes, it makes the code more
   readable and more like the shell script from which someone is
   likely porting.

2. Build execution environment

   This is the set of environment variables, like PATH, CC, CXX,
   etc. that control the build.  There are also a number of
   environment variables used to pass information (like RPATHs and
   other information about dependencies) to Spack's compiler wrappers.
   All of these env vars are also set up here.

Skimming this module is a nice way to get acquainted with the types of
calls you can make from within the install() function.
"""
import inspect
import multiprocessing
import os
import shutil
import sys
import traceback
from six import iteritems
from six import StringIO

import llnl.util.tty as tty
from llnl.util.tty.color import colorize
from llnl.util.filesystem import *

import spack
import spack.store
from spack.environment import EnvironmentModifications, validate
from spack.util.environment import *
from spack.util.executable import Executable
from spack.util.module_cmd import load_module, get_path_from_module
from spack.util.log_parse import *


#
# This can be set by the user to globally disable parallel builds.
#
SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE'

#
# These environment variables are set by
# set_build_environment_variables and used to pass parameters to
# Spack's compiler wrappers.
#
SPACK_ENV_PATH = 'SPACK_ENV_PATH'
SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES'
SPACK_RPATH_DEPS = 'SPACK_RPATH_DEPS'
SPACK_LINK_DEPS = 'SPACK_LINK_DEPS'
SPACK_PREFIX = 'SPACK_PREFIX'
SPACK_INSTALL = 'SPACK_INSTALL'
SPACK_DEBUG = 'SPACK_DEBUG'
SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'


# Platform-specific library suffix.
dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so'


class MakeExecutable(Executable):
    """Special callable executable object for make so the user can
       specify parallel or not on a per-invocation basis.  Using
       'parallel' as a kwarg will override whatever the package's
       global setting is, so you can either default to true or false
       and override particular calls.

       Note that if the SPACK_NO_PARALLEL_MAKE env var is set it overrides
       everything.
    """

    def __init__(self, name, jobs):
        super(MakeExecutable, self).__init__(name)
        self.jobs = jobs

    def __call__(self, *args, **kwargs):
        disable = env_flag(SPACK_NO_PARALLEL_MAKE)
        parallel = not disable and kwargs.get('parallel', self.jobs > 1)

        if parallel:
            jobs = "-j%d" % self.jobs
            args = (jobs,) + args

        return super(MakeExecutable, self).__call__(*args, **kwargs)


def set_compiler_environment_variables(pkg, env):
    assert(pkg.spec.concrete)
    compiler = pkg.compiler
    flags = pkg.spec.compiler_flags

    # Set compiler variables used by CMake and autotools
    assert all(key in compiler.link_paths for key in (
        'cc', 'cxx', 'f77', 'fc'))

    # Populate an object with the list of environment modifications
    # and return it
    # TODO : add additional kwargs for better diagnostics, like requestor,
    # ttyout, ttyerr, etc.
    link_dir = spack.build_env_path

    # Set SPACK compiler variables so that our wrapper knows what to call
    if compiler.cc:
        env.set('SPACK_CC', compiler.cc)
        env.set('CC', join_path(link_dir, compiler.link_paths['cc']))
    if compiler.cxx:
        env.set('SPACK_CXX', compiler.cxx)
        env.set('CXX', join_path(link_dir, compiler.link_paths['cxx']))
    if compiler.f77:
        env.set('SPACK_F77', compiler.f77)
        env.set('F77', join_path(link_dir, compiler.link_paths['f77']))
    if compiler.fc:
        env.set('SPACK_FC',  compiler.fc)
        env.set('FC', join_path(link_dir, compiler.link_paths['fc']))

    # Set SPACK compiler rpath flags so that our wrapper knows what to use
    env.set('SPACK_CC_RPATH_ARG',  compiler.cc_rpath_arg)
    env.set('SPACK_CXX_RPATH_ARG', compiler.cxx_rpath_arg)
    env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg)
    env.set('SPACK_FC_RPATH_ARG',  compiler.fc_rpath_arg)

    # Add every valid compiler flag to the environment, prefixed with "SPACK_"
    for flag in spack.spec.FlagMap.valid_compiler_flags():
        # Concreteness guarantees key safety here
        if flags[flag] != []:
            env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag]))

    env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))

    for mod in compiler.modules:
        # Fixes issue https://github.com/LLNL/spack/issues/3153
        if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
            load_module("cce")
        load_module(mod)

    compiler.setup_custom_environment(pkg, env)

    return env


def set_build_environment_variables(pkg, env, dirty):
    """Ensure a clean install environment when we build packages.

    This involves unsetting pesky environment variables that may
    affect the build. It also involves setting environment variables
    used by Spack's compiler wrappers.

    Args:
        pkg: The package we are building
        env: The build environment
        dirty (bool): Skip unsetting the user's environment settings
    """
    # Gather information about various types of dependencies
    build_deps      = set(pkg.spec.dependencies(deptype=('build', 'test')))
    link_deps       = set(pkg.spec.traverse(root=False, deptype=('link')))
    build_link_deps = build_deps | link_deps
    rpath_deps      = get_rpath_deps(pkg)

    build_prefixes      = [dep.prefix for dep in build_deps]
    link_prefixes       = [dep.prefix for dep in link_deps]
    build_link_prefixes = [dep.prefix for dep in build_link_deps]
    rpath_prefixes      = [dep.prefix for dep in rpath_deps]

    # add run-time dependencies of direct build-time dependencies:
    for build_dep in build_deps:
        for run_dep in build_dep.traverse(deptype='run'):
            build_prefixes.append(run_dep.prefix)

    # Filter out system paths: ['/', '/usr', '/usr/local']
    # These paths can be introduced into the build when an external package
    # is added as a dependency. The problem with these paths is that they often
{'fraction_non_alphanumeric': 0.09699470378125385, 'fraction_numerical': 0.007266904791230447, 'mean_word_length': 3.70422943221321, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4994646', 'n_tokens_mistral': 4897, 'n_tokens_neox': 4688, 'n_words': 1734}