<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import os.path
import sys


class Mvapich2(AutotoolsPackage):
    """Mvapich2 is a High-Performance MPI Library for clusters with diverse
    networks (InfiniBand, Omni-Path, Ethernet/iWARP, and RoCE) and computing
    platforms (x86 (Intel and AMD), ARM and OpenPOWER)"""

    homepage = "http://mvapich.cse.ohio-state.edu/"
    url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.3.4.tar.gz"
    list_url = "http://mvapich.cse.ohio-state.edu/downloads/"

    maintainers = ['nithintsk', 'harisubramoni']

    # Prefer the latest stable release
    version('2.3.4', sha256='7226a45c7c98333c8e5d2888119cce186199b430c13b7b1dca1769909e68ea7a')
    version('2.3.3', sha256='41d3261be57e5bc8aabf4e32981543c015c5443ff032a26f18205985e18c2b73')
    version('2.3.2', sha256='30cc0d7bcaa075d204692f76bca4d65a539e0f661c7460ffa9f835d6249e1ebf')
    version('2.3.1', sha256='314e12829f75f3ed83cd4779a972572d1787aac6543a3d024ea7c6080e0ee3bf')
    version('2.3', sha256='01d5fb592454ddd9ecc17e91c8983b6aea0e7559aa38f410b111c8ef385b50dd')
    version('2.3rc2', sha256='dc3801f879a54358d17002a56afd45186e2e83edc5b8367b5c317e282eb6d6bf')
    version('2.3rc1', sha256='607d309c864a6d57f5fa78fe6dd02368919736b8be0f4ddb938aba303ef9c45c')
    version('2.3a', sha256='7f0bc94265de9f66af567a263b1be6ef01755f7f6aedd25303d640cc4d8b1cff')
    version('2.2', sha256='791a6fc2b23de63b430b3e598bf05b1b25b82ba8bf7e0622fc81ba593b3bb131')
    version('2.1', sha256='49f3225ad17d2f3b6b127236a0abdc979ca8a3efb8d47ab4b6cd4f5252d05d29')

    provides('mpi')
    provides('mpi@:3.1', when='@2.3:')
    provides('mpi@:3.0', when='@2.1:')

    variant('wrapperrpath', default=True, description='Enable wrapper rpath')
    variant('debug', default=False,
            description='Enable debug info and error messages at run-time')

    variant('cuda', default=False,
            description='Enable CUDA extension')

    variant('regcache', default=True,
            description='Enable memory registration cache')

    # Accepted values are:
    #   single      - No threads (MPI_THREAD_SINGLE)
    #   funneled    - Only the main thread calls MPI (MPI_THREAD_FUNNELED)
    #   serialized  - User serializes calls to MPI (MPI_THREAD_SERIALIZED)
    #   multiple    - Fully multi-threaded (MPI_THREAD_MULTIPLE)
    #   runtime     - Alias to "multiple"
    variant(
        'threads',
        default='multiple',
        values=('single', 'funneled', 'serialized', 'multiple'),
        multi=False,
        description='Control the level of thread support'
    )

    # 32 is needed when job size exceeds 32768 cores
    variant(
        'ch3_rank_bits',
        default='32',
        values=('16', '32'),
        multi=False,
        description='Number of bits allocated to the rank field (16 or 32)'
    )

    variant(
        'process_managers',
        description='List of the process managers to activate',
        values=disjoint_sets(
            ('auto',), ('slurm',), ('hydra', 'gforker', 'remshell')
        ).prohibit_empty_set().with_error(
            "'slurm' or 'auto' cannot be activated along with "
            "other process managers"
        ).with_default('auto').with_non_feature_values('auto'),
    )

    variant(
        'fabrics',
        description='The fabric enabled for this build',
        default='mrail',
        values=(
            'psm', 'psm2', 'sock', 'nemesisib', 'nemesis', 'mrail',
            'nemesisibtcp', 'nemesistcpib', 'nemesisofi'
        )
    )

    variant(
        'alloca',
        default=False,
        description='Use alloca to allocate temporary memory if available'
    )

    variant(
        'file_systems',
        description='List of the ROMIO file systems to activate',
        values=auto_or_any_combination_of('lustre', 'gpfs', 'nfs', 'ufs'),
    )

    depends_on('findutils', type='build')
    depends_on('bison', type='build')
    depends_on('pkgconfig', type='build')
    depends_on('zlib')
    depends_on('libpciaccess', when=(sys.platform != 'darwin'))
    depends_on('libxml2')
    depends_on('cuda', when='+cuda')
    depends_on('psm', when='fabrics=psm')
    depends_on('opa-psm2', when='fabrics=psm2')
    depends_on('rdma-core', when='fabrics=mrail')
    depends_on('rdma-core', when='fabrics=nemesisib')
    depends_on('rdma-core', when='fabrics=nemesistcpib')
    depends_on('rdma-core', when='fabrics=nemesisibtcp')
    depends_on('libfabric', when='fabrics=nemesisofi')
    depends_on('slurm', when='process_managers=slurm')

    conflicts('fabrics=psm2', when='@:2.1')  # psm2 support was added at version 2.2

    filter_compiler_wrappers(
        'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin'
    )

    @property
    def libs(self):
        query_parameters = self.spec.last_query.extra_parameters
        libraries = ['libmpi']

        if 'cxx' in query_parameters:
            libraries = ['libmpicxx'] + libraries

        return find_libraries(
            libraries, root=self.prefix, shared=True, recursive=True
        )

    @property
    def process_manager_options(self):
        spec = self.spec

        other_pms = []
        for x in ('hydra', 'gforker', 'remshell'):
            if 'process_managers={0}'.format(x) in spec:
                other_pms.append(x)

        opts = []<commit_msg>mvapich2: Adding external find support. (#18177)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import re
import os.path
import sys


class Mvapich2(AutotoolsPackage):
    """Mvapich2 is a High-Performance MPI Library for clusters with diverse
    networks (InfiniBand, Omni-Path, Ethernet/iWARP, and RoCE) and computing
    platforms (x86 (Intel and AMD), ARM and OpenPOWER)"""

    homepage = "http://mvapich.cse.ohio-state.edu/"
    url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.3.4.tar.gz"
    list_url = "http://mvapich.cse.ohio-state.edu/downloads/"

    maintainers = ['nithintsk', 'harisubramoni']

    executables = ['^mpiname$']

    # Prefer the latest stable release
    version('2.3.4', sha256='7226a45c7c98333c8e5d2888119cce186199b430c13b7b1dca1769909e68ea7a')
    version('2.3.3', sha256='41d3261be57e5bc8aabf4e32981543c015c5443ff032a26f18205985e18c2b73')
    version('2.3.2', sha256='30cc0d7bcaa075d204692f76bca4d65a539e0f661c7460ffa9f835d6249e1ebf')
    version('2.3.1', sha256='314e12829f75f3ed83cd4779a972572d1787aac6543a3d024ea7c6080e0ee3bf')
    version('2.3', sha256='01d5fb592454ddd9ecc17e91c8983b6aea0e7559aa38f410b111c8ef385b50dd')
    version('2.3rc2', sha256='dc3801f879a54358d17002a56afd45186e2e83edc5b8367b5c317e282eb6d6bf')
    version('2.3rc1', sha256='607d309c864a6d57f5fa78fe6dd02368919736b8be0f4ddb938aba303ef9c45c')
    version('2.3a', sha256='7f0bc94265de9f66af567a263b1be6ef01755f7f6aedd25303d640cc4d8b1cff')
    version('2.2', sha256='791a6fc2b23de63b430b3e598bf05b1b25b82ba8bf7e0622fc81ba593b3bb131')
    version('2.1', sha256='49f3225ad17d2f3b6b127236a0abdc979ca8a3efb8d47ab4b6cd4f5252d05d29')

    provides('mpi')
    provides('mpi@:3.1', when='@2.3:')
    provides('mpi@:3.0', when='@2.1:')

    variant('wrapperrpath', default=True, description='Enable wrapper rpath')
    variant('debug', default=False,
            description='Enable debug info and error messages at run-time')

    variant('cuda', default=False,
            description='Enable CUDA extension')

    variant('regcache', default=True,
            description='Enable memory registration cache')

    # Accepted values are:
    #   single      - No threads (MPI_THREAD_SINGLE)
    #   funneled    - Only the main thread calls MPI (MPI_THREAD_FUNNELED)
    #   serialized  - User serializes calls to MPI (MPI_THREAD_SERIALIZED)
    #   multiple    - Fully multi-threaded (MPI_THREAD_MULTIPLE)
    #   runtime     - Alias to "multiple"
    variant(
        'threads',
        default='multiple',
        values=('single', 'funneled', 'serialized', 'multiple'),
        multi=False,
        description='Control the level of thread support'
    )

    # 32 is needed when job size exceeds 32768 cores
    variant(
        'ch3_rank_bits',
        default='32',
        values=('16', '32'),
        multi=False,
        description='Number of bits allocated to the rank field (16 or 32)'
    )

    variant(
        'process_managers',
        description='List of the process managers to activate',
        values=disjoint_sets(
            ('auto',), ('slurm',), ('hydra', 'gforker', 'remshell')
        ).prohibit_empty_set().with_error(
            "'slurm' or 'auto' cannot be activated along with "
            "other process managers"
        ).with_default('auto').with_non_feature_values('auto'),
    )

    variant(
        'fabrics',
        description='The fabric enabled for this build',
        default='mrail',
        values=(
            'psm', 'psm2', 'sock', 'nemesisib', 'nemesis', 'mrail',
            'nemesisibtcp', 'nemesistcpib', 'nemesisofi'
        )
    )

    variant(
        'alloca',
        default=False,
        description='Use alloca to allocate temporary memory if available'
    )

    variant(
        'file_systems',
        description='List of the ROMIO file systems to activate',
        values=auto_or_any_combination_of('lustre', 'gpfs', 'nfs', 'ufs'),
    )

    depends_on('findutils', type='build')
    depends_on('bison', type='build')
    depends_on('pkgconfig', type='build')
    depends_on('zlib')
    depends_on('libpciaccess', when=(sys.platform != 'darwin'))
    depends_on('libxml2')
    depends_on('cuda', when='+cuda')
    depends_on('psm', when='fabrics=psm')
    depends_on('opa-psm2', when='fabrics=psm2')
    depends_on('rdma-core', when='fabrics=mrail')
    depends_on('rdma-core', when='fabrics=nemesisib')
    depends_on('rdma-core', when='fabrics=nemesistcpib')
    depends_on('rdma-core', when='fabrics=nemesisibtcp')
    depends_on('libfabric', when='fabrics=nemesisofi')
    depends_on('slurm', when='process_managers=slurm')

    conflicts('fabrics=psm2', when='@:2.1')  # psm2 support was added at version 2.2

    filter_compiler_wrappers(
        'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin'
    )

    @classmethod
    def determine_version(cls, exe):
        output = Executable(exe)('-a', output=str, error=str)
        match = re.search(r'^MVAPICH2 (\S+)', output)
        return match.group(1) if match else None

    @classmethod
    def determine_variants(cls, exes, version):
        def get_spack_compiler_spec(path):
            spack_compilers = spack.compilers.find_compilers([path])
            for spack_compiler in spack_compilers:
                if os.path.dirname(spack_compiler.cc) == path:
                    return spack_compiler.spec
            return None
        results = []
        for exe in exes:
            variants = ''
            output = Executable(exe)('-a', output=str, error=str)

            if re.search(r'--enable-wrapper-rpath=yes', output):
                variants += '+wrapperrpath'
            else:
                variants += '~wrapperrpath'

            if (re.search(r'--disable-fast', output)
                and re.search(r'--enable-error-checking=runtime', output)
                and re.search(r'--enable-error-messages', output)
                and re.search(r'--enable-g', output)
                and re.search(r'--enable-debuginfo', output)):
                variants += '+debug'
            else:
                variants += '~debug'

            if re.search('--enable-cuda', output):
                variants += '+cuda'
            else:
                variants += '~cuda'

            if re.search('--enable-registration-cache', output):
                variants += '+regcache'
            else:
                variants += '~regcache'

            match = re.search(r'--enable-threads=(\S+)', output)
            if match:
                variants += " threads=" + match.group(1)

            match = re.search(r'--with-ch3-rank-bits=(\S+)', output)
            if match:
                variants += " ch3_rank_bits=" + match.group(1)

            pms = []
            if re.search(r'--with-pm=slurm', output):
                pms.append('slurm')
            if re.search(r'--with-pm=[A-Za-z0-9:]*hydra', output):
                pms.append('hydra')
            if re.search(r'--with-pm=[A-Za-z0-9:]*gforker', output):
                pms.append('gforker')
            if re.search(r'--with-pm=[A-Za-z0-9:]*remshell', output):
                pms.append('remshell')
            if pms:
                variants += " process_managers=" + ",".join(pms)

            fabrics = {
                'sock': 'ch3:sock',
                'nemesistcpib': 'ch3:nemesis:tcp,ib',
                'nemesisibtcp': 'ch3:nemesis:ib,tcp',
                'nemesisib': 'ch3:nemesis:ib',
                'nemesis': 'ch3:nemesis',
                'mrail': 'ch3:mrail',
                'nemesisofi': 'ch3:nemesis:ofi',
            }
            for fabric_name, conf_flag in fabrics.items():
                if re.search(r'--with-device=' + conf_flag, output):
                    variants += ' fabrics=' + fabric_name
                    break
            else:
                if re.search(r'--with-device=psm', output):
                    if re.search(r'--with-psm=', output):
                        variants += ' fabrics=psm'
                    elif re.search(r'--with-psm2=', output):
                        variants += ' fabrics=psm2'

            used_fs = []
            for fs in ('lustre', 'gpfs', 'nfs', 'ufs'):
                if re.search(
                        '--with-file-system=[a-zA-Z0-9+]*' + fs,
                        output):
                    used_fs.append(fs)
            if used_fs:
                variants += ' file_systems=' + ",".join(used_fs)

            match = re.search(r'CC: (\S+)', output)
            if match:
                comp_spec = get_spack_compiler_spec(
                    os.path.dirname(match.group(1)))
                if comp_spec:
                    variants += " %" + str(comp_spec)
            results.append(variants)
        return results

    @property
    def libs(self):
        query_parameters = self.spec.last_query.extra_parameters
        libraries = ['libmpi']

        if 'cxx' in query_parameters:
            libraries = ['libmpicxx'] + libraries

        return find_libraries(
            libraries, root=self.prefix, shared=True, recursive=True
        )

    @property
    def process_manager_options(self):
        spec = self.spec

        other_pms = []
        for x in ('hydra', 'gforker', 'remshell'):
            if 'process_managers={0}'.format(x) in spec:
                other_pms.append(x)

        opts = []
{'fraction_non_alphanumeric': 0.13221329437545654, 'fraction_numerical': 0.07012417823228634, 'mean_word_length': 2.661560904449307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '8709', 'n_tokens_mistral': 5734, 'n_tokens_neox': 5151, 'n_words': 910}