<commit_before>#!/usr/bin/env python
# -*- coding: latin-1 -*-
#
# Copyright 2009-2012 Ghent University
# Copyright 2009-2012 <NAME>
# Copyright 2012 <NAME>
#
# This file is part of VSC-tools,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/VSC-tools
#
# VSC-tools is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# VSC-tools is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with VSC-tools. If not, see <http://www.gnu.org/licenses/>.
#
"""
Shared module for VSC-tools setup
"""
import glob
import os
import shutil
import sys
from distutils import log  # also for setuptools
from distutils.dir_util import remove_tree

# 0 : WARN (default), 1 : INFO, 2 : DEBUG
log.set_verbosity(2)

has_setuptools = None

EXTRA_SDIST_FILES = ['shared_setup.py', 'setup.py']

def find_extra_sdist_files():
    filelist = []
    for fn in EXTRA_SDIST_FILES:
        if os.path.isfile(fn):
            filelist.append(fn)
        else:
            print "sdist add_defaults Failed to find %s" % fn
            sys.exit(1)
    log.info("find_extra_sdist_files: %s" % filelist)
    return filelist

try:
    # setuptools makes copies of the scripts, does not preserve symlinks
    # raise("no setuptools")  # to try distutils, uncomment
    from setuptools import setup
    from setuptools.command.install_scripts import install_scripts
    from setuptools.command.build_py import build_py
    from setuptools.command.sdist import sdist
    # egg_info uses sdist directly through manifest_maker
    from setuptools.command.egg_info import egg_info
    class vsc_egg_info(egg_info):
        def find_sources(self):
            egg_info.find_sources(self)
            self.filelist.extend(find_extra_sdist_files())
    has_setuptools = True
except:
    from distutils.core import setup
    from distutils.command.install_scripts import install_scripts
    from distutils.command.build_py import build_py
    from distutils.command.sdist import sdist

    class vsc_egg_info():
        pass  # dummy class for distutils

    has_setuptools = False


# authors
ag = ('<NAME>', '<EMAIL>')
jt = ('<NAME>', '<EMAIL>')
kh = ('<NAME>', '<EMAIL>')
lm = ('<NAME>', '<EMAIL>')
sdw = ('<NAME>', '<EMAIL>')
wdp = ('<NAME>', '<EMAIL>')


class vsc_install_scripts(install_scripts):
    """Create the (fake) links for mympirun
        also remove .sh and .py extensions from the scripts
    """
    def __init__(self, *args):
        install_scripts.__init__(self, *args)
        self.original_outfiles = None

    def run(self):
        # old-style class
        install_scripts.run(self)

        self.original_outfiles = self.get_outputs()[:]  # make a copy
        self.outfiles = []  # reset it
        for script in self.original_outfiles:
            # remove suffixes for .py and .sh
            if script.endswith(".py") or script.endswith(".sh"):
                shutil.move(script, script[:-3])
                script = script[:-3]
            self.outfiles.append(script)

class vsc_build_py(build_py):
    def find_package_modules (self, package, package_dir):
        """Extend build_by (not used for now)"""
        result = build_py.find_package_modules(self, package, package_dir)
        return result

class vsc_sdist(sdist):
    def add_defaults(self):
        """Add shared_setup.py"""
        sdist.add_defaults()
        self.filelist.extend(find_extra_sdist_files())

# shared target config
SHARED_TARGET = {
    'url': 'http://hpcugent.github.com/VSC-tools',
    'download_url': 'https://github.com/hpcugent/VSC-tools',
    'package_dir': {'': 'lib'},
    'cmdclass': {
        "install_scripts": vsc_install_scripts,
        # "build_py":vsc_build_py,
        "sdist":vsc_sdist,
        "egg_info":vsc_egg_info,
    },
}

def cleanup(prefix=''):
    dirs=[prefix+'build']+glob.glob(prefix + 'lib/*.egg-info')
    for d in dirs:
        if os.path.isdir(d):
            log.warn("cleanup %s" % d)
            try:
                remove_tree(d, verbose=False)
            except OSError, _:
                log.error("cleanup failed for %s" % d)

    for fn in ('setup.cfg',):
        ffn = prefix + fn
        if os.path.isfile(ffn):
            os.remove(ffn)

    make_setup(prefix=prefix)

def make_setup(name='base',prefix=''):
    """Create the setup.py
        - default is base
    """
    fn = '%ssetup_%s.py' % (prefix, name)
    if os.path.isfile(fn):
        shutil.copyfile(fn, 'setup.py')
    else:
        log.error("setup file %s for name %s not found" % (fn, name))

def sanitize(v):
    """Transforms v into a sensible string for use in setup.cfg."""
    if isinstance(v, str):
        return v

    if isinstance(v, list):
        return ",".join(v)

def parse_target(target):
    """Add some fields"""
    new_target = {}
    new_target.update(SHARED_TARGET)
    for k, v in target.items():
        if k in ('author', 'maintainer'):
            if not isinstance(v, list):
                log.error("%s of config %s needs to be a list (not tuple or string)" % (k, target['name']))
                sys.exit(1)
            new_target[k] = ";".join([x[0] for x in v])
            new_target["%s_email" % k] = ";".join([x[1] for x in v])
        else:
            if isinstance(v, dict):
                # eg command_class
                if not k in new_target:
                    new_target[k] = type(v)()
                new_target[k].update(v)
            else:
                new_target[k] = type(v)()
                new_target[k] += v<commit_msg>Fix bug in the shared_setup.py<commit_after>#!/usr/bin/env python
# -*- coding: latin-1 -*-
#
# Copyright 2009-2012 Ghent University
# Copyright 2009-2012 <NAME>
# Copyright 2012 <NAME>
#
# This file is part of VSC-tools,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/VSC-tools
#
# VSC-tools is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# VSC-tools is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with VSC-tools. If not, see <http://www.gnu.org/licenses/>.
#
"""
Shared module for VSC-tools setup
"""
import glob
import os
import shutil
import sys
from distutils import log  # also for setuptools
from distutils.dir_util import remove_tree

# 0 : WARN (default), 1 : INFO, 2 : DEBUG
log.set_verbosity(2)

has_setuptools = None

EXTRA_SDIST_FILES = ['shared_setup.py', 'setup.py']

def find_extra_sdist_files():
    filelist = []
    for fn in EXTRA_SDIST_FILES:
        if os.path.isfile(fn):
            filelist.append(fn)
        else:
            print "sdist add_defaults Failed to find %s" % fn
            sys.exit(1)
    log.info("find_extra_sdist_files: %s" % filelist)
    return filelist

try:
    # setuptools makes copies of the scripts, does not preserve symlinks
    # raise("no setuptools")  # to try distutils, uncomment
    from setuptools import setup
    from setuptools.command.install_scripts import install_scripts
    from setuptools.command.build_py import build_py
    from setuptools.command.sdist import sdist
    # egg_info uses sdist directly through manifest_maker
    from setuptools.command.egg_info import egg_info
    class vsc_egg_info(egg_info):
        def find_sources(self):
            egg_info.find_sources(self)
            self.filelist.extend(find_extra_sdist_files())
    has_setuptools = True
except:
    from distutils.core import setup
    from distutils.command.install_scripts import install_scripts
    from distutils.command.build_py import build_py
    from distutils.command.sdist import sdist

    class vsc_egg_info():
        pass  # dummy class for distutils

    has_setuptools = False


# authors
ag = ('<NAME>', '<EMAIL>')
jt = ('<NAME>', '<EMAIL>')
kh = ('<NAME>', '<EMAIL>')
lm = ('<NAME>', '<EMAIL>')
sdw = ('<NAME>', '<EMAIL>')
wdp = ('<NAME>', '<EMAIL>')


class vsc_install_scripts(install_scripts):
    """Create the (fake) links for mympirun
        also remove .sh and .py extensions from the scripts
    """
    def __init__(self, *args):
        install_scripts.__init__(self, *args)
        self.original_outfiles = None

    def run(self):
        # old-style class
        install_scripts.run(self)

        self.original_outfiles = self.get_outputs()[:]  # make a copy
        self.outfiles = []  # reset it
        for script in self.original_outfiles:
            # remove suffixes for .py and .sh
            if script.endswith(".py") or script.endswith(".sh"):
                shutil.move(script, script[:-3])
                script = script[:-3]
            self.outfiles.append(script)

class vsc_build_py(build_py):
    def find_package_modules (self, package, package_dir):
        """Extend build_by (not used for now)"""
        result = build_py.find_package_modules(self, package, package_dir)
        return result

class vsc_sdist(sdist):
    def add_defaults(self):
        """Add shared_setup.py"""
        sdist.add_defaults()
        self.filelist.extend(find_extra_sdist_files())

# shared target config
SHARED_TARGET = {
    'url': 'http://hpcugent.github.com/VSC-tools',
    'download_url': 'https://github.com/hpcugent/VSC-tools',
    'package_dir': {'': 'lib'},
    'cmdclass': {
        "install_scripts": vsc_install_scripts,
        # "build_py":vsc_build_py,
        "sdist":vsc_sdist,
        "egg_info":vsc_egg_info,
    },
}

def cleanup(prefix=''):
    dirs=[prefix+'build']+glob.glob(prefix + 'lib/*.egg-info')
    for d in dirs:
        if os.path.isdir(d):
            log.warn("cleanup %s" % d)
            try:
                remove_tree(d, verbose=False)
            except OSError, _:
                log.error("cleanup failed for %s" % d)

    for fn in ('setup.cfg',):
        ffn = prefix + fn
        if os.path.isfile(ffn):
            os.remove(ffn)

    make_setup(prefix=prefix)

def make_setup(name='base',prefix=''):
    """Create the setup.py
        - default is base

        Unneeded and obsolete
    """
    pass

def sanitize(v):
    """Transforms v into a sensible string for use in setup.cfg."""
    if isinstance(v, str):
        return v

    if isinstance(v, list):
        return ",".join(v)

def parse_target(target):
    """Add some fields"""
    new_target = {}
    new_target.update(SHARED_TARGET)
    for k, v in target.items():
        if k in ('author', 'maintainer'):
            if not isinstance(v, list):
                log.error("%s of config %s needs to be a list (not tuple or string)" % (k, target['name']))
                sys.exit(1)
            new_target[k] = ";".join([x[0] for x in v])
            new_target["%s_email" % k] = ";".join([x[1] for x in v])
        else:
            if isinstance(v, dict):
                # eg command_class
                if not k in new_target:
                    new_target[k] = type(v)()
                new_target[k].update(v)
            else:
                new_target[k] = type(v)()
                new_target[k] += v
{'fraction_non_alphanumeric': 0.12169441723800196, 'fraction_numerical': 0.005223636957231472, 'mean_word_length': 2.7029313992142643, 'pattern_counts': {'":': 8, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 4, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '604665', 'n_tokens_mistral': 3992, 'n_tokens_neox': 3881, 'n_words': 1088}