<commit_before>import os
import sys
import shutil
import logging
import collections

if os.environ.has_key("WAFDIR"):
    WAFDIR = os.path.join(os.environ["WAFDIR"], "waflib")
else:
    WAFDIR = os.path.join(os.getcwd(), "waflib")
if not os.path.exists(WAFDIR):
    raise RuntimeError("%r not found: required when using waf extras !" % WAFDIR)
sys.path.insert(0, os.path.dirname(WAFDIR))

from waflib.Context \
    import \
        create_context
from waflib.Options \
    import \
        OptionsContext
from waflib import Options
from waflib import Context
from waflib import Logs
from waflib import Build

from bento.commands.context \
    import \
        ConfigureContext, BuildContext
from bento.installed_package_description \
    import \
        InstalledSection
from bento.core.utils \
    import \
        normalize_path

WAF_TOP = os.path.join(WAFDIR, os.pardir)

WAF_CONFIG_LOG = 'config.log'

__USE_NO_OUTPUT_LOGGING = False
def disable_output():
    # Make Betty proud...
    global __USE_NO_OUTPUT_LOGGING
    __USE_NO_OUTPUT_LOGGING = True

def make_stream_logger(name, stream):
    # stream should be a file-like object supporting write/read/?
    logger = logging.getLogger(name)
    hdlr = logging.StreamHandler(stream)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)
    return logger

def _init_log_no_output():
    # XXX: all this heavily plays with waf internals - only use for unit
    # testing bento where waf output screws up nose own output/logging magic
    from cStringIO import StringIO
    Logs.got_tty = False
    Logs.get_term_cols = lambda: 80
    Logs.get_color = lambda cl: ''
    Logs.colors = Logs.color_dict()

    fake_output = StringIO()
    def fake_pprint(col, str, label='', sep='\n'):
        fake_output.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))

    Logs.pprint = fake_pprint

    log = logging.getLogger('waflib')
    log.handlers = []
    log.filters = []
    hdlr = logging.StreamHandler(StringIO())
    hdlr.setFormatter(Logs.formatter())
    log.addHandler(hdlr)
    log.addFilter(Logs.log_filter())
    log.setLevel(logging.DEBUG)
    Logs.log = log

def _init():
    tooldir = os.path.join(WAFDIR, "Tools")

    sys.path.insert(0, tooldir)
    cwd = os.getcwd()

    if __USE_NO_OUTPUT_LOGGING is True:
        _init_log_no_output()
    else:
        Logs.init_log()

    class FakeModule(object):
        pass
    Context.g_module = FakeModule
    Context.g_module.root_path = os.path.abspath(__file__)
    Context.g_module.top = os.getcwd()
    Context.g_module.out = os.path.join(os.getcwd(), "build")

    Context.top_dir = os.getcwd()
    Context.run_dir = os.getcwd()
    Context.out_dir = os.path.join(os.getcwd(), "build")
    Context.waf_dir = WAF_TOP

    opts = OptionsContext()
    opts.parse_args([])
    opts.load("compiler_c")
    Options.options.check_c_compiler = "gcc"

class ConfigureWafContext(ConfigureContext):
    def __init__(self, cmd_argv, options_context, pkg, top_node):
        super(ConfigureWafContext, self).__init__(cmd_argv, options_context, pkg, top_node)

        _init()
        waf_context = create_context("configure")
        waf_context.options = Options.options
        waf_context.init_dirs()
        waf_context.cachedir = waf_context.bldnode.make_node(Build.CACHE_DIR)
        waf_context.cachedir.mkdir()

        path = os.path.join(waf_context.bldnode.abspath(), WAF_CONFIG_LOG)
        waf_context.logger = Logs.make_logger(path, 'cfg')
        self.waf_context = waf_context

        # FIXME: this is wrong (not taking into account sub packages)
        has_compiled_code = len(pkg.extensions) > 0 or len(pkg.compiled_libraries) > 0
        conf = self.waf_context
        try:
            if has_compiled_code:
                conf.load("compiler_c")
                conf.load("python")
                conf.check_python_version((2,4,2))
                conf.check_python_headers()

                # HACK for mac os x
                if sys.platform == "darwin":
                    conf.env["CC"] = ["/usr/bin/gcc-4.0"]
        finally:
            conf.store()

        self._old_path = None

    def pre_recurse(self, local_node):
        ConfigureContext.pre_recurse(self, local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.path_from(self.top_node))

    def post_recurse(self):
        self.waf_context.path = self._old_path
        ConfigureContext.post_recurse(self)

def ext_name_to_path(name):
    """Convert extension name to path - the path does not include the
    file extension

    Example: foo.bar -> foo/bar
    """
    return name.replace('.', os.path.sep)

class BuildWafContext(BuildContext):
    def pre_recurse(self, local_node):
        super(BuildWafContext, self).pre_recurse(local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.path_from(self.top_node))

    def post_recurse(self):
        self.waf_context.path = self._old_path
        super(BuildWafContext, self).post_recurse()

    def __init__(self, cmd_argv, options_context, pkg, top_node):
        super(BuildWafContext, self).__init__(cmd_argv, options_context, pkg, top_node)

        o, a = options_context.parser.parse_args(cmd_argv)
        if o.jobs:
            jobs = int(o.jobs)
        else:
            jobs = 1
        if o.verbose:
            verbose = int(o.verbose)
            zones = ["runner"]
        else:
            verbose = 0
            zones = []
        if o.inplace:
            self.inplace = 1
        else:
            self.inplace = 0

        Logs.verbose = verbose
        Logs.init_log()
        if zones is None:
            Logs.zones = []
        else:
            Logs.zones = zones

        _init()
        waf_context = create_context("build")
        waf_context.restore()
        if not waf_context.all_envs:
            waf_context.load_envs()
        waf_context.jobs = jobs
        self.waf_context = waf_context

        def _default_extension_builder(extension):
            # FIXME: should be handled in the waf builder itself maybe ?
            target = extension.name.replace(".", os.sep)
            return self.waf_context(features='c cshlib pyext',
                                    source=extension.sources, target=target,
                                    name=extension.name)

        def _default_library_builder(library):
            return self.waf_context(features='c cstlib pyext', source=library.sources, target=library.name)

        self._extension_callbacks = collections.defaultdict(lambda : _default_extension_builder)
        self._compiled_library_callbacks = collections.defaultdict(lambda : _default_library_builder)

    def compile(self):
        self._outputs["extensions"] = {}
        self._outputs["compiled_libraries"] = {}

        for name, extension in self._node_pkg.iter_category("extensions"):<commit_msg>ENH: add customizable build_dir for waf.<commit_after>import os
import sys
import shutil
import logging
import collections

if os.environ.has_key("WAFDIR"):
    WAFDIR = os.path.join(os.environ["WAFDIR"], "waflib")
else:
    WAFDIR = os.path.join(os.getcwd(), "waflib")
if not os.path.exists(WAFDIR):
    raise RuntimeError("%r not found: required when using waf extras !" % WAFDIR)
sys.path.insert(0, os.path.dirname(WAFDIR))

from waflib.Context \
    import \
        create_context
from waflib.Options \
    import \
        OptionsContext
from waflib import Options
from waflib import Context
from waflib import Logs
from waflib import Build

from bento.commands.context \
    import \
        ConfigureContext, BuildContext
from bento.installed_package_description \
    import \
        InstalledSection
from bento.core.utils \
    import \
        normalize_path

WAF_TOP = os.path.join(WAFDIR, os.pardir)

WAF_CONFIG_LOG = 'config.log'

__USE_NO_OUTPUT_LOGGING = False
def disable_output():
    # Make Betty proud...
    global __USE_NO_OUTPUT_LOGGING
    __USE_NO_OUTPUT_LOGGING = True

def make_stream_logger(name, stream):
    # stream should be a file-like object supporting write/read/?
    logger = logging.getLogger(name)
    hdlr = logging.StreamHandler(stream)
    formatter = logging.Formatter('%(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.DEBUG)
    return logger

def _init_log_no_output():
    # XXX: all this heavily plays with waf internals - only use for unit
    # testing bento where waf output screws up nose own output/logging magic
    from cStringIO import StringIO
    Logs.got_tty = False
    Logs.get_term_cols = lambda: 80
    Logs.get_color = lambda cl: ''
    Logs.colors = Logs.color_dict()

    fake_output = StringIO()
    def fake_pprint(col, str, label='', sep='\n'):
        fake_output.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))

    Logs.pprint = fake_pprint

    log = logging.getLogger('waflib')
    log.handlers = []
    log.filters = []
    hdlr = logging.StreamHandler(StringIO())
    hdlr.setFormatter(Logs.formatter())
    log.addHandler(hdlr)
    log.addFilter(Logs.log_filter())
    log.setLevel(logging.DEBUG)
    Logs.log = log

def _init(build_path="build"):
    tooldir = os.path.join(WAFDIR, "Tools")

    sys.path.insert(0, tooldir)
    cwd = os.getcwd()

    if __USE_NO_OUTPUT_LOGGING is True:
        _init_log_no_output()
    else:
        Logs.init_log()

    class FakeModule(object):
        pass
    Context.g_module = FakeModule
    Context.g_module.root_path = os.path.abspath(__file__)
    Context.g_module.top = os.getcwd()
    Context.g_module.out = os.path.join(os.getcwd(), build_path)

    Context.top_dir = os.getcwd()
    Context.run_dir = os.getcwd()
    Context.out_dir = os.path.join(os.getcwd(), build_path)
    Context.waf_dir = WAF_TOP

    opts = OptionsContext()
    opts.parse_args([])
    opts.load("compiler_c")
    Options.options.check_c_compiler = "gcc"

class ConfigureWafContext(ConfigureContext):
    def __init__(self, cmd_argv, options_context, pkg, top_node):
        super(ConfigureWafContext, self).__init__(cmd_argv, options_context, pkg, top_node)

        build_path = top_node.bldnode.srcpath()
        _init(build_path=build_path)
        waf_context = create_context("configure")
        waf_context.options = Options.options
        waf_context.init_dirs()
        waf_context.cachedir = waf_context.bldnode.make_node(Build.CACHE_DIR)
        waf_context.cachedir.mkdir()

        path = os.path.join(waf_context.bldnode.abspath(), WAF_CONFIG_LOG)
        waf_context.logger = Logs.make_logger(path, 'cfg')
        self.waf_context = waf_context

        # FIXME: this is wrong (not taking into account sub packages)
        has_compiled_code = len(pkg.extensions) > 0 or len(pkg.compiled_libraries) > 0
        conf = self.waf_context
        try:
            if has_compiled_code:
                conf.load("compiler_c")
                conf.load("python")
                conf.check_python_version((2,4,2))
                conf.check_python_headers()

                # HACK for mac os x
                if sys.platform == "darwin":
                    conf.env["CC"] = ["/usr/bin/gcc-4.0"]
        finally:
            conf.store()

        self._old_path = None

    def pre_recurse(self, local_node):
        ConfigureContext.pre_recurse(self, local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.path_from(self.top_node))

    def post_recurse(self):
        self.waf_context.path = self._old_path
        ConfigureContext.post_recurse(self)

def ext_name_to_path(name):
    """Convert extension name to path - the path does not include the
    file extension

    Example: foo.bar -> foo/bar
    """
    return name.replace('.', os.path.sep)

class BuildWafContext(BuildContext):
    def pre_recurse(self, local_node):
        super(BuildWafContext, self).pre_recurse(local_node)
        self._old_path = self.waf_context.path
        # Gymnastic to make a *waf* node from a *bento* node
        self.waf_context.path = self.waf_context.path.make_node(self.local_node.path_from(self.top_node))

    def post_recurse(self):
        self.waf_context.path = self._old_path
        super(BuildWafContext, self).post_recurse()

    def __init__(self, cmd_argv, options_context, pkg, top_node):
        super(BuildWafContext, self).__init__(cmd_argv, options_context, pkg, top_node)

        o, a = options_context.parser.parse_args(cmd_argv)
        if o.jobs:
            jobs = int(o.jobs)
        else:
            jobs = 1
        if o.verbose:
            verbose = int(o.verbose)
            zones = ["runner"]
        else:
            verbose = 0
            zones = []
        if o.inplace:
            self.inplace = 1
        else:
            self.inplace = 0

        Logs.verbose = verbose
        Logs.init_log()
        if zones is None:
            Logs.zones = []
        else:
            Logs.zones = zones

        build_path = top_node.bldnode.srcpath()
        _init(build_path=build_path)
        waf_context = create_context("build")
        waf_context.restore()
        if not waf_context.all_envs:
            waf_context.load_envs()
        waf_context.jobs = jobs
        self.waf_context = waf_context

        def _default_extension_builder(extension):
            # FIXME: should be handled in the waf builder itself maybe ?
            target = extension.name.replace(".", os.sep)
            return self.waf_context(features='c cshlib pyext',
                                    source=extension.sources, target=target,
                                    name=extension.name)

        def _default_library_builder(library):
            return self.waf_context(features='c cstlib pyext', source=library.sources, target=library.name)

        self._extension_callbacks = collections.defaultdict(lambda : _default_extension_builder)
        self._compiled_library_callbacks = collections.defaultdict(lambda : _default_library_builder)

    def compile(self):
        self._outputs["extensions"] = {}
        self._outputs["compiled_libraries"] = {}

        for name, extension in self._node_pkg.iter_category("extensions"):
{'fraction_non_alphanumeric': 0.10261844534080809, 'fraction_numerical': 0.002051001572434539, 'mean_word_length': 2.6542593055208594, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2946577', 'n_tokens_mistral': 4816, 'n_tokens_neox': 4696, 'n_words': 940}