<commit_before>from collections import defaultdict
import waflib.Build

from waflib import Utils
import os.path
import re

variants = defaultdict(set)
cmds = defaultdict(set)


def declare_variants(args=None, subdir=''):
    if not args:
        args = ['']
    for a in args:
        var = a
        if subdir != '':
            var = subdir + '/' + a
        ctxts = []
        ctxt_re = re.compile('(\w+)Context')
        for key, ctxt in waflib.Build.__dict__.items():
            m = ctxt_re.match(key)
            if m is not None:
                t = m.group(1).lower()
                cmd_name = '{0}_{1}'.format(t, var)
                cmds[t].add(cmd_name)

                class new_variant(ctxt):
                    cmd = cmd_name
                    variant = var

                variants[a].add(new_variant)


def run_all(kind):
    from waflib.Scripting import run_command
    for cmd in cmds[kind]:
        run_command(cmd)


def clone_env(conf, old_name, new_name):
    conf.setenv(old_name)
    new_env = conf.env.copy()
    new_env.detach()
    conf.setenv(new_name, new_env)


class sub_conf(object):
    def __init__(self, conf, name):
        if conf.variant != '':
            name = conf.variant + '/' + name
        self.conf = conf
        self.name = name
        self.old_env = conf.variant

    def __enter__(self):
        new_env = self.conf.env.derive()
        new_env.detach()
        self.conf.setenv(self.name, new_env)

    def __exit__(self, *args):
        self.conf.setenv(self.old_env)


def add_dependency(bld, tgt, src):
    # Expand configuration variables (e.g.
    # "${root}/tools" -> "/home/.../tenshi/tools")
    tgt = Utils.subst_vars(tgt, bld.env)
    src = Utils.subst_vars(src, bld.env)
    # Handle paths relative the current wscript
    src_node = bld.path.find_resource(src)
    if src_node is None:
        # Handle paths relative to the root and absolute paths.
        src_node = bld.root.find_resource(src)
    if src_node is None:
        bld.fatal("Could not find manual dependency '{}'".format(src))
    bld.add_manual_dependency(tgt, src_node)


def configure_emscripten(conf):
    conf.env['CC'] = 'emcc'
    conf.env['CC_NAME'] = 'emcc'
    conf.env['COMPILER_CC'] = 'emcc'
    conf.env['LINK_CC'] = 'emcc'
    conf.env.append_value('CFLAGS', '-Wno-warn-absolute-paths')
    conf.load('compiler_c')
    conf.env['cprogram_PATTERN'] = '%s.js'
    conf.env['AR'] = 'llvm-ar'
    conf.load('ar')


def export_str_from_filenames(filenames):
    func_decl = re.compile(r'''
    # Begin at the start of the line
    ^
    # Do not capture the next group
    (?:
    # Require some words with possible * and ().
    [*()]*\w+[*()]*\s+
    # There must be at least one such word
    )+
    # Then allow * and (
    [*(]*
    # There may be some whitespace
    \s*
    # Capture the function name.<commit_msg>Fix emscripten not building asm.js<commit_after>from collections import defaultdict
import waflib.Build

from waflib import Utils
import os.path
import re

variants = defaultdict(set)
cmds = defaultdict(set)


def declare_variants(args=None, subdir=''):
    if not args:
        args = ['']
    for a in args:
        var = a
        if subdir != '':
            var = subdir + '/' + a
        ctxts = []
        ctxt_re = re.compile('(\w+)Context')
        for key, ctxt in waflib.Build.__dict__.items():
            m = ctxt_re.match(key)
            if m is not None:
                t = m.group(1).lower()
                cmd_name = '{0}_{1}'.format(t, var)
                cmds[t].add(cmd_name)

                class new_variant(ctxt):
                    cmd = cmd_name
                    variant = var

                variants[a].add(new_variant)


def run_all(kind):
    from waflib.Scripting import run_command
    for cmd in cmds[kind]:
        run_command(cmd)


def clone_env(conf, old_name, new_name):
    conf.setenv(old_name)
    new_env = conf.env.copy()
    new_env.detach()
    conf.setenv(new_name, new_env)


class sub_conf(object):
    def __init__(self, conf, name):
        if conf.variant != '':
            name = conf.variant + '/' + name
        self.conf = conf
        self.name = name
        self.old_env = conf.variant

    def __enter__(self):
        new_env = self.conf.env.derive()
        new_env.detach()
        self.conf.setenv(self.name, new_env)

    def __exit__(self, *args):
        self.conf.setenv(self.old_env)


def add_dependency(bld, tgt, src):
    # Expand configuration variables (e.g.
    # "${root}/tools" -> "/home/.../tenshi/tools")
    tgt = Utils.subst_vars(tgt, bld.env)
    src = Utils.subst_vars(src, bld.env)
    # Handle paths relative the current wscript
    src_node = bld.path.find_resource(src)
    if src_node is None:
        # Handle paths relative to the root and absolute paths.
        src_node = bld.root.find_resource(src)
    if src_node is None:
        bld.fatal("Could not find manual dependency '{}'".format(src))
    bld.add_manual_dependency(tgt, src_node)


def configure_emscripten(conf):
    conf.env['CC'] = 'emcc'
    conf.env['CC_NAME'] = 'emcc'
    conf.env['COMPILER_CC'] = 'emcc'
    conf.env['LINK_CC'] = 'emcc'
    conf.env.append_value('CFLAGS', '-Wno-warn-absolute-paths')
    conf.env.append_value('CFLAGS', '-O3')
    conf.load('compiler_c')
    conf.env['cprogram_PATTERN'] = '%s.js'
    conf.env['AR'] = 'llvm-ar'
    conf.load('ar')


def export_str_from_filenames(filenames):
    func_decl = re.compile(r'''
    # Begin at the start of the line
    ^
    # Do not capture the next group
    (?:
    # Require some words with possible * and ().
    [*()]*\w+[*()]*\s+
    # There must be at least one such word
    )+
    # Then allow * and (
    [*(]*
    # There may be some whitespace
    \s*
    # Capture the function name.
{'fraction_non_alphanumeric': 0.13327586206896552, 'fraction_numerical': 0.001206896551724138, 'mean_word_length': 2.294151050539466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4847760', 'n_tokens_mistral': 2030, 'n_tokens_neox': 1973, 'n_words': 459}