<commit_before>#!/usr/bin/env python
# encoding: utf-8

import os
import waflib
from waflib import Task
from waflib.Errors import WafError
from waflib.TaskGen import extension
from waflib.Configure import conf

def configure(ctx):
    # Note: This tool will also work if zcm-gen is a tool built by this build if:
    #       configuration:
    #           ctx.env.ZCMGEN is set to the programs path prior to loading the zcm-gen tool
    #       build:
    #           all zcmtype generation happens after the tool is compiled.
    #           That is, use ctx.add_group() while ctx.post_mode = waflib.Build.POST_LAZY
    #           to separate zcm-gen compilation and zcmtype generation.
    #
    # TODO: could improve this to actually use the waf node of the zcm-gen binary as in the
    #       ex. of building a compiler in the waf book (https://waf.io/book/#_advanced_scenarios)
    # XXX:  I believe because we are not doing the above, changes to the zcm-gen binary do not
    #       force the regeneration of the zcmtypes. This is generally not a huge problem because
    #       zcm-gen stays pretty static, but it is technically a build error.
    # TODO: The user MUST have uselib variables for zcm and zcmjar defined through their own
    #       configuration. We should add them here or move them into a different waf module

    # find program leaves the program itself in an array (probably for if there are multiple),
    # but we only expect / want one
    if not getattr(ctx.env, 'ZCMGEN', []):
        ctx.find_program('zcm-gen', var='ZCMGEN', mandatory=True)
        ctx.env.ZCMGEN = ctx.env.ZCMGEN[0]

# Context method used to set up proper include paths and simplify user code for zcmtype builds
# keywords:
#   name:         identifier for uselib linking (e.g. c/c++ builds may simply add 'name'
#                 to their 'use' list to get proper include path and library linking)
#                 default = 'zcmtypes' (though it is encouraged to name it something more unique
#                                       to avoid library naming conflicts)
#   source:       list of zcmtype source files to be interpreted
#   build:        False if only zcmtype generation (no compiling or linking) is desired
#                 Defaults to true.
#   lang:         list of languages for which zcmtypes should be generated, options are:
#                 ['c', 'cpp', 'java', 'python', 'nodejs']. Can also be a space separated string.
#   littleEndian  True or false based on desired endianess of output. Should almost always
#                 be false. Don't use this option unless you really know what you're doing
#   javapkg:      name of the java package
#                 default = 'zcmtypes' (though it is encouraged to name it something more unique
#                                       to avoid library naming conflicts)
#
#   TODO: add support for changing c/c++ include directory. Currently defaults so that the
#         description below works
#
# Using the output zcmtypes:
#   For the following explanations, assume:
#     ${name} = value entered for the "name" keyword,
#     ${dir}  = dir containing the zcmtypes (eg for ~/zcm/examples/types/*.zcm, ${dir} = "types")
#     ${pkg}  = value entered for the "javapkg" keyword
#
#   C:            types will be compiled into a static library and linked into targets
#     wscript:    add '${name}_c' to the list of "use" dependencies
#     c files:    target include directives at "${dir}/type_name_t.h"
#
#   C++:          types are header only and will be included directly in source
#     wscript:    add '${name}_cpp' to the list of "use" dependencies
#     cpp files:  target include directives at "${dir}/type_name_t.hpp"
#
#   Java:         types will be compiled into a .jar, see details below on execution
#     wscript:    add '${name}_java' to the list of "use" dependencies
#     java files: target import directives at "$pkg/type_name_t"
#
#   Python:       types will be compiled into .py python files and are imported
#                 into final python script.
#     wscript:    add '${name}_python' to the list of "use" dependencies if you
#                 have any build time dependencies (most python dependencies,
#                 however, are runtime so this will often, if not always, go unused).
#     py files:   add directory containing .py files to your sys path and target
#                 import directives at "$pkg/type_name_t"
#
#   NodeJS:       types will be compiled into a zcmtypes.js file which is
#                 'require'd in your nodejs main app.js file and passed into
#                 nodes zcm constructor
#     wscript:    add '${name}_nodejs to the list of "use" dependencies if you
#                 have any build time dependencies (most nodejs dependencies,
#                 however, are runtime so this will often, if not always, go unused).
#
# Note on running the output java classes:
#   Because of the way that java's CLASSPATH works, even though waf will link the appropriate jar
#   files for the build, their location is not maintained in the output jar. This means that to
#   run a java program compiled in this manner, the final output jar AND the zcmtype jar MUST be
#   in the CLASSPATH. Indeed, this waf build system will even be able to find the primary zcm.jar
#   if it is not in the CLASSPATH and will be able to build without issue, but any attempt to run
#   the output java classes without having zcm.jar in the CLASSPATH will fail. The zcmtype jar
#   will be called ${name}.jar and will be located in the mirrored build path to wherever the
#   waf zcmgen tool was invoked.

def outFileName(ctx, inp, lang, absPath=False):
    fileparts = getFileParts(ctx, inp)

    def defaultOutFileName(fileparts, absPath):
        ret = ""
        if absPath:
            if fileparts[3] != "":
                ret = fileparts[3]
        if fileparts[1] != "":
            if ret != "":
                ret = ret + "/"
            ret = ret + fileparts[1]
        if fileparts[2] != "":
            if ret != "":
                ret = ret + "/"
            ret = ret + fileparts[2]
        return ret

    if lang == 'zcm':
        return defaultOutFileName(fileparts, absPath)
    if lang == 'c':
        hfileparts = fileparts[:]
        cfileparts = fileparts[:]
        hfileparts[2] = fileparts[2].replace('.zcm', '.h')
        cfileparts[2] = fileparts[2].replace('.zcm', '.c')
        if fileparts[1] != "":
            hfileparts[2] = fileparts[1] + "_" + hfileparts[2]
            cfileparts[2] = fileparts[1] + "_" + cfileparts[2]
        return [defaultOutFileName(hfileparts, absPath).replace('.zcm', '.h'),
                defaultOutFileName(cfileparts, absPath).replace('.zcm', '.c')]
    if lang == 'cpp':
        return defaultOutFileName(fileparts, absPath).replace('.zcm', '.hpp')
    if lang == 'python':
        return defaultOutFileName(fileparts, absPath).replace('.zcm', '.py')

    raise WafError('This should not be possible')


def getFileParts(ctx, path):
    package = ctx.cmd_and_log('zcm-gen --package %s' % (path),
                                  output=waflib.Context.STDOUT,
                                  quiet=waflib.Context.BOTH).strip()
    pathparts = path.split('/')
    absdirparts = '/'.join(pathparts[:-1])
    nameparts = '/'.join(pathparts[-1:])
    reldirparts = absdirparts.replace(ctx.path.abspath() + '/', '')
    return [reldirparts, package, nameparts, absdirparts]


@conf
def zcmgen(ctx, **kw):
    if not getattr(ctx.env, 'ZCMGEN', []):
        raise WafError('zcmgen requires ctx.env.ZCMGEN set to the zcm-gen executable')<commit_msg>Fixed waftool handling of multiple packages<commit_after>#!/usr/bin/env python
# encoding: utf-8

import os
import waflib
from waflib import Task
from waflib.Errors import WafError
from waflib.TaskGen import extension
from waflib.Configure import conf

def configure(ctx):
    # Note: This tool will also work if zcm-gen is a tool built by this build if:
    #       configuration:
    #           ctx.env.ZCMGEN is set to the programs path prior to loading the zcm-gen tool
    #       build:
    #           all zcmtype generation happens after the tool is compiled.
    #           That is, use ctx.add_group() while ctx.post_mode = waflib.Build.POST_LAZY
    #           to separate zcm-gen compilation and zcmtype generation.
    #
    # TODO: could improve this to actually use the waf node of the zcm-gen binary as in the
    #       ex. of building a compiler in the waf book (https://waf.io/book/#_advanced_scenarios)
    # XXX:  I believe because we are not doing the above, changes to the zcm-gen binary do not
    #       force the regeneration of the zcmtypes. This is generally not a huge problem because
    #       zcm-gen stays pretty static, but it is technically a build error.
    # TODO: The user MUST have uselib variables for zcm and zcmjar defined through their own
    #       configuration. We should add them here or move them into a different waf module

    # find program leaves the program itself in an array (probably for if there are multiple),
    # but we only expect / want one
    if not getattr(ctx.env, 'ZCMGEN', []):
        ctx.find_program('zcm-gen', var='ZCMGEN', mandatory=True)
        ctx.env.ZCMGEN = ctx.env.ZCMGEN[0]

# Context method used to set up proper include paths and simplify user code for zcmtype builds
# keywords:
#   name:         identifier for uselib linking (e.g. c/c++ builds may simply add 'name'
#                 to their 'use' list to get proper include path and library linking)
#                 default = 'zcmtypes' (though it is encouraged to name it something more unique
#                                       to avoid library naming conflicts)
#   source:       list of zcmtype source files to be interpreted
#   build:        False if only zcmtype generation (no compiling or linking) is desired
#                 Defaults to true.
#   lang:         list of languages for which zcmtypes should be generated, options are:
#                 ['c', 'cpp', 'java', 'python', 'nodejs']. Can also be a space separated string.
#   littleEndian  True or false based on desired endianess of output. Should almost always
#                 be false. Don't use this option unless you really know what you're doing
#   javapkg:      name of the java package
#                 default = 'zcmtypes' (though it is encouraged to name it something more unique
#                                       to avoid library naming conflicts)
#
#   TODO: add support for changing c/c++ include directory. Currently defaults so that the
#         description below works
#
# Using the output zcmtypes:
#   For the following explanations, assume:
#     ${name} = value entered for the "name" keyword,
#     ${dir}  = dir containing the zcmtypes (eg for ~/zcm/examples/types/*.zcm, ${dir} = "types")
#     ${pkg}  = value entered for the "javapkg" keyword
#
#   C:            types will be compiled into a static library and linked into targets
#     wscript:    add '${name}_c' to the list of "use" dependencies
#     c files:    target include directives at "${dir}/type_name_t.h"
#
#   C++:          types are header only and will be included directly in source
#     wscript:    add '${name}_cpp' to the list of "use" dependencies
#     cpp files:  target include directives at "${dir}/type_name_t.hpp"
#
#   Java:         types will be compiled into a .jar, see details below on execution
#     wscript:    add '${name}_java' to the list of "use" dependencies
#     java files: target import directives at "$pkg/type_name_t"
#
#   Python:       types will be compiled into .py python files and are imported
#                 into final python script.
#     wscript:    add '${name}_python' to the list of "use" dependencies if you
#                 have any build time dependencies (most python dependencies,
#                 however, are runtime so this will often, if not always, go unused).
#     py files:   add directory containing .py files to your sys path and target
#                 import directives at "$pkg/type_name_t"
#
#   NodeJS:       types will be compiled into a zcmtypes.js file which is
#                 'require'd in your nodejs main app.js file and passed into
#                 nodes zcm constructor
#     wscript:    add '${name}_nodejs to the list of "use" dependencies if you
#                 have any build time dependencies (most nodejs dependencies,
#                 however, are runtime so this will often, if not always, go unused).
#
# Note on running the output java classes:
#   Because of the way that java's CLASSPATH works, even though waf will link the appropriate jar
#   files for the build, their location is not maintained in the output jar. This means that to
#   run a java program compiled in this manner, the final output jar AND the zcmtype jar MUST be
#   in the CLASSPATH. Indeed, this waf build system will even be able to find the primary zcm.jar
#   if it is not in the CLASSPATH and will be able to build without issue, but any attempt to run
#   the output java classes without having zcm.jar in the CLASSPATH will fail. The zcmtype jar
#   will be called ${name}.jar and will be located in the mirrored build path to wherever the
#   waf zcmgen tool was invoked.

def outFileName(ctx, inp, lang, absPath=False):
    fileparts = getFileParts(ctx, inp)
    print fileparts

    def defaultOutFileName(fileparts, absPath):
        ret = ""
        if absPath:
            if fileparts[3] != "":
                ret = fileparts[3]
        if fileparts[1] != "":
            if ret != "":
                ret = ret + "/"
            ret = ret + '/'.join(fileparts[1].split('.'))
        if fileparts[2] != "":
            if ret != "":
                ret = ret + "/"
            ret = ret + fileparts[2]
        return ret

    if lang == 'zcm':
        return defaultOutFileName(fileparts, absPath)
    if lang == 'c':
        hfileparts = fileparts[:]
        cfileparts = fileparts[:]
        hfileparts[2] = fileparts[2].replace('.zcm', '.h')
        cfileparts[2] = fileparts[2].replace('.zcm', '.c')
        if fileparts[1] != "":
            hfileparts[2] = '_'.join(fileparts[1].split('.')) + "_" + hfileparts[2]
            cfileparts[2] = '_'.join(fileparts[1].split('.')) + "_" + cfileparts[2]
        return [defaultOutFileName(hfileparts, absPath).replace('.zcm', '.h'),
                defaultOutFileName(cfileparts, absPath).replace('.zcm', '.c')]
    if lang == 'cpp':
        return defaultOutFileName(fileparts, absPath).replace('.zcm', '.hpp')
    if lang == 'python':
        return defaultOutFileName(fileparts, absPath).replace('.zcm', '.py')

    raise WafError('This should not be possible')


def getFileParts(ctx, path):
    package = ctx.cmd_and_log('zcm-gen --package %s' % (path),
                                  output=waflib.Context.STDOUT,
                                  quiet=waflib.Context.BOTH).strip()
    pathparts = path.split('/')
    absdirparts = '/'.join(pathparts[:-1])
    nameparts = '/'.join(pathparts[-1:])
    reldirparts = absdirparts.replace(ctx.path.abspath() + '/', '')
    return [reldirparts, package, nameparts, absdirparts]


@conf
def zcmgen(ctx, **kw):
    if not getattr(ctx.env, 'ZCMGEN', []):
        raise WafError('zcmgen requires ctx.env.ZCMGEN set to the zcm-gen executable')
{'fraction_non_alphanumeric': 0.09274534000129896, 'fraction_numerical': 0.0027278041176852633, 'mean_word_length': 2.4742779783393503, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2985124', 'n_tokens_mistral': 4345, 'n_tokens_neox': 4266, 'n_words': 1766}