<commit_before>#
# Copyright 2014, NICTA
#
# This software may be distributed and modified according to the terms of
# the BSD 2-Clause license. Note that NO WARRANTY is provided.
# See "LICENSE_BSD2.txt" for details.
#
# @TAG(NICTA_BSD)
#

'''Entry point for the runner (template instantiator).'''

# Excuse this horrible prelude. When running under a different interpreter we
# have an import path that doesn't include dependencies like elftools and
# Jinja. We need to juggle the import path prior to importing them. Note, this
# code has no effect when running under the standard Python interpreter.
import platform, subprocess, sys
if platform.python_implementation() != 'CPython':
    path = eval(subprocess.check_output(['python', '-c', 'import sys; print sys.path']))
    for p in path:
        if p not in sys.path:
            sys.path.append(p)

from camkes.templates import Templates, PLATFORMS
from camkes.internal.argumentparsing import parse_args
from camkes.internal.profile import get_profiler
from camkes.internal.cache import Cache
from camkes.internal.FileSet import FileSet
import camkes.internal.log as log
import camkes.internal.constants as constants
from camkes.internal.version import version
from NameMangling import Perspective, RUNNER
from Renderer import Renderer
from Filters import CAPDL_FILTERS
from Transforms import AST_TRANSFORMS, PRE_RESOLUTION, POST_RESOLUTION
import Context

import functools, os, traceback
from copy import deepcopy

from capdl import seL4_CapTableObject, ObjectAllocator, CSpaceAllocator, \
    ELF, seL4_PageDirectoryObject

import camkes.ast as AST
import camkes.parser as parser

# Items that should never be cached as AST_keyed entries in the compilation
# cache.
NEVER_AST_CACHE = [
    'capdl', # Can't cache because it depends on ELF contents.
]

def cache_relevant_options(opts):
    '''Return a list of tuples representing the cache-relevant command line
    options. That is, the current command line options that have a relevant
    effect on code generation.'''
    # Command line options that do not influence code generation. These are
    # used to exclude options from influencing the template cache key and,
    # incorrectly, cause the cache to miss. Each of these options has one of
    # the following justifications for being in this set:
    #  1. It is already accounted for in the cache key (e.g. platform);
    #  2. It affects the AST, which is already in the cache key (e.g. cpp); or
    #  3. It has no affect on code generation (e.g. profiler).
    # We do this as an exclude list, rather than an include list so a
    # mistakenly missing entry will cause an (safe) unnecesary cache miss, as
    # opposed to an incorrect cache hit.
    CACHE_IRRELEVANT_OPTIONS = frozenset([
        'allow_forward_references',
        'cache',
        'cache_dir',
        'cpp',
        'cpp_flag',
        'elf',
        'file',
        'import_path',
        'item',
        'outfile',
        'platform',
        'ply-optimise',
        'post_render_edit',
        'profile_log',
        'profiler',
        'verbosity',
    ])
    return sorted([x for x in opts.__dict__.items() if \
        x[0] not in CACHE_IRRELEVANT_OPTIONS])

def _die(debug, s):
    log.error(str(s))
    log.debug('\n --- Python traceback ---\n%s ------------------------\n' % \
        traceback.format_exc())
    sys.exit(-1)

def main():
    options = parse_args(constants.TOOL_RUNNER)

    # Save us having to pass debugging everywhere.
    die = functools.partial(_die, options.verbosity >= 3)

    log.set_verbosity(options.verbosity)

    def done(s):
        ret = 0
        if s:
            print >>options.outfile, s
            options.outfile.close()
            if options.post_render_edit and \
                    raw_input('Edit rendered template %s [y/N]? ' % \
                    options.outfile.name) == 'y':
                editor = os.environ.get('EDITOR', 'vim')
                ret = subprocess.call([editor, options.outfile.name])
        sys.exit(ret)

    if not options.platform or options.platform in ['?', 'help'] \
            or options.platform not in PLATFORMS:
        die('Valid --platform arguments are %s' % ', '.join(PLATFORMS))

    if not options.file or len(options.file) > 1:
        die('A single input file must be provided for this operation')

    try:
        profiler = get_profiler(options.profiler, options.profile_log)
    except Exception as inst:
        die('Failed to create profiler: %s' % str(inst))

    # Construct the compilation cache if requested.
    cache = None
    if options.cache in ['on', 'readonly', 'writeonly']:
        cache = Cache(options.cache_dir)

    f = options.file[0]
    try:
        with profiler('Reading input'):
            s = f.read()
        # Try to find this output in the compilation cache if possible. This is
        # one of two places that we check in the cache. This check will 'hit'
        # if the source files representing the input spec are identical to some
        # previous execution.
        if options.cache in ['on', 'readonly']:
            with profiler('Looking for a cached version of this output'):
                key = [version(), os.path.abspath(f.name), s,
                    cache_relevant_options(options), options.platform,
                    options.item]
                value = cache.get(key)
                if value is not None and value.valid():
                    # Cache hit.
                    assert isinstance(value, FileSet), \
                        'illegally cached a value for %s that is not a FileSet' % options.item
                    log.debug('Retrieved %(platform)s.%(item)s from cache' % \
                        options.__dict__)
                    done(value.output)
        with profiler('Parsing input'):
            ast = parser.parse_to_ast(s, options.cpp, options.cpp_flag, options.ply_optimise)
            parser.assign_filenames(ast, f.name)
    except parser.CAmkESSyntaxError as e:
        e.set_column(s)
        die('%s:%s' % (f.name, str(e)))
    except Exception as inst:
        die('While parsing \'%s\': %s' % (f.name, str(inst)))

    try:
        for t in AST_TRANSFORMS[PRE_RESOLUTION]:
            with profiler('Running AST transform %s' % t.__name__):
                ast = t(ast)
    except Exception as inst:
        die('While transforming AST: %s' % str(inst))

    try:
        with profiler('Resolving imports'):
            ast, imported = parser.resolve_imports(ast, \
                os.path.dirname(os.path.abspath(f.name)), options.import_path,
                options.cpp, options.cpp_flag, options.ply_optimise)
    except Exception as inst:
        die('While resolving imports of \'%s\': %s' % (f.name, str(inst)))

    # if there are multiple assemblies, combine them now
    ast = compose_assemblies(ast)

    with profiler('Caching original AST'):
        orig_ast = deepcopy(ast)
    with profiler('Deduping AST'):
        ast = parser.dedupe(ast)
    try:
        with profiler('Resolving references'):
            ast = parser.resolve_references(ast, False)
    except Exception as inst:
        die('While resolving references of \'%s\': %s' % (f.name, str(inst)))

    try:
        with profiler('Collapsing references'):
            parser.collapse_references(ast)
    except Exception as inst:
        die('While collapsing references of \'%s\': %s' % (f.name, str(inst)))

    try:
        for t in AST_TRANSFORMS[POST_RESOLUTION]:
            with profiler('Running AST transform %s' % t.__name__):
                ast = t(ast)
    except Exception as inst:
        die('While transforming AST: %s' % str(inst))

    ast = resolve_hierarchy(ast)

    # If we have a readable cache check if our current target is in the cache.<commit_msg>wrapped the call to compose_assemblies in a try/except and profiling<commit_after>#
# Copyright 2014, NICTA
#
# This software may be distributed and modified according to the terms of
# the BSD 2-Clause license. Note that NO WARRANTY is provided.
# See "LICENSE_BSD2.txt" for details.
#
# @TAG(NICTA_BSD)
#

'''Entry point for the runner (template instantiator).'''

# Excuse this horrible prelude. When running under a different interpreter we
# have an import path that doesn't include dependencies like elftools and
# Jinja. We need to juggle the import path prior to importing them. Note, this
# code has no effect when running under the standard Python interpreter.
import platform, subprocess, sys
if platform.python_implementation() != 'CPython':
    path = eval(subprocess.check_output(['python', '-c', 'import sys; print sys.path']))
    for p in path:
        if p not in sys.path:
            sys.path.append(p)

from camkes.templates import Templates, PLATFORMS
from camkes.internal.argumentparsing import parse_args
from camkes.internal.profile import get_profiler
from camkes.internal.cache import Cache
from camkes.internal.FileSet import FileSet
import camkes.internal.log as log
import camkes.internal.constants as constants
from camkes.internal.version import version
from NameMangling import Perspective, RUNNER
from Renderer import Renderer
from Filters import CAPDL_FILTERS
from Transforms import AST_TRANSFORMS, PRE_RESOLUTION, POST_RESOLUTION
import Context

import functools, os, traceback
from copy import deepcopy

from capdl import seL4_CapTableObject, ObjectAllocator, CSpaceAllocator, \
    ELF, seL4_PageDirectoryObject

import camkes.ast as AST
import camkes.parser as parser

# Items that should never be cached as AST_keyed entries in the compilation
# cache.
NEVER_AST_CACHE = [
    'capdl', # Can't cache because it depends on ELF contents.
]

def cache_relevant_options(opts):
    '''Return a list of tuples representing the cache-relevant command line
    options. That is, the current command line options that have a relevant
    effect on code generation.'''
    # Command line options that do not influence code generation. These are
    # used to exclude options from influencing the template cache key and,
    # incorrectly, cause the cache to miss. Each of these options has one of
    # the following justifications for being in this set:
    #  1. It is already accounted for in the cache key (e.g. platform);
    #  2. It affects the AST, which is already in the cache key (e.g. cpp); or
    #  3. It has no affect on code generation (e.g. profiler).
    # We do this as an exclude list, rather than an include list so a
    # mistakenly missing entry will cause an (safe) unnecesary cache miss, as
    # opposed to an incorrect cache hit.
    CACHE_IRRELEVANT_OPTIONS = frozenset([
        'allow_forward_references',
        'cache',
        'cache_dir',
        'cpp',
        'cpp_flag',
        'elf',
        'file',
        'import_path',
        'item',
        'outfile',
        'platform',
        'ply-optimise',
        'post_render_edit',
        'profile_log',
        'profiler',
        'verbosity',
    ])
    return sorted([x for x in opts.__dict__.items() if \
        x[0] not in CACHE_IRRELEVANT_OPTIONS])

def _die(debug, s):
    log.error(str(s))
    log.debug('\n --- Python traceback ---\n%s ------------------------\n' % \
        traceback.format_exc())
    sys.exit(-1)

def main():
    options = parse_args(constants.TOOL_RUNNER)

    # Save us having to pass debugging everywhere.
    die = functools.partial(_die, options.verbosity >= 3)

    log.set_verbosity(options.verbosity)

    def done(s):
        ret = 0
        if s:
            print >>options.outfile, s
            options.outfile.close()
            if options.post_render_edit and \
                    raw_input('Edit rendered template %s [y/N]? ' % \
                    options.outfile.name) == 'y':
                editor = os.environ.get('EDITOR', 'vim')
                ret = subprocess.call([editor, options.outfile.name])
        sys.exit(ret)

    if not options.platform or options.platform in ['?', 'help'] \
            or options.platform not in PLATFORMS:
        die('Valid --platform arguments are %s' % ', '.join(PLATFORMS))

    if not options.file or len(options.file) > 1:
        die('A single input file must be provided for this operation')

    try:
        profiler = get_profiler(options.profiler, options.profile_log)
    except Exception as inst:
        die('Failed to create profiler: %s' % str(inst))

    # Construct the compilation cache if requested.
    cache = None
    if options.cache in ['on', 'readonly', 'writeonly']:
        cache = Cache(options.cache_dir)

    f = options.file[0]
    try:
        with profiler('Reading input'):
            s = f.read()
        # Try to find this output in the compilation cache if possible. This is
        # one of two places that we check in the cache. This check will 'hit'
        # if the source files representing the input spec are identical to some
        # previous execution.
        if options.cache in ['on', 'readonly']:
            with profiler('Looking for a cached version of this output'):
                key = [version(), os.path.abspath(f.name), s,
                    cache_relevant_options(options), options.platform,
                    options.item]
                value = cache.get(key)
                if value is not None and value.valid():
                    # Cache hit.
                    assert isinstance(value, FileSet), \
                        'illegally cached a value for %s that is not a FileSet' % options.item
                    log.debug('Retrieved %(platform)s.%(item)s from cache' % \
                        options.__dict__)
                    done(value.output)
        with profiler('Parsing input'):
            ast = parser.parse_to_ast(s, options.cpp, options.cpp_flag, options.ply_optimise)
            parser.assign_filenames(ast, f.name)
    except parser.CAmkESSyntaxError as e:
        e.set_column(s)
        die('%s:%s' % (f.name, str(e)))
    except Exception as inst:
        die('While parsing \'%s\': %s' % (f.name, str(inst)))

    try:
        for t in AST_TRANSFORMS[PRE_RESOLUTION]:
            with profiler('Running AST transform %s' % t.__name__):
                ast = t(ast)
    except Exception as inst:
        die('While transforming AST: %s' % str(inst))

    try:
        with profiler('Resolving imports'):
            ast, imported = parser.resolve_imports(ast, \
                os.path.dirname(os.path.abspath(f.name)), options.import_path,
                options.cpp, options.cpp_flag, options.ply_optimise)
    except Exception as inst:
        die('While resolving imports of \'%s\': %s' % (f.name, str(inst)))

    try:
        with profiler('Combining assemblies'):
            # if there are multiple assemblies, combine them now
            ast = compose_assemblies(ast)
    except Exception as inst:
        die('While combining assemblies: %s' % str(inst))

    with profiler('Caching original AST'):
        orig_ast = deepcopy(ast)
    with profiler('Deduping AST'):
        ast = parser.dedupe(ast)
    try:
        with profiler('Resolving references'):
            ast = parser.resolve_references(ast, False)
    except Exception as inst:
        die('While resolving references of \'%s\': %s' % (f.name, str(inst)))

    try:
        with profiler('Collapsing references'):
            parser.collapse_references(ast)
    except Exception as inst:
        die('While collapsing references of \'%s\': %s' % (f.name, str(inst)))

    try:
        for t in AST_TRANSFORMS[POST_RESOLUTION]:
            with profiler('Running AST transform %s' % t.__name__):
                ast = t(ast)
    except Exception as inst:
        die('While transforming AST: %s' % str(inst))

    ast = resolve_hierarchy(ast)

    # If we have a readable cache check if our current target is in the cache.
{'fraction_non_alphanumeric': 0.0993989243910155, 'fraction_numerical': 0.002151217968997153, 'mean_word_length': 2.5226209048361934, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5905756', 'n_tokens_mistral': 4633, 'n_tokens_neox': 4430, 'n_words': 1557}