<commit_before>#!/usr/bin/env python
__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Stephens lab"
__email__ = "<EMAIL>"
__license__ = "MIT"

import os, sys, re, glob
import warnings
warnings.filterwarnings("ignore")
from sos.utils import env, get_traceback
from .utils import get_slice, flatten_list, workflow2html, dsc2html, transcript2html, dotdict, Timer
from . import VERSION

class Silencer:
    def __init__(self, verbosity):
        self.verbosity = verbosity
        self.env_verbosity = env.verbosity

    def __enter__(self):
        env.verbosity = self.verbosity

    def __exit__(self, etype, value, traceback):
        env.verbosity = self.env_verbosity

def prepare_args(args, db, script, workflow, mode):
    out = dotdict()
    out.__max_running_jobs__ = out.__max_procs__ = args.__max_jobs__
    # FIXME: should wait when local host
    # no-wait when extern task
    out.__wait__ = True
    out.__no_wait__ = False
    out.__targets__ = []
    # FIXME: add bin dir here
    out.__bin_dirs__ = []
    # FIXME: add more options here
    out.__queue__ = 'localhost'
    # FIXME: when remote is used should make it `no_wait`
    # Yet to observe the behavior there
    out.__remote__ = None
    out.dryrun = False
    out.__sig_mode__ = mode
    out.verbosity = env.verbosity
    # FIXME
    out.__dag__ = '.sos/.dsc/{}.dot'.format(db)
    # FIXME: use config info
    out.__config__ = '.sos/.dsc/{}.conf.yml'.format(db)
    # FIXME: port the entire resume related features
    out.__resume__ = False
    out.script = script
    out.workflow = workflow
    return out

def remove(workflows, steps, db, debug, replace = False):
    import pickle
    from sos.__main__ import cmd_remove
    to_remove = [x for x in steps if os.path.isfile(x)]
    steps = [x for x in steps if x not in to_remove]
    filename = '{}/{}.db'.format(db, os.path.basename(db))
    if not os.path.isfile(filename):
        env.logger.warning('Cannot remove ``{}``, due to missing output database ``{}``.'.format(repr(steps), filename))
    else:
        remove_steps = []
        for item in steps:
            name, step_idx = get_slice(item, mismatch_quit = False)
            removed = False
            for workflow in workflows:
                for block in workflow:
                    for step in workflow[block].steps:
                        proceed = False
                        if step.group == name:
                            if step_idx is None:
                                proceed = True
                            else:
                                for idx in step_idx:
                                    if idx + 1 == step.exe_id:
                                        proceed = True
                                        break
                        if proceed:
                            tmp = re.sub(r'[^\w' + '_.' + ']', '_', step.name)
                            if tmp not in remove_steps:
                                remove_steps.append(tmp)
                            removed = True
            if removed is False:
                env.logger.warning("Cannot remove target ``{}`` because it is neither files nor " \
                                   "modules defined in \"DSC::run\".".format(item))
        #
        data = pickle.load(open(filename, 'rb'))
        to_remove.extend(flatten_list([[glob.glob(os.path.join(db, '{}.*'.format(x)))
                                        for x in data[item]['return']]
                                       for item in remove_steps if item in data]))
    if len(to_remove):
        cmd_remove(dotdict({"tracked": False, "untracked": False,
                            "targets": to_remove, "external": True,
                            "__confirm__": True, "signature": False,
                            "verbosity": env.verbosity, "zap": True if replace else False,
                            "size": None, "age": None, "dryrun": debug}), [])
    else:
        env.logger.warning("No files found to {}. Please check your ``--target`` option".\
                           format('replace' if replace else 'purge'))

def env_init(args, output):
    os.makedirs('.sos/.dsc', exist_ok = True)
    if os.path.dirname(output):
        os.makedirs(os.path.dirname(output), exist_ok = True)
    # FIXME: need to utilize this to properly make global configs
    with open('.sos/.dsc/{}.conf.yml'.format(os.path.basename(output)), 'w') as f:
        f.write('name: dsc')
    env.logfile = os.path.basename(output) + '.log'
    if os.path.isfile(env.logfile):
        os.remove(env.logfile)
    if os.path.isfile('.sos/transcript.txt'):
        os.remove('.sos/transcript.txt')

def execute(args):
    if args.to_remove:
        if args.target is None:
            raise ValueError("``--remove`` must be specified with ``--target``.")
        rm_objects = args.target
        args.target = None
    if args.target:
        env.logger.info("Load command line DSC sequence: ``{}``".\
                        format(' '.join(', '.join(args.target).split())))
    from .dsc_parser import DSC_Script
    from .dsc_analyzer import DSC_Analyzer
    from .dsc_translator import DSC_Translator
    from .dsc_database import ResultDB
    script = DSC_Script(args.dsc_file, output = args.output, sequence = args.target, seeds = args.seeds,
                        extern = args.host)
    db = os.path.basename(script.runtime.output)
    env_init(args, script.runtime.output)
    workflow = DSC_Analyzer(script)
    if args.debug:
        workflow2html('.sos/.dsc/{}.workflow.html'.format(db), workflow.workflows, script.dump().values())
    pipeline = DSC_Translator(workflow.workflows, script.runtime, args.__construct__ == "no",
                              args.__max_jobs__, args.try_catch)
    # Apply clean-up
    if args.to_remove:
        remove(workflow.workflows, rm_objects, script.runtime.output, args.debug, args.to_remove == 'replace')
        return
    # Recover DSC from existing files
    if args.__construct__ in ["full", "partial"] and not \
       (os.path.isfile('{1}/{0}.map.mpk'.format(db, script.runtime.output)) \
        and os.path.isfile('{1}/{0}.io.mpk'.format(db, script.runtime.output))):
        raise RuntimeError('Project cannot be safely recovered because no meta-data can be found under\n``{}``'.\
                           format(os.path.abspath(script.runtime.output)))
    if args.__construct__ == "partial":
        # FIXME: need test
        master = list(set([x[list(x.keys())[-1]].name for x in workflow.workflows]))
        ResultDB('{}/{}'.format(script.runtime.output, db), master).Build(script = open(args.dsc_file).read())
        return
    # Archive scripts
    from collections import OrderedDict
    exec_content = OrderedDict([(k, [step.exe for step in script.blocks[k].steps])
                                for k in script.runtime.sequence_ordering])
    dsc2html(open(args.dsc_file).read(), None, script.runtime.output, section_content = exec_content)
    env.logger.info("DSC script exported to ``{}.html``".format(script.runtime.output))
    env.logger.info("Constructing DSC from ``{}`` ...".format(args.dsc_file))
    # Setup
    from sos.__main__ import cmd_run
    from sos.converter import script_to_html
    script_prepare = pipeline.write_pipeline(1)
    if args.debug:
        script_to_html(script_prepare, '.sos/.dsc/{}.prepare.html'.format(db))
    mode = "default"
    if args.__construct__ == "no":
        mode = "force"
    with Silencer(env.verbosity if args.debug else 0):
        cmd_run(prepare_args(args, db, script_prepare, "default", mode), [])
    # Run
    env.logger.debug("Running command ``{}``".format(' '.join(sys.argv)))
    env.logger.info("Building execution graph ...")
    pipeline.filter_execution()
    script_run = pipeline.write_pipeline(2)
    if args.debug:
        script_to_html(script_run, '.sos/.dsc/{}.run.html'.format(db))
        return
    if args.__construct__ == "full":
        # For this mode, since file names are properly determined by the `cmd_run` above,
        # then the fact that file name remains the same should be a result of same unique parameter + code
        # in that case it is safe to simply build signatures for them
        # For files having different parameter + code new file names should be generated and used for them
        # The new files should not conflict with existing files, due to the use of `remove_obsolete_output`
        mode = "build"
    try:
        with Silencer(args.verbosity if args.host else min(1, args.verbosity)):
            cmd_run(prepare_args(args, db, script_run, "DSC", mode), [])
    except Exception as e:
        if env.verbosity > 2:
            sys.stderr.write(get_traceback())
        if args.host is None:
            transcript2html('.sos/transcript.txt', '{}.transcript.html'.format(db), title = db)
            env.logger.error(e)
            env.logger.warning("If needed, you can open ``{}.transcript.html`` and "\
                               "use ``ctrl-F`` to search by ``output file name`` "\
                               "for the problematic chunk of code.".\
                               format(db))
        sys.exit(1)
    # Build database
    master = list(set([x[list(x.keys())[-1]].name for x in workflow.workflows]))
    env.logger.info("Writing output database ...")
    ResultDB('{}/{}'.format(script.runtime.output, db), master).\
        Build(script = open(script.runtime.output + '.html').read())
    env.logger.info("DSC complete!")

def main():
    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS
    class ArgumentParserError(Exception): pass
    class MyArgParser(ArgumentParser):
        def error(self, message):
            raise ArgumentParserError(message)
    #
    p = MyArgParser(description = __doc__, allow_abbrev = False, formatter_class = ArgumentDefaultsHelpFormatter)
    p.add_argument('--debug', action='store_true', help = SUPPRESS)
    p.add_argument('--version', action = 'version', version = '{}'.format(VERSION))
    p.add_argument('dsc_file', metavar = "DSC script", help = 'DSC script to execute.')<commit_msg>Improve internal argument parser<commit_after>#!/usr/bin/env python
__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Stephens lab"
__email__ = "<EMAIL>"
__license__ = "MIT"

import os, sys, re, glob
import warnings
warnings.filterwarnings("ignore")
from sos.utils import env, get_traceback
from .utils import get_slice, flatten_list, workflow2html, dsc2html, transcript2html, dotdict, Timer
from . import VERSION

class Silencer:
    def __init__(self, verbosity):
        self.verbosity = verbosity
        self.env_verbosity = env.verbosity

    def __enter__(self):
        env.verbosity = self.verbosity

    def __exit__(self, etype, value, traceback):
        env.verbosity = self.env_verbosity

def prepare_args(name, content):
    out = dotdict()
    out.verbosity = env.verbosity
    # FIXME: should wait when local host
    # no-wait when extern task
    out.__wait__ = True
    out.__no_wait__ = False
    out.__targets__ = []
    # FIXME: add bin dir here
    out.__bin_dirs__ = []
    # FIXME: add more options here
    out.__queue__ = 'localhost'
    # FIXME: when remote is used should make it `no_wait`
    # Yet to observe the behavior there
    out.__remote__ = None
    out.dryrun = False
    # FIXME
    out.__dag__ = '.sos/.dsc/{}.dot'.format(name)
    # FIXME: use config info
    out.__config__ = '.sos/.dsc/{}.conf.yml'.format(name)
    if not os.path.isfile(out.__config__):
        with open(out.__config__, 'w') as f:
            f.write('name: dsc')
    # FIXME: port the entire resume related features
    out.__resume__ = False
    out.update(content)
    return out

def remove(workflows, steps, db, debug, replace = False):
    import pickle
    from sos.__main__ import cmd_remove
    to_remove = [x for x in steps if os.path.isfile(x)]
    steps = [x for x in steps if x not in to_remove]
    filename = '{}/{}.db'.format(db, os.path.basename(db))
    if not os.path.isfile(filename):
        env.logger.warning('Cannot remove ``{}``, due to missing output database ``{}``.'.format(repr(steps), filename))
    else:
        remove_steps = []
        for item in steps:
            name, step_idx = get_slice(item, mismatch_quit = False)
            removed = False
            for workflow in workflows:
                for block in workflow:
                    for step in workflow[block].steps:
                        proceed = False
                        if step.group == name:
                            if step_idx is None:
                                proceed = True
                            else:
                                for idx in step_idx:
                                    if idx + 1 == step.exe_id:
                                        proceed = True
                                        break
                        if proceed:
                            tmp = re.sub(r'[^\w' + '_.' + ']', '_', step.name)
                            if tmp not in remove_steps:
                                remove_steps.append(tmp)
                            removed = True
            if removed is False:
                env.logger.warning("Cannot remove target ``{}`` because it is neither files nor " \
                                   "modules defined in \"DSC::run\".".format(item))
        #
        data = pickle.load(open(filename, 'rb'))
        to_remove.extend(flatten_list([[glob.glob(os.path.join(db, '{}.*'.format(x)))
                                        for x in data[item]['return']]
                                       for item in remove_steps if item in data]))
    if len(to_remove):
        cmd_remove(dotdict({"tracked": False, "untracked": False,
                            "targets": to_remove, "external": True,
                            "__confirm__": True, "signature": False,
                            "verbosity": env.verbosity, "zap": True if replace else False,
                            "size": None, "age": None, "dryrun": debug}), [])
    else:
        env.logger.warning("No files found to {}. Please check your ``--target`` option".\
                           format('replace' if replace else 'purge'))

def env_init(args, output):
    os.makedirs('.sos/.dsc', exist_ok = True)
    if os.path.dirname(output):
        os.makedirs(os.path.dirname(output), exist_ok = True)
    env.logfile = os.path.basename(output) + '.log'
    if os.path.isfile(env.logfile):
        os.remove(env.logfile)
    if os.path.isfile('.sos/transcript.txt'):
        os.remove('.sos/transcript.txt')

def execute(args):
    if args.to_remove:
        if args.target is None:
            raise ValueError("``--remove`` must be specified with ``--target``.")
        rm_objects = args.target
        args.target = None
    if args.target:
        env.logger.info("Load command line DSC sequence: ``{}``".\
                        format(' '.join(', '.join(args.target).split())))
    from .dsc_parser import DSC_Script
    from .dsc_analyzer import DSC_Analyzer
    from .dsc_translator import DSC_Translator
    from .dsc_database import ResultDB
    script = DSC_Script(args.dsc_file, output = args.output, sequence = args.target, seeds = args.seeds,
                        extern = args.host)
    db = os.path.basename(script.runtime.output)
    env_init(args, script.runtime.output)
    workflow = DSC_Analyzer(script)
    if args.debug:
        workflow2html('.sos/.dsc/{}.workflow.html'.format(db), workflow.workflows, script.dump().values())
    pipeline = DSC_Translator(workflow.workflows, script.runtime, args.__construct__ == "no",
                              args.__max_jobs__, args.try_catch)
    # Apply clean-up
    if args.to_remove:
        remove(workflow.workflows, rm_objects, script.runtime.output, args.debug, args.to_remove == 'replace')
        return
    # Recover DSC from existing files
    if args.__construct__ in ["full", "partial"] and not \
       (os.path.isfile('{1}/{0}.map.mpk'.format(db, script.runtime.output)) \
        and os.path.isfile('{1}/{0}.io.mpk'.format(db, script.runtime.output))):
        raise RuntimeError('Project cannot be safely recovered because no meta-data can be found under\n``{}``'.\
                           format(os.path.abspath(script.runtime.output)))
    if args.__construct__ == "partial":
        # FIXME: need test
        master = list(set([x[list(x.keys())[-1]].name for x in workflow.workflows]))
        ResultDB('{}/{}'.format(script.runtime.output, db), master).Build(script = open(args.dsc_file).read())
        return
    # Archive scripts
    from collections import OrderedDict
    exec_content = OrderedDict([(k, [step.exe for step in script.blocks[k].steps])
                                for k in script.runtime.sequence_ordering])
    dsc2html(open(args.dsc_file).read(), None, script.runtime.output, section_content = exec_content)
    env.logger.info("DSC script exported to ``{}.html``".format(script.runtime.output))
    env.logger.info("Constructing DSC from ``{}`` ...".format(args.dsc_file))
    # Setup
    from sos.__main__ import cmd_run
    from sos.converter import script_to_html
    script_prepare = pipeline.write_pipeline(1)
    if args.debug:
        script_to_html(script_prepare, '.sos/.dsc/{}.prepare.html'.format(db))
    mode = "default"
    if args.__construct__ == "no":
        mode = "force"
    with Silencer(env.verbosity if args.debug else 0):
        content = {'__max_running_jobs__': args.__max_jobs__,
                   '__max_procs__': args.__max_jobs__,
                   '__sig_mode__': mode,
                   'script': script_prepare,
                   'workflow': "default"}
        cmd_run(prepare_args(db + '.prepare', content), [])
    # Run
    env.logger.debug("Running command ``{}``".format(' '.join(sys.argv)))
    env.logger.info("Building execution graph ...")
    pipeline.filter_execution()
    script_run = pipeline.write_pipeline(2)
    if args.debug:
        script_to_html(script_run, '.sos/.dsc/{}.run.html'.format(db))
        return
    if args.__construct__ == "full":
        # For this mode, since file names are properly determined by the `cmd_run` above,
        # then the fact that file name remains the same should be a result of same unique parameter + code
        # in that case it is safe to simply build signatures for them
        # For files having different parameter + code new file names should be generated and used for them
        # The new files should not conflict with existing files, due to the use of `remove_obsolete_output`
        mode = "build"
    try:
        with Silencer(args.verbosity if args.host else min(1, args.verbosity)):
            content = {'__max_running_jobs__': args.__max_jobs__,
                       '__max_procs__': args.__max_jobs__,
                       '__sig_mode__': mode,
                       'script': script_run,
                       'workflow': "DSC"}
            cmd_run(prepare_args(db + '.run', content), [])
    except Exception as e:
        if env.verbosity > 2:
            sys.stderr.write(get_traceback())
        if args.host is None:
            transcript2html('.sos/transcript.txt', '{}.transcript.html'.format(db), title = db)
            env.logger.error(e)
            env.logger.warning("If needed, you can open ``{}.transcript.html`` and "\
                               "use ``ctrl-F`` to search by ``output file name`` "\
                               "for the problematic chunk of code.".\
                               format(db))
        sys.exit(1)
    # Build database
    master = list(set([x[list(x.keys())[-1]].name for x in workflow.workflows]))
    env.logger.info("Writing output database ...")
    ResultDB('{}/{}'.format(script.runtime.output, db), master).\
        Build(script = open(script.runtime.output + '.html').read())
    env.logger.info("DSC complete!")

def main():
    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS
    class ArgumentParserError(Exception): pass
    class MyArgParser(ArgumentParser):
        def error(self, message):
            raise ArgumentParserError(message)
    #
    p = MyArgParser(description = __doc__, allow_abbrev = False, formatter_class = ArgumentDefaultsHelpFormatter)
    p.add_argument('--debug', action='store_true', help = SUPPRESS)
    p.add_argument('--version', action = 'version', version = '{}'.format(VERSION))
    p.add_argument('dsc_file', metavar = "DSC script", help = 'DSC script to execute.')
{'fraction_non_alphanumeric': 0.12360369111219038, 'fraction_numerical': 0.002234094220495386, 'mean_word_length': 2.21734375, 'pattern_counts': {'":': 28, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '756022', 'n_tokens_mistral': 6026, 'n_tokens_neox': 5911, 'n_words': 1550}