<commit_before>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle command line args and props files
"""
import os.path
import re
import argparse
import uuid
import smv.jprops as jprops
from smv.error import SmvRuntimeError

class SmvConfig(object):
    """Smv configurations 
        Including:
            - command line parsing
            - read in property files
            - dynamic configuration handling
    """
    def __init__(self, arglist, _jvm):
        self._jvm = _jvm
        self.cmdline = self._create_cmdline_conf(arglist)
        
        DEFAULT_SMV_HOME_CONF_FILE = os.getenv('HOME', '') + "/.smv/smv-user-conf.props"

        self.app_dir = self.cmdline.pop('smvAppDir')
        self.static_props = {}
        self.dynamic_props = {}

        self.app_conf_path = self.cmdline.pop('smvAppConfFile')
        self.home_conf_path = DEFAULT_SMV_HOME_CONF_FILE
        self.user_conf_path = self.cmdline.pop('smvUserConfFile')
        self.cmdline_props = dict(self.cmdline.pop('smvProps'))

        self.read_props_from_app_dir(self.app_dir)

        self.mods_to_run = self.cmdline.pop('modsToRun')
        self.stages_to_run = self.cmdline.pop('stagesToRun')

        # Scaffolding: SmvConfig2 is a dummy Scala class to test the 
        # approach of passing in conf to scala side
        from py4j.java_gateway import java_import
        java_import(self._jvm, "org.tresamigos.smv.SmvConfig2")

        self.j_smvconf = self._jvm.SmvConfig2(
            self.mods_to_run,
            self.stages_to_run,
            self.cmdline, 
            self.merged_props(), 
            self.all_data_dirs()
        )

    def reset_j_smvconf():
        """Reset scala side conf - for dynamic conf
        """
        self.j_smvconf.reset(self.j_merged_props(), self.j_all_data_dirs())

    def read_props_from_app_dir(self, _app_dir):
        """For a given app dir, read in the prop files
        """
        self.app_dir = _app_dir
        self.static_props = self._read_props()

    def merged_props(self):
        """All the props (static + dynamic)
        """
        res = self.static_props.copy()
        res.update(self.dynamic_props)
        return res

    def all_data_dirs(self):
        """Create all the data dir configs
        """
        props = self.merged_props()
        if (self.cmdline.get('dataDir')):
            data_dir = self.cmdline.get('dataDir')
        elif (props.get('smv.dataDir')):
            data_dir = props.get('smv.dataDir')
        elif(os.getenv('DATA_DIR', None)):
            data_dir = os.getenv('DATA_DIR')
            print("WARNING: use of DATA_DIR environment variable is deprecated. use smv.dataDir instead!!!")
        else:
            raise SmvRuntimeError("Must specify a data-dir either on command line or in conf.")

        def get_sub_dir(name, default):
            res = "{}/{}".format(data_dir, default)
            if (self.cmdline.get(name)):
                res = self.cmdline.get(name)
            elif (props.get('smv.' + name)):
                res = props.get('smv.' + name)
            return res

        return {
            'dataDir': data_dir,
            'inputDir': get_sub_dir('inputDir', "input"),
            'outputDir': get_sub_dir('outputDir', "output"),
            'historyDir': get_sub_dir('historyDir', "history"),
            'publishDir': get_sub_dir('publishDir', 'publish')
        }

    def stage_names(self):
        return self._split_prop("smv.stages")

    def infer_full_stage_name(self, part_name):
        all_stages = self.stage_names()
        candidates = [s for s in all_stages if s.endswith(part_name)]

        if (len(candidates) == 0):
            raise SmvRuntimeError("Can't find stage {}".format(part_name))
        elif(len(candidates) == 1):
            return candidates[0]
        else:
            raise SmvRuntimeError("Stage name {} is ambiguous".format(part_name))

    def print_conf(self):
        print(self.cmdline)
        print(self.merged_props())

    def _split_prop(self, prop_name):
        prop_val = self.merged_props().get(prop_name)
        return [f.strip() for f in re.split("[:,]", prop_val)]

    def _create_cmdline_conf(self, arglist):
        """Parse arglist to a config dictionary
        """
        parser = argparse.ArgumentParser(
            usage="smv-run -m ModuleToRun\n       smv-run --run-app",
            description="For additional usage information, please refer to the user guide and API docs at: \nhttp://tresamigossd.github.io/SMV"
        )

        DEFAULT_SMV_APP_CONF_FILE  = "conf/smv-app-conf.props"
        DEFAULT_SMV_USER_CONF_FILE = "conf/smv-user-conf.props"
    
        parser.add_argument('--cbs-port', dest='cbsPort', type=int, help="python callback server port")

        # Where to find props files
        parser.add_argument('--smv-app-dir', dest='smvAppDir', default=".", help="SMV app directory")
        parser.add_argument('--smv-app-conf', dest='smvAppConfFile', default=DEFAULT_SMV_APP_CONF_FILE, help="app level (static) SMV configuration file path")
        parser.add_argument('--smv-user-conf', dest='smvUserConfFile', default=DEFAULT_SMV_USER_CONF_FILE, help="user level (dynamic) SMV configuration file path")

        # Where to find/store data
        parser.add_argument('--data-dir', dest='dataDir', help="specify the top level data directory")
        parser.add_argument('--input-dir', dest='inputDir', help="specify the input directory (default: datadir/input)")
        parser.add_argument('--output-dir', dest='outputDir', help="specify the output directory (default: datadir/output)")
        parser.add_argument('--hostory-dir', dest='historyDir', help="specify the history directory (default: datadir/history)")
        parser.add_argument('--publish-dir', dest='publishDir', help="specify the publish directory (default: datadir/publish)")

        # All app run flags
        parser.add_argument('--force-run-all', dest='forceRunAll', action="store_true", help="ignore persisted data and force all modules to run")
        parser.add_argument('--publish-jdbc', dest='publishJDBC', action="store_true", help="publish the given modules/stage/app through JDBC connection")
        parser.add_argument('--publish-hive', dest='publishHive', action="store_true", help="publish|export given modules/stage/app to hive tables")
        parser.add_argument('--dead', dest='printDeadModules', action="store_true", help="print a list of the dead modules in this application")
        parser.add_argument('--graph', dest='graph', action="store_true", help="generate a dot dependency graph of the given modules (modules are not run)")
        parser.add_argument('--purge-old-output', dest='purgeOldOutput', action="store_true", help="remove all old output files in output dir")
        parser.add_argument('--dry-run', dest='dryRun', action="store_true", help="determine which modules do not have persisted data and will need to be run")

        # Where to output CSVs
        parser.add_argument('--publish', dest='publish', help="publish the given modules/stage/app as given version")
        parser.add_argument('--export-csv', dest='exportCsv', help="publish|export given modules/stage/app to a CSV file at the given path on the local file system")

        # What modules to run
        parser.add_argument('--run-app', dest='runAllApp', action='store_true', help="run all output modules in all stages in app")<commit_msg> #1384 add the edd command flag<commit_after>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handle command line args and props files
"""
import os.path
import re
import argparse
import uuid
import smv.jprops as jprops
from smv.error import SmvRuntimeError

class SmvConfig(object):
    """Smv configurations 
        Including:
            - command line parsing
            - read in property files
            - dynamic configuration handling
    """
    def __init__(self, arglist, _jvm):
        self._jvm = _jvm
        self.cmdline = self._create_cmdline_conf(arglist)
        
        DEFAULT_SMV_HOME_CONF_FILE = os.getenv('HOME', '') + "/.smv/smv-user-conf.props"

        self.app_dir = self.cmdline.pop('smvAppDir')
        self.static_props = {}
        self.dynamic_props = {}

        self.app_conf_path = self.cmdline.pop('smvAppConfFile')
        self.home_conf_path = DEFAULT_SMV_HOME_CONF_FILE
        self.user_conf_path = self.cmdline.pop('smvUserConfFile')
        self.cmdline_props = dict(self.cmdline.pop('smvProps'))

        self.read_props_from_app_dir(self.app_dir)

        self.mods_to_run = self.cmdline.pop('modsToRun')
        self.stages_to_run = self.cmdline.pop('stagesToRun')

        # Scaffolding: SmvConfig2 is a dummy Scala class to test the 
        # approach of passing in conf to scala side
        from py4j.java_gateway import java_import
        java_import(self._jvm, "org.tresamigos.smv.SmvConfig2")

        self.j_smvconf = self._jvm.SmvConfig2(
            self.mods_to_run,
            self.stages_to_run,
            self.cmdline, 
            self.merged_props(), 
            self.all_data_dirs()
        )

    def reset_j_smvconf():
        """Reset scala side conf - for dynamic conf
        """
        self.j_smvconf.reset(self.j_merged_props(), self.j_all_data_dirs())

    def read_props_from_app_dir(self, _app_dir):
        """For a given app dir, read in the prop files
        """
        self.app_dir = _app_dir
        self.static_props = self._read_props()

    def merged_props(self):
        """All the props (static + dynamic)
        """
        res = self.static_props.copy()
        res.update(self.dynamic_props)
        return res

    def all_data_dirs(self):
        """Create all the data dir configs
        """
        props = self.merged_props()
        if (self.cmdline.get('dataDir')):
            data_dir = self.cmdline.get('dataDir')
        elif (props.get('smv.dataDir')):
            data_dir = props.get('smv.dataDir')
        elif(os.getenv('DATA_DIR', None)):
            data_dir = os.getenv('DATA_DIR')
            print("WARNING: use of DATA_DIR environment variable is deprecated. use smv.dataDir instead!!!")
        else:
            raise SmvRuntimeError("Must specify a data-dir either on command line or in conf.")

        def get_sub_dir(name, default):
            res = "{}/{}".format(data_dir, default)
            if (self.cmdline.get(name)):
                res = self.cmdline.get(name)
            elif (props.get('smv.' + name)):
                res = props.get('smv.' + name)
            return res

        return {
            'dataDir': data_dir,
            'inputDir': get_sub_dir('inputDir', "input"),
            'outputDir': get_sub_dir('outputDir', "output"),
            'historyDir': get_sub_dir('historyDir', "history"),
            'publishDir': get_sub_dir('publishDir', 'publish')
        }

    def stage_names(self):
        return self._split_prop("smv.stages")

    def infer_full_stage_name(self, part_name):
        all_stages = self.stage_names()
        candidates = [s for s in all_stages if s.endswith(part_name)]

        if (len(candidates) == 0):
            raise SmvRuntimeError("Can't find stage {}".format(part_name))
        elif(len(candidates) == 1):
            return candidates[0]
        else:
            raise SmvRuntimeError("Stage name {} is ambiguous".format(part_name))

    def print_conf(self):
        print(self.cmdline)
        print(self.merged_props())

    def _split_prop(self, prop_name):
        prop_val = self.merged_props().get(prop_name)
        return [f.strip() for f in re.split("[:,]", prop_val)]

    def _create_cmdline_conf(self, arglist):
        """Parse arglist to a config dictionary
        """
        parser = argparse.ArgumentParser(
            usage="smv-run -m ModuleToRun\n       smv-run --run-app",
            description="For additional usage information, please refer to the user guide and API docs at: \nhttp://tresamigossd.github.io/SMV"
        )

        DEFAULT_SMV_APP_CONF_FILE  = "conf/smv-app-conf.props"
        DEFAULT_SMV_USER_CONF_FILE = "conf/smv-user-conf.props"
    
        parser.add_argument('--cbs-port', dest='cbsPort', type=int, help="python callback server port")

        # Where to find props files
        parser.add_argument('--smv-app-dir', dest='smvAppDir', default=".", help="SMV app directory")
        parser.add_argument('--smv-app-conf', dest='smvAppConfFile', default=DEFAULT_SMV_APP_CONF_FILE, help="app level (static) SMV configuration file path")
        parser.add_argument('--smv-user-conf', dest='smvUserConfFile', default=DEFAULT_SMV_USER_CONF_FILE, help="user level (dynamic) SMV configuration file path")

        # Where to find/store data
        parser.add_argument('--data-dir', dest='dataDir', help="specify the top level data directory")
        parser.add_argument('--input-dir', dest='inputDir', help="specify the input directory (default: datadir/input)")
        parser.add_argument('--output-dir', dest='outputDir', help="specify the output directory (default: datadir/output)")
        parser.add_argument('--hostory-dir', dest='historyDir', help="specify the history directory (default: datadir/history)")
        parser.add_argument('--publish-dir', dest='publishDir', help="specify the publish directory (default: datadir/publish)")

        # All app run flags
        parser.add_argument('--force-run-all', dest='forceRunAll', action="store_true", help="ignore persisted data and force all modules to run")
        parser.add_argument('--publish-jdbc', dest='publishJDBC', action="store_true", help="publish the given modules/stage/app through JDBC connection")
        parser.add_argument('--publish-hive', dest='publishHive', action="store_true", help="publish|export given modules/stage/app to hive tables")
        parser.add_argument('--dead', dest='printDeadModules', action="store_true", help="print a list of the dead modules in this application")
        parser.add_argument('--graph', dest='graph', action="store_true", help="generate a dot dependency graph of the given modules (modules are not run)")
        parser.add_argument('--purge-old-output', dest='purgeOldOutput', action="store_true", help="remove all old output files in output dir")
        parser.add_argument('--dry-run', dest='dryRun', action="store_true", help="determine which modules do not have persisted data and will need to be run")
        parser.add_argument('--edd', dest='genEdd', action="store_true", help="summarize data and generate an edd file in the same directory as csv and schema")

        # Where to output CSVs
        parser.add_argument('--publish', dest='publish', help="publish the given modules/stage/app as given version")
        parser.add_argument('--export-csv', dest='exportCsv', help="publish|export given modules/stage/app to a CSV file at the given path on the local file system")

        # What modules to run
        parser.add_argument('--run-app', dest='runAllApp', action='store_true', help="run all output modules in all stages in app")
{'fraction_non_alphanumeric': 0.11975377728035815, 'fraction_numerical': 0.0016166138158303798, 'mean_word_length': 2.949901768172888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7227248', 'n_tokens_mistral': 4817, 'n_tokens_neox': 4717, 'n_words': 1233}