<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import re
import ConfigParser

from devstack import env
from devstack import exceptions as excp
from devstack import log as logging
from devstack import shell as sh
from devstack import utils

LOG = logging.getLogger("devstack.cfg")
PW_TMPL = "Enter a password for %s: "
ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$")
SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)")
CACHE_MSG = "(value will now be internally cached)"


class IgnoreMissingConfigParser(ConfigParser.RawConfigParser):
    DEF_INT = 0
    DEF_FLOAT = 0.0
    DEF_BOOLEAN = False

    def __init__(self):
        ConfigParser.RawConfigParser.__init__(self, allow_no_value=True)

    def get(self, section, option):
        value = None
        try:
            value = ConfigParser.RawConfigParser.get(self, section, option)
        except ConfigParser.NoSectionError, e:
            pass
        except ConfigParser.NoOptionError, e:
            pass
        return value

    def getboolean(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_BOOLEAN
        return ConfigParser.RawConfigParser.getboolean(self, section, option)

    def getfloat(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_FLOAT
        return ConfigParser.RawConfigParser.getfloat(self, section, option)

    def getint(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_INT
        return ConfigParser.RawConfigParser.getint(self, section, option)


class EnvConfigParser(ConfigParser.RawConfigParser):
    def __init__(self):
        ConfigParser.RawConfigParser.__init__(self, allow_no_value=True)
        self.pws = dict()
        self.configs_fetched = dict()
        self.db_dsns = dict()

    def _makekey(self, section, option):
        parts = [section, option]
        return "/".join(parts)

    def _resolve_special(self, section, option, value_gotten):
        if(value_gotten and len(value_gotten)):
            #you already got something, so no need to figure it out
            return value_gotten
        if(section == 'host' and option == 'ip'):
            LOG.debug("Host ip from configuration/environment was empty, programatically attempting to determine it.")
            host_ip = utils.get_host_ip()
            LOG.debug("Determined host ip to be: \"%s\"" % (host_ip))
            return host_ip
        elif(section == 'passwords'):
            key = self._makekey(section, option)
            LOG.debug("Being forced to ask for password for \"%s\" since the configuration/environment value is empty.", key)
            pw = sh.password(PW_TMPL % (key))
            self.pws[key] = pw
            return pw
        else:
            return value_gotten

    def get(self, section, option):
        key = self._makekey(section, option)
        value = None
        if(key in self.configs_fetched):
            value = self.configs_fetched.get(key)
            LOG.debug("Fetched cached value \"%s\" for param \"%s\"" % (value, key))
        else:
            LOG.debug("Fetching value for param \"%s\"" % (key))
            gotten_value = self._get_special(section, option)
            value = self._resolve_special(section, option, gotten_value)
            LOG.debug("Fetched \"%s\" for \"%s\"" % (value, key))
            self.configs_fetched[key] = value
        return value

    def _extract_default(self, default_value):
        if(not SUB_MATCH.search(default_value)):
            return default_value

        LOG.debug("Performing simple replacement on %s", default_value)

        #allow for our simple replacement to occur
        def replacer(match):
            section = match.group(1)
            option = match.group(2)
            return self.get(section, option)

        return SUB_MATCH.sub(replacer, default_value)

    def _get_special(self, section, option):
        key = self._makekey(section, option)
        parent_val = ConfigParser.RawConfigParser.get(self, section, option)
        if(parent_val == None):
            #parent didn't have anything, we are unable to do anything with it then
            return None
        extracted_val = None
        mtch = ENV_PAT.match(parent_val)
        if(mtch):
            env_key = mtch.group(1).strip()
            def_val = mtch.group(2)
            if(len(def_val) == 0 and len(env_key) == 0):
                msg = "Invalid bash-like value \"%s\" for \"%s\"" % (parent_val, key)
                raise excp.BadParamException(msg)
            if(len(env_key) == 0 or env.get_key(env_key) == None):
                LOG.debug("Extracting default value from config provided default value \"%s\" for \"%s\"" % (def_val, key))
                actual_def_val = self._extract_default(def_val)
                LOG.debug("Using config provided default value \"%s\" for \"%s\" (no environment key)" % (actual_def_val, key))
                extracted_val = actual_def_val
            else:
                env_val = env.get_key(env_key)
                LOG.debug("Using enviroment provided value \"%s\" for \"%s\"" % (env_val, key))
                extracted_val = env_val
        else:
            LOG.debug("Using raw config provided value \"%s\" for \"%s\"" % (parent_val, key))
            extracted_val = parent_val
        return extracted_val

    def get_dbdsn(self, dbname):
        user = self.get("db", "sql_user")
        host = self.get("db", "sql_host")
        port = self.get("db", "port")
        pw = self.get("passwords", "sql")
        #check the dsn cache
        if(dbname in self.db_dsns):
            return self.db_dsns[dbname]
        #form the dsn (from components we have...)
        #dsn = "<driver>://<username>:<password>@<host>:<port>/<database>"
        if(not host):
            msg = "Unable to fetch a database dsn - no host found"
            raise excp.BadParamException(msg)
        driver = self.get("db", "type")
        if(not driver):
            msg = "Unable to fetch a database dsn - no driver type found"
            raise excp.BadParamException(msg)
        dsn = driver + "://"
        if(user):
            dsn += user
        if(pw):
            dsn += ":" + pw
        if(user or pw):
            dsn += "@"
        dsn += host
        if(port):
            dsn += ":" + port
        if(dbname):
            dsn += "/" + dbname
        else:
            dsn += "/"
        LOG.debug("For database \"%s\" fetched dsn \"%s\" %s" % (dbname, dsn, CACHE_MSG))
        #store for later...
        self.db_dsns[dbname] = dsn
        return dsn
<commit_msg>Ensured password added to cache if value found without asking.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import re
import ConfigParser

from devstack import env
from devstack import exceptions as excp
from devstack import log as logging
from devstack import shell as sh
from devstack import utils

LOG = logging.getLogger("devstack.cfg")
PW_TMPL = "Enter a password for %s: "
ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$")
SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)")
CACHE_MSG = "(value will now be internally cached)"


class IgnoreMissingConfigParser(ConfigParser.RawConfigParser):
    DEF_INT = 0
    DEF_FLOAT = 0.0
    DEF_BOOLEAN = False

    def __init__(self):
        ConfigParser.RawConfigParser.__init__(self, allow_no_value=True)

    def get(self, section, option):
        value = None
        try:
            value = ConfigParser.RawConfigParser.get(self, section, option)
        except ConfigParser.NoSectionError, e:
            pass
        except ConfigParser.NoOptionError, e:
            pass
        return value

    def getboolean(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_BOOLEAN
        return ConfigParser.RawConfigParser.getboolean(self, section, option)

    def getfloat(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_FLOAT
        return ConfigParser.RawConfigParser.getfloat(self, section, option)

    def getint(self, section, option):
        value = self.get(section, option)
        if(value == None):
            #not there so don't let the parent blowup
            return IgnoreMissingConfigParser.DEF_INT
        return ConfigParser.RawConfigParser.getint(self, section, option)


class EnvConfigParser(ConfigParser.RawConfigParser):
    def __init__(self):
        ConfigParser.RawConfigParser.__init__(self, allow_no_value=True)
        self.pws = dict()
        self.configs_fetched = dict()
        self.db_dsns = dict()

    def _makekey(self, section, option):
        parts = [section, option]
        return "/".join(parts)

    def _resolve_special(self, section, option, value_gotten):
        if(value_gotten and len(value_gotten)):
            if(section == 'passwords'):
                #ensure we store it as a password
                key = self._makekey(section, option)
                self.pws[key] = value_gotten
            return value_gotten
        if(section == 'host' and option == 'ip'):
            LOG.debug("Host ip from configuration/environment was empty, programatically attempting to determine it.")
            host_ip = utils.get_host_ip()
            LOG.debug("Determined host ip to be: \"%s\"" % (host_ip))
            return host_ip
        elif(section == 'passwords'):
            key = self._makekey(section, option)
            LOG.debug("Being forced to ask for password for \"%s\" since the configuration/environment value is empty.", key)
            pw = sh.password(PW_TMPL % (key))
            self.pws[key] = pw
            return pw
        else:
            return value_gotten

    def get(self, section, option):
        key = self._makekey(section, option)
        value = None
        if(key in self.configs_fetched):
            value = self.configs_fetched.get(key)
            LOG.debug("Fetched cached value \"%s\" for param \"%s\"" % (value, key))
        else:
            LOG.debug("Fetching value for param \"%s\"" % (key))
            gotten_value = self._get_special(section, option)
            value = self._resolve_special(section, option, gotten_value)
            LOG.debug("Fetched \"%s\" for \"%s\"" % (value, key))
            self.configs_fetched[key] = value
        return value

    def _extract_default(self, default_value):
        if(not SUB_MATCH.search(default_value)):
            return default_value

        LOG.debug("Performing simple replacement on %s", default_value)

        #allow for our simple replacement to occur
        def replacer(match):
            section = match.group(1)
            option = match.group(2)
            return self.get(section, option)

        return SUB_MATCH.sub(replacer, default_value)

    def _get_special(self, section, option):
        key = self._makekey(section, option)
        parent_val = ConfigParser.RawConfigParser.get(self, section, option)
        if(parent_val == None):
            #parent didn't have anything, we are unable to do anything with it then
            return None
        extracted_val = None
        mtch = ENV_PAT.match(parent_val)
        if(mtch):
            env_key = mtch.group(1).strip()
            def_val = mtch.group(2)
            if(len(def_val) == 0 and len(env_key) == 0):
                msg = "Invalid bash-like value \"%s\" for \"%s\"" % (parent_val, key)
                raise excp.BadParamException(msg)
            if(len(env_key) == 0 or env.get_key(env_key) == None):
                LOG.debug("Extracting default value from config provided default value \"%s\" for \"%s\"" % (def_val, key))
                actual_def_val = self._extract_default(def_val)
                LOG.debug("Using config provided default value \"%s\" for \"%s\" (no environment key)" % (actual_def_val, key))
                extracted_val = actual_def_val
            else:
                env_val = env.get_key(env_key)
                LOG.debug("Using enviroment provided value \"%s\" for \"%s\"" % (env_val, key))
                extracted_val = env_val
        else:
            LOG.debug("Using raw config provided value \"%s\" for \"%s\"" % (parent_val, key))
            extracted_val = parent_val
        return extracted_val

    def get_dbdsn(self, dbname):
        user = self.get("db", "sql_user")
        host = self.get("db", "sql_host")
        port = self.get("db", "port")
        pw = self.get("passwords", "sql")
        #check the dsn cache
        if(dbname in self.db_dsns):
            return self.db_dsns[dbname]
        #form the dsn (from components we have...)
        #dsn = "<driver>://<username>:<password>@<host>:<port>/<database>"
        if(not host):
            msg = "Unable to fetch a database dsn - no host found"
            raise excp.BadParamException(msg)
        driver = self.get("db", "type")
        if(not driver):
            msg = "Unable to fetch a database dsn - no driver type found"
            raise excp.BadParamException(msg)
        dsn = driver + "://"
        if(user):
            dsn += user
        if(pw):
            dsn += ":" + pw
        if(user or pw):
            dsn += "@"
        dsn += host
        if(port):
            dsn += ":" + port
        if(dbname):
            dsn += "/" + dbname
        else:
            dsn += "/"
        LOG.debug("For database \"%s\" fetched dsn \"%s\" %s" % (dbname, dsn, CACHE_MSG))
        #store for later...
        self.db_dsns[dbname] = dsn
        return dsn

{'fraction_non_alphanumeric': 0.11308859263150982, 'fraction_numerical': 0.002232875812701123, 'mean_word_length': 2.2379332341058897, 'pattern_counts': {'":': 6, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5474038', 'n_tokens_mistral': 4483, 'n_tokens_neox': 4399, 'n_words': 1282}