<commit_before>import pymysql.cursors
import argparse
import prettytable
import logging
import os
import re
import ConfigParser
from errors import *
import pprint
import _mysql_exceptions
from version import *
import boto
from amslib.core.manager import BaseManager

class Config:

    def __init__(self, override_values={}):
        self.AMS_LOGLEVEL = 'WARNING'
        self.GLOBAL_LOGLEVEL = 'CRITICAL'

        self.override_values = override_values
        for k,v in override_values.iteritems():
            setattr(self, k, v)

        if self.env('_ARGCOMPLETE', 0):
            self.AMS_LOGLEVEL = 'CRITICAL'
            self.GLOBAL_LOGLEVEL = 'CRITICAL'

        self.NEED_INSTALL = False
        self.NEED_UPGRADE = False
        self.DISABLE_OPERATIONS = False
        self.DATABASE_VERSION = 0

        self._logger = self.get_logger()

        self._iniconfigs = {}
        self._dbconfigs = {}
        self._legacyConfigs = {}
        self._sources = {}
        self._env_overrides = {}
        self._inifile = None
        self._regionalOverrides = {}

        self.load_legacy()
        self.load_ini()
        self.load_database()

        self.combine()
        if self.NEED_INSTALL or self.NEED_UPGRADE:
            self.DISABLE_OPERATIONS = True

        self._check_versions()
        self.modules = {}

    def getRegionalSetting(self, region, item):
        if region in self._regionalOverrides and item in self._regionalOverrides[region]:
            return self._regionalOverrides[region][item]
        return getattr(self, item)

    def register_module(self, ModuleInstance):
        if not isinstance(ModuleInstance, BaseManager):
            raise InvalidModule("Incompatible module {0}".format(ModuleInstance))

        self.modules[ModuleInstance.argparse_stub()] = ModuleInstance


    def _check_versions(self):
        boto_version_parts = boto.__version__.split('.')
        min_boto_version_parts = MINIMUM_BOTO_VERSION.split('.')
        for i in range(len(boto_version_parts)):
            if boto_version_parts[i] > min_boto_version_parts[i]:
                break
            elif boto_version_parts[i] == min_boto_version_parts[i]:
                continue
            else:
                raise DependencyRequired("boto library out of date, (installed = {0}) (required >= {1}). Run 'pip install -r requirements.txt' to update dependencies.".format(boto.__version__, MINIMUM_BOTO_VERSION))

    def load_database(self):
        dbconn = pymysql.connect(host=self.TRACKING_DB['host'],
                             port=self.TRACKING_DB['port'],
                             user=self.TRACKING_DB['user'],
                             password=self.TRACKING_DB['pass'],
                             db=self.TRACKING_DB['dbname'])
        db = dbconn.cursor()
        try:
            try:
                db.execute("select var, value, type, env_overrides from config")
                rows = db.fetchall()
            except _mysql_exceptions.OperationalError as e:
                # this handles the case where the old version of the software is running database version < 16 and identifies that the database needs upgrade rather than install
                if e.args[0] == 1054:
                    db.execute("select var, value, null, null from config")
                    rows = db.fetchall()
                    self.NEED_UPGRADE = True
                else:
                    raise
            if not rows:
                self._logger.critical("Database has not been installed, before continuing you must run: ams internals database install")
                self.NEED_INSTALL = True
                return
            for row in rows:
                name, value, vartype, env_overrides = row
                if str(name).startswith('TRACKING_DB'):
                    continue
                if value is not None:
                    if vartype == 'int':
                        value = int(value)
                    if vartype == 'string':
                        value = str(value)
                    elif vartype == 'bool':
                        value = bool(int(value))
                    elif vartype == 'path':
                        value = os.path.realpath(os.path.expanduser(value))
                # ensure at least database version is properly typed for the legacy upgrade case
                if vartype is None and name == 'DATABASE_VERSION':
                    value = int(value)

                self._dbconfigs[str(name)] = value
                if env_overrides:
                    self._env_overrides[str(name)] = self.env(str(env_overrides).split(','), None)




        except _mysql_exceptions.ProgrammingError as e:
            if e.args[0] == 1146:
                self.NEED_INSTALL = True
                self._logger.critical("Database has not been installed, before continuing you must run: ams internals database install")
                return
            else:
                raise

        except _mysql_exceptions.OperationalError as e:
            if e.args[0] == 1054:
                self.NEED_UPGRADE = True
                self._logger.critical("Database needs to be updated, before continuing you must run: ams internals database upgrade")
                return
            else:
                raise


        pass


    def load_legacy(self):
        try:
            import settings
            for k in dir(settings):
                if k.startswith('__') or k == 'os' or k == 'env':<commit_msg>Missed references to _mysql_exceptions module, those are now removed and replaced with pymysql equivalents<commit_after>import pymysql.cursors
from pymysql.err import ProgrammingError, OperationalError
import argparse
import prettytable
import logging
import os
import re
import ConfigParser
from errors import *
import pprint
from version import *
import boto
from amslib.core.manager import BaseManager

class Config:

    def __init__(self, override_values={}):
        self.AMS_LOGLEVEL = 'WARNING'
        self.GLOBAL_LOGLEVEL = 'CRITICAL'

        self.override_values = override_values
        for k,v in override_values.iteritems():
            setattr(self, k, v)

        if self.env('_ARGCOMPLETE', 0):
            self.AMS_LOGLEVEL = 'CRITICAL'
            self.GLOBAL_LOGLEVEL = 'CRITICAL'

        self.NEED_INSTALL = False
        self.NEED_UPGRADE = False
        self.DISABLE_OPERATIONS = False
        self.DATABASE_VERSION = 0

        self._logger = self.get_logger()

        self._iniconfigs = {}
        self._dbconfigs = {}
        self._legacyConfigs = {}
        self._sources = {}
        self._env_overrides = {}
        self._inifile = None
        self._regionalOverrides = {}

        self.load_legacy()
        self.load_ini()
        self.load_database()

        self.combine()
        if self.NEED_INSTALL or self.NEED_UPGRADE:
            self.DISABLE_OPERATIONS = True

        self._check_versions()
        self.modules = {}

    def getRegionalSetting(self, region, item):
        if region in self._regionalOverrides and item in self._regionalOverrides[region]:
            return self._regionalOverrides[region][item]
        return getattr(self, item)

    def register_module(self, ModuleInstance):
        if not isinstance(ModuleInstance, BaseManager):
            raise InvalidModule("Incompatible module {0}".format(ModuleInstance))

        self.modules[ModuleInstance.argparse_stub()] = ModuleInstance


    def _check_versions(self):
        boto_version_parts = boto.__version__.split('.')
        min_boto_version_parts = MINIMUM_BOTO_VERSION.split('.')
        for i in range(len(boto_version_parts)):
            if boto_version_parts[i] > min_boto_version_parts[i]:
                break
            elif boto_version_parts[i] == min_boto_version_parts[i]:
                continue
            else:
                raise DependencyRequired("boto library out of date, (installed = {0}) (required >= {1}). Run 'pip install -r requirements.txt' to update dependencies.".format(boto.__version__, MINIMUM_BOTO_VERSION))

    def load_database(self):
        dbconn = pymysql.connect(host=self.TRACKING_DB['host'],
                             port=self.TRACKING_DB['port'],
                             user=self.TRACKING_DB['user'],
                             password=self.TRACKING_DB['pass'],
                             db=self.TRACKING_DB['dbname'])
        db = dbconn.cursor()
        try:
            try:
                db.execute("select var, value, type, env_overrides from config")
                rows = db.fetchall()
            except OperationalError as e:
                # this handles the case where the old version of the software is running database version < 16 and identifies that the database needs upgrade rather than install
                if e.args[0] == 1054:
                    db.execute("select var, value, null, null from config")
                    rows = db.fetchall()
                    self.NEED_UPGRADE = True
                else:
                    raise
            if not rows:
                self._logger.critical("Database has not been installed, before continuing you must run: ams internals database install")
                self.NEED_INSTALL = True
                return
            for row in rows:
                name, value, vartype, env_overrides = row
                if str(name).startswith('TRACKING_DB'):
                    continue
                if value is not None:
                    if vartype == 'int':
                        value = int(value)
                    if vartype == 'string':
                        value = str(value)
                    elif vartype == 'bool':
                        value = bool(int(value))
                    elif vartype == 'path':
                        value = os.path.realpath(os.path.expanduser(value))
                # ensure at least database version is properly typed for the legacy upgrade case
                if vartype is None and name == 'DATABASE_VERSION':
                    value = int(value)

                self._dbconfigs[str(name)] = value
                if env_overrides:
                    self._env_overrides[str(name)] = self.env(str(env_overrides).split(','), None)




        except ProgrammingError as e:
            if e.args[0] == 1146:
                self.NEED_INSTALL = True
                self._logger.critical("Database has not been installed, before continuing you must run: ams internals database install")
                return
            else:
                raise

        except OperationalError as e:
            if e.args[0] == 1054:
                self.NEED_UPGRADE = True
                self._logger.critical("Database needs to be updated, before continuing you must run: ams internals database upgrade")
                return
            else:
                raise


        pass


    def load_legacy(self):
        try:
            import settings
            for k in dir(settings):
                if k.startswith('__') or k == 'os' or k == 'env':
{'fraction_non_alphanumeric': 0.08320548686941612, 'fraction_numerical': 0.00397076076166411, 'mean_word_length': 1.800606520090978, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4080313', 'n_tokens_mistral': 3057, 'n_tokens_neox': 2889, 'n_words': 767}