<commit_before># ----- Info ------------------------------------------------------------------

__author__ = '<NAME> <<EMAIL>>'

# ----- Imports ---------------------------------------------------------------

from .exception import RDBMSBuilderException
from tinyAPI.base.config import ConfigManager
from tinyAPI.base.data_store.exception import DataStoreDuplicateKeyException
from tinyAPI.base.utils import find_dirs, find_files

import hashlib
import importlib.machinery
import os
import re
import subprocess
import sys
import tempfile
import tinyAPI

__all__ = [
    'Manager'
]

# ----- Protected Classes -----------------------------------------------------

class _RDBMSBuilderModuleSQL(object):
    '''Simple container for all of the SQL related assets for a module.'''

    def __init__(self, name, prefix):
        self.__name = name
        self.__prefix = prefix
        self.__dml_files = []
        self.__build_file = None
        self.__definitions = []
        self.__indexes = []
        self.__inserts = []


    def add_definition(self, db_name, statement):
        self.__definitions.append([db_name, statement])
        return self


    def add_dml_file(self, dml_file):
        self.__dml_files.append(dml_file)
        return self


    def add_index(self, db_name, statement):
        self.__indexes.append([db_name, statement])
        return self


    def add_insert(self, db_name, statement):
        self.__inserts.append([db_name, statement])
        return self


    def get_build_file(self):
        return self.__build_file


    def get_definitions(self):
        return self.__definitions


    def get_dml_files(self):
        return self.__dml_files


    def get_indexes(self):
        return self.__indexes


    def get_inserts(self):
        return self.__inserts


    def get_name(self):
        return self.__name


    def get_prefix(self):
        return self.__prefix


    def set_build_file(self, build_file):
        self.__build_file = build_file
        return self

# ----- Public Classes --------------------------------------------------------

class Manager(object):
    '''Determines which database objects to build and builds them.'''

    def __init__(self, cli=None):
        self.__cli = cli
        self.__managed_schema = None
        self.__modules = {}
        self.__num_rdbms_objects = 0
        self.__num_rdbms_tables = 0
        self.__num_rdbms_indexes = 0
        self.__num_rdbms_routines = 0
        self.__connection_name = None
        self.__exec_sql_command = None
        self.__dependencies_map = {}
        self.__dependents_map = {}
        self.__modules_to_build = {}
        self.__modules_to_build_prefix = {}
        self.__prefix_to_module = {}
        self.__foreign_keys = {}
        self.__unindexed_foreign_keys = []


    def __add_foreign_key_constraints(self):
        self.__notice("Adding foreign key constraints...")

        for module_name in self.__foreign_keys.keys():
            if module_name in self.__modules_to_build:
                for fk in self.__foreign_keys[module_name]:
                    db_name = fk[0]
                    foreign_key = fk[1]

                    matches = re.search('add constraint (.*?)$',
                                        foreign_key,
                                        re.M | re.I | re.S)
                    if matches is None:
                        raise RDBMSBuilderException(
                                'could not find name of constraint in '
                                + 'statement:\n'
                                + foreign_key)

                    self.__execute_statement(foreign_key, db_name)
                    self.__notice('(+) ' + matches.group(1), 1)

                    self.__num_rdbms_objects += 1


    def __assemble_all_modules(self):
        '''Finds all of the modules in the application and puts their
           respective parts together in a centralize module object.'''
        self.__notice('Assembling all modules...')

        ##
        # Step 1
        #
        # Take an initial pass at building the modules with the minimum set of
        # information required.  This is necessary so that the second pass can
        # properly assign dependencies based on knowing all of the available
        # modules.
        ##

        for path in ConfigManager.value('application dirs'):
            files = find_files(path + '/*', 'build.py')
            for file in files:
                if file != '':
                    module_name = None
                    prefix = None

                    matches = re.search('(.*)/sql/ddl/build.py$', file)
                    if matches is not None:
                        module_name = matches.group(1).split('/')[-1]

                        with open(file) as f:
                            contents = f.read()

                        if contents != '':
                            matches = re.search('def (.*?)_build\s?\(',
                                                contents,
                                                re.M | re.S | re.I)
                            if matches is None:
                                raise RDBMSBuilderException(
                                        'found build.py file but could not '
                                        + 'find build function in "'
                                        + file
                                        + '"')<commit_msg>UTF-8 codec integration.<commit_after># ----- Info ------------------------------------------------------------------

__author__ = '<NAME> <<EMAIL>>'

# ----- Imports ---------------------------------------------------------------

from .exception import RDBMSBuilderException
from tinyAPI.base.config import ConfigManager
from tinyAPI.base.data_store.exception import DataStoreDuplicateKeyException
from tinyAPI.base.utils import find_dirs, find_files

import codecs
import hashlib
import importlib.machinery
import os
import re
import subprocess
import sys
import tempfile
import tinyAPI

__all__ = [
    'Manager'
]

# ----- Protected Classes -----------------------------------------------------

class _RDBMSBuilderModuleSQL(object):
    '''Simple container for all of the SQL related assets for a module.'''

    def __init__(self, name, prefix):
        self.__name = name
        self.__prefix = prefix
        self.__dml_files = []
        self.__build_file = None
        self.__definitions = []
        self.__indexes = []
        self.__inserts = []


    def add_definition(self, db_name, statement):
        self.__definitions.append([db_name, statement])
        return self


    def add_dml_file(self, dml_file):
        self.__dml_files.append(dml_file)
        return self


    def add_index(self, db_name, statement):
        self.__indexes.append([db_name, statement])
        return self


    def add_insert(self, db_name, statement):
        self.__inserts.append([db_name, statement])
        return self


    def get_build_file(self):
        return self.__build_file


    def get_definitions(self):
        return self.__definitions


    def get_dml_files(self):
        return self.__dml_files


    def get_indexes(self):
        return self.__indexes


    def get_inserts(self):
        return self.__inserts


    def get_name(self):
        return self.__name


    def get_prefix(self):
        return self.__prefix


    def set_build_file(self, build_file):
        self.__build_file = build_file
        return self

# ----- Public Classes --------------------------------------------------------

class Manager(object):
    '''Determines which database objects to build and builds them.'''

    def __init__(self, cli=None):
        self.__cli = cli
        self.__managed_schema = None
        self.__modules = {}
        self.__num_rdbms_objects = 0
        self.__num_rdbms_tables = 0
        self.__num_rdbms_indexes = 0
        self.__num_rdbms_routines = 0
        self.__connection_name = None
        self.__exec_sql_command = None
        self.__dependencies_map = {}
        self.__dependents_map = {}
        self.__modules_to_build = {}
        self.__modules_to_build_prefix = {}
        self.__prefix_to_module = {}
        self.__foreign_keys = {}
        self.__unindexed_foreign_keys = []


    def __add_foreign_key_constraints(self):
        self.__notice("Adding foreign key constraints...")

        for module_name in self.__foreign_keys.keys():
            if module_name in self.__modules_to_build:
                for fk in self.__foreign_keys[module_name]:
                    db_name = fk[0]
                    foreign_key = fk[1]

                    matches = re.search('add constraint (.*?)$',
                                        foreign_key,
                                        re.M | re.I | re.S)
                    if matches is None:
                        raise RDBMSBuilderException(
                                'could not find name of constraint in '
                                + 'statement:\n'
                                + foreign_key)

                    self.__execute_statement(foreign_key, db_name)
                    self.__notice('(+) ' + matches.group(1), 1)

                    self.__num_rdbms_objects += 1


    def __assemble_all_modules(self):
        '''Finds all of the modules in the application and puts their
           respective parts together in a centralize module object.'''
        self.__notice('Assembling all modules...')

        ##
        # Step 1
        #
        # Take an initial pass at building the modules with the minimum set of
        # information required.  This is necessary so that the second pass can
        # properly assign dependencies based on knowing all of the available
        # modules.
        ##

        for path in ConfigManager.value('application dirs'):
            files = find_files(path + '/*', 'build.py')
            for file in files:
                if file != '':
                    module_name = None
                    prefix = None

                    matches = re.search('(.*)/sql/ddl/build.py$', file)
                    if matches is not None:
                        module_name = matches.group(1).split('/')[-1]

                        with codecs.open(file, 'r', 'utf-8') as f:
                            contents = f.read()

                        if contents != '':
                            matches = re.search('def (.*?)_build\s?\(',
                                                contents,
                                                re.M | re.S | re.I)
                            if matches is None:
                                raise RDBMSBuilderException(
                                        'found build.py file but could not '
                                        + 'find build function in "'
                                        + file
                                        + '"')
{'fraction_non_alphanumeric': 0.12490882567469001, 'fraction_numerical': 0.0023705324580598104, 'mean_word_length': 1.3934104298494436, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5309760', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2623, 'n_words': 702}