<commit_before>"""
    atlas.instance_operations
    ~~~~
    Commands that run on servers to deploy instances.

    Instance methods:
    Create - Local - All symlinks are in place, DB exists, NFS mount is attached
    Update - Local and Update code or configuration;
    Delete - Local - Remove instance symlinks, delete settings file, delete NFS files.; Remote - Delete database.

    # TODO After `util` and `ops` servers are combined
    # Repair - Local - Check that only intended code exists in instance, add any missing code. If extra code is found, raise an exception and open a ticket.
    # Install - Remote - Drupal install command runs
    # Update - Remote - Run optional clear caches, rebuild registry, and/or updb.
    # Backup - Remote - Create a database and NFS files backup of the instance.
    # Restore - Local - Restore files on an new instance; Remote - Restore database on instance.
"""
import logging
import os
import re
import stat

from grp import getgrnam
from shutil import copyfile, rmtree
from pwd import getpwuid

from jinja2 import Environment, PackageLoader

from atlas import utilities
from atlas.config import (ENVIRONMENT, INSTANCE_ROOT, WEB_ROOT, CORE_WEB_ROOT_SYMLINKS,
                          NFS_MOUNT_FILES_DIR, NFS_MOUNT_LOCATION, SAML_AUTH,
                          SERVICE_ACCOUNT_USERNAME, SERVICE_ACCOUNT_PASSWORD, VARNISH_CONTROL_KEY,
                          SMTP_PASSWORD, WEBSERVER_USER_GROUP, ATLAS_LOCATION, SITE_DOWN_PATH,
                          SSH_USER, SERVICENOW_KEY)
from atlas.config_servers import (SERVERDEFS, ATLAS_LOGGING_URLS, API_URLS,
                                  VARNISH_CONTROL_TERMINALS, BASE_URLS)

# Setup a sub-logger. See tasks.py for longer comment.
log = logging.getLogger('atlas.instance_operations')


def instance_create(instance, nfs_preserve=False):
    """Create symlink structure, settings file, and NFS space for an instance.

    Arguments:
        instance {dict} -- complete instance dict from POST request
    """
    log.info('Instance | Provision | Instance ID - %s', instance['_id'])
    log.debug('Instance | Provision | Instance ID - %s | Instance - %s', instance['_id'], instance)
    # Setup path variables
    instance_code_path_sid = '{0}/{1}/{1}'.format(INSTANCE_ROOT, instance['sid'])
    instance_code_path_current = '{0}/{1}/current'.format(INSTANCE_ROOT, instance['sid'])
    instance_web_path_sid = '{0}/{1}'.format(WEB_ROOT, instance['sid'])
    log.debug('Instance | Provision | Instance sid path - %s', instance_code_path_sid)
    # Create structure in INSTANCE_ROOT
    if os.path.exists(instance_code_path_sid):
        raise Exception('Destinaton directory already exists')
    os.makedirs(instance_code_path_sid)
    # Add Core
    switch_core(instance)
    # Add profile
    switch_profile(instance)
    # Add packages
    switch_packages(instance)
    # Add NFS mounted files directory
    if NFS_MOUNT_FILES_DIR:
        # Setup paths
        nfs_files_dir = NFS_MOUNT_LOCATION[ENVIRONMENT] + '/' + instance['sid']
        site_files_dir = instance_code_path_sid + '/sites/default/files'
        nfs_src = nfs_files_dir + '/files'
        # Make dir on mount if it does not exist or we are not preserving a previous mount.
        if not os.access(nfs_src, os.F_OK) or not nfs_preserve:
            nfs_directories_to_create = [nfs_files_dir, nfs_src, nfs_files_dir + '/tmp']
            for directory in nfs_directories_to_create:
                os.mkdir(directory)
        # Replace default files dir with one from NFS mount
        # Will error if there are files in the directory
        # Check is path exists, is a directory (matches symlink)
        if os.path.exists(site_files_dir) and os.path.isdir(site_files_dir):
            # Check for symlink
            if not os.path.islink(site_files_dir):
                # Check if directory is empty and remove it if it is
                if not os.listdir(site_files_dir):
                    os.rmdir(site_files_dir)
            else:
                # Remove symlink
                os.remove(site_files_dir)
            os.symlink(nfs_src, site_files_dir)
    # Create setttings file
    switch_settings_files(instance)
    # Correct file permissions
    correct_fs_permissions(instance)
    # Create symlinks for current in instance root, 'sid' and 'path' (if needed) in web root.
    log.info('Instance | Provision | Instance ID - %s | Symlink current - %s | Symlink web - %s',
             instance['_id'], instance_code_path_current, instance_web_path_sid)
    utilities.relative_symlink(instance_code_path_sid, instance_code_path_current)
    utilities.relative_symlink(instance_code_path_current, instance_web_path_sid)
    if instance['status'] in ['launched', 'launching']:
        switch_web_root_symlinks(instance)


def instance_delete(instance, nfs_preserve=False):
    """Delete symlink structure, settings file, and NFS space for an instance.

    Arguments:
        instance {dict} -- full instance record
    """
    log.info('Instance | Delete | Instance ID - %s', instance['_id'])
    log.debug('Instance | Delete | Instance ID - %s | Instance - %s', instance['_id'], instance)
    # Setup path variables
    instance_code_path = '{0}/{1}'.format(INSTANCE_ROOT, instance['sid'])
    instance_code_path_current = '{0}/{1}/current'.format(INSTANCE_ROOT, instance['sid'])
    instance_web_path_sid = '{0}/{1}'.format(WEB_ROOT, instance['sid'])
    instance_web_path_path = '{0}/{1}'.format(WEB_ROOT, instance['path'])
    # Remove symlinks and directories
    symlinks_to_remove = [instance_code_path_current, instance_web_path_sid, instance_web_path_path]
    # Directories to remove
    directories_to_remove = [instance_code_path]<commit_msg>Move the file permission correction to later in the process so that if it fails, it is the last thing to happen and the current symlinks are still created.<commit_after>"""
    atlas.instance_operations
    ~~~~
    Commands that run on servers to deploy instances.

    Instance methods:
    Create - Local - All symlinks are in place, DB exists, NFS mount is attached
    Update - Local and Update code or configuration;
    Delete - Local - Remove instance symlinks, delete settings file, delete NFS files.; Remote - Delete database.

    # TODO After `util` and `ops` servers are combined
    # Repair - Local - Check that only intended code exists in instance, add any missing code. If extra code is found, raise an exception and open a ticket.
    # Install - Remote - Drupal install command runs
    # Update - Remote - Run optional clear caches, rebuild registry, and/or updb.
    # Backup - Remote - Create a database and NFS files backup of the instance.
    # Restore - Local - Restore files on an new instance; Remote - Restore database on instance.
"""
import logging
import os
import re
import stat

from grp import getgrnam
from shutil import copyfile, rmtree
from pwd import getpwuid

from jinja2 import Environment, PackageLoader

from atlas import utilities
from atlas.config import (ENVIRONMENT, INSTANCE_ROOT, WEB_ROOT, CORE_WEB_ROOT_SYMLINKS,
                          NFS_MOUNT_FILES_DIR, NFS_MOUNT_LOCATION, SAML_AUTH,
                          SERVICE_ACCOUNT_USERNAME, SERVICE_ACCOUNT_PASSWORD, VARNISH_CONTROL_KEY,
                          SMTP_PASSWORD, WEBSERVER_USER_GROUP, ATLAS_LOCATION, SITE_DOWN_PATH,
                          SSH_USER, SERVICENOW_KEY)
from atlas.config_servers import (SERVERDEFS, ATLAS_LOGGING_URLS, API_URLS,
                                  VARNISH_CONTROL_TERMINALS, BASE_URLS)

# Setup a sub-logger. See tasks.py for longer comment.
log = logging.getLogger('atlas.instance_operations')


def instance_create(instance, nfs_preserve=False):
    """Create symlink structure, settings file, and NFS space for an instance.

    Arguments:
        instance {dict} -- complete instance dict from POST request
    """
    log.info('Instance | Provision | Instance ID - %s', instance['_id'])
    log.debug('Instance | Provision | Instance ID - %s | Instance - %s', instance['_id'], instance)
    # Setup path variables
    instance_code_path_sid = '{0}/{1}/{1}'.format(INSTANCE_ROOT, instance['sid'])
    instance_code_path_current = '{0}/{1}/current'.format(INSTANCE_ROOT, instance['sid'])
    instance_web_path_sid = '{0}/{1}'.format(WEB_ROOT, instance['sid'])
    log.debug('Instance | Provision | Instance sid path - %s', instance_code_path_sid)
    # Create structure in INSTANCE_ROOT
    if os.path.exists(instance_code_path_sid):
        raise Exception('Destinaton directory already exists')
    os.makedirs(instance_code_path_sid)
    # Add Core
    switch_core(instance)
    # Add profile
    switch_profile(instance)
    # Add packages
    switch_packages(instance)
    # Add NFS mounted files directory
    if NFS_MOUNT_FILES_DIR:
        # Setup paths
        nfs_files_dir = NFS_MOUNT_LOCATION[ENVIRONMENT] + '/' + instance['sid']
        site_files_dir = instance_code_path_sid + '/sites/default/files'
        nfs_src = nfs_files_dir + '/files'
        # Make dir on mount if it does not exist or we are not preserving a previous mount.
        if not os.access(nfs_src, os.F_OK) or not nfs_preserve:
            nfs_directories_to_create = [nfs_files_dir, nfs_src, nfs_files_dir + '/tmp']
            for directory in nfs_directories_to_create:
                os.mkdir(directory)
        # Replace default files dir with one from NFS mount
        # Will error if there are files in the directory
        # Check is path exists, is a directory (matches symlink)
        if os.path.exists(site_files_dir) and os.path.isdir(site_files_dir):
            # Check for symlink
            if not os.path.islink(site_files_dir):
                # Check if directory is empty and remove it if it is
                if not os.listdir(site_files_dir):
                    os.rmdir(site_files_dir)
            else:
                # Remove symlink
                os.remove(site_files_dir)
            os.symlink(nfs_src, site_files_dir)
    # Create setttings file
    switch_settings_files(instance)
    # Create symlinks for current in instance root, 'sid' and 'path' (if needed) in web root.
    log.info('Instance | Provision | Instance ID - %s | Symlink current - %s | Symlink web - %s',
             instance['_id'], instance_code_path_current, instance_web_path_sid)
    utilities.relative_symlink(instance_code_path_sid, instance_code_path_current)
    utilities.relative_symlink(instance_code_path_current, instance_web_path_sid)
    if instance['status'] in ['launched', 'launching']:
        switch_web_root_symlinks(instance)
    # Correct file permissions
    correct_fs_permissions(instance)


def instance_delete(instance, nfs_preserve=False):
    """Delete symlink structure, settings file, and NFS space for an instance.

    Arguments:
        instance {dict} -- full instance record
    """
    log.info('Instance | Delete | Instance ID - %s', instance['_id'])
    log.debug('Instance | Delete | Instance ID - %s | Instance - %s', instance['_id'], instance)
    # Setup path variables
    instance_code_path = '{0}/{1}'.format(INSTANCE_ROOT, instance['sid'])
    instance_code_path_current = '{0}/{1}/current'.format(INSTANCE_ROOT, instance['sid'])
    instance_web_path_sid = '{0}/{1}'.format(WEB_ROOT, instance['sid'])
    instance_web_path_path = '{0}/{1}'.format(WEB_ROOT, instance['path'])
    # Remove symlinks and directories
    symlinks_to_remove = [instance_code_path_current, instance_web_path_sid, instance_web_path_path]
    # Directories to remove
    directories_to_remove = [instance_code_path]
{'fraction_non_alphanumeric': 0.08767405879319237, 'fraction_numerical': 0.0027505587072374077, 'mean_word_length': 3.2432530999270606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4093763', 'n_tokens_mistral': 3458, 'n_tokens_neox': 3432, 'n_words': 999}