<commit_before>"""
Fabric Commands

Commands that run on servers to do the actual work.
"""
import sys
import requests
import re
import os

from fabric.contrib.files import append, exists, sed
from fabric.api import *
from fabric.network import disconnect_all
from jinja2 import Environment, PackageLoader
from random import randint
from time import time
from datetime import datetime
from atlas.config import *
from atlas import utilities

path = '/data/code'
if path not in sys.path:
    sys.path.append(path)

# Tell Jinja where our templates live.
jinja_env = Environment(loader=PackageLoader('atlas', 'templates'))

# Fabric environmental settings.
env.user = ssh_user
# env.key_filename =

# Allow ~/.ssh/config to be utilized.
env.use_ssh_config = True
env.roledefs = serverdefs[environment]


# TODO: Figure out a better way to deal with the output. Calling functions via
# 'var = execute(func)' seems to suppress a lot of the output.


# Code Commands.
@roles('webservers')
def code_deploy(item):
    """
    Responds to POSTs to deploy code to the right places on the server.

    :param item:
    :return:
    """
    print('Code - Deploy\n{0}'.format(item))
    if item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        item['meta']['name'],
        item['meta']['version'])
    _create_directory_structure(code_folder)
    _clone_repo(item["git_url"], item["commit_hash"], code_folder)
    if item['meta']['is_current']:
        code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
            code_root,
            code_type_dir,
            item['meta']['name'])
        _update_symlink(code_folder, code_folder_current)


@roles('webservers')
def code_update(updated_item, original_item):
    """
    Responds to PATCHes to update code in the right places on the server.

    :param updated_item:
    :param original_item:
    :return:
    """
    print('Code - Update\nUpdated Item\n{0}\n\nOriginal Item\n{1}'.format(
        updated_item,
        original_item))
    if updated_item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = updated_item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        updated_item['meta']['name'],
        updated_item['meta']['version'])
    if (updated_item['meta']['name'] != original_item['meta']['name']) or (updated_item['meta']['version'] != original_item['meta']['version']) or (updated_item['meta']['code_type'] != original_item['meta']['code_type']):
        code_remove(original_item)
        code_deploy(updated_item)
    else:
        _checkout_repo(updated_item["commit_hash"], code_folder)
        if updated_item['meta']['is_current']:
            code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
                code_root,
                code_type_dir,
                updated_item['meta']['name'])
            _update_symlink(code_folder, code_folder_current)


@roles('webservers')
def code_remove(item):
    """
    Responds to DELETEs to remove code from the server.

    :param item: Item to remove
    :return:
    """
    print('Code - Remove\n{0}'.format(item))
    if item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        item['meta']['name'],
        item['meta']['version'])
    _remove_directory(code_folder)
    if item['meta']['is_current']:
        code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
            code_root,
            code_type_dir,
            item['meta']['name'])
        _remove_symlink(code_folder_current)


@roles('webservers')
def site_provision(site):
    """
    Responds to POSTs to provision a site to the right places on the server.

    :param site: The flask.request object, JSON encoded
    :param install: Boolean. Indicates if the install command will run.
    :return:
    """
    print('Site Provision\n{0}'.format(site))

    code_directory = '{0}/{1}'.format(sites_code_root, site['sid'])
    code_directory_sid = '{0}/{1}'.format(code_directory, site['sid'])
    code_directory_current = '{0}/current'.format(code_directory)
    web_directory_type = '{0}/{1}'.format(
        sites_web_root,
        site['type'])
    web_directory_sid = '{0}/{1}'.format(
        web_directory_type,
        site['sid'])
    profile = utilities.get_single_eve('code', site['code']['profile'])
    profile_name = profile['meta']['name']

    _create_database(site)

    _create_settings_files(site, profile_name)

    _create_directory_structure(code_directory)
    _create_directory_structure(web_directory_type)

    with cd(code_directory):
        core = _get_code_name_version(site['code']['core'])
        run('drush dslm-new {0} {1}'.format(site['sid'], core))

    _update_symlink(code_directory_sid, code_directory_current)

    with cd(code_directory_current):
        profile = _get_code_name_version(site['code']['profile'])
        run('drush dslm-add-profile {0}'.format(profile))

    if nfs_mount_files_dir:
        nfs_dir = nfs_mount_location[environment]
        nfs_files_dir = '{0}/sitefiles/{1}/files'.format(nfs_dir, site['sid'])
        _create_nfs_files_dir(nfs_dir, site['sid'])
        # Replace default files dir with this one
        site_files_dir = code_directory_current + '/sites/default/files'
        _replace_files_directory(nfs_files_dir, site_files_dir)

    _push_settings_files(site, code_directory_current)

    _update_symlink(code_directory_current, web_directory_sid)
    correct_file_directory_permissions(site)



@roles('webserver_single')
def site_install(site):
    code_directory = '{0}/{1}'.format(sites_code_root, site['sid'])
    code_directory_current = '{0}/current'.format(code_directory)
    profile = utilities.get_single_eve('code', site['code']['profile'])
    profile_name = profile['meta']['name']

    _install_site(profile_name, code_directory_current)


@roles('webservers')
def site_package_update(site):
    print('Site Package Update\n{0}'.format(site))
    code_directory_sid = '{0}/{1}/{1}'.format(sites_code_root, site['sid'])
    packages_directory = '{0}/sites/all'.format(code_directory_sid)

    package_name_string = ""
    for package in site['code']['package']:
        # Append the package name and a space.
        package_name_string += _get_code_name_version(package) + " "
    # Strip the trailing space off the end.
    package_name_string = package_name_string.rstrip()
    print("Ready to add packages - {0}\n{1}".format(
        site['sid'],
        package_name_string))
<commit_msg>Add file perm check.<commit_after>"""
Fabric Commands

Commands that run on servers to do the actual work.
"""
import sys
import requests
import re
import os

from fabric.contrib.files import append, exists, sed
from fabric.api import *
from fabric.network import disconnect_all
from jinja2 import Environment, PackageLoader
from random import randint
from time import time
from datetime import datetime
from atlas.config import *
from atlas import utilities

path = '/data/code'
if path not in sys.path:
    sys.path.append(path)

# Tell Jinja where our templates live.
jinja_env = Environment(loader=PackageLoader('atlas', 'templates'))

# Fabric environmental settings.
env.user = ssh_user
# env.key_filename =

# Allow ~/.ssh/config to be utilized.
env.use_ssh_config = True
env.roledefs = serverdefs[environment]


# TODO: Figure out a better way to deal with the output. Calling functions via
# 'var = execute(func)' seems to suppress a lot of the output.


# Code Commands.
@roles('webservers')
def code_deploy(item):
    """
    Responds to POSTs to deploy code to the right places on the server.

    :param item:
    :return:
    """
    print('Code - Deploy\n{0}'.format(item))
    if item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        item['meta']['name'],
        item['meta']['version'])
    _create_directory_structure(code_folder)
    _clone_repo(item["git_url"], item["commit_hash"], code_folder)
    if item['meta']['is_current']:
        code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
            code_root,
            code_type_dir,
            item['meta']['name'])
        _update_symlink(code_folder, code_folder_current)


@roles('webservers')
def code_update(updated_item, original_item):
    """
    Responds to PATCHes to update code in the right places on the server.

    :param updated_item:
    :param original_item:
    :return:
    """
    print('Code - Update\nUpdated Item\n{0}\n\nOriginal Item\n{1}'.format(
        updated_item,
        original_item))
    if updated_item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = updated_item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        updated_item['meta']['name'],
        updated_item['meta']['version'])
    if (updated_item['meta']['name'] != original_item['meta']['name']) or (updated_item['meta']['version'] != original_item['meta']['version']) or (updated_item['meta']['code_type'] != original_item['meta']['code_type']):
        code_remove(original_item)
        code_deploy(updated_item)
    else:
        _checkout_repo(updated_item["commit_hash"], code_folder)
        if updated_item['meta']['is_current']:
            code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
                code_root,
                code_type_dir,
                updated_item['meta']['name'])
            _update_symlink(code_folder, code_folder_current)


@roles('webservers')
def code_remove(item):
    """
    Responds to DELETEs to remove code from the server.

    :param item: Item to remove
    :return:
    """
    print('Code - Remove\n{0}'.format(item))
    if item['meta']['code_type'] == 'library':
        code_type_dir = 'libraries'
    else:
        code_type_dir = item['meta']['code_type'] + 's'
    code_folder = '{0}/{1}/{2}/{2}-{3}'.format(
        code_root,
        code_type_dir,
        item['meta']['name'],
        item['meta']['version'])
    _remove_directory(code_folder)
    if item['meta']['is_current']:
        code_folder_current = '{0}/{1}/{2}/{2}-current'.format(
            code_root,
            code_type_dir,
            item['meta']['name'])
        _remove_symlink(code_folder_current)


@roles('webservers')
def site_provision(site):
    """
    Responds to POSTs to provision a site to the right places on the server.

    :param site: The flask.request object, JSON encoded
    :param install: Boolean. Indicates if the install command will run.
    :return:
    """
    print('Site Provision\n{0}'.format(site))

    code_directory = '{0}/{1}'.format(sites_code_root, site['sid'])
    code_directory_sid = '{0}/{1}'.format(code_directory, site['sid'])
    code_directory_current = '{0}/current'.format(code_directory)
    web_directory_type = '{0}/{1}'.format(
        sites_web_root,
        site['type'])
    web_directory_sid = '{0}/{1}'.format(
        web_directory_type,
        site['sid'])
    profile = utilities.get_single_eve('code', site['code']['profile'])
    profile_name = profile['meta']['name']

    _create_database(site)

    _create_settings_files(site, profile_name)

    _create_directory_structure(code_directory)
    _create_directory_structure(web_directory_type)

    with cd(code_directory):
        core = _get_code_name_version(site['code']['core'])
        run('drush dslm-new {0} {1}'.format(site['sid'], core))

    _update_symlink(code_directory_sid, code_directory_current)

    with cd(code_directory_current):
        profile = _get_code_name_version(site['code']['profile'])
        run('drush dslm-add-profile {0}'.format(profile))

    if nfs_mount_files_dir:
        nfs_dir = nfs_mount_location[environment]
        nfs_files_dir = '{0}/sitefiles/{1}/files'.format(nfs_dir, site['sid'])
        _create_nfs_files_dir(nfs_dir, site['sid'])
        # Replace default files dir with this one
        site_files_dir = code_directory_current + '/sites/default/files'
        _replace_files_directory(nfs_files_dir, site_files_dir)

    _push_settings_files(site, code_directory_current)

    _update_symlink(code_directory_current, web_directory_sid)
    correct_file_directory_permissions(site)



@roles('webserver_single')
def site_install(site):
    code_directory = '{0}/{1}'.format(sites_code_root, site['sid'])
    code_directory_current = '{0}/current'.format(code_directory)
    profile = utilities.get_single_eve('code', site['code']['profile'])
    profile_name = profile['meta']['name']

    _install_site(profile_name, code_directory_current)
    correct_file_directory_permissions(site)


@roles('webservers')
def site_package_update(site):
    print('Site Package Update\n{0}'.format(site))
    code_directory_sid = '{0}/{1}/{1}'.format(sites_code_root, site['sid'])
    packages_directory = '{0}/sites/all'.format(code_directory_sid)

    package_name_string = ""
    for package in site['code']['package']:
        # Append the package name and a space.
        package_name_string += _get_code_name_version(package) + " "
    # Strip the trailing space off the end.
    package_name_string = package_name_string.rstrip()
    print("Ready to add packages - {0}\n{1}".format(
        site['sid'],
        package_name_string))

{'fraction_non_alphanumeric': 0.14292943898207056, 'fraction_numerical': 0.008241758241758242, 'mean_word_length': 3.6575757575757577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6394707', 'n_tokens_mistral': 4447, 'n_tokens_neox': 4352, 'n_words': 880}