<commit_before>from fabric.api import *
from fabric.operations import put
from fabric.contrib.files import *
import random
import string
# Custom Code Enigma modules
import common.Utils
import common.MySQL


# Generate a drush alias for this site
@task
@roles('app_all')
def generate_drush_alias(repo, url, branch):
  print "===> Generating Drush alias"
  # Copy drush script to server(s)
  script_dir = os.path.dirname(os.path.realpath(__file__))
  if put(script_dir + '/../util/drush_alias.sh', '/home/jenkins', mode=0755).failed:
    raise SystemExit("Could not copy the drush script to the application server, aborting because we won't be able to make a drush alias")
  else:
    print "===> Drush alias preparation script copied to %s:/home/jenkins/drush_alias.sh" % env.host
  sudo("/home/jenkins/drush_alias.sh %s %s %s" % (repo, url, branch))


@task
@roles('app_all')
def initial_build_create_live_symlink(repo, branch, build):
  print "===> Setting the live document root symlink"
  # We need to force this to avoid a repeat of https://redmine.codeenigma.net/issues/20779
  sudo("ln -nsf /var/www/%s_%s_%s /var/www/live.%s.%s" % (repo, branch, build, repo, branch))

# If composer was run beforehand because the site is Drupal 8, it'll have created a sites/default/files directory with 777 perms. Move it aside and fix perms.
@task
@roles('app_all')
def initial_build_create_files_symlink(repo, branch, build):
  with settings(warn_only=True):
    if run("stat /var/www/%s_%s_%s/www/sites/default/files" % (repo, branch, build)).return_code == 0:
      print "===> Found a files directory, probably Drupal 8, making it safe"
      sudo("mv /var/www/%s_%s_%s/www/sites/default/files /var/www/%s_%s_%s/www/sites/default/files_bak" % (repo, branch, build, repo, branch, build))
      sudo("chmod 775 /var/www/%s_%s_%s/www/sites/default/files_bak" % (repo, branch, build))
      sudo("find /var/www/%s_%s_%s/www/sites/default/files_bak -type d -print0 | xargs -r -0 chmod 775" % (repo, branch, build))
      sudo("find /var/www/%s_%s_%s/www/sites/default/files_bak -type f -print0 | xargs -r -0 chmod 664" % (repo, branch, build))
      sudo("mv /var/www/%s_%s_%s/www/sites/default/files_bak/* /var/www/shared/%s_%s_files/" % (repo, branch, build, repo, branch))
  print "===> Creating files symlink"
  sudo("ln -s /var/www/shared/%s_%s_files /var/www/%s_%s_%s/www/sites/default/files" % (repo, branch, repo, branch, build))


# Run database updates, just in case. Separate function to the main Drupal one
# as we cannot revert database or settings.php during an initial build.
@task
@roles('app_primary')
def initial_build_updatedb(repo, branch, build, drupal_version):
  print "===> Running any database hook updates"
  with settings(warn_only=True):
    if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y updatedb'" % (repo, branch, build)).failed:
      raise SystemExit("Could not apply database updates! Everything else has been done, but failing the build to alert to the fact database updates could not be run.")
    if drupal_version == '8':
      if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y entity-updates'" % (repo, branch, build)).failed:
        print "Could not carry out entity updates! Continuing anyway, as this probably isn't a major issue."
  print "===> Database updates applied"


# Function used by Drupal 8 builds to import site config
@task
@roles('app_primary')
def initial_build_config_import(repo, branch, build, drupal_version):
  with settings(warn_only=True):
    # Check to see if this is a Drupal 8 build
    if drupal_version == '8':
      print "===> Importing configuration for Drupal 8 site..."
      if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y cim'" % (repo, branch, build)).failed:
        raise SystemExit("Could not import configuration! Failing the initial build.")
      else:
        print "===> Configuration imported."


# Stuff to do when this is the initial build
@task
@roles('app_primary')
def initial_build(repo, url, branch, build, profile, buildtype, sanitise, config, db_name, db_username, db_password, mysql_version, mysql_config, dump_file, sanitised_password, sanitised_email, cluster=False, rds=False):
  print "===> This looks like the first build! We have some things to do.."

  print "===> Making the shared files dir and setting symlink"
  sudo("mkdir -p /var/www/shared/%s_%s_files" % (repo, branch))
  sudo("chown jenkins:www-data /var/www/shared/%s_%s_files" % (repo, branch))
  sudo("chmod 775 /var/www/shared/%s_%s_files" % (repo, branch))

  print "===> Making the private files dir"
  sudo("mkdir -p /var/www/shared/%s_%s_private_files" % (repo, branch))
  sudo("chown jenkins:www-data /var/www/shared/%s_%s_private_files" % (repo, branch))
  sudo("chmod 775 /var/www/shared/%s_%s_private_files" % (repo, branch))

  print "===> Preparing the database"

  # This process creates a database, database user/pass, imports the db dump from the repo
  # and generates the settings.php database credential string via a drush site-install.

  # Check if a db/ directory exists first.
  db_dir = False
  with settings(warn_only=True):
    if run("find /var/www/%s_%s_%s -maxdepth 1 -type d -name db | egrep '.*'" % (repo, branch, build)).return_code == 0:
      db_dir = True

  # We can default these to None, mysql_new_database() will sort itself out
  list_of_app_servers = None
  db_host = None

  # For clusters we need to do some extra things
  if cluster:
    # This is the Database host that we need to insert into Drupal settings.php. It is different from the main db host because it might be a floating IP
    db_host = config.get('DrupalDBHost', 'dbhost')
    # Convert a list of apps back into a string, to pass to the mysqlprepare script for setting appropriate GRANTs to the database
    list_of_app_servers = env.roledefs['app_all']

  if config.has_section('AppIPs'):
    list_of_app_servers = env.roledefs['app_ip_all']

  # Prepare the database
  site_root = "/var/www/%s_%s_%s/www" % (repo, branch, build)

  # We'll get back db_name, db_username, db_password and db_host from this call as a list in new_database
  new_database = common.MySQL.mysql_new_database(repo, buildtype, site_root, rds, db_name, db_host, db_username, mysql_version, db_password, mysql_config, list_of_app_servers)

  # Now install Drupal

  # We need to actually run a drush si first, then drop the tables and import
  # the database in the db/ directory.
  with cd("%s/sites/default" % site_root):
    run("cp default.settings.php settings.php")
    db_url = "mysql://%s:%s@%s/%s" % (new_database[1], new_database[2], new_database[3], new_database[0])
    print "===> Installing Drupal with MySQL string of %s" % db_url
    run ("drush si %s -y --db-url=%s" % (profile, db_url))
    # Append the necessary include and other settings
    append_string = """$config_directories['sync'] = '../config/sync';
$file = '/var/www/%s_%s_%s/www/sites/default/%s.settings.php';
if (file_exists($file)) {
  include_once($file);
}""" % (repo, branch, build, buildtype)
    append("settings.php", append_string, use_sudo=True)

  # Now if we have a database to import we can do that
  if db_dir and dump_file:
    with cd("%s/sites/default" % site_root):<commit_msg>Updating comment referencing mysqlprepare.sh which no longer exists.<commit_after>from fabric.api import *
from fabric.operations import put
from fabric.contrib.files import *
import random
import string
# Custom Code Enigma modules
import common.Utils
import common.MySQL


# Generate a drush alias for this site
@task
@roles('app_all')
def generate_drush_alias(repo, url, branch):
  print "===> Generating Drush alias"
  # Copy drush script to server(s)
  script_dir = os.path.dirname(os.path.realpath(__file__))
  if put(script_dir + '/../util/drush_alias.sh', '/home/jenkins', mode=0755).failed:
    raise SystemExit("Could not copy the drush script to the application server, aborting because we won't be able to make a drush alias")
  else:
    print "===> Drush alias preparation script copied to %s:/home/jenkins/drush_alias.sh" % env.host
  sudo("/home/jenkins/drush_alias.sh %s %s %s" % (repo, url, branch))


@task
@roles('app_all')
def initial_build_create_live_symlink(repo, branch, build):
  print "===> Setting the live document root symlink"
  # We need to force this to avoid a repeat of https://redmine.codeenigma.net/issues/20779
  sudo("ln -nsf /var/www/%s_%s_%s /var/www/live.%s.%s" % (repo, branch, build, repo, branch))

# If composer was run beforehand because the site is Drupal 8, it'll have created a sites/default/files directory with 777 perms. Move it aside and fix perms.
@task
@roles('app_all')
def initial_build_create_files_symlink(repo, branch, build):
  with settings(warn_only=True):
    if run("stat /var/www/%s_%s_%s/www/sites/default/files" % (repo, branch, build)).return_code == 0:
      print "===> Found a files directory, probably Drupal 8, making it safe"
      sudo("mv /var/www/%s_%s_%s/www/sites/default/files /var/www/%s_%s_%s/www/sites/default/files_bak" % (repo, branch, build, repo, branch, build))
      sudo("chmod 775 /var/www/%s_%s_%s/www/sites/default/files_bak" % (repo, branch, build))
      sudo("find /var/www/%s_%s_%s/www/sites/default/files_bak -type d -print0 | xargs -r -0 chmod 775" % (repo, branch, build))
      sudo("find /var/www/%s_%s_%s/www/sites/default/files_bak -type f -print0 | xargs -r -0 chmod 664" % (repo, branch, build))
      sudo("mv /var/www/%s_%s_%s/www/sites/default/files_bak/* /var/www/shared/%s_%s_files/" % (repo, branch, build, repo, branch))
  print "===> Creating files symlink"
  sudo("ln -s /var/www/shared/%s_%s_files /var/www/%s_%s_%s/www/sites/default/files" % (repo, branch, repo, branch, build))


# Run database updates, just in case. Separate function to the main Drupal one
# as we cannot revert database or settings.php during an initial build.
@task
@roles('app_primary')
def initial_build_updatedb(repo, branch, build, drupal_version):
  print "===> Running any database hook updates"
  with settings(warn_only=True):
    if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y updatedb'" % (repo, branch, build)).failed:
      raise SystemExit("Could not apply database updates! Everything else has been done, but failing the build to alert to the fact database updates could not be run.")
    if drupal_version == '8':
      if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y entity-updates'" % (repo, branch, build)).failed:
        print "Could not carry out entity updates! Continuing anyway, as this probably isn't a major issue."
  print "===> Database updates applied"


# Function used by Drupal 8 builds to import site config
@task
@roles('app_primary')
def initial_build_config_import(repo, branch, build, drupal_version):
  with settings(warn_only=True):
    # Check to see if this is a Drupal 8 build
    if drupal_version == '8':
      print "===> Importing configuration for Drupal 8 site..."
      if sudo("su -s /bin/bash www-data -c 'cd /var/www/%s_%s_%s/www/sites/default && drush -y cim'" % (repo, branch, build)).failed:
        raise SystemExit("Could not import configuration! Failing the initial build.")
      else:
        print "===> Configuration imported."


# Stuff to do when this is the initial build
@task
@roles('app_primary')
def initial_build(repo, url, branch, build, profile, buildtype, sanitise, config, db_name, db_username, db_password, mysql_version, mysql_config, dump_file, sanitised_password, sanitised_email, cluster=False, rds=False):
  print "===> This looks like the first build! We have some things to do.."

  print "===> Making the shared files dir and setting symlink"
  sudo("mkdir -p /var/www/shared/%s_%s_files" % (repo, branch))
  sudo("chown jenkins:www-data /var/www/shared/%s_%s_files" % (repo, branch))
  sudo("chmod 775 /var/www/shared/%s_%s_files" % (repo, branch))

  print "===> Making the private files dir"
  sudo("mkdir -p /var/www/shared/%s_%s_private_files" % (repo, branch))
  sudo("chown jenkins:www-data /var/www/shared/%s_%s_private_files" % (repo, branch))
  sudo("chmod 775 /var/www/shared/%s_%s_private_files" % (repo, branch))

  print "===> Preparing the database"

  # This process creates a database, database user/pass, imports the db dump from the repo
  # and generates the settings.php database credential string via a drush site-install.

  # Check if a db/ directory exists first.
  db_dir = False
  with settings(warn_only=True):
    if run("find /var/www/%s_%s_%s -maxdepth 1 -type d -name db | egrep '.*'" % (repo, branch, build)).return_code == 0:
      db_dir = True

  # We can default these to None, mysql_new_database() will sort itself out
  list_of_app_servers = None
  db_host = None

  # For clusters we need to do some extra things
  if cluster:
    # This is the Database host that we need to insert into Drupal settings.php. It is different from the main db host because it might be a floating IP
    db_host = config.get('DrupalDBHost', 'dbhost')
    # Convert a list of apps back into a string, to pass to the MySQL new database function for setting appropriate GRANTs to the database
    list_of_app_servers = env.roledefs['app_all']

  if config.has_section('AppIPs'):
    list_of_app_servers = env.roledefs['app_ip_all']

  # Prepare the database
  site_root = "/var/www/%s_%s_%s/www" % (repo, branch, build)

  # We'll get back db_name, db_username, db_password and db_host from this call as a list in new_database
  new_database = common.MySQL.mysql_new_database(repo, buildtype, site_root, rds, db_name, db_host, db_username, mysql_version, db_password, mysql_config, list_of_app_servers)

  # Now install Drupal

  # We need to actually run a drush si first, then drop the tables and import
  # the database in the db/ directory.
  with cd("%s/sites/default" % site_root):
    run("cp default.settings.php settings.php")
    db_url = "mysql://%s:%s@%s/%s" % (new_database[1], new_database[2], new_database[3], new_database[0])
    print "===> Installing Drupal with MySQL string of %s" % db_url
    run ("drush si %s -y --db-url=%s" % (profile, db_url))
    # Append the necessary include and other settings
    append_string = """$config_directories['sync'] = '../config/sync';
$file = '/var/www/%s_%s_%s/www/sites/default/%s.settings.php';
if (file_exists($file)) {
  include_once($file);
}""" % (repo, branch, build, buildtype)
    append("settings.php", append_string, use_sudo=True)

  # Now if we have a database to import we can do that
  if db_dir and dump_file:
    with cd("%s/sites/default" % site_root):
{'fraction_non_alphanumeric': 0.1311453176917339, 'fraction_numerical': 0.006102936190411609, 'mean_word_length': 4.1315240083507305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 84, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475962', 'n_tokens_mistral': 5037, 'n_tokens_neox': 4864, 'n_words': 1566}