<commit_before>"""Stack Plugin for EasyEngine."""

from cement.core.controller import CementBaseController, expose
from cement.core import handler, hook
from ee.core.variables import EEVariables
from ee.core.aptget import EEAptGet
from ee.core.download import EEDownload
from ee.core.shellexec import EEShellExec
from ee.core.fileutils import EEFileUtils
from ee.core.apt_repo import EERepo
from ee.core.extract import EEExtract
from ee.core.mysql import EEMysql
from ee.core.addswap import EESwap
from ee.core.git import EEGit
from ee.core.checkfqdn import check_fqdn
from pynginxconfig import NginxConfig
from ee.core.services import EEService
from ee.core.variables import EEVariables
import random
import string
import configparser
import time
import shutil
import os
import pwd
import grp
import codecs
from ee.cli.plugins.stack_services import EEStackStatusController
from ee.cli.plugins.stack_migrate import EEStackMigrateController
from ee.core.logging import Log


def ee_stack_hook(app):
    # do something with the ``app`` object here.
    pass


class EEStackController(CementBaseController):
    class Meta:
        label = 'stack'
        stacked_on = 'base'
        stacked_type = 'nested'
        description = 'Stack command manages stack operations'
        arguments = [
            (['--all'],
                dict(help='Install all stack', action='store_true')),
            (['--web'],
                dict(help='Install web stack', action='store_true')),
            (['--admin'],
                dict(help='Install admin tools stack', action='store_true')),
            (['--mail'],
                dict(help='Install mail server stack', action='store_true')),
            (['--mailscanner'],
                dict(help='Install mail scanner stack', action='store_true')),
            (['--nginx'],
                dict(help='Install Nginx stack', action='store_true')),
            (['--php'],
                dict(help='Install PHP stack', action='store_true')),
            (['--mysql'],
                dict(help='Install MySQL stack', action='store_true')),
            (['--postfix'],
                dict(help='Install Postfix stack', action='store_true')),
            (['--wpcli'],
                dict(help='Install WPCLI stack', action='store_true')),
            (['--phpmyadmin'],
                dict(help='Install PHPMyAdmin stack', action='store_true')),
            (['--adminer'],
                dict(help='Install Adminer stack', action='store_true')),
            (['--utils'],
                dict(help='Install Utils stack', action='store_true')),
            ]

    @expose(hide=True)
    def default(self):
        """default action of ee stack command"""
        self.app.args.print_help()

    @expose(hide=True)
    def pre_pref(self, apt_packages):
        """Pre settings to do before installation packages"""
        if set(EEVariables.ee_postfix).issubset(set(apt_packages)):
            Log.debug(self, "Pre-seeding Postfix")
            EEShellExec.cmd_exec(self, "echo \"postfix postfix"
                                 "/main_mailer_type string \'Internet Site\'\""
                                 " | debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"postfix postfix/mailname string"
                                 " $(hostname -f)\" | debconf-set-selections")

        if set(EEVariables.ee_mysql).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for MySQL, please wait ...")
            mysql_pref = ("Package: *\nPin: origin mirror.aarnet.edu.au"
                          "\nPin-Priority: 1000\n")
            with open('/etc/apt/preferences.d/'
                      'MariaDB.pref', 'w') as mysql_pref_file:
                mysql_pref_file.write(mysql_pref)
            EERepo.add(self, repo_url=EEVariables.ee_mysql_repo)
            Log.debug(self, 'Adding key for {0}'
                      .format(EEVariables.ee_mysql_repo))
            EERepo.add_key(self, '<KEY>',
                           keyserver="key<EMAIL>")
            chars = ''.join(random.sample(string.ascii_letters, 8))
            Log.debug(self, "Pre-seeding MySQL")
            Log.debug(self, "echo \"mariadb-server-10.0 "
                      "mysql-server/root_password "
                      "password \" | "
                      "debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
                                 "mysql-server/root_password "
                                 "password {chars}\" | "
                                 "debconf-set-selections".format(chars=chars),
                                 log=False)
            Log.debug(self, "echo \"mariadb-server-10.0 "
                      "mysql-server/root_password_again "
                      "password \" | "
                      "debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
                                 "mysql-server/root_password_again "
                                 "password {chars}\" | "
                                 "debconf-set-selections".format(chars=chars),
                                 log=False)
            mysql_config = """
            [client]
            user = root
            password = {<PASSWORD>}
            """.format(chars=chars)
            config = configparser.ConfigParser()
            config.read_string(mysql_config)
            Log.debug(self, 'Writting configuration into MySQL file')
            with open(os.path.expanduser("~")+'/.my.cnf', encoding='utf-8',
                      mode='w') as configfile:
                config.write(configfile)

        if set(EEVariables.ee_nginx).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for NGINX, please wait ...")
            if EEVariables.ee_platform_distro == 'debian':
                Log.debug(self, 'Adding Dotdeb/nginx GPG key')
                EERepo.add(self, repo_url=EEVariables.ee_nginx_repo)
            else:
                EERepo.add(self, ppa=EEVariables.ee_nginx_repo)
                Log.debug(self, 'Adding ppa of Nginx')

        if set(EEVariables.ee_php).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for PHP, please wait ...")
            if EEVariables.ee_platform_distro == 'debian':
                Log.debug(self, 'Adding repo_url of php for debian')
                EERepo.add(self, repo_url=EEVariables.ee_php_repo)
                Log.debug(self, 'Adding Dotdeb/php GPG key')
                EERepo.add_key(self, '89DF5277')
            else:
                Log.debug(self, 'Adding ppa for PHP')
                EERepo.add(self, ppa=EEVariables.ee_php_repo)

        if set(EEVariables.ee_mail).issubset(set(apt_packages)):
            if EEVariables.ee_platform_codename == 'squeeze':
                Log.info(self, "Adding repository for dovecot, "
                         "please wait ...")
                EERepo.add(self, repo_url=EEVariables.ee_dovecot_repo)
            Log.debug(self, 'Executing the command debconf-set-selections.')
            EEShellExec.cmd_exec(self, "echo \"dovecot-core dovecot-core/"
                                 "create-ssl-cert boolean yes\" "
                                 "| debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"dovecot-core dovecot-core"
                                 "/ssl-cert-name string $(hostname -f)\""
                                 " | debconf-set-selections")

    @expose(hide=True)
    def post_pref(self, apt_packages, packages):
        """Post activity after installation of packages"""
        if len(apt_packages):
            if set(EEVariables.ee_postfix).issubset(set(apt_packages)):
                EEGit.add(self, ["/etc/postfix"],
                          msg="Adding Postfix into Git")
                EEService.reload_service(self, 'postfix')

            if set(EEVariables.ee_nginx).issubset(set(apt_packages)):
                if ((not EEShellExec.cmd_exec(self, "grep -Hr EasyEngine "
                   "/etc/nginx")) and os.path.isfile('/etc/nginx/nginx.conf')):
                    nc = NginxConfig()
                    Log.debug(self, 'Loading file /etc/nginx/nginx.conf ')
                    nc.loadf('/etc/nginx/nginx.conf')
                    nc.set('worker_processes', 'auto')
                    nc.append(('worker_rlimit_nofile', '100000'), position=2)
                    nc.remove(('events', ''))
                    nc.append({'name': 'events', 'param': '', 'value':
                              [('worker_connections', '4096'),
                               ('multi_accept', 'on')]}, position=4)
                    nc.set([('http',), 'keepalive_timeout'], '30')
                    Log.debug(self, "Writting nginx configuration to "
                              "file /etc/nginx/nginx.conf ")
                    nc.savef('/etc/nginx/nginx.conf')

                    # Custom Nginx configuration by EasyEngine
                    data = dict(version=EEVariables.ee_version)
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/ee-nginx.conf ')
                    ee_nginx = open('/etc/nginx/conf.d/ee-nginx.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'nginx-core.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    data = dict()
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/blockips.conf')
                    ee_nginx = open('/etc/nginx/conf.d/blockips.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'blockips.mustache', out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/fastcgi.conf')
                    ee_nginx = open('/etc/nginx/conf.d/fastcgi.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'fastcgi.mustache', out=ee_nginx)
                    ee_nginx.close()

                    data = dict(php="9000", debug="9001")
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/upstream.conf ')
                    ee_nginx = open('/etc/nginx/conf.d/upstream.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'upstream.mustache', out=ee_nginx)
                    ee_nginx.close()

                    # Setup Nginx common directory
                    if not os.path.exists('/etc/nginx/common'):
                        Log.debug(self, 'Creating directory'
                                  '/etc/nginx/common')
                        os.makedirs('/etc/nginx/common')

                    data = dict(webroot=EEVariables.ee_webroot)
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/acl.conf')
                    ee_nginx = open('/etc/nginx/common/acl.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'acl.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/locations.conf')
                    ee_nginx = open('/etc/nginx/common/locations.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'locations.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/ php.conf')
                    ee_nginx = open('/etc/nginx/common/php.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'php.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/w3tc.conf')
                    ee_nginx = open('/etc/nginx/common/w3tc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'w3tc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpcommon.conf')
                    ee_nginx = open('/etc/nginx/common/wpcommon.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpcommon.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpfc.conf')
                    ee_nginx = open('/etc/nginx/common/wpfc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpfc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpsc.conf')
                    ee_nginx = open('/etc/nginx/common/wpsc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpsc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpsubdir.conf')
                    ee_nginx = open('/etc/nginx/common/wpsubdir.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpsubdir.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    # 22222 port settings
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/sites-available/'
                              '22222')
                    ee_nginx = open('/etc/nginx/sites-available/22222',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), '22222.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    passwd = ''.join([random.choice
                                     (string.ascii_letters + string.digits)
                                     for n in range(6)])
                    EEShellExec.cmd_exec(self, "printf \"easyengine:"
                                         "$(openssl passwd -crypt "
                                         "{password} 2> /dev/null)\n\""
                                         "> /etc/nginx/htpasswd-ee 2>/dev/null"
                                         .format(password=<PASSWORD>))

                    # Create Symbolic link for 22222
                    EEFileUtils.create_symlink(self, ['/etc/nginx/'
                                                      'sites-available/'
                                                      '22222',
                                                      '/etc/nginx/'
                                                      'sites-enabled/'
                                                      '22222'])
                    # Create log and cert folder and softlinks
                    if not os.path.exists('{0}22222/logs'
                                          .format(EEVariables.ee_webroot)):
                        Log.debug(self, "Creating directory "
                                  "{0}22222/logs "
                                  .format(EEVariables.ee_webroot))
                        os.makedirs('{0}22222/logs'
                                    .format(EEVariables.ee_webroot))

                    if not os.path.exists('{0}22222/cert'
                                          .format(EEVariables.ee_webroot)):
                        Log.debug(self, "Creating directory "
                                  "{0}22222/cert"
                                  .format(EEVariables.ee_webroot))
                        os.makedirs('{0}22222/cert'
                                    .format(EEVariables.ee_webroot))

                    EEFileUtils.create_symlink(self, ['/var/log/nginx/'
                                                      '22222.access.log',
                                                      '{0}22222/'
                                                      'logs/access.log'
                                               .format(EEVariables.ee_webroot)]
                                               )

                    EEFileUtils.create_symlink(self, ['/var/log/nginx/'
                                                      '22222.error.log',
                                                      '{0}22222/'
                                                      'logs/error.log'
                                               .format(EEVariables.ee_webroot)]
                                               )

                    EEShellExec.cmd_exec(self, "openssl genrsa -out "
                                         "{0}22222/cert/22222.key 2048"
                                         .format(EEVariables.ee_webroot))
                    EEShellExec.cmd_exec(self, "openssl req -new -batch -subj "
                                               "/commonName=127.0.0.1/ -key "
                                               "{0}22222/cert/22222.key "
                                               "-out {0}22222/cert/"
                                               "22222.csr"
                                         .format(EEVariables.ee_webroot))

                    EEFileUtils.mvfile(self, "{0}22222/cert/22222.key"
                                       .format(EEVariables.ee_webroot),
                                             "{0}22222/cert/"
                                             "22222.key.org"
                                       .format(EEVariables.ee_webroot))

                    EEShellExec.cmd_exec(self, "openssl rsa -in "
                                               "{0}22222/cert/"
                                               "22222.key.org -out "
                                               "{0}22222/cert/22222.key"
                                         .format(EEVariables.ee_webroot))

                    EEShellExec.cmd_exec(self, "openssl x509 -req -days 3652 "
                                               "-in {0}22222/cert/"
                                               "22222.csr -signkey {0}"
                                               "22222/cert/22222.key -out "
                                               "{0}22222/cert/22222.crt"
                                         .format(EEVariables.ee_webroot))
                    # Nginx Configation into GIT
                    EEGit.add(self,
                              ["/etc/nginx"], msg="Adding Nginx into Git")
                    EEService.reload_service(self, 'nginx')
                    self.msg = (self.msg + ["HTTP Auth User Name: easyengine"]
                                + ["HTTP Auth Password : {0}".format(passwd)])
            if set(EEVariables.ee_php).issubset(set(apt_packages)):
                # Create log directories
                if not os.path.exists('/var/log/php5/'):
                    Log.debug(self, 'Creating directory /var/log/php5/')
                    os.makedirs('/var/log/php5/')

                # Parse etc/php5/fpm/php.ini
                config = configparser.ConfigParser()
                Log.debug(self, "configuring php file /etc/php5/fpm/php.ini")
                config.read('/etc/php5/fpm/php.ini')
                config['PHP']['expose_php'] = 'Off'
                config['PHP']['post_max_size'] = '100M'
                config['PHP']['upload_max_filesize'] = '100M'
                config['PHP']['max_execution_time'] = '300'
                config['PHP']['date.timezone'] = time.tzname[time.daylight]
                with open('/etc/php5/fpm/php.ini',
                          encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "Writting php configuration into "
                              "/etc/php5/fpm/php.ini")
                    config.write(configfile)

                # Prase /etc/php5/fpm/php-fpm.conf
                config = configparser.ConfigParser()
                Log.debug(self, "configuring php file"
                          "/etc/php5/fpm/php-fpm.conf")
                config.read_file(codecs.open("/etc/php5/fpm/php-fpm.conf",
                                             "r", "utf8"))
                config['global']['error_log'] = '/var/log/php5/fpm.log'
                config.remove_option('global', 'include')
                config['global']['log_level'] = 'notice'
                config['global']['include'] = '/etc/php5/fpm/pool.d/*.conf'
                with codecs.open('/etc/php5/fpm/php-fpm.conf',
                                 encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "writting php5 configuration into "
                              "/etc/php5/fpm/php-fpm.conf")
                    config.write(configfile)

                # Parse /etc/php5/fpm/pool.d/www.conf
                config = configparser.ConfigParser()
                config.read_file(codecs.open('/etc/php5/fpm/pool.d/www.conf',
                                             "r", "utf8"))
                config['www']['ping.path'] = '/ping'
                config['www']['pm.status_path'] = '/status'
                config['www']['pm.max_requests'] = '500'
                config['www']['pm.max_children'] = '100'
                config['www']['pm.start_servers'] = '20'
                config['www']['pm.min_spare_servers'] = '10'
                config['www']['pm.max_spare_servers'] = '30'
                config['www']['request_terminate_timeout'] = '300'
                config['www']['pm'] = 'ondemand'
                config['www']['listen'] = '127.0.0.1:9000'
                with codecs.open('/etc/php5/fpm/pool.d/www.conf',
                                 encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "writting PHP5 configuration into "
                              "/etc/php5/fpm/pool.d/www.conf")
                    config.write(configfile)

                # Generate /etc/php5/fpm/pool.d/debug.conf
                EEFileUtils.copyfile(self, "/etc/php5/fpm/pool.d/www.conf",
                                     "/etc/php5/fpm/pool.d/debug.conf")
                EEFileUtils.searchreplace(self, "/etc/php5/fpm/pool.d/"
                                          "debug.conf", "[www]", "[debug]")
                config = configparser.ConfigParser()
                config.read('/etc/php5/fpm/pool.d/debug.conf')
                config['debug']['listen'] = '127.0.0.1:9001'
                config['debug']['rlimit_core'] = 'unlimited'
                with open('/etc/php5/fpm/pool.d/debug.conf',
                          encoding='utf-8', mode='w') as confifile:
                    Log.debug(self, "writting PHP5 configuration into "
                              "/etc/php5/fpm/pool.d/debug.conf")
                    config.write(confifile)

                with open("/etc/php5/fpm/pool.d/debug.conf",
                          encoding='utf-8', mode='a') as myfile:
                    myfile.write("php_admin_value[xdebug.profiler_output_dir] "
                                 "= /tmp/ \nphp_admin_value[xdebug.profiler_"
                                 "output_name] = cachegrind.out.%p-%H-%R "
                                 "\nphp_admin_flag[xdebug.profiler_enable"
                                 "_trigger] = on \nphp_admin_flag[xdebug."
                                 "profiler_enable] = off\n")

                # PHP and Debug pull configuration
                if not os.path.exists('{0}22222/htdocs/fpm/status/'
                                      .format(EEVariables.ee_webroot)):
                    Log.debug(self, 'Creating directory '
                              '{0}22222/htdocs/fpm/status/ '
                              .format(EEVariables.ee_webroot))
                    os.makedirs('{0}22222/htdocs/fpm/status/'
                                .format(EEVariables.ee_webroot))
                open('{0}22222/htdocs/fpm/status/debug'
                     .format(EEVariables.ee_webroot),
                     encoding='utf-8', mode='a').close()
                open('{0}22222/htdocs/fpm/status/php'
                     .format(EEVariables.ee_webroot),
                     encoding='utf-8', mode='a').close()

                # Write info.php
                if not os.path.exists('{0}22222/htdocs/php/'
                                      .format(EEVariables.ee_webroot)):
                    Log.debug(self, 'Creating directory '
                              '{0}22222/htdocs/php/ '
                              .format(EEVariables.ee_webroot))
                    os.makedirs('{0}22222/htdocs/php'
                                .format(EEVariables.ee_webroot))

                with open("{0}22222/htdocs/php/info.php"
                          .format(EEVariables.ee_webroot),
                          encoding='utf-8', mode='w') as myfile:
                    myfile.write("<?php\nphpinfo();\n?>")

                EEFileUtils.chown(self, "{0}22222"
                                  .format(EEVariables.ee_webroot),
                                  EEVariables.ee_php_user,
                                  EEVariables.ee_php_user, recursive=True)

                EEGit.add(self, ["/etc/php5"], msg="Adding PHP into Git")
                EEService.reload_service(self, 'php5-fpm')

            if set(EEVariables.ee_mysql).issubset(set(apt_packages)):
                # TODO: Currently we are using, we need to remove it in future
                # config = configparser.ConfigParser()
                # config.read('/etc/mysql/my.cnf')
                # config['mysqld']['wait_timeout'] = 30
                # config['mysqld']['interactive_timeout'] = 60
                # config['mysqld']['performance_schema'] = 0
                # with open('/etc/mysql/my.cnf', 'w') as configfile:
                #     config.write(configfile)
                if not os.path.isfile("/etc/mysql/my.cnf"):
                    config = ("[mysqld]\nwait_timeout = 30\n"
                              "interactive_timeout=60\nperformance_schema = 0")
                    config_file = open("/etc/mysql/my.cnf",
                                       encoding='utf-8', mode='w')
                    config_file.write(config)
                    config_file.close()
                else:
                    EEShellExec.cmd_exec(self, "sed -i \"/#max_connections/a "
                                         "wait_timeout = 30 \\n"
                                         "interactive_timeout = 60 \\n"
                                         "performance_schema = 0\" "
                                         "/etc/mysql/my.cnf")

                EEGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git")
                EEService.reload_service(self, 'mysql')

            if set(EEVariables.ee_mail).issubset(set(apt_packages)):
                Log.debug(self, "Adding user")
                EEShellExec.cmd_exec(self, "adduser --uid 5000 --home /var"
                                     "/vmail --disabled-password --gecos ''"
                                     " vmail")
                EEShellExec.cmd_exec(self, "openssl req -new -x509 -days 3650 "
                                     "-nodes -subj /commonName={HOSTNAME}"
                                     "/emailAddress={EMAIL} -out /etc/ssl"
                                     "/certs/dovecot."
                                     "pem -keyout /etc/ssl/private/dovecot.pem"
                                     .format(HOSTNAME=EEVariables.ee_fqdn,
                                             EMAIL=EEVariables.ee_email))
                Log.debug(self, "Setting Privileges to "
                          "/etc/ssl/private/dovecot.pem file ")
                EEShellExec.cmd_exec(self, "chmod 0600 /etc/ssl/private"
                                     "/dovecot.pem")

                # Custom Dovecot configuration by EasyEngine
                data = dict(email=EEVariables.ee_email)
                Log.debug(self, "Writting configuration into file"
                          "/etc/dovecot/conf.d/99-ee.conf ")
                ee_dovecot = open('/etc/dovecot/conf.d/99-ee.conf',
                                  encoding='utf-8', mode='w')
                self.app.render((data), 'dovecot.mustache', out=ee_dovecot)
                ee_dovecot.close()<commit_msg>Fixed auth-sql-conf issue in mail<commit_after>"""Stack Plugin for EasyEngine."""

from cement.core.controller import CementBaseController, expose
from cement.core import handler, hook
from ee.core.variables import EEVariables
from ee.core.aptget import EEAptGet
from ee.core.download import EEDownload
from ee.core.shellexec import EEShellExec
from ee.core.fileutils import EEFileUtils
from ee.core.apt_repo import EERepo
from ee.core.extract import EEExtract
from ee.core.mysql import EEMysql
from ee.core.addswap import EESwap
from ee.core.git import EEGit
from ee.core.checkfqdn import check_fqdn
from pynginxconfig import NginxConfig
from ee.core.services import EEService
from ee.core.variables import EEVariables
import random
import string
import configparser
import time
import shutil
import os
import pwd
import grp
import codecs
from ee.cli.plugins.stack_services import EEStackStatusController
from ee.cli.plugins.stack_migrate import EEStackMigrateController
from ee.core.logging import Log


def ee_stack_hook(app):
    # do something with the ``app`` object here.
    pass


class EEStackController(CementBaseController):
    class Meta:
        label = 'stack'
        stacked_on = 'base'
        stacked_type = 'nested'
        description = 'Stack command manages stack operations'
        arguments = [
            (['--all'],
                dict(help='Install all stack', action='store_true')),
            (['--web'],
                dict(help='Install web stack', action='store_true')),
            (['--admin'],
                dict(help='Install admin tools stack', action='store_true')),
            (['--mail'],
                dict(help='Install mail server stack', action='store_true')),
            (['--mailscanner'],
                dict(help='Install mail scanner stack', action='store_true')),
            (['--nginx'],
                dict(help='Install Nginx stack', action='store_true')),
            (['--php'],
                dict(help='Install PHP stack', action='store_true')),
            (['--mysql'],
                dict(help='Install MySQL stack', action='store_true')),
            (['--postfix'],
                dict(help='Install Postfix stack', action='store_true')),
            (['--wpcli'],
                dict(help='Install WPCLI stack', action='store_true')),
            (['--phpmyadmin'],
                dict(help='Install PHPMyAdmin stack', action='store_true')),
            (['--adminer'],
                dict(help='Install Adminer stack', action='store_true')),
            (['--utils'],
                dict(help='Install Utils stack', action='store_true')),
            ]

    @expose(hide=True)
    def default(self):
        """default action of ee stack command"""
        self.app.args.print_help()

    @expose(hide=True)
    def pre_pref(self, apt_packages):
        """Pre settings to do before installation packages"""
        if set(EEVariables.ee_postfix).issubset(set(apt_packages)):
            Log.debug(self, "Pre-seeding Postfix")
            EEShellExec.cmd_exec(self, "echo \"postfix postfix"
                                 "/main_mailer_type string \'Internet Site\'\""
                                 " | debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"postfix postfix/mailname string"
                                 " $(hostname -f)\" | debconf-set-selections")

        if set(EEVariables.ee_mysql).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for MySQL, please wait ...")
            mysql_pref = ("Package: *\nPin: origin mirror.aarnet.edu.au"
                          "\nPin-Priority: 1000\n")
            with open('/etc/apt/preferences.d/'
                      'MariaDB.pref', 'w') as mysql_pref_file:
                mysql_pref_file.write(mysql_pref)
            EERepo.add(self, repo_url=EEVariables.ee_mysql_repo)
            Log.debug(self, 'Adding key for {0}'
                      .format(EEVariables.ee_mysql_repo))
            EERepo.add_key(self, '<KEY>',
                           keyserver="keyserver.ubuntu.com")
            chars = ''.join(random.sample(string.ascii_letters, 8))
            Log.debug(self, "Pre-seeding MySQL")
            Log.debug(self, "echo \"mariadb-server-10.0 "
                      "mysql-server/root_password "
                      "password \" | "
                      "debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
                                 "mysql-server/root_password "
                                 "password {chars}\" | "
                                 "debconf-set-selections".format(chars=chars),
                                 log=False)
            Log.debug(self, "echo \"mariadb-server-10.0 "
                      "mysql-server/root_password_again "
                      "password \" | "
                      "debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
                                 "mysql-server/root_password_again "
                                 "password {chars}\" | "
                                 "debconf-set-selections".format(chars=chars),
                                 log=False)
            mysql_config = """
            [client]
            user = root
            password = {chars}
            """.format(chars=chars)
            config = configparser.ConfigParser()
            config.read_string(mysql_config)
            Log.debug(self, 'Writting configuration into MySQL file')
            with open(os.path.expanduser("~")+'/.my.cnf', encoding='utf-8',
                      mode='w') as configfile:
                config.write(configfile)

        if set(EEVariables.ee_nginx).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for NGINX, please wait ...")
            if EEVariables.ee_platform_distro == 'debian':
                Log.debug(self, 'Adding Dotdeb/nginx GPG key')
                EERepo.add(self, repo_url=EEVariables.ee_nginx_repo)
            else:
                EERepo.add(self, ppa=EEVariables.ee_nginx_repo)
                Log.debug(self, 'Adding ppa of Nginx')

        if set(EEVariables.ee_php).issubset(set(apt_packages)):
            Log.info(self, "Adding repository for PHP, please wait ...")
            if EEVariables.ee_platform_distro == 'debian':
                Log.debug(self, 'Adding repo_url of php for debian')
                EERepo.add(self, repo_url=EEVariables.ee_php_repo)
                Log.debug(self, 'Adding Dotdeb/php GPG key')
                EERepo.add_key(self, '89DF5277')
            else:
                Log.debug(self, 'Adding ppa for PHP')
                EERepo.add(self, ppa=EEVariables.ee_php_repo)

        if set(EEVariables.ee_mail).issubset(set(apt_packages)):
            if EEVariables.ee_platform_codename == 'squeeze':
                Log.info(self, "Adding repository for dovecot, "
                         "please wait ...")
                EERepo.add(self, repo_url=EEVariables.ee_dovecot_repo)
            Log.debug(self, 'Executing the command debconf-set-selections.')
            EEShellExec.cmd_exec(self, "echo \"dovecot-core dovecot-core/"
                                 "create-ssl-cert boolean yes\" "
                                 "| debconf-set-selections")
            EEShellExec.cmd_exec(self, "echo \"dovecot-core dovecot-core"
                                 "/ssl-cert-name string $(hostname -f)\""
                                 " | debconf-set-selections")

    @expose(hide=True)
    def post_pref(self, apt_packages, packages):
        """Post activity after installation of packages"""
        if len(apt_packages):
            if set(EEVariables.ee_postfix).issubset(set(apt_packages)):
                EEGit.add(self, ["/etc/postfix"],
                          msg="Adding Postfix into Git")
                EEService.reload_service(self, 'postfix')

            if set(EEVariables.ee_nginx).issubset(set(apt_packages)):
                if ((not EEShellExec.cmd_exec(self, "grep -Hr EasyEngine "
                   "/etc/nginx")) and os.path.isfile('/etc/nginx/nginx.conf')):
                    nc = NginxConfig()
                    Log.debug(self, 'Loading file /etc/nginx/nginx.conf ')
                    nc.loadf('/etc/nginx/nginx.conf')
                    nc.set('worker_processes', 'auto')
                    nc.append(('worker_rlimit_nofile', '100000'), position=2)
                    nc.remove(('events', ''))
                    nc.append({'name': 'events', 'param': '', 'value':
                              [('worker_connections', '4096'),
                               ('multi_accept', 'on')]}, position=4)
                    nc.set([('http',), 'keepalive_timeout'], '30')
                    Log.debug(self, "Writting nginx configuration to "
                              "file /etc/nginx/nginx.conf ")
                    nc.savef('/etc/nginx/nginx.conf')

                    # Custom Nginx configuration by EasyEngine
                    data = dict(version=EEVariables.ee_version)
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/ee-nginx.conf ')
                    ee_nginx = open('/etc/nginx/conf.d/ee-nginx.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'nginx-core.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    data = dict()
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/blockips.conf')
                    ee_nginx = open('/etc/nginx/conf.d/blockips.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'blockips.mustache', out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/fastcgi.conf')
                    ee_nginx = open('/etc/nginx/conf.d/fastcgi.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'fastcgi.mustache', out=ee_nginx)
                    ee_nginx.close()

                    data = dict(php="9000", debug="9001")
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/conf.d/upstream.conf ')
                    ee_nginx = open('/etc/nginx/conf.d/upstream.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'upstream.mustache', out=ee_nginx)
                    ee_nginx.close()

                    # Setup Nginx common directory
                    if not os.path.exists('/etc/nginx/common'):
                        Log.debug(self, 'Creating directory'
                                  '/etc/nginx/common')
                        os.makedirs('/etc/nginx/common')

                    data = dict(webroot=EEVariables.ee_webroot)
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/acl.conf')
                    ee_nginx = open('/etc/nginx/common/acl.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'acl.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/locations.conf')
                    ee_nginx = open('/etc/nginx/common/locations.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'locations.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/ php.conf')
                    ee_nginx = open('/etc/nginx/common/php.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'php.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/w3tc.conf')
                    ee_nginx = open('/etc/nginx/common/w3tc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'w3tc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpcommon.conf')
                    ee_nginx = open('/etc/nginx/common/wpcommon.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpcommon.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpfc.conf')
                    ee_nginx = open('/etc/nginx/common/wpfc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpfc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpsc.conf')
                    ee_nginx = open('/etc/nginx/common/wpsc.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpsc.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/common/wpsubdir.conf')
                    ee_nginx = open('/etc/nginx/common/wpsubdir.conf',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), 'wpsubdir.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    # 22222 port settings
                    Log.debug(self, 'Writting the nginx configuration to '
                              'file /etc/nginx/sites-available/'
                              '22222')
                    ee_nginx = open('/etc/nginx/sites-available/22222',
                                    encoding='utf-8', mode='w')
                    self.app.render((data), '22222.mustache',
                                    out=ee_nginx)
                    ee_nginx.close()

                    passwd = ''.join([random.choice
                                     (string.ascii_letters + string.digits)
                                     for n in range(6)])
                    EEShellExec.cmd_exec(self, "printf \"easyengine:"
                                         "$(openssl passwd -crypt "
                                         "{password} 2> /dev/null)\n\""
                                         "> /etc/nginx/htpasswd-ee 2>/dev/null"
                                         .format(password=passwd))

                    # Create Symbolic link for 22222
                    EEFileUtils.create_symlink(self, ['/etc/nginx/'
                                                      'sites-available/'
                                                      '22222',
                                                      '/etc/nginx/'
                                                      'sites-enabled/'
                                                      '22222'])
                    # Create log and cert folder and softlinks
                    if not os.path.exists('{0}22222/logs'
                                          .format(EEVariables.ee_webroot)):
                        Log.debug(self, "Creating directory "
                                  "{0}22222/logs "
                                  .format(EEVariables.ee_webroot))
                        os.makedirs('{0}22222/logs'
                                    .format(EEVariables.ee_webroot))

                    if not os.path.exists('{0}22222/cert'
                                          .format(EEVariables.ee_webroot)):
                        Log.debug(self, "Creating directory "
                                  "{0}22222/cert"
                                  .format(EEVariables.ee_webroot))
                        os.makedirs('{0}22222/cert'
                                    .format(EEVariables.ee_webroot))

                    EEFileUtils.create_symlink(self, ['/var/log/nginx/'
                                                      '22222.access.log',
                                                      '{0}22222/'
                                                      'logs/access.log'
                                               .format(EEVariables.ee_webroot)]
                                               )

                    EEFileUtils.create_symlink(self, ['/var/log/nginx/'
                                                      '22222.error.log',
                                                      '{0}22222/'
                                                      'logs/error.log'
                                               .format(EEVariables.ee_webroot)]
                                               )

                    EEShellExec.cmd_exec(self, "openssl genrsa -out "
                                         "{0}22222/cert/22222.key 2048"
                                         .format(EEVariables.ee_webroot))
                    EEShellExec.cmd_exec(self, "openssl req -new -batch -subj "
                                               "/commonName=127.0.0.1/ -key "
                                               "{0}22222/cert/22222.key "
                                               "-out {0}22222/cert/"
                                               "22222.csr"
                                         .format(EEVariables.ee_webroot))

                    EEFileUtils.mvfile(self, "{0}22222/cert/22222.key"
                                       .format(EEVariables.ee_webroot),
                                             "{0}22222/cert/"
                                             "22222.key.org"
                                       .format(EEVariables.ee_webroot))

                    EEShellExec.cmd_exec(self, "openssl rsa -in "
                                               "{0}22222/cert/"
                                               "22222.key.org -out "
                                               "{0}22222/cert/22222.key"
                                         .format(EEVariables.ee_webroot))

                    EEShellExec.cmd_exec(self, "openssl x509 -req -days 3652 "
                                               "-in {0}22222/cert/"
                                               "22222.csr -signkey {0}"
                                               "22222/cert/22222.key -out "
                                               "{0}22222/cert/22222.crt"
                                         .format(EEVariables.ee_webroot))
                    # Nginx Configation into GIT
                    EEGit.add(self,
                              ["/etc/nginx"], msg="Adding Nginx into Git")
                    EEService.reload_service(self, 'nginx')
                    self.msg = (self.msg + ["HTTP Auth User Name: easyengine"]
                                + ["HTTP Auth Password : {0}".format(passwd)])
            if set(EEVariables.ee_php).issubset(set(apt_packages)):
                # Create log directories
                if not os.path.exists('/var/log/php5/'):
                    Log.debug(self, 'Creating directory /var/log/php5/')
                    os.makedirs('/var/log/php5/')

                # Parse etc/php5/fpm/php.ini
                config = configparser.ConfigParser()
                Log.debug(self, "configuring php file /etc/php5/fpm/php.ini")
                config.read('/etc/php5/fpm/php.ini')
                config['PHP']['expose_php'] = 'Off'
                config['PHP']['post_max_size'] = '100M'
                config['PHP']['upload_max_filesize'] = '100M'
                config['PHP']['max_execution_time'] = '300'
                config['PHP']['date.timezone'] = time.tzname[time.daylight]
                with open('/etc/php5/fpm/php.ini',
                          encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "Writting php configuration into "
                              "/etc/php5/fpm/php.ini")
                    config.write(configfile)

                # Prase /etc/php5/fpm/php-fpm.conf
                config = configparser.ConfigParser()
                Log.debug(self, "configuring php file"
                          "/etc/php5/fpm/php-fpm.conf")
                config.read_file(codecs.open("/etc/php5/fpm/php-fpm.conf",
                                             "r", "utf8"))
                config['global']['error_log'] = '/var/log/php5/fpm.log'
                config.remove_option('global', 'include')
                config['global']['log_level'] = 'notice'
                config['global']['include'] = '/etc/php5/fpm/pool.d/*.conf'
                with codecs.open('/etc/php5/fpm/php-fpm.conf',
                                 encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "writting php5 configuration into "
                              "/etc/php5/fpm/php-fpm.conf")
                    config.write(configfile)

                # Parse /etc/php5/fpm/pool.d/www.conf
                config = configparser.ConfigParser()
                config.read_file(codecs.open('/etc/php5/fpm/pool.d/www.conf',
                                             "r", "utf8"))
                config['www']['ping.path'] = '/ping'
                config['www']['pm.status_path'] = '/status'
                config['www']['pm.max_requests'] = '500'
                config['www']['pm.max_children'] = '100'
                config['www']['pm.start_servers'] = '20'
                config['www']['pm.min_spare_servers'] = '10'
                config['www']['pm.max_spare_servers'] = '30'
                config['www']['request_terminate_timeout'] = '300'
                config['www']['pm'] = 'ondemand'
                config['www']['listen'] = '127.0.0.1:9000'
                with codecs.open('/etc/php5/fpm/pool.d/www.conf',
                                 encoding='utf-8', mode='w') as configfile:
                    Log.debug(self, "writting PHP5 configuration into "
                              "/etc/php5/fpm/pool.d/www.conf")
                    config.write(configfile)

                # Generate /etc/php5/fpm/pool.d/debug.conf
                EEFileUtils.copyfile(self, "/etc/php5/fpm/pool.d/www.conf",
                                     "/etc/php5/fpm/pool.d/debug.conf")
                EEFileUtils.searchreplace(self, "/etc/php5/fpm/pool.d/"
                                          "debug.conf", "[www]", "[debug]")
                config = configparser.ConfigParser()
                config.read('/etc/php5/fpm/pool.d/debug.conf')
                config['debug']['listen'] = '127.0.0.1:9001'
                config['debug']['rlimit_core'] = 'unlimited'
                with open('/etc/php5/fpm/pool.d/debug.conf',
                          encoding='utf-8', mode='w') as confifile:
                    Log.debug(self, "writting PHP5 configuration into "
                              "/etc/php5/fpm/pool.d/debug.conf")
                    config.write(confifile)

                with open("/etc/php5/fpm/pool.d/debug.conf",
                          encoding='utf-8', mode='a') as myfile:
                    myfile.write("php_admin_value[xdebug.profiler_output_dir] "
                                 "= /tmp/ \nphp_admin_value[xdebug.profiler_"
                                 "output_name] = cachegrind.out.%p-%H-%R "
                                 "\nphp_admin_flag[xdebug.profiler_enable"
                                 "_trigger] = on \nphp_admin_flag[xdebug."
                                 "profiler_enable] = off\n")

                # PHP and Debug pull configuration
                if not os.path.exists('{0}22222/htdocs/fpm/status/'
                                      .format(EEVariables.ee_webroot)):
                    Log.debug(self, 'Creating directory '
                              '{0}22222/htdocs/fpm/status/ '
                              .format(EEVariables.ee_webroot))
                    os.makedirs('{0}22222/htdocs/fpm/status/'
                                .format(EEVariables.ee_webroot))
                open('{0}22222/htdocs/fpm/status/debug'
                     .format(EEVariables.ee_webroot),
                     encoding='utf-8', mode='a').close()
                open('{0}22222/htdocs/fpm/status/php'
                     .format(EEVariables.ee_webroot),
                     encoding='utf-8', mode='a').close()

                # Write info.php
                if not os.path.exists('{0}22222/htdocs/php/'
                                      .format(EEVariables.ee_webroot)):
                    Log.debug(self, 'Creating directory '
                              '{0}22222/htdocs/php/ '
                              .format(EEVariables.ee_webroot))
                    os.makedirs('{0}22222/htdocs/php'
                                .format(EEVariables.ee_webroot))

                with open("{0}22222/htdocs/php/info.php"
                          .format(EEVariables.ee_webroot),
                          encoding='utf-8', mode='w') as myfile:
                    myfile.write("<?php\nphpinfo();\n?>")

                EEFileUtils.chown(self, "{0}22222"
                                  .format(EEVariables.ee_webroot),
                                  EEVariables.ee_php_user,
                                  EEVariables.ee_php_user, recursive=True)

                EEGit.add(self, ["/etc/php5"], msg="Adding PHP into Git")
                EEService.reload_service(self, 'php5-fpm')

            if set(EEVariables.ee_mysql).issubset(set(apt_packages)):
                # TODO: Currently we are using, we need to remove it in future
                # config = configparser.ConfigParser()
                # config.read('/etc/mysql/my.cnf')
                # config['mysqld']['wait_timeout'] = 30
                # config['mysqld']['interactive_timeout'] = 60
                # config['mysqld']['performance_schema'] = 0
                # with open('/etc/mysql/my.cnf', 'w') as configfile:
                #     config.write(configfile)
                if not os.path.isfile("/etc/mysql/my.cnf"):
                    config = ("[mysqld]\nwait_timeout = 30\n"
                              "interactive_timeout=60\nperformance_schema = 0")
                    config_file = open("/etc/mysql/my.cnf",
                                       encoding='utf-8', mode='w')
                    config_file.write(config)
                    config_file.close()
                else:
                    EEShellExec.cmd_exec(self, "sed -i \"/#max_connections/a "
                                         "wait_timeout = 30 \\n"
                                         "interactive_timeout = 60 \\n"
                                         "performance_schema = 0\" "
                                         "/etc/mysql/my.cnf")

                EEGit.add(self, ["/etc/mysql"], msg="Adding MySQL into Git")
                EEService.reload_service(self, 'mysql')

            if set(EEVariables.ee_mail).issubset(set(apt_packages)):
                Log.debug(self, "Adding user")
                EEShellExec.cmd_exec(self, "adduser --uid 5000 --home /var"
                                     "/vmail --disabled-password --gecos ''"
                                     " vmail")
                EEShellExec.cmd_exec(self, "openssl req -new -x509 -days 3650 "
                                     "-nodes -subj /commonName={HOSTNAME}"
                                     "/emailAddress={EMAIL} -out /etc/ssl"
                                     "/certs/dovecot."
                                     "pem -keyout /etc/ssl/private/dovecot.pem"
                                     .format(HOSTNAME=EEVariables.ee_fqdn,
                                             EMAIL=EEVariables.ee_email))
                Log.debug(self, "Setting Privileges to "
                          "/etc/ssl/private/dovecot.pem file ")
                EEShellExec.cmd_exec(self, "chmod 0600 /etc/ssl/private"
                                     "/dovecot.pem")

                # Custom Dovecot configuration by EasyEngine
                Log.debug(self, "Writting configuration into file"
                          "/etc/dovecot/conf.d/auth-sql.conf.ext ")
                ee_dovecot = open('/etc/dovecot/conf.d/auth-sql.conf.ext',
                                  encoding='utf-8', mode='w')
                self.app.render((data), 'auth-sql-conf.mustache',
                                out=ee_dovecot)
                ee_dovecot.close()

                data = dict(email=EEVariables.ee_email)
                Log.debug(self, "Writting configuration into file"
                          "/etc/dovecot/conf.d/99-ee.conf ")
                ee_dovecot = open('/etc/dovecot/conf.d/99-ee.conf',
                                  encoding='utf-8', mode='w')
                self.app.render((data), 'dovecot.mustache', out=ee_dovecot)
                ee_dovecot.close()
{'fraction_non_alphanumeric': 0.11771313605499203, 'fraction_numerical': 0.015743146952804092, 'mean_word_length': 1.171156086196855, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 32, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 343, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6503499', 'n_tokens_mistral': 16715, 'n_tokens_neox': 15254, 'n_words': 2489}