<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_backup.py

    Manage backups
"""
import os
import re
import sys
import json
import errno
import time
import tarfile
import shutil
import subprocess
from collections import OrderedDict

from moulinette.core import MoulinetteError
from moulinette.utils import filesystem
from moulinette.utils.log import getActionLogger

backup_path   = '/home/yunohost.backup'
archives_path = '%s/archives' % backup_path

logger = getActionLogger('yunohost.backup')


def backup_create(name=None, description=None, output_directory=None,
                  no_compress=False, ignore_hooks=False, hooks=[],
                  ignore_apps=False, apps=[]):
    """
    Create a backup local archive

    Keyword arguments:
        name -- Name of the backup archive
        description -- Short description of the backup
        output_directory -- Output directory for the backup
        no_compress -- Do not create an archive file
        hooks -- List of backup hooks names to execute
        ignore_hooks -- Do not execute backup hooks
        apps -- List of application names to backup
        ignore_apps -- Do not backup apps

    """
    # TODO: Add a 'clean' argument to clean output directory
    from yunohost.hook import hook_callback, hook_exec

    tmp_dir = None

    # Validate what to backup
    if ignore_hooks and ignore_apps:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_action_required'))

    # Validate and define backup name
    timestamp = int(time.time())
    if not name:
        name = str(timestamp)
    if name in backup_list()['archives']:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_archive_name_exists'))

    # Validate additional arguments
    if no_compress and not output_directory:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_output_directory_required'))
    if output_directory:
        output_directory = os.path.abspath(output_directory)

        # Check for forbidden folders
        if output_directory.startswith(archives_path) or \
           re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
                    output_directory):
            logger.error("forbidden output directory '%'", output_directory)
            raise MoulinetteError(errno.EINVAL,
                                  m18n.n('backup_output_directory_forbidden'))

        # Create the output directory
        if not os.path.isdir(output_directory):
            logger.info("creating output directory '%s'", output_directory)
            os.makedirs(output_directory, 0750)
        # Check that output directory is empty
        elif no_compress and os.listdir(output_directory):
            logger.error("not empty output directory '%'", output_directory)
            raise MoulinetteError(errno.EIO,
                                  m18n.n('backup_output_directory_not_empty'))

        # Define temporary directory
        if no_compress:
            tmp_dir = output_directory
    else:
        output_directory = archives_path

    # Create temporary directory
    if not tmp_dir:
        tmp_dir = "%s/tmp/%s" % (backup_path, name)
        if os.path.isdir(tmp_dir):
            logger.warning("temporary directory for backup '%s' already exists",
                           tmp_dir)
            filesystem.rm(tmp_dir, recursive=True)
        filesystem.mkdir(tmp_dir, 0750, parents=True, uid='admin')

    def _clean_tmp_dir(retcode=0):
        ret = hook_callback('post_backup_create', args=[tmp_dir, retcode])
        if not ret['failed']:
            filesystem.rm(tmp_dir, True, True)
        else:
            msignals.display(m18n.n('backup_cleaning_failed'), 'warning')

    # Initialize backup info
    info = {
        'description': description or '',
        'created_at': timestamp,
        'apps': {},
        'hooks': {},
    }

    # Run system hooks
    if not ignore_hooks:
        msignals.display(m18n.n('backup_running_hooks'))
        hooks_ret = hook_callback('backup', hooks, args=[tmp_dir])
        info['hooks'] = hooks_ret['succeed']

    # Backup apps
    if not ignore_apps:
        from yunohost.app import app_info

        # Filter applications to backup
        apps_list = set(os.listdir('/etc/yunohost/apps'))
        apps_filtered = set()
        if apps:
            for a in apps:
                if a not in apps_list:
                    logger.warning("app '%s' not found", a)
                    msignals.display(m18n.n('unbackup_app', a), 'warning')
                else:
                    apps_filtered.add(a)
        else:
            apps_filtered = apps_list

        # Run apps backup scripts
        tmp_script = '/tmp/backup_' + str(timestamp)
        for app_id in apps_filtered:
            app_setting_path = '/etc/yunohost/apps/' + app_id

            # Check if the app has a backup script
            app_script = app_setting_path + '/scripts/backup'
            if not os.path.isfile(app_script):
                logger.warning("backup script '%s' not found", app_script)
                msignals.display(m18n.n('unbackup_app', app_id),
                                 'warning')
                continue

            tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id)
            tmp_app_bkp_dir = tmp_app_dir + '/backup'
            msignals.display(m18n.n('backup_running_app_script', app_id))
            try:
                # Prepare backup directory for the app
                filesystem.mkdir(tmp_app_bkp_dir, 0750, True, uid='admin')
                shutil.copytree(app_setting_path, tmp_app_dir + '/settings')

                # Copy app backup script in a temporary folder and execute it
                subprocess.call(['install', '-Dm555', app_script, tmp_script])
                hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_id])
            except:
                logger.exception("error while executing backup of '%s'", app_id)
                msignals.display(m18n.n('backup_app_failed', app=app_id),
                                 'error')
                # Cleaning app backup directory
                shutil.rmtree(tmp_app_dir, ignore_errors=True)
            else:
                # Add app info
                i = app_info(app_id)
                info['apps'][app_id] = {
                    'version': i['version'],
                }
            finally:
                filesystem.rm(tmp_script, force=True)<commit_msg>[enh] Add name and description in info.json of backup<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_backup.py

    Manage backups
"""
import os
import re
import sys
import json
import errno
import time
import tarfile
import shutil
import subprocess
from collections import OrderedDict

from moulinette.core import MoulinetteError
from moulinette.utils import filesystem
from moulinette.utils.log import getActionLogger

backup_path   = '/home/yunohost.backup'
archives_path = '%s/archives' % backup_path

logger = getActionLogger('yunohost.backup')


def backup_create(name=None, description=None, output_directory=None,
                  no_compress=False, ignore_hooks=False, hooks=[],
                  ignore_apps=False, apps=[]):
    """
    Create a backup local archive

    Keyword arguments:
        name -- Name of the backup archive
        description -- Short description of the backup
        output_directory -- Output directory for the backup
        no_compress -- Do not create an archive file
        hooks -- List of backup hooks names to execute
        ignore_hooks -- Do not execute backup hooks
        apps -- List of application names to backup
        ignore_apps -- Do not backup apps

    """
    # TODO: Add a 'clean' argument to clean output directory
    from yunohost.hook import hook_callback, hook_exec

    tmp_dir = None

    # Validate what to backup
    if ignore_hooks and ignore_apps:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_action_required'))

    # Validate and define backup name
    timestamp = int(time.time())
    if not name:
        name = str(timestamp)
    if name in backup_list()['archives']:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_archive_name_exists'))

    # Validate additional arguments
    if no_compress and not output_directory:
        raise MoulinetteError(errno.EINVAL,
                              m18n.n('backup_output_directory_required'))
    if output_directory:
        output_directory = os.path.abspath(output_directory)

        # Check for forbidden folders
        if output_directory.startswith(archives_path) or \
           re.match(r'^/(|(bin|boot|dev|etc|lib|root|run|sbin|sys|usr|var)(|/.*))$',
                    output_directory):
            logger.error("forbidden output directory '%'", output_directory)
            raise MoulinetteError(errno.EINVAL,
                                  m18n.n('backup_output_directory_forbidden'))

        # Create the output directory
        if not os.path.isdir(output_directory):
            logger.info("creating output directory '%s'", output_directory)
            os.makedirs(output_directory, 0750)
        # Check that output directory is empty
        elif no_compress and os.listdir(output_directory):
            logger.error("not empty output directory '%'", output_directory)
            raise MoulinetteError(errno.EIO,
                                  m18n.n('backup_output_directory_not_empty'))

        # Define temporary directory
        if no_compress:
            tmp_dir = output_directory
    else:
        output_directory = archives_path

    # Create temporary directory
    if not tmp_dir:
        tmp_dir = "%s/tmp/%s" % (backup_path, name)
        if os.path.isdir(tmp_dir):
            logger.warning("temporary directory for backup '%s' already exists",
                           tmp_dir)
            filesystem.rm(tmp_dir, recursive=True)
        filesystem.mkdir(tmp_dir, 0750, parents=True, uid='admin')

    def _clean_tmp_dir(retcode=0):
        ret = hook_callback('post_backup_create', args=[tmp_dir, retcode])
        if not ret['failed']:
            filesystem.rm(tmp_dir, True, True)
        else:
            msignals.display(m18n.n('backup_cleaning_failed'), 'warning')

    # Initialize backup info
    info = {
        'description': description or '',
        'created_at': timestamp,
        'apps': {},
        'hooks': {},
    }

    # Run system hooks
    if not ignore_hooks:
        msignals.display(m18n.n('backup_running_hooks'))
        hooks_ret = hook_callback('backup', hooks, args=[tmp_dir])
        info['hooks'] = hooks_ret['succeed']

    # Backup apps
    if not ignore_apps:
        from yunohost.app import app_info

        # Filter applications to backup
        apps_list = set(os.listdir('/etc/yunohost/apps'))
        apps_filtered = set()
        if apps:
            for a in apps:
                if a not in apps_list:
                    logger.warning("app '%s' not found", a)
                    msignals.display(m18n.n('unbackup_app', a), 'warning')
                else:
                    apps_filtered.add(a)
        else:
            apps_filtered = apps_list

        # Run apps backup scripts
        tmp_script = '/tmp/backup_' + str(timestamp)
        for app_id in apps_filtered:
            app_setting_path = '/etc/yunohost/apps/' + app_id

            # Check if the app has a backup script
            app_script = app_setting_path + '/scripts/backup'
            if not os.path.isfile(app_script):
                logger.warning("backup script '%s' not found", app_script)
                msignals.display(m18n.n('unbackup_app', app_id),
                                 'warning')
                continue

            tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_id)
            tmp_app_bkp_dir = tmp_app_dir + '/backup'
            msignals.display(m18n.n('backup_running_app_script', app_id))
            try:
                # Prepare backup directory for the app
                filesystem.mkdir(tmp_app_bkp_dir, 0750, True, uid='admin')
                shutil.copytree(app_setting_path, tmp_app_dir + '/settings')

                # Copy app backup script in a temporary folder and execute it
                subprocess.call(['install', '-Dm555', app_script, tmp_script])
                hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_id])
            except:
                logger.exception("error while executing backup of '%s'", app_id)
                msignals.display(m18n.n('backup_app_failed', app=app_id),
                                 'error')
                # Cleaning app backup directory
                shutil.rmtree(tmp_app_dir, ignore_errors=True)
            else:
                # Add app info
                i = app_info(app_id)
                info['apps'][app_id] = {
                    'version': i['version'],
                    'name': i['name'],
                    'description': i['description'],
                }
            finally:
                filesystem.rm(tmp_script, force=True)
{'fraction_non_alphanumeric': 0.08792184724689166, 'fraction_numerical': 0.006011750239103703, 'mean_word_length': 2.1535975872468764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4786300', 'n_tokens_mistral': 4121, 'n_tokens_neox': 3971, 'n_words': 1117}