<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2019 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

import os
import yaml
import json
import subprocess
import shutil
import hashlib

from difflib import unified_diff
from datetime import datetime

from moulinette import m18n
from moulinette.utils import log, filesystem
from moulinette.utils.filesystem import read_file

from yunohost.utils.error import YunohostError
from yunohost.log import is_unit_operation
from yunohost.hook import hook_callback, hook_list

BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, 'pending')
REGEN_CONF_FILE = '/etc/yunohost/regenconf.yml'

logger = log.getActionLogger('yunohost.regenconf')


# FIXME : those ain't just services anymore ... what are we supposed to do with this ...
# FIXME : check for all reference of 'service' close to operation_logger stuff
@is_unit_operation([('names', 'service')])
def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
                       list_pending=False):
    """
    Regenerate the configuration file(s)

    Keyword argument:
        names -- Categories to regenerate configuration of
        with_diff -- Show differences in case of configuration changes
        force -- Override all manual modifications in configuration files
        dry_run -- Show what would have been regenerated
        list_pending -- List pending configuration files and exit

    """

    # Legacy code to automatically run the migration
    # This is required because regen_conf is called before the migration call
    # in debian's postinst script
    if os.path.exists("/etc/yunohost/installed") \
       and ("conffiles" in read_file("/etc/yunohost/services.yml") \
            or not os.path.exists(REGEN_CONF_FILE)):
        from yunohost.tools import _get_migration_by_name
        migration = _get_migration_by_name("decouple_regenconf_from_services")
        migration.migrate()

    result = {}

    # Return the list of pending conf
    if list_pending:
        pending_conf = _get_pending_conf(names)

        if not with_diff:
            return pending_conf

        for category, conf_files in pending_conf.items():
            for system_path, pending_path in conf_files.items():

                pending_conf[category][system_path] = {
                    'pending_conf': pending_path,
                    'diff': _get_files_diff(
                        system_path, pending_path, True),
                }

        return pending_conf

    if not dry_run:
        operation_logger.related_to = [('service', x) for x in names]
        if not names:
            operation_logger.name_parameter_override = 'all'
        elif len(names) != 1:
            operation_logger.name_parameter_override = str(len(operation_logger.related_to)) + '_services'
        operation_logger.start()

    # Clean pending conf directory
    if os.path.isdir(PENDING_CONF_DIR):
        if not names:
            shutil.rmtree(PENDING_CONF_DIR, ignore_errors=True)
        else:
            for name in names:
                shutil.rmtree(os.path.join(PENDING_CONF_DIR, name),
                              ignore_errors=True)
    else:
        filesystem.mkdir(PENDING_CONF_DIR, 0o755, True)

    # Format common hooks arguments
    common_args = [1 if force else 0, 1 if dry_run else 0]

    # Execute hooks for pre-regen
    pre_args = ['pre', ] + common_args

    def _pre_call(name, priority, path, args):
        # create the pending conf directory for the category
        category_pending_path = os.path.join(PENDING_CONF_DIR, name)
        filesystem.mkdir(category_pending_path, 0o755, True, uid='root')

        # return the arguments to pass to the script
        return pre_args + [category_pending_path, ]

    # Don't regen SSH if not specifically specified
    if not names:
        names = hook_list('conf_regen', list_by='name',
                          show_info=False)['hooks']
        names.remove('ssh')

    pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)

    # Keep only the hook names with at least one success
    names = [hook for hook, infos in pre_result.items()
             if any(result["state"] == "succeed" for result in infos.values())]

    # FIXME : what do in case of partial success/failure ...
    if not names:
        ret_failed = [hook for hook, infos in pre_result.items()
                      if any(result["state"] == "failed" for result in infos.values())]
        raise YunohostError('regenconf_failed',
                            categories=', '.join(ret_failed))

    # Set the processing method
    _regen = _process_regen_conf if not dry_run else lambda *a, **k: True

    operation_logger.related_to = []

    # Iterate over categorys and process pending conf
    for category, conf_files in _get_pending_conf(names).items():
        if not dry_run:
            operation_logger.related_to.append(('service', category))

        logger.debug(m18n.n(
            'regenconf_pending_applying' if not dry_run else
            'regenconf_dry_pending_applying',
            category=category))<commit_msg>Fix naming for operation logger<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2019 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

import os
import yaml
import json
import subprocess
import shutil
import hashlib

from difflib import unified_diff
from datetime import datetime

from moulinette import m18n
from moulinette.utils import log, filesystem
from moulinette.utils.filesystem import read_file

from yunohost.utils.error import YunohostError
from yunohost.log import is_unit_operation
from yunohost.hook import hook_callback, hook_list

BASE_CONF_PATH = '/home/yunohost.conf'
BACKUP_CONF_DIR = os.path.join(BASE_CONF_PATH, 'backup')
PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, 'pending')
REGEN_CONF_FILE = '/etc/yunohost/regenconf.yml'

logger = log.getActionLogger('yunohost.regenconf')


# FIXME : those ain't just services anymore ... what are we supposed to do with this ...
# FIXME : check for all reference of 'service' close to operation_logger stuff
@is_unit_operation([('names', 'configuration')])
def regen_conf(operation_logger, names=[], with_diff=False, force=False, dry_run=False,
                       list_pending=False):
    """
    Regenerate the configuration file(s)

    Keyword argument:
        names -- Categories to regenerate configuration of
        with_diff -- Show differences in case of configuration changes
        force -- Override all manual modifications in configuration files
        dry_run -- Show what would have been regenerated
        list_pending -- List pending configuration files and exit

    """

    # Legacy code to automatically run the migration
    # This is required because regen_conf is called before the migration call
    # in debian's postinst script
    if os.path.exists("/etc/yunohost/installed") \
       and ("conffiles" in read_file("/etc/yunohost/services.yml") \
            or not os.path.exists(REGEN_CONF_FILE)):
        from yunohost.tools import _get_migration_by_name
        migration = _get_migration_by_name("decouple_regenconf_from_services")
        migration.migrate()

    result = {}

    # Return the list of pending conf
    if list_pending:
        pending_conf = _get_pending_conf(names)

        if not with_diff:
            return pending_conf

        for category, conf_files in pending_conf.items():
            for system_path, pending_path in conf_files.items():

                pending_conf[category][system_path] = {
                    'pending_conf': pending_path,
                    'diff': _get_files_diff(
                        system_path, pending_path, True),
                }

        return pending_conf

    if not dry_run:
        operation_logger.related_to = [('configuration', x) for x in names]
        if not names:
            operation_logger.name_parameter_override = 'all'
        elif len(names) != 1:
            operation_logger.name_parameter_override = str(len(operation_logger.related_to)) + '_categories'
        operation_logger.start()

    # Clean pending conf directory
    if os.path.isdir(PENDING_CONF_DIR):
        if not names:
            shutil.rmtree(PENDING_CONF_DIR, ignore_errors=True)
        else:
            for name in names:
                shutil.rmtree(os.path.join(PENDING_CONF_DIR, name),
                              ignore_errors=True)
    else:
        filesystem.mkdir(PENDING_CONF_DIR, 0o755, True)

    # Format common hooks arguments
    common_args = [1 if force else 0, 1 if dry_run else 0]

    # Execute hooks for pre-regen
    pre_args = ['pre', ] + common_args

    def _pre_call(name, priority, path, args):
        # create the pending conf directory for the category
        category_pending_path = os.path.join(PENDING_CONF_DIR, name)
        filesystem.mkdir(category_pending_path, 0o755, True, uid='root')

        # return the arguments to pass to the script
        return pre_args + [category_pending_path, ]

    # Don't regen SSH if not specifically specified
    if not names:
        names = hook_list('conf_regen', list_by='name',
                          show_info=False)['hooks']
        names.remove('ssh')

    pre_result = hook_callback('conf_regen', names, pre_callback=_pre_call)

    # Keep only the hook names with at least one success
    names = [hook for hook, infos in pre_result.items()
             if any(result["state"] == "succeed" for result in infos.values())]

    # FIXME : what do in case of partial success/failure ...
    if not names:
        ret_failed = [hook for hook, infos in pre_result.items()
                      if any(result["state"] == "failed" for result in infos.values())]
        raise YunohostError('regenconf_failed',
                            categories=', '.join(ret_failed))

    # Set the processing method
    _regen = _process_regen_conf if not dry_run else lambda *a, **k: True

    operation_logger.related_to = []

    # Iterate over categories and process pending conf
    for category, conf_files in _get_pending_conf(names).items():
        if not dry_run:
            operation_logger.related_to.append(('configuration', category))

        logger.debug(m18n.n(
            'regenconf_pending_applying' if not dry_run else
            'regenconf_dry_pending_applying',
            category=category))
{'fraction_non_alphanumeric': 0.07773911567512518, 'fraction_numerical': 0.003903929389798863, 'mean_word_length': 2.8272166287755764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5078004', 'n_tokens_mistral': 3375, 'n_tokens_neox': 3265, 'n_words': 1059}