<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2018 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_log.py

    Manage debug logs
"""

import os
import re
import yaml
import glob
import psutil

from datetime import datetime, timedelta
from logging import FileHandler, getLogger, Formatter
from io import IOBase

from moulinette import m18n, Moulinette
from moulinette.core import MoulinetteError
from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.packages import get_ynh_package_version
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, read_yaml

CATEGORIES_PATH = "/var/log/yunohost/categories/"
OPERATIONS_PATH = "/var/log/yunohost/categories/operation/"
METADATA_FILE_EXT = ".yml"
LOG_FILE_EXT = ".log"

logger = getActionLogger("yunohost.log")


def log_list(limit=None, with_details=False, with_suboperations=False):
    """
    List available logs

    Keyword argument:
        limit -- Maximum number of logs
        with_details -- Include details (e.g. if the operation was a success).
        Likely to increase the command time as it needs to open and parse the
        metadata file for each log...
        with_suboperations -- Include operations that are not the "main"
        operation but are sub-operations triggered by another ongoing operation
        ... (e.g. initializing groups/permissions when installing an app)
    """

    operations = {}

    logs = [x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT)]
    logs = list(reversed(sorted(logs)))

    if limit is not None:
        logs = logs[:limit]

    for log in logs:

        base_filename = log[: -len(METADATA_FILE_EXT)]
        md_path = os.path.join(OPERATIONS_PATH, log)

        entry = {
            "name": base_filename,
            "path": md_path,
            "description": _get_description_from_name(base_filename),
        }

        try:
            entry["started_at"] = _get_datetime_from_name(base_filename)
        except ValueError:
            pass

        try:
            metadata = (
                read_yaml(md_path) or {}
            )  # Making sure this is a dict and not  None..?
        except Exception as e:
            # If we can't read the yaml for some reason, report an error and ignore this entry...
            logger.error(m18n.n("log_corrupted_md_file", md_file=md_path, error=e))
            continue

        if with_details:
            entry["success"] = metadata.get("success", "?")
            entry["parent"] = metadata.get("parent")

        if with_suboperations:
            entry["parent"] = metadata.get("parent")
            entry["suboperations"] = []
        elif metadata.get("parent") is not None:
            continue

        operations[base_filename] = entry

    # When displaying suboperations, we build a tree-like structure where
    # "suboperations" is a list of suboperations (each of them may also have a list of
    # "suboperations" suboperations etc...
    if with_suboperations:
        suboperations = [o for o in operations.values() if o["parent"] is not None]
        for suboperation in suboperations:
            parent = operations.get(suboperation["parent"])
            if not parent:
                continue
            parent["suboperations"].append(suboperation)
        operations = [o for o in operations.values() if o["parent"] is None]
    else:
        operations = [o for o in operations.values()]

    operations = list(reversed(sorted(operations, key=lambda o: o["name"])))
    # Reverse the order of log when in cli, more comfortable to read (avoid
    # unecessary scrolling)
    is_api = Moulinette.interface.type == "api"
    if not is_api:
        operations = list(reversed(operations))

    return {"operation": operations}


def log_show(
    path, number=None, share=False, filter_irrelevant=False, with_suboperations=False
):
    """
    Display a log file enriched with metadata if any.

    If the file_name is not an absolute path, it will try to search the file in
    the unit operations log path (see OPERATIONS_PATH).

    Argument:
        file_name
        number
        share
    """

    if share:
        filter_irrelevant = True

    if filter_irrelevant:
<commit_msg>log: log list was displaying much less entries than required on webadmin<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2018 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_log.py

    Manage debug logs
"""

import os
import re
import yaml
import glob
import psutil

from datetime import datetime, timedelta
from logging import FileHandler, getLogger, Formatter
from io import IOBase

from moulinette import m18n, Moulinette
from moulinette.core import MoulinetteError
from yunohost.utils.error import YunohostError, YunohostValidationError
from yunohost.utils.packages import get_ynh_package_version
from moulinette.utils.log import getActionLogger
from moulinette.utils.filesystem import read_file, read_yaml

CATEGORIES_PATH = "/var/log/yunohost/categories/"
OPERATIONS_PATH = "/var/log/yunohost/categories/operation/"
METADATA_FILE_EXT = ".yml"
LOG_FILE_EXT = ".log"

logger = getActionLogger("yunohost.log")


def log_list(limit=None, with_details=False, with_suboperations=False):
    """
    List available logs

    Keyword argument:
        limit -- Maximum number of logs
        with_details -- Include details (e.g. if the operation was a success).
        Likely to increase the command time as it needs to open and parse the
        metadata file for each log...
        with_suboperations -- Include operations that are not the "main"
        operation but are sub-operations triggered by another ongoing operation
        ... (e.g. initializing groups/permissions when installing an app)
    """

    operations = {}

    logs = [x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT)]
    logs = list(reversed(sorted(logs)))

    if limit is not None:
        if with_suboperations:
            logs = logs[:limit]
        else:
            # If we displaying only parent, we are still gonna load up to limit * 5 logs
            # because many of them are suboperations which are not gonna be kept
            # Yet we still want to obtain ~limit number of logs
            logs = logs[:limit * 5]

    for log in logs:

        base_filename = log[: -len(METADATA_FILE_EXT)]
        md_path = os.path.join(OPERATIONS_PATH, log)

        entry = {
            "name": base_filename,
            "path": md_path,
            "description": _get_description_from_name(base_filename),
        }

        try:
            entry["started_at"] = _get_datetime_from_name(base_filename)
        except ValueError:
            pass

        try:
            metadata = (
                read_yaml(md_path) or {}
            )  # Making sure this is a dict and not  None..?
        except Exception as e:
            # If we can't read the yaml for some reason, report an error and ignore this entry...
            logger.error(m18n.n("log_corrupted_md_file", md_file=md_path, error=e))
            continue

        if with_details:
            entry["success"] = metadata.get("success", "?")
            entry["parent"] = metadata.get("parent")

        if with_suboperations:
            entry["parent"] = metadata.get("parent")
            entry["suboperations"] = []
        elif metadata.get("parent") is not None:
            continue

        operations[base_filename] = entry

    # When displaying suboperations, we build a tree-like structure where
    # "suboperations" is a list of suboperations (each of them may also have a list of
    # "suboperations" suboperations etc...
    if with_suboperations:
        suboperations = [o for o in operations.values() if o["parent"] is not None]
        for suboperation in suboperations:
            parent = operations.get(suboperation["parent"])
            if not parent:
                continue
            parent["suboperations"].append(suboperation)
        operations = [o for o in operations.values() if o["parent"] is None]
    else:
        operations = [o for o in operations.values()]

    if limit:
        operations = operations[:limit]

    operations = list(reversed(sorted(operations, key=lambda o: o["name"])))
    # Reverse the order of log when in cli, more comfortable to read (avoid
    # unecessary scrolling)
    is_api = Moulinette.interface.type == "api"
    if not is_api:
        operations = list(reversed(operations))

    return {"operation": operations}


def log_show(
    path, number=None, share=False, filter_irrelevant=False, with_suboperations=False
):
    """
    Display a log file enriched with metadata if any.

    If the file_name is not an absolute path, it will try to search the file in
    the unit operations log path (see OPERATIONS_PATH).

    Argument:
        file_name
        number
        share
    """

    if share:
        filter_irrelevant = True

    if filter_irrelevant:

{'fraction_non_alphanumeric': 0.0791686805219913, 'fraction_numerical': 0.002126631222812953, 'mean_word_length': 2.7256031688872886, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6301996', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2786, 'n_words': 1036}