<commit_before>/**
 * @file modinfo.c
 * @author <NAME> <<EMAIL>>
 * @brief routines for working with modinfo structure
 *
 * @copyright
 * Copyright (c) 2018 - 2021 Deutsche Telekom AG.
 * Copyright (c) 2018 - 2021 CESNET, z.s.p.o.
 *
 * This source code is licensed under BSD 3-Clause License (the "License").
 * You may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://opensource.org/licenses/BSD-3-Clause
 */

#define _GNU_SOURCE

#include "modinfo.h"

#include <assert.h>
#include <ctype.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

#include <libyang/libyang.h>

#include "common.h"
#include "compat.h"
#include "edit_diff.h"
#include "log.h"
#include "lyd_mods.h"
#include "plugins_datastore.h"
#include "replay.h"
#include "shm_ext.h"
#include "shm_mod.h"
#include "shm_sub.h"
#include "utils/nacm.h"

sr_error_info_t *
sr_modinfo_add(const struct lys_module *ly_mod, const char *xpath, int no_dup_check, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod = NULL;
    uint32_t i;
    void *mem;

    if (!no_dup_check) {
        /* try to find the module */
        for (i = 0; i < mod_info->mod_count; ++i) {
            if (mod_info->mods[i].ly_mod == ly_mod) {
                mod = &mod_info->mods[i];
                break;
            }
        }
    }

    if (!mod) {
        /* add new mod */
        mem = realloc(mod_info->mods, (mod_info->mod_count + 1) * sizeof *mod_info->mods);
        SR_CHECK_MEM_RET(!mem, err_info);
        mod_info->mods = mem;

        mod = &mod_info->mods[mod_info->mod_count];
        memset(mod, 0, sizeof *mod);
        ++mod_info->mod_count;

        mod->ly_mod = ly_mod;
        mod->state |= MOD_INFO_NEW;
    } else if (!mod->xpath_count) {
        /* mod is present with no xpaths (full data tree), nothing to add */
        return NULL;
    }

    if (xpath) {
        for (i = 0; i < mod->xpath_count; ++i) {
            if (!strcmp(mod->xpaths[i], xpath)) {
                /* xpath has already been added */
                return NULL;
            }
        }

        /* add xpath for mod */
        mem = realloc(mod->xpaths, (mod->xpath_count + 1) * sizeof *mod->xpaths);
        SR_CHECK_MEM_RET(!mem, err_info);
        mod->xpaths = mem;

        mod->xpaths[mod->xpath_count] = xpath;
        ++mod->xpath_count;
        mod->state |= MOD_INFO_NEW;
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_add_all_modules_with_data(const struct ly_ctx *ly_ctx, int state_data, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    uint32_t i = 0;

    while ((ly_mod = ly_ctx_get_module_iter(ly_ctx, &i))) {
        if (!ly_mod->implemented) {
            continue;
        } else if (!strcmp(ly_mod->name, "sysrepo")) {
            /* sysrepo module cannot be locked because it is not in SHM with other modules */
            continue;
        } else if (!strcmp(ly_mod->name, "ietf-netconf")) {
            /* ietf-netconf defines data but only internal that should be ignored */
            continue;
        } else if (!sr_module_has_data(ly_mod, state_data)) {
            /* no data */
            continue;
        }

        if ((err_info = sr_modinfo_add(ly_mod, NULL, 1, mod_info))) {
            return err_info;
        }
    }

    return err_info;
}

sr_error_info_t *
sr_modinfo_collect_edit(const struct lyd_node *edit, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    const struct lyd_node *root;

    /* add all the modules from the edit into our array */
    ly_mod = NULL;
    LY_LIST_FOR(edit, root) {
        if (lyd_owner_module(root) == ly_mod) {
            continue;
        } else if (!strcmp(lyd_owner_module(root)->name, "sysrepo")) {
            sr_errinfo_new(&err_info, SR_ERR_UNSUPPORTED, "Data of internal module \"sysrepo\" cannot be modified.");
            return err_info;
        }

        /* remember last mod, good chance it will also be the module of some next data nodes */
        ly_mod = lyd_owner_module(root);

        /* remember the module */
        if ((err_info = sr_modinfo_add(ly_mod, NULL, 0, mod_info))) {
            return err_info;
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_collect_xpath(const struct ly_ctx *ly_ctx, const char *xpath, sr_datastore_t ds, int store_xpath,
        struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *prev_ly_mod, *ly_mod;
    const struct lysc_node *snode;
    struct ly_set *set = NULL;
    uint32_t i;

    /* learn what nodes are needed for evaluation */
    if (lys_find_xpath_atoms(ly_ctx, NULL, xpath, 0, &set)) {
        sr_errinfo_new_ly(&err_info, (struct ly_ctx *)ly_ctx);
        goto cleanup;
    }

    /* add all the modules of the nodes */
    prev_ly_mod = NULL;
    for (i = 0; i < set->count; ++i) {
        snode = set->snodes[i];

        /* skip uninteresting nodes */
        if ((snode->nodetype & (LYS_RPC | LYS_NOTIF)) || ((snode->flags & LYS_CONFIG_R) && SR_IS_CONVENTIONAL_DS(ds))) {
            continue;
        }

        ly_mod = lysc_owner_module(snode);
        if (ly_mod == prev_ly_mod) {
            /* skip already-added modules */
            continue;
        }
        prev_ly_mod = ly_mod;

        if (!ly_mod->implemented || !strcmp(ly_mod->name, "sysrepo") || !strcmp(ly_mod->name, "ietf-netconf")) {
            /* skip import-only modules, the internal sysrepo module, and ietf-netconf (as it has no data, only in libyang) */
            continue;
        }

        if ((err_info = sr_modinfo_add(ly_mod, store_xpath ? xpath : NULL, 0, mod_info))) {
            goto cleanup;
        }
    }

cleanup:
    ly_set_free(set, NULL);
    return err_info;
}

sr_error_info_t *
sr_modinfo_collect_deps(struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        switch (mod->state & MOD_INFO_TYPE_MASK) {
        case MOD_INFO_REQ:
            if (!(mod->state & MOD_INFO_CHANGED)) {
                /* data were not changed so no reason to validate them */
                break;
            }
        /* fallthrough */
        case MOD_INFO_INV_DEP:
            /* this module data will be validated */
            assert(mod->state & MOD_INFO_DATA);
            if ((err_info = sr_shmmod_collect_deps(SR_CONN_MOD_SHM(mod_info->conn),
                    (sr_dep_t *)(mod_info->conn->mod_shm.addr + mod->shm_mod->deps),
                    mod->shm_mod->dep_count, mod_info->conn->ly_ctx, mod_info->data, mod_info))) {
                return err_info;
            }
            break;
        case MOD_INFO_DEP:
            /* this module will not be validated */
            break;
        default:
            SR_CHECK_INT_RET(0, err_info);
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_perm_check(struct sr_mod_info_s *mod_info, int wr, int strict)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;
    int has_access;

    /* it is simply not covered because we would have to also remove the failed permission check module data */
    assert(!mod_info->data || strict);

    i = 0;
    while (i < mod_info->mod_count) {
        mod = &mod_info->mods[i];

        /* check also modules additionally modified by validation */
        if (mod->state & (MOD_INFO_REQ | MOD_INFO_CHANGED)) {
            /* check perm */
            if ((err_info = sr_perm_check(mod_info->conn, mod->ly_mod, mod_info->ds, wr, strict ? NULL : &has_access))) {
                return err_info;
            }

            if (!strict && !has_access) {
                /* remove this module from mod_info by moving all succeding modules */
                SR_LOG_INF("No %s permission for the module \"%s\", skipping.", wr ? "write" : "read", mod->ly_mod->name);
                free(mod->xpaths);
                --mod_info->mod_count;
                if (!mod_info->mod_count) {
                    free(mod_info->mods);
                    mod_info->mods = NULL;
                } else if (i < mod_info->mod_count) {
                    memmove(&mod_info->mods[i], &mod_info->mods[i + 1], (mod_info->mod_count - i) * sizeof *mod);
                }
                continue;
            }
        }

        ++i;
    }

    return NULL;
}

struct sr_mod_info_mod_s *
sr_modinfo_next_mod(struct sr_mod_info_mod_s *last, struct sr_mod_info_s *mod_info, const struct lyd_node *data,
        uint32_t **aux)
{
    struct sr_mod_info_mod_s *mod;
    const struct lyd_node *node;
    uint32_t i;

    if (!last) {
        node = data;

        /* allocate aux array */
        *aux = calloc(mod_info->mod_count, sizeof **aux);
    } else {
        assert(data);

        /* find the last edit node */
        for (node = data; lyd_owner_module(node) != last->ly_mod; node = node->next) {}

next_mod:
        /* skip all edit nodes from this module */
        for ( ; node && (lyd_owner_module(node) == last->ly_mod); node = node->next) {}
    }

    if (node) {
        /* find mod of this edit node */
        mod = NULL;
        for (i = 0; i < mod_info->mod_count; ++i) {
            if (mod_info->mods[i].ly_mod == lyd_owner_module(node)) {
                mod = &mod_info->mods[i];
                break;
            }
        }

        if (!mod) {
            /* possible only for unknown opaque nodes, free the auxiliary array */
            free(*aux);
            *aux = NULL;
        } else if ((*aux)[i]) {
            /* already returned, continue search */
            last = mod;
            goto next_mod;
        } else {
            /* mark this mod as returned */
            (*aux)[i] = 1;
        }
    } else {
        mod = NULL;

        /* free the auxiliary array */
        free(*aux);
        *aux = NULL;
    }

    return mod;
}

sr_error_info_t *
sr_modinfo_edit_apply(struct sr_mod_info_s *mod_info, const struct lyd_node *edit, int create_diff)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    struct sr_mod_info_mod_s *mod = NULL;
    const struct lyd_node *node;
    uint32_t *aux = NULL;
    int change;
    struct sr_lyd_merge_cb_data cb_data;

    assert(!mod_info->data_cached);

    LY_LIST_FOR(edit, node) {
        ly_mod = lyd_owner_module(node);
        if (ly_mod && !strcmp(ly_mod->name, "sysrepo")) {
            sr_errinfo_new(&err_info, SR_ERR_UNSUPPORTED, "Data of internal module \"sysrepo\" cannot be modified.");
            return err_info;
        }
    }

    while ((mod = sr_modinfo_next_mod(mod, mod_info, edit, &aux))) {
        assert(mod->state & MOD_INFO_REQ);

        if (mod_info->ds == SR_DS_OPERATIONAL) {
            /* prepare callback data */
            cb_data.cid = mod_info->conn->cid;
            cb_data.diff = create_diff ? &mod_info->diff : NULL;
            cb_data.changed = 0;
            cb_data.err_info = NULL;

            /* merge edit */
            if (lyd_merge_module(&mod_info->data, edit, mod->ly_mod, sr_lyd_merge_cb, &cb_data, 0)) {
                if (cb_data.err_info) {
                    err_info = cb_data.err_info;
                } else {
                    sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                }
                goto cleanup;
            }

            if (cb_data.changed) {
                /* there is a diff for this module */
                mod->state |= MOD_INFO_CHANGED;
            }
        } else {
            /* apply relevant edit changes */
            if ((err_info = sr_edit_mod_apply(edit, mod->ly_mod, 0, &mod_info->data, create_diff ? &mod_info->diff : NULL,
                    &change))) {
                goto cleanup;
            }

            if (change) {
                /* there is a diff for this module */
                mod->state |= MOD_INFO_CHANGED;
            }
        }
    }

cleanup:
    free(aux);
    return err_info;
}

sr_error_info_t *
sr_modinfo_diff_merge(struct sr_mod_info_s *mod_info, const struct lyd_node *new_diff)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_REQ) {
            /* merge relevant diff part */
            if (lyd_diff_merge_module(&mod_info->diff, new_diff, mod->ly_mod, NULL, NULL, 0)) {
                sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                return err_info;
            }
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_replace(struct sr_mod_info_s *mod_info, struct lyd_node **src_data)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    struct lyd_node *src_mod_data, *dst_mod_data, *diff;
    uint32_t i;

    assert(!mod_info->diff && !mod_info->data_cached);

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_REQ) {
            dst_mod_data = sr_module_data_unlink(&mod_info->data, mod->ly_mod);
            src_mod_data = sr_module_data_unlink(src_data, mod->ly_mod);

            /* get diff on only this module's data */
            if (lyd_diff_siblings(dst_mod_data, src_mod_data, LYD_DIFF_DEFAULTS, &diff)) {
                sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                lyd_free_all(dst_mod_data);
                lyd_free_all(src_mod_data);
                return err_info;
            }

            if (diff) {
                /* there is a diff */
                mod->state |= MOD_INFO_CHANGED;

                /* merge the diff */
                lyd_insert_sibling(mod_info->diff, diff, &mod_info->diff);

                /* update data */
                if (src_mod_data) {
                    lyd_insert_sibling(mod_info->data, src_mod_data, &mod_info->data);
                }
                lyd_free_all(dst_mod_data);
            } else {
                /* keep old data (for validation) */
                if (dst_mod_data) {
                    lyd_insert_sibling(mod_info->data, dst_mod_data, &mod_info->data);
                }
                lyd_free_all(src_mod_data);
            }
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_changesub_rdlock(struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i, j;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ LOCK */
            if ((err_info = sr_rwlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT,
                    SR_LOCK_READ, mod_info->conn->cid, __func__, NULL, NULL))) {
                goto error;
            }
        }
    }

    return NULL;

error:
    for (j = 0; j < i; ++j) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ UNLOCK */
            sr_rwunlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ,
                    mod_info->conn->cid, __func__);
        }
    }
    return err_info;
}

void
sr_modinfo_changesub_rdunlock(struct sr_mod_info_s *mod_info)
{
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ UNLOCK */
            sr_rwunlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ,
                    mod_info->conn->cid, __func__);
        }
    }
}

/**
 * @brief Check whether operational data are required based on single request and subscription atom.
 *
 * @param[in] request_atom Request text atom.
 * @param[in] sub_atom Subscription text atom.
 * @return 0 data are not required based on the atoms;
 * @return 1 data are required;
 * @return 2 data are not required (would be filtered out).
 */
static int
sr_xpath_oper_data_text_atoms_required(const char *request_atom, const char *sub_atom)
{
    const char *req_ptr, *sub_ptr, *mod1, *name1, *val1, *mod2, *name2, *val2;
    int mlen1, mlen2, len1, len2, wildc1, wildc2;

    req_ptr = request_atom;
    sub_ptr = sub_atom;

    do {
        /* parse nodes */
        req_ptr = sr_xpath_next_qname(req_ptr + 1, &mod1, &mlen1, &name1, &len1);
        sub_ptr = sr_xpath_next_qname(sub_ptr + 1, &mod2, &mlen2, &name2, &len2);

        /* wildcards */
        if ((len1 == 1) && (name1[0] == '*')) {
            wildc1 = 1;
        } else {
            wildc1 = 0;
        }
        if ((len2 == 1) && (name2[0] == '*')) {
            wildc2 = 1;
        } else {
            wildc2 = 0;
        }

        /* module name */
        if ((mlen1 && mlen2) && ((mlen1 != mlen2) || strncmp(mod1, mod2, mlen1))) {
            /* different modules */
            return 0;
        }

        /* node name */
        if (!wildc1 && !wildc2 && ((len1 != len2) || strncmp(name1, name2, len1))) {
            /* different node names */
            return 0;
        }

        /* value */
        if ((req_ptr[0] == '[') && (sub_ptr[0] == '[')) {
            val1 = req_ptr + 4;
            len1 = strchr(val1, req_ptr[3]) - val1;
            val2 = sub_ptr + 4;
            len2 = strchr(val2, sub_ptr[3]) - val2;
            if ((len1 != len2) || strncmp(val1, val2, len1)) {
                /* different values, filtered out */
                return 2;
            }
        }

        /* parse until the subscription path ends */
    } while (req_ptr[0] && sub_ptr[0]);

    /* atom match */
    return 1;
}

/**
 * @brief Check whether operational data are required.
 *
 * @param[in] request_xpath Get request full XPath.
 * @param[in] sub_xpath Operational subscription XPath.
 * @param[out] required Whether the oper data are required or not.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_xpath_oper_data_required(const char *request_xpath, const char *sub_xpath, int *required)
{
    sr_error_info_t *err_info = NULL;
    sr_xp_atoms_t *req_atoms = NULL, *sub_atoms = NULL;
    uint32_t i, j, k;
    int r;

    assert(sub_xpath);

    *required = 1;

    if (!request_xpath) {
        /* we do not know, say it is required */
        goto cleanup;
    }

    /* get text atoms for both xpaths */
    if ((err_info = sr_xpath_get_text_atoms(request_xpath, &req_atoms)) || !req_atoms) {
        goto cleanup;
    }
    if ((err_info = sr_xpath_get_text_atoms(sub_xpath, &sub_atoms)) || !sub_atoms) {
        goto cleanup;
    }
    assert(sub_atoms->union_count == 1);

    /* check whether any atoms match */
    *required = 0;
    for (i = 0; i < req_atoms->union_count; ++i) {
        for (j = 0; j < req_atoms->unions[i].atom_count; ++j) {
            for (k = 0; k < sub_atoms->unions[0].atom_count; ++k) {
                r = sr_xpath_oper_data_text_atoms_required(req_atoms->unions[i].atoms[j], sub_atoms->unions[0].atoms[k]);
                if (r == 1) {
                    /* required but need to check all atoms */
                    *required = 1;
                } else if (r == 2) {
                    /* not required for the union */
                    *required = 0;
                    break;
                }
            }
            if (k < sub_atoms->unions[0].atom_count) {
                break;
            }
        }

        if (*required) {
            /* required for a union */
            goto cleanup;
        }
    }

cleanup:
    sr_xpath_atoms_free(req_atoms);
    sr_xpath_atoms_free(sub_atoms);
    return err_info;
}

/**
 * @brief Get specific operational data from a subscriber.
 *
 * @param[in] ly_mod libyang module of the data.
 * @param[in] xpath XPath of the provided data.
 * @param[in] request_xpaths XPaths based on which these data are required, if NULL the complete module data are needed.
 * @param[in] req_xpath_count Count of @p request_xpaths.
 * @param[in] orig_name Event originator name.
 * @param[in] orig_data Event originator data.
 * @param[in] evpipe_num Subscriber event pipe number.
 * @param[in] parent Data parent required for the subscription, NULL if top-level.
 * @param[in] timeout_ms Operational callback timeout in milliseconds.
 * @param[in] cid Connection ID.
 * @param[out] data Data tree with appended operational data.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_xpath_oper_data_get(const struct lys_module *ly_mod, const char *xpath, const char **request_xpaths,
        uint32_t req_xpath_count, const char *orig_name, const void *orig_data, uint32_t evpipe_num,
        const struct lyd_node *parent, uint32_t timeout_ms, sr_cid_t cid, struct lyd_node **oper_data)
{
    sr_error_info_t *err_info = NULL, *cb_err_info = NULL;
    struct lyd_node *parent_dup = NULL, *last_parent;
    const char *request_xpath;
    char *parent_path = NULL;
    uint32_t i;
    int required;

    *oper_data = NULL;

    if (parent) {
        /* duplicate parent so that it is a stand-alone subtree */
        if (lyd_dup_single(parent, NULL, LYD_DUP_WITH_PARENTS, &last_parent)) {
            sr_errinfo_new_ly(&err_info, ly_mod->ctx);
            return err_info;
        }

        /* go top-level */
        for (parent_dup = last_parent; parent_dup->parent; parent_dup = lyd_parent(parent_dup)) {}

        if (req_xpath_count) {
            /* check whether the parent would not be filtered out */
            parent_path = lyd_path(last_parent, LYD_PATH_STD, NULL, 0);
            SR_CHECK_MEM_GOTO(!parent_path, err_info, cleanup);

            for (i = 0; i < req_xpath_count; ++i) {
                if ((err_info = sr_xpath_oper_data_required(request_xpaths[i], parent_path, &required))) {
                    goto cleanup;
                }
                if (required) {
                    break;
                }
            }
            if (i == req_xpath_count) {
                goto cleanup;
            }
        }
    }

    /* provide request XPath for the client, if possible */
    request_xpath = (req_xpath_count == 1) ? request_xpaths[0] : NULL;

    /* get data from client */
    if ((err_info = sr_shmsub_oper_notify(ly_mod, xpath, request_xpath, parent_dup, orig_name, orig_data, evpipe_num,
            timeout_ms, cid, oper_data, &cb_err_info))) {
        goto cleanup;
    }

    /* return callback error if some was generated */
    if (cb_err_info) {
        sr_errinfo_merge(&err_info, cb_err_info);
        sr_errinfo_new(&err_info, SR_ERR_CALLBACK_FAILED, "User callback failed.");
        goto cleanup;
    }

    if (*oper_data) {
        /* add any missing NP containers, redundant to add top-level containers */
        if (lyd_new_implicit_tree(*oper_data, LYD_IMPLICIT_NO_DEFAULTS, NULL)) {
            sr_errinfo_new_ly(&err_info, ly_mod->ctx);
            goto cleanup;
        }
    }

cleanup:
    lyd_free_tree(parent_dup);
    free(parent_path);
    return err_info;
}

/**
 * @brief Update (replace or append) operational data for a specific module.
 *
 * @param[in] mod Mod info module to process.
 * @param[in] oper_mode Current lock mode of @p mod for ::SR_DS_OPERATIONAL.
 * @param[in] orig_name Event originator name.
 * @param[in] orig_data Event originator data.
 * @param[in] conn Connection to use.
 * @param[in] timeout_ms Operational callback timeout in milliseconds.
 * @param[in] opts Get oper data options.
 * @param[in,out] data Operational data tree.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_module_oper_data_update(struct sr_mod_info_mod_s *mod, const char *orig_name, const void *orig_data, sr_conn_ctx_t *conn,
        uint32_t timeout_ms, sr_get_oper_options_t opts, struct lyd_node **data)
{
    sr_error_info_t *err_info = NULL;
    sr_mod_oper_sub_t *shm_sub;
    const char *sub_xpath, **request_xpaths = NULL;
    char *parent_xpath = NULL;
    uint32_t i, j, req_xpath_count = 0;
    int required;
    struct ly_set *set = NULL;
    struct lyd_node *edit = NULL, *oper_data;

    if (!(opts & SR_OPER_NO_STORED)) {
        /* apply stored operational edit */
        if ((err_info = sr_module_file_oper_data_load(mod, &edit))) {
            return err_info;
        }
        err_info = sr_edit_mod_apply(edit, mod->ly_mod, 1, data, NULL, NULL);
        lyd_free_all(edit);
        if (err_info) {
            return err_info;
        }
    }

    if (opts & SR_OPER_NO_SUBS) {
        /* do not get data from subscribers */
        return NULL;
    }

    assert(timeout_ms);

    /* OPER SUB READ LOCK */
    if ((err_info = sr_rwlock(&mod->shm_mod->oper_lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ, conn->cid, __func__,
            NULL, NULL))) {
        return err_info;<commit_msg>modinfo BUGFIX add NP containers to push oper data<commit_after>/**
 * @file modinfo.c
 * @author <NAME> <<EMAIL>>
 * @brief routines for working with modinfo structure
 *
 * @copyright
 * Copyright (c) 2018 - 2021 Deutsche Telekom AG.
 * Copyright (c) 2018 - 2021 CESNET, z.s.p.o.
 *
 * This source code is licensed under BSD 3-Clause License (the "License").
 * You may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://opensource.org/licenses/BSD-3-Clause
 */

#define _GNU_SOURCE

#include "modinfo.h"

#include <assert.h>
#include <ctype.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

#include <libyang/libyang.h>

#include "common.h"
#include "compat.h"
#include "edit_diff.h"
#include "log.h"
#include "lyd_mods.h"
#include "plugins_datastore.h"
#include "replay.h"
#include "shm_ext.h"
#include "shm_mod.h"
#include "shm_sub.h"
#include "utils/nacm.h"

sr_error_info_t *
sr_modinfo_add(const struct lys_module *ly_mod, const char *xpath, int no_dup_check, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod = NULL;
    uint32_t i;
    void *mem;

    if (!no_dup_check) {
        /* try to find the module */
        for (i = 0; i < mod_info->mod_count; ++i) {
            if (mod_info->mods[i].ly_mod == ly_mod) {
                mod = &mod_info->mods[i];
                break;
            }
        }
    }

    if (!mod) {
        /* add new mod */
        mem = realloc(mod_info->mods, (mod_info->mod_count + 1) * sizeof *mod_info->mods);
        SR_CHECK_MEM_RET(!mem, err_info);
        mod_info->mods = mem;

        mod = &mod_info->mods[mod_info->mod_count];
        memset(mod, 0, sizeof *mod);
        ++mod_info->mod_count;

        mod->ly_mod = ly_mod;
        mod->state |= MOD_INFO_NEW;
    } else if (!mod->xpath_count) {
        /* mod is present with no xpaths (full data tree), nothing to add */
        return NULL;
    }

    if (xpath) {
        for (i = 0; i < mod->xpath_count; ++i) {
            if (!strcmp(mod->xpaths[i], xpath)) {
                /* xpath has already been added */
                return NULL;
            }
        }

        /* add xpath for mod */
        mem = realloc(mod->xpaths, (mod->xpath_count + 1) * sizeof *mod->xpaths);
        SR_CHECK_MEM_RET(!mem, err_info);
        mod->xpaths = mem;

        mod->xpaths[mod->xpath_count] = xpath;
        ++mod->xpath_count;
        mod->state |= MOD_INFO_NEW;
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_add_all_modules_with_data(const struct ly_ctx *ly_ctx, int state_data, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    uint32_t i = 0;

    while ((ly_mod = ly_ctx_get_module_iter(ly_ctx, &i))) {
        if (!ly_mod->implemented) {
            continue;
        } else if (!strcmp(ly_mod->name, "sysrepo")) {
            /* sysrepo module cannot be locked because it is not in SHM with other modules */
            continue;
        } else if (!strcmp(ly_mod->name, "ietf-netconf")) {
            /* ietf-netconf defines data but only internal that should be ignored */
            continue;
        } else if (!sr_module_has_data(ly_mod, state_data)) {
            /* no data */
            continue;
        }

        if ((err_info = sr_modinfo_add(ly_mod, NULL, 1, mod_info))) {
            return err_info;
        }
    }

    return err_info;
}

sr_error_info_t *
sr_modinfo_collect_edit(const struct lyd_node *edit, struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    const struct lyd_node *root;

    /* add all the modules from the edit into our array */
    ly_mod = NULL;
    LY_LIST_FOR(edit, root) {
        if (lyd_owner_module(root) == ly_mod) {
            continue;
        } else if (!strcmp(lyd_owner_module(root)->name, "sysrepo")) {
            sr_errinfo_new(&err_info, SR_ERR_UNSUPPORTED, "Data of internal module \"sysrepo\" cannot be modified.");
            return err_info;
        }

        /* remember last mod, good chance it will also be the module of some next data nodes */
        ly_mod = lyd_owner_module(root);

        /* remember the module */
        if ((err_info = sr_modinfo_add(ly_mod, NULL, 0, mod_info))) {
            return err_info;
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_collect_xpath(const struct ly_ctx *ly_ctx, const char *xpath, sr_datastore_t ds, int store_xpath,
        struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *prev_ly_mod, *ly_mod;
    const struct lysc_node *snode;
    struct ly_set *set = NULL;
    uint32_t i;

    /* learn what nodes are needed for evaluation */
    if (lys_find_xpath_atoms(ly_ctx, NULL, xpath, 0, &set)) {
        sr_errinfo_new_ly(&err_info, (struct ly_ctx *)ly_ctx);
        goto cleanup;
    }

    /* add all the modules of the nodes */
    prev_ly_mod = NULL;
    for (i = 0; i < set->count; ++i) {
        snode = set->snodes[i];

        /* skip uninteresting nodes */
        if ((snode->nodetype & (LYS_RPC | LYS_NOTIF)) || ((snode->flags & LYS_CONFIG_R) && SR_IS_CONVENTIONAL_DS(ds))) {
            continue;
        }

        ly_mod = lysc_owner_module(snode);
        if (ly_mod == prev_ly_mod) {
            /* skip already-added modules */
            continue;
        }
        prev_ly_mod = ly_mod;

        if (!ly_mod->implemented || !strcmp(ly_mod->name, "sysrepo") || !strcmp(ly_mod->name, "ietf-netconf")) {
            /* skip import-only modules, the internal sysrepo module, and ietf-netconf (as it has no data, only in libyang) */
            continue;
        }

        if ((err_info = sr_modinfo_add(ly_mod, store_xpath ? xpath : NULL, 0, mod_info))) {
            goto cleanup;
        }
    }

cleanup:
    ly_set_free(set, NULL);
    return err_info;
}

sr_error_info_t *
sr_modinfo_collect_deps(struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        switch (mod->state & MOD_INFO_TYPE_MASK) {
        case MOD_INFO_REQ:
            if (!(mod->state & MOD_INFO_CHANGED)) {
                /* data were not changed so no reason to validate them */
                break;
            }
        /* fallthrough */
        case MOD_INFO_INV_DEP:
            /* this module data will be validated */
            assert(mod->state & MOD_INFO_DATA);
            if ((err_info = sr_shmmod_collect_deps(SR_CONN_MOD_SHM(mod_info->conn),
                    (sr_dep_t *)(mod_info->conn->mod_shm.addr + mod->shm_mod->deps),
                    mod->shm_mod->dep_count, mod_info->conn->ly_ctx, mod_info->data, mod_info))) {
                return err_info;
            }
            break;
        case MOD_INFO_DEP:
            /* this module will not be validated */
            break;
        default:
            SR_CHECK_INT_RET(0, err_info);
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_perm_check(struct sr_mod_info_s *mod_info, int wr, int strict)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;
    int has_access;

    /* it is simply not covered because we would have to also remove the failed permission check module data */
    assert(!mod_info->data || strict);

    i = 0;
    while (i < mod_info->mod_count) {
        mod = &mod_info->mods[i];

        /* check also modules additionally modified by validation */
        if (mod->state & (MOD_INFO_REQ | MOD_INFO_CHANGED)) {
            /* check perm */
            if ((err_info = sr_perm_check(mod_info->conn, mod->ly_mod, mod_info->ds, wr, strict ? NULL : &has_access))) {
                return err_info;
            }

            if (!strict && !has_access) {
                /* remove this module from mod_info by moving all succeding modules */
                SR_LOG_INF("No %s permission for the module \"%s\", skipping.", wr ? "write" : "read", mod->ly_mod->name);
                free(mod->xpaths);
                --mod_info->mod_count;
                if (!mod_info->mod_count) {
                    free(mod_info->mods);
                    mod_info->mods = NULL;
                } else if (i < mod_info->mod_count) {
                    memmove(&mod_info->mods[i], &mod_info->mods[i + 1], (mod_info->mod_count - i) * sizeof *mod);
                }
                continue;
            }
        }

        ++i;
    }

    return NULL;
}

struct sr_mod_info_mod_s *
sr_modinfo_next_mod(struct sr_mod_info_mod_s *last, struct sr_mod_info_s *mod_info, const struct lyd_node *data,
        uint32_t **aux)
{
    struct sr_mod_info_mod_s *mod;
    const struct lyd_node *node;
    uint32_t i;

    if (!last) {
        node = data;

        /* allocate aux array */
        *aux = calloc(mod_info->mod_count, sizeof **aux);
    } else {
        assert(data);

        /* find the last edit node */
        for (node = data; lyd_owner_module(node) != last->ly_mod; node = node->next) {}

next_mod:
        /* skip all edit nodes from this module */
        for ( ; node && (lyd_owner_module(node) == last->ly_mod); node = node->next) {}
    }

    if (node) {
        /* find mod of this edit node */
        mod = NULL;
        for (i = 0; i < mod_info->mod_count; ++i) {
            if (mod_info->mods[i].ly_mod == lyd_owner_module(node)) {
                mod = &mod_info->mods[i];
                break;
            }
        }

        if (!mod) {
            /* possible only for unknown opaque nodes, free the auxiliary array */
            free(*aux);
            *aux = NULL;
        } else if ((*aux)[i]) {
            /* already returned, continue search */
            last = mod;
            goto next_mod;
        } else {
            /* mark this mod as returned */
            (*aux)[i] = 1;
        }
    } else {
        mod = NULL;

        /* free the auxiliary array */
        free(*aux);
        *aux = NULL;
    }

    return mod;
}

sr_error_info_t *
sr_modinfo_edit_apply(struct sr_mod_info_s *mod_info, const struct lyd_node *edit, int create_diff)
{
    sr_error_info_t *err_info = NULL;
    const struct lys_module *ly_mod;
    struct sr_mod_info_mod_s *mod = NULL;
    const struct lyd_node *node;
    uint32_t *aux = NULL;
    int change;
    struct sr_lyd_merge_cb_data cb_data;

    assert(!mod_info->data_cached);

    LY_LIST_FOR(edit, node) {
        ly_mod = lyd_owner_module(node);
        if (ly_mod && !strcmp(ly_mod->name, "sysrepo")) {
            sr_errinfo_new(&err_info, SR_ERR_UNSUPPORTED, "Data of internal module \"sysrepo\" cannot be modified.");
            return err_info;
        }
    }

    while ((mod = sr_modinfo_next_mod(mod, mod_info, edit, &aux))) {
        assert(mod->state & MOD_INFO_REQ);

        if (mod_info->ds == SR_DS_OPERATIONAL) {
            /* prepare callback data */
            cb_data.cid = mod_info->conn->cid;
            cb_data.diff = create_diff ? &mod_info->diff : NULL;
            cb_data.changed = 0;
            cb_data.err_info = NULL;

            /* merge edit */
            if (lyd_merge_module(&mod_info->data, edit, mod->ly_mod, sr_lyd_merge_cb, &cb_data, 0)) {
                if (cb_data.err_info) {
                    err_info = cb_data.err_info;
                } else {
                    sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                }
                goto cleanup;
            }

            if (cb_data.changed) {
                /* there is a diff for this module */
                mod->state |= MOD_INFO_CHANGED;
            }
        } else {
            /* apply relevant edit changes */
            if ((err_info = sr_edit_mod_apply(edit, mod->ly_mod, 0, &mod_info->data, create_diff ? &mod_info->diff : NULL,
                    &change))) {
                goto cleanup;
            }

            if (change) {
                /* there is a diff for this module */
                mod->state |= MOD_INFO_CHANGED;
            }
        }
    }

cleanup:
    free(aux);
    return err_info;
}

sr_error_info_t *
sr_modinfo_diff_merge(struct sr_mod_info_s *mod_info, const struct lyd_node *new_diff)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_REQ) {
            /* merge relevant diff part */
            if (lyd_diff_merge_module(&mod_info->diff, new_diff, mod->ly_mod, NULL, NULL, 0)) {
                sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                return err_info;
            }
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_replace(struct sr_mod_info_s *mod_info, struct lyd_node **src_data)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    struct lyd_node *src_mod_data, *dst_mod_data, *diff;
    uint32_t i;

    assert(!mod_info->diff && !mod_info->data_cached);

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_REQ) {
            dst_mod_data = sr_module_data_unlink(&mod_info->data, mod->ly_mod);
            src_mod_data = sr_module_data_unlink(src_data, mod->ly_mod);

            /* get diff on only this module's data */
            if (lyd_diff_siblings(dst_mod_data, src_mod_data, LYD_DIFF_DEFAULTS, &diff)) {
                sr_errinfo_new_ly(&err_info, mod_info->conn->ly_ctx);
                lyd_free_all(dst_mod_data);
                lyd_free_all(src_mod_data);
                return err_info;
            }

            if (diff) {
                /* there is a diff */
                mod->state |= MOD_INFO_CHANGED;

                /* merge the diff */
                lyd_insert_sibling(mod_info->diff, diff, &mod_info->diff);

                /* update data */
                if (src_mod_data) {
                    lyd_insert_sibling(mod_info->data, src_mod_data, &mod_info->data);
                }
                lyd_free_all(dst_mod_data);
            } else {
                /* keep old data (for validation) */
                if (dst_mod_data) {
                    lyd_insert_sibling(mod_info->data, dst_mod_data, &mod_info->data);
                }
                lyd_free_all(src_mod_data);
            }
        }
    }

    return NULL;
}

sr_error_info_t *
sr_modinfo_changesub_rdlock(struct sr_mod_info_s *mod_info)
{
    sr_error_info_t *err_info = NULL;
    struct sr_mod_info_mod_s *mod;
    uint32_t i, j;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ LOCK */
            if ((err_info = sr_rwlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT,
                    SR_LOCK_READ, mod_info->conn->cid, __func__, NULL, NULL))) {
                goto error;
            }
        }
    }

    return NULL;

error:
    for (j = 0; j < i; ++j) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ UNLOCK */
            sr_rwunlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ,
                    mod_info->conn->cid, __func__);
        }
    }
    return err_info;
}

void
sr_modinfo_changesub_rdunlock(struct sr_mod_info_s *mod_info)
{
    struct sr_mod_info_mod_s *mod;
    uint32_t i;

    for (i = 0; i < mod_info->mod_count; ++i) {
        mod = &mod_info->mods[i];
        if (mod->state & MOD_INFO_CHANGED) {
            /* CHANGE SUB READ UNLOCK */
            sr_rwunlock(&mod->shm_mod->change_sub[mod_info->ds].lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ,
                    mod_info->conn->cid, __func__);
        }
    }
}

/**
 * @brief Check whether operational data are required based on single request and subscription atom.
 *
 * @param[in] request_atom Request text atom.
 * @param[in] sub_atom Subscription text atom.
 * @return 0 data are not required based on the atoms;
 * @return 1 data are required;
 * @return 2 data are not required (would be filtered out).
 */
static int
sr_xpath_oper_data_text_atoms_required(const char *request_atom, const char *sub_atom)
{
    const char *req_ptr, *sub_ptr, *mod1, *name1, *val1, *mod2, *name2, *val2;
    int mlen1, mlen2, len1, len2, wildc1, wildc2;

    req_ptr = request_atom;
    sub_ptr = sub_atom;

    do {
        /* parse nodes */
        req_ptr = sr_xpath_next_qname(req_ptr + 1, &mod1, &mlen1, &name1, &len1);
        sub_ptr = sr_xpath_next_qname(sub_ptr + 1, &mod2, &mlen2, &name2, &len2);

        /* wildcards */
        if ((len1 == 1) && (name1[0] == '*')) {
            wildc1 = 1;
        } else {
            wildc1 = 0;
        }
        if ((len2 == 1) && (name2[0] == '*')) {
            wildc2 = 1;
        } else {
            wildc2 = 0;
        }

        /* module name */
        if ((mlen1 && mlen2) && ((mlen1 != mlen2) || strncmp(mod1, mod2, mlen1))) {
            /* different modules */
            return 0;
        }

        /* node name */
        if (!wildc1 && !wildc2 && ((len1 != len2) || strncmp(name1, name2, len1))) {
            /* different node names */
            return 0;
        }

        /* value */
        if ((req_ptr[0] == '[') && (sub_ptr[0] == '[')) {
            val1 = req_ptr + 4;
            len1 = strchr(val1, req_ptr[3]) - val1;
            val2 = sub_ptr + 4;
            len2 = strchr(val2, sub_ptr[3]) - val2;
            if ((len1 != len2) || strncmp(val1, val2, len1)) {
                /* different values, filtered out */
                return 2;
            }
        }

        /* parse until the subscription path ends */
    } while (req_ptr[0] && sub_ptr[0]);

    /* atom match */
    return 1;
}

/**
 * @brief Check whether operational data are required.
 *
 * @param[in] request_xpath Get request full XPath.
 * @param[in] sub_xpath Operational subscription XPath.
 * @param[out] required Whether the oper data are required or not.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_xpath_oper_data_required(const char *request_xpath, const char *sub_xpath, int *required)
{
    sr_error_info_t *err_info = NULL;
    sr_xp_atoms_t *req_atoms = NULL, *sub_atoms = NULL;
    uint32_t i, j, k;
    int r;

    assert(sub_xpath);

    *required = 1;

    if (!request_xpath) {
        /* we do not know, say it is required */
        goto cleanup;
    }

    /* get text atoms for both xpaths */
    if ((err_info = sr_xpath_get_text_atoms(request_xpath, &req_atoms)) || !req_atoms) {
        goto cleanup;
    }
    if ((err_info = sr_xpath_get_text_atoms(sub_xpath, &sub_atoms)) || !sub_atoms) {
        goto cleanup;
    }
    assert(sub_atoms->union_count == 1);

    /* check whether any atoms match */
    *required = 0;
    for (i = 0; i < req_atoms->union_count; ++i) {
        for (j = 0; j < req_atoms->unions[i].atom_count; ++j) {
            for (k = 0; k < sub_atoms->unions[0].atom_count; ++k) {
                r = sr_xpath_oper_data_text_atoms_required(req_atoms->unions[i].atoms[j], sub_atoms->unions[0].atoms[k]);
                if (r == 1) {
                    /* required but need to check all atoms */
                    *required = 1;
                } else if (r == 2) {
                    /* not required for the union */
                    *required = 0;
                    break;
                }
            }
            if (k < sub_atoms->unions[0].atom_count) {
                break;
            }
        }

        if (*required) {
            /* required for a union */
            goto cleanup;
        }
    }

cleanup:
    sr_xpath_atoms_free(req_atoms);
    sr_xpath_atoms_free(sub_atoms);
    return err_info;
}

/**
 * @brief Get specific operational data from a subscriber.
 *
 * @param[in] ly_mod libyang module of the data.
 * @param[in] xpath XPath of the provided data.
 * @param[in] request_xpaths XPaths based on which these data are required, if NULL the complete module data are needed.
 * @param[in] req_xpath_count Count of @p request_xpaths.
 * @param[in] orig_name Event originator name.
 * @param[in] orig_data Event originator data.
 * @param[in] evpipe_num Subscriber event pipe number.
 * @param[in] parent Data parent required for the subscription, NULL if top-level.
 * @param[in] timeout_ms Operational callback timeout in milliseconds.
 * @param[in] cid Connection ID.
 * @param[out] data Data tree with appended operational data.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_xpath_oper_data_get(const struct lys_module *ly_mod, const char *xpath, const char **request_xpaths,
        uint32_t req_xpath_count, const char *orig_name, const void *orig_data, uint32_t evpipe_num,
        const struct lyd_node *parent, uint32_t timeout_ms, sr_cid_t cid, struct lyd_node **oper_data)
{
    sr_error_info_t *err_info = NULL, *cb_err_info = NULL;
    struct lyd_node *parent_dup = NULL, *last_parent;
    const char *request_xpath;
    char *parent_path = NULL;
    uint32_t i;
    int required;

    *oper_data = NULL;

    if (parent) {
        /* duplicate parent so that it is a stand-alone subtree */
        if (lyd_dup_single(parent, NULL, LYD_DUP_WITH_PARENTS, &last_parent)) {
            sr_errinfo_new_ly(&err_info, ly_mod->ctx);
            return err_info;
        }

        /* go top-level */
        for (parent_dup = last_parent; parent_dup->parent; parent_dup = lyd_parent(parent_dup)) {}

        if (req_xpath_count) {
            /* check whether the parent would not be filtered out */
            parent_path = lyd_path(last_parent, LYD_PATH_STD, NULL, 0);
            SR_CHECK_MEM_GOTO(!parent_path, err_info, cleanup);

            for (i = 0; i < req_xpath_count; ++i) {
                if ((err_info = sr_xpath_oper_data_required(request_xpaths[i], parent_path, &required))) {
                    goto cleanup;
                }
                if (required) {
                    break;
                }
            }
            if (i == req_xpath_count) {
                goto cleanup;
            }
        }
    }

    /* provide request XPath for the client, if possible */
    request_xpath = (req_xpath_count == 1) ? request_xpaths[0] : NULL;

    /* get data from client */
    if ((err_info = sr_shmsub_oper_notify(ly_mod, xpath, request_xpath, parent_dup, orig_name, orig_data, evpipe_num,
            timeout_ms, cid, oper_data, &cb_err_info))) {
        goto cleanup;
    }

    /* return callback error if some was generated */
    if (cb_err_info) {
        sr_errinfo_merge(&err_info, cb_err_info);
        sr_errinfo_new(&err_info, SR_ERR_CALLBACK_FAILED, "User callback failed.");
        goto cleanup;
    }

    if (*oper_data) {
        /* add any missing NP containers, redundant to add top-level containers */
        if (lyd_new_implicit_tree(*oper_data, LYD_IMPLICIT_NO_DEFAULTS, NULL)) {
            sr_errinfo_new_ly(&err_info, ly_mod->ctx);
            goto cleanup;
        }
    }

cleanup:
    lyd_free_tree(parent_dup);
    free(parent_path);
    return err_info;
}

/**
 * @brief Update (replace or append) operational data for a specific module.
 *
 * @param[in] mod Mod info module to process.
 * @param[in] oper_mode Current lock mode of @p mod for ::SR_DS_OPERATIONAL.
 * @param[in] orig_name Event originator name.
 * @param[in] orig_data Event originator data.
 * @param[in] conn Connection to use.
 * @param[in] timeout_ms Operational callback timeout in milliseconds.
 * @param[in] opts Get oper data options.
 * @param[in,out] data Operational data tree.
 * @return err_info, NULL on success.
 */
static sr_error_info_t *
sr_module_oper_data_update(struct sr_mod_info_mod_s *mod, const char *orig_name, const void *orig_data, sr_conn_ctx_t *conn,
        uint32_t timeout_ms, sr_get_oper_options_t opts, struct lyd_node **data)
{
    sr_error_info_t *err_info = NULL;
    sr_mod_oper_sub_t *shm_sub;
    const char *sub_xpath, **request_xpaths = NULL;
    char *parent_xpath = NULL;
    uint32_t i, j, req_xpath_count = 0;
    int required;
    struct ly_set *set = NULL;
    struct lyd_node *edit = NULL, *oper_data;

    if (!(opts & SR_OPER_NO_STORED)) {
        /* get stored operational edit */
        if ((err_info = sr_module_file_oper_data_load(mod, &edit))) {
            return err_info;
        }
    }
    if (edit) {
        /* apply the edit */
        err_info = sr_edit_mod_apply(edit, mod->ly_mod, 1, data, NULL, NULL);
        lyd_free_all(edit);
        if (err_info) {
            return err_info;
        }

        /* add any missing NP containers in the data */
        if (lyd_new_implicit_module(data, mod->ly_mod, LYD_IMPLICIT_NO_DEFAULTS, NULL)) {
            sr_errinfo_new_ly(&err_info, mod->ly_mod->ctx);
            return err_info;
        }
    }

    if (opts & SR_OPER_NO_SUBS) {
        /* do not get data from subscribers */
        return NULL;
    }

    assert(timeout_ms);

    /* OPER SUB READ LOCK */
    if ((err_info = sr_rwlock(&mod->shm_mod->oper_lock, SR_SHMEXT_SUB_LOCK_TIMEOUT, SR_LOCK_READ, conn->cid, __func__,
            NULL, NULL))) {
        return err_info;
{'fraction_non_alphanumeric': 0.1168551320575676, 'fraction_numerical': 0.007156413095049818, 'mean_word_length': 2.117335305355272, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 338, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169380', 'n_tokens_mistral': 17782, 'n_tokens_neox': 17050, 'n_words': 4091}