<commit_before>/**
 * @file attrmap.cpp
 * @brief Class for manipulating node attributes
 *
 * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
 *
 * This file is part of the MEGA SDK - Client Access Engine.
 *
 * Applications using the MEGA API must present a valid application key
 * and comply with the the rules set forth in the Terms of Service.
 *
 * The MEGA SDK is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * @copyright Simplified (2-clause) BSD License.
 *
 * You should have received a copy of the license along with this
 * program.
 */

#include "mega/attrmap.h"

namespace mega {
// approximate raw storage size of serialized AttrMap, not taking JSON escaping
// or name length into account
unsigned AttrMap::storagesize(int perrecord) const
{
    assert(perrecord >= 0);
    unsigned t = 0;

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        t += static_cast<unsigned>(perrecord + it->second.size());
    }

    return t;
}

int AttrMap::nameid2string(nameid id, char* buf)
{
    char* ptr = buf;

    for (int i = 64; (i -= 8) >= 0;)
    {
        if ((*ptr = static_cast<char>( (id >> i) & 0xff)))
        {
            ptr++;
        }
    }

    return static_cast<int>(ptr - buf);
}

string AttrMap::nameid2string(nameid id)
{
    string s;
    s.resize(10);
    s.resize(nameid2string(id, const_cast<char*>(s.data())));
    return s;
}


nameid AttrMap::string2nameid(const char *a)
{
    if (!a)
    {
        return 0;
    }

    size_t len = strlen(a);
    if (len > 8)
    {
        return 0;
    }

    switch (len)
    {
        case 1:
            return *a;
        case 2:
            return MAKENAMEID2(a[0], a[1]);
        case 3:
            return MAKENAMEID3(a[0], a[1], a[2]);
        case 4:
            return MAKENAMEID4(a[0], a[1], a[2], a[3]);
        case 5:
            return MAKENAMEID5(a[0], a[1], a[2], a[3], a[4]);
        case 6:
            return MAKENAMEID6(a[0], a[1], a[2], a[3], a[4], a[5]);
        case 7:
            return MAKENAMEID7(a[0], a[1], a[2], a[3], a[4], a[5], a[6]);
        case 8:
            return MAKENAMEID8(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7]);
        default:
            break;
    }
    return 0;
}

// generate binary serialize of attr_map name-value pairs
void AttrMap::serialize(string* d) const
{
    char buf[8];
    unsigned char l;
    unsigned short ll;

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        if ((l = (unsigned char)nameid2string(it->first, buf)))
        {
            d->append((char*)&l, sizeof l);
            d->append(buf, l);
            ll = (unsigned short)it->second.size();
            d->append((char*)&ll, sizeof ll);
            d->append(it->second.data(), ll);
        }
    }

    d->append("", 1);
}

// read binary serialize, return final offset
const char* AttrMap::unserialize(const char* ptr , const char *end)
{
    unsigned char l;
    unsigned short ll;
    nameid id;

    while ((ptr < end) && (l = *ptr++))
    {
        id = 0;

        if (ptr + l + sizeof ll > end)
        {
            return NULL;
        }

        while (l--)
        {
            id = (id << 8) + (unsigned char)*ptr++;
        }

        ll = MemAccess::get<short>(ptr);
        ptr += sizeof ll;

        if (ptr + ll > end)
        {
            return NULL;
        }

        map[id].assign(ptr, ll);
        ptr += ll;
    }

    return ptr;
}

bool AttrMap::hasUpdate(const attr_map& updates, nameid attrId) const
{
    auto curIt = map.find(attrId);
    auto updIt = updates.find(attrId);
    return curIt != map.end() && (updIt == updates.end() || curIt->second != updIt->second);
}

void AttrMap::applyUpdates(const attr_map& updates)
{
    for (auto& u : updates)
    {
        if (u.second.empty()) map.erase(u.first);
        else map[u.first] = u.second;
    }
}

// generate JSON object containing attr_map
void AttrMap::getjson(string* s) const
{
    nameid id;
    char buf[8];
    const char* ptr;
    const char* pptr;

    // reserve estimated size of final string
    s->erase();
    s->reserve(storagesize(20));

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        s->append(s->size() ? ",\"" : "\"");
<commit_msg>Add comment to explain confusing update of AttrMap<commit_after>/**
 * @file attrmap.cpp
 * @brief Class for manipulating node attributes
 *
 * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
 *
 * This file is part of the MEGA SDK - Client Access Engine.
 *
 * Applications using the MEGA API must present a valid application key
 * and comply with the the rules set forth in the Terms of Service.
 *
 * The MEGA SDK is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * @copyright Simplified (2-clause) BSD License.
 *
 * You should have received a copy of the license along with this
 * program.
 */

#include "mega/attrmap.h"

namespace mega {
// approximate raw storage size of serialized AttrMap, not taking JSON escaping
// or name length into account
unsigned AttrMap::storagesize(int perrecord) const
{
    assert(perrecord >= 0);
    unsigned t = 0;

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        t += static_cast<unsigned>(perrecord + it->second.size());
    }

    return t;
}

int AttrMap::nameid2string(nameid id, char* buf)
{
    char* ptr = buf;

    for (int i = 64; (i -= 8) >= 0;)
    {
        if ((*ptr = static_cast<char>( (id >> i) & 0xff)))
        {
            ptr++;
        }
    }

    return static_cast<int>(ptr - buf);
}

string AttrMap::nameid2string(nameid id)
{
    string s;
    s.resize(10);
    s.resize(nameid2string(id, const_cast<char*>(s.data())));
    return s;
}


nameid AttrMap::string2nameid(const char *a)
{
    if (!a)
    {
        return 0;
    }

    size_t len = strlen(a);
    if (len > 8)
    {
        return 0;
    }

    switch (len)
    {
        case 1:
            return *a;
        case 2:
            return MAKENAMEID2(a[0], a[1]);
        case 3:
            return MAKENAMEID3(a[0], a[1], a[2]);
        case 4:
            return MAKENAMEID4(a[0], a[1], a[2], a[3]);
        case 5:
            return MAKENAMEID5(a[0], a[1], a[2], a[3], a[4]);
        case 6:
            return MAKENAMEID6(a[0], a[1], a[2], a[3], a[4], a[5]);
        case 7:
            return MAKENAMEID7(a[0], a[1], a[2], a[3], a[4], a[5], a[6]);
        case 8:
            return MAKENAMEID8(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7]);
        default:
            break;
    }
    return 0;
}

// generate binary serialize of attr_map name-value pairs
void AttrMap::serialize(string* d) const
{
    char buf[8];
    unsigned char l;
    unsigned short ll;

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        if ((l = (unsigned char)nameid2string(it->first, buf)))
        {
            d->append((char*)&l, sizeof l);
            d->append(buf, l);
            ll = (unsigned short)it->second.size();
            d->append((char*)&ll, sizeof ll);
            d->append(it->second.data(), ll);
        }
    }

    d->append("", 1);
}

// read binary serialize, return final offset
const char* AttrMap::unserialize(const char* ptr , const char *end)
{
    unsigned char l;
    unsigned short ll;
    nameid id;

    while ((ptr < end) && (l = *ptr++))
    {
        id = 0;

        if (ptr + l + sizeof ll > end)
        {
            return NULL;
        }

        while (l--)
        {
            id = (id << 8) + (unsigned char)*ptr++;
        }

        ll = MemAccess::get<short>(ptr);
        ptr += sizeof ll;

        if (ptr + ll > end)
        {
            return NULL;
        }

        map[id].assign(ptr, ll);
        ptr += ll;
    }

    return ptr;
}

bool AttrMap::hasUpdate(const attr_map& updates, nameid attrId) const
{
    // An attribute will only be touched if it was included in `updates` map.
    // Even for removing it, it should be present there with an empty value.
    auto curIt = map.find(attrId);
    auto updIt = updates.find(attrId);
    return curIt != map.end() && (updIt == updates.end() || curIt->second != updIt->second);
}

void AttrMap::applyUpdates(const attr_map& updates)
{
    for (auto& u : updates)
    {
        if (u.second.empty()) map.erase(u.first);
        else map[u.first] = u.second;
    }
}

// generate JSON object containing attr_map
void AttrMap::getjson(string* s) const
{
    nameid id;
    char buf[8];
    const char* ptr;
    const char* pptr;

    // reserve estimated size of final string
    s->erase();
    s->reserve(storagesize(20));

    for (attr_map::const_iterator it = map.begin(); it != map.end(); it++)
    {
        s->append(s->size() ? ",\"" : "\"");

{'fraction_non_alphanumeric': 0.14944444444444444, 'fraction_numerical': 0.018666666666666668, 'mean_word_length': 2.182814710042433, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823852', 'n_tokens_mistral': 3128, 'n_tokens_neox': 3001, 'n_words': 901}