<commit_before>#!/usr/bin/env python
#
# esp-idf NVS partition generation tool. Tool helps in generating NVS-compatible
# partition binary, with key-value pair entries provided via a CSV file.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import argparse
import binascii
import getopt
import struct
import os
import array
import csv
import zlib
from os import path

""" Class for standard NVS page structure """
class Page(object):
    PAGE_PARAMS = {
        "max_size": 4096,
        "max_blob_size": 4000,
        "max_entries": 126
    }

    # Item type codes
    U8   = 0x01
    I8   = 0x11
    U16  = 0x02
    I16  = 0x12
    U32  = 0x04
    I32  = 0x14
    SZ   = 0x21
    BLOB = 0x41
    BLOB_DATA = 0x42
    BLOB_IDX = 0x48

    # Few Page constants
    HEADER_SIZE = 32
    BITMAPARRAY_OFFSET = 32
    BITMAPARRAY_SIZE_IN_BYTES = 32
    FIRST_ENTRY_OFFSET = 64
    SINGLE_ENTRY_SIZE = 32
    CHUNK_ANY = 0xFF
    ACTIVE = 0xFFFFFFFE
    FULL = 0xFFFFFFFC

    def __init__(self, page_num):
        self.entry_num = 0
        self.bitmap_array = array.array('B')
        self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
        self.bitmap_array = self.create_bitmap_array()
        self.set_header(page_num)

    def set_header(self, page_num):
        global page_header

        # set page state to active
        page_header= bytearray(b'\xff')*32
        page_state_active_seq = Page.ACTIVE
        page_header[0:4] = struct.pack('<I', page_state_active_seq)
        # set page sequence number
        page_header[4:8] = struct.pack('<I', page_num)
        # set header's CRC
        crc_data = page_header[4:28]
        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
        page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
        self.page_buf[0:len(page_header)] = page_header

    def create_bitmap_array(self):
        bitarray = array.array('B')
        charsize = 32 # bitmaparray has 256 bits, hence 32 bytes
        fill = 255 # Fill all 8 bits with 1's
        bitarray.extend((fill,) * charsize)
        return bitarray

    def write_bitmaparray(self):
        bitnum = self.entry_num * 2
        byte_idx = bitnum / 8 # Find byte index in the array
        bit_offset = bitnum & 7 # Find bit offset in given byte index
        mask = ~(1 << bit_offset)
        self.bitmap_array[byte_idx] &= mask
        start_idx = Page.BITMAPARRAY_OFFSET
        end_idx = Page.BITMAPARRAY_OFFSET + Page.BITMAPARRAY_SIZE_IN_BYTES
        self.page_buf[start_idx:end_idx] = self.bitmap_array

    def write_entry_to_buf(self, data, entrycount):
        data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
        start_idx = data_offset
        end_idx = data_offset + len(data)
        self.page_buf[start_idx:end_idx]  = data
        # Set bitmap array for entries in current page
        for i in range(0, entrycount):
            self.write_bitmaparray()
            self.entry_num += 1

    def set_crc_header(self, entry_struct):
        crc_data = bytearray(28)
        crc_data[0:4] = entry_struct[0:4]
        crc_data[4:28] = entry_struct[8:32]
        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
        return entry_struct

    def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, nvs_obj):
        chunk_start = 0
        chunk_count = 0
        chunk_index = Page.CHUNK_ANY
        offset = 0
        remaining_size = data_size
        tailroom = None

        while True:
            chunk_size = 0

            # Get the size available in current page
            if self.entry_num < (Page.PAGE_PARAMS["max_entries"] - 1):
                tailroom = (Page.PAGE_PARAMS["max_entries"] - self.entry_num - 1) * Page.SINGLE_ENTRY_SIZE

            # Split the binary data into two and store a chunk of available size onto curr page
            if tailroom < remaining_size:
                chunk_size = tailroom
            else:
                chunk_size = remaining_size

            remaining_size = remaining_size - chunk_size

            # Change type of data to BLOB_DATA
            entry_struct[1] = Page.BLOB_DATA

            # Calculate no. of entries data chunk will require
            datachunk_rounded_size = (chunk_size + 31) & ~31
            datachunk_entry_count = datachunk_rounded_size / 32
            datachunk_total_entry_count = datachunk_entry_count + 1 # +1 for the entry header

            # Set Span
            entry_struct[2] = datachunk_total_entry_count

            # Update the chunkIndex
            chunk_index = chunk_start + chunk_count
            entry_struct[3] = chunk_index

            # Set data chunk
            data_chunk =  data[offset:offset + chunk_size]

            # Compute CRC of data chunk
            entry_struct[24:26] = struct.pack('<H', chunk_size)
            crc = zlib.crc32(data_chunk, 0xFFFFFFFF)<commit_msg>Add nvs part gen utility multi page blob support update<commit_after>#!/usr/bin/env python
#
# esp-idf NVS partition generation tool. Tool helps in generating NVS-compatible
# partition binary, with key-value pair entries provided via a CSV file.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import argparse
import binascii
import getopt
import struct
import os
import array
import csv
import zlib
from os import path

""" Class for standard NVS page structure """
class Page(object):
    PAGE_PARAMS = {
        "max_size": 4096,
        "max_blob_size": 4000,
        "max_entries": 126
    }

    # Item type codes
    U8   = 0x01
    I8   = 0x11
    U16  = 0x02
    I16  = 0x12
    U32  = 0x04
    I32  = 0x14
    SZ   = 0x21
    BLOB = 0x41
    BLOB_DATA = 0x42
    BLOB_IDX = 0x48

    # Few Page constants
    HEADER_SIZE = 32
    BITMAPARRAY_OFFSET = 32
    BITMAPARRAY_SIZE_IN_BYTES = 32
    FIRST_ENTRY_OFFSET = 64
    SINGLE_ENTRY_SIZE = 32
    CHUNK_ANY = 0xFF
    ACTIVE = 0xFFFFFFFE
    FULL = 0xFFFFFFFC

    def __init__(self, page_num):
        self.entry_num = 0
        self.bitmap_array = array.array('B')
        self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
        self.bitmap_array = self.create_bitmap_array()
        self.set_header(page_num)

    def set_header(self, page_num):
        global page_header

        # set page state to active
        page_header= bytearray(b'\xff')*32
        page_state_active_seq = Page.ACTIVE
        page_header[0:4] = struct.pack('<I', page_state_active_seq)
        # set page sequence number
        page_header[4:8] = struct.pack('<I', page_num)
        # set header's CRC
        crc_data = page_header[4:28]
        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
        page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
        self.page_buf[0:len(page_header)] = page_header

    def create_bitmap_array(self):
        bitarray = array.array('B')
        charsize = 32 # bitmaparray has 256 bits, hence 32 bytes
        fill = 255 # Fill all 8 bits with 1's
        bitarray.extend((fill,) * charsize)
        return bitarray

    def write_bitmaparray(self):
        bitnum = self.entry_num * 2
        byte_idx = bitnum / 8 # Find byte index in the array
        bit_offset = bitnum & 7 # Find bit offset in given byte index
        mask = ~(1 << bit_offset)
        self.bitmap_array[byte_idx] &= mask
        start_idx = Page.BITMAPARRAY_OFFSET
        end_idx = Page.BITMAPARRAY_OFFSET + Page.BITMAPARRAY_SIZE_IN_BYTES
        self.page_buf[start_idx:end_idx] = self.bitmap_array

    def write_entry_to_buf(self, data, entrycount):
        data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
        start_idx = data_offset
        end_idx = data_offset + len(data)
        self.page_buf[start_idx:end_idx]  = data
        # Set bitmap array for entries in current page
        for i in range(0, entrycount):
            self.write_bitmaparray()
            self.entry_num += 1

    def set_crc_header(self, entry_struct):
        crc_data = bytearray(28)
        crc_data[0:4] = entry_struct[0:4]
        crc_data[4:28] = entry_struct[8:32]
        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
        return entry_struct

    def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, nvs_obj):
        chunk_start = 0
        chunk_count = 0
        chunk_index = Page.CHUNK_ANY
        offset = 0
        remaining_size = data_size
        tailroom = None

        while True:
            chunk_size = 0

            # Get the size available in current page
            tailroom = (Page.PAGE_PARAMS["max_entries"] - self.entry_num - 1) * Page.SINGLE_ENTRY_SIZE
            assert tailroom >=0, "Page overflow!!"

            # Split the binary data into two and store a chunk of available size onto curr page
            if tailroom < remaining_size:
                chunk_size = tailroom
            else:
                chunk_size = remaining_size

            remaining_size = remaining_size - chunk_size

            # Change type of data to BLOB_DATA
            entry_struct[1] = Page.BLOB_DATA

            # Calculate no. of entries data chunk will require
            datachunk_rounded_size = (chunk_size + 31) & ~31
            datachunk_entry_count = datachunk_rounded_size / 32
            datachunk_total_entry_count = datachunk_entry_count + 1 # +1 for the entry header

            # Set Span
            entry_struct[2] = datachunk_total_entry_count

            # Update the chunkIndex
            chunk_index = chunk_start + chunk_count
            entry_struct[3] = chunk_index

            # Set data chunk
            data_chunk =  data[offset:offset + chunk_size]

            # Compute CRC of data chunk
            entry_struct[24:26] = struct.pack('<H', chunk_size)
            crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
{'fraction_non_alphanumeric': 0.07875292845557758, 'fraction_numerical': 0.02721211029014237, 'mean_word_length': 2.4727784730913642, 'pattern_counts': {'":': 6, '<': 20, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 14, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5395767', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3422, 'n_words': 959}