<commit_before>#if HAVE_CONFIG_H
#include <config.h>
#endif
#include "maxminddb.h"
#include "maxminddb-compat-util.h"
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>

#ifdef _WIN32
#include <windows.h>
#include <ws2ipdef.h>
#else
#include <arpa/inet.h>
#include <sys/mman.h>
#include <unistd.h>
#endif

#define MMDB_DATA_SECTION_SEPARATOR (16)
#define MAXIMUM_DATA_STRUCTURE_DEPTH (512)

#ifdef MMDB_DEBUG
#define LOCAL
#define NO_PROTO
#define DEBUG_FUNC
#define DEBUG_MSG(msg) fprintf(stderr, msg "\n")
#define DEBUG_MSGF(fmt, ...) fprintf(stderr, fmt "\n", __VA_ARGS__)
#define DEBUG_BINARY(fmt, byte)                                 \
    do {                                                        \
        char *binary = byte_to_binary(byte);                    \
        if (NULL == binary) {                                   \
            fprintf(stderr, "Malloc failed in DEBUG_BINARY\n"); \
            abort();                                            \
        }                                                       \
        fprintf(stderr, fmt "\n", binary);                      \
        free(binary);                                           \
    } while (0)
#define DEBUG_NL fprintf(stderr, "\n")
#else
#define LOCAL static
#define NO_PROTO static
#define DEBUG_MSG(...)
#define DEBUG_MSGF(...)
#define DEBUG_BINARY(...)
#define DEBUG_NL
#endif

#ifdef MMDB_DEBUG
DEBUG_FUNC char *byte_to_binary(uint8_t byte)
{
    char *bits = malloc(sizeof(char) * 9);
    if (NULL == bits) {
        return bits;
    }

    for (uint8_t i = 0; i < 8; i++) {
        bits[i] = byte & (128 >> i) ? '1' : '0';
    }
    bits[8] = '\0';

    return bits;
}

DEBUG_FUNC char *type_num_to_name(uint8_t num)
{
    switch (num) {
    case 0:
        return "extended";
    case 1:
        return "pointer";
    case 2:
        return "utf8_string";
    case 3:
        return "double";
    case 4:
        return "bytes";
    case 5:
        return "uint16";
    case 6:
        return "uint32";
    case 7:
        return "map";
    case 8:
        return "int32";
    case 9:
        return "uint64";
    case 10:
        return "uint128";
    case 11:
        return "array";
    case 12:
        return "container";
    case 13:
        return "end_marker";
    case 14:
        return "boolean";
    case 15:
        return "float";
    default:
        return "unknown type";
    }
}
#endif

typedef struct record_info_s {
    uint16_t record_length;
    uint32_t (*left_record_getter)(const uint8_t *);
    uint32_t (*right_record_getter)(const uint8_t *);
    uint8_t right_record_offset;
} record_info_s;

#define METADATA_MARKER "\xab\xcd\xefMaxMind.com"
/* This is 128kb */
#define METADATA_BLOCK_MAX_SIZE 131072

/* *INDENT-OFF* */
/* --prototypes automatically generated by dev-bin/regen-prototypes.pl - don't remove this comment */
LOCAL int map_file(MMDB_s *const mmdb);
LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
                                   ssize_t file_size, uint32_t *metadata_size);
LOCAL int read_metadata(MMDB_s *mmdb);
LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb);
LOCAL uint16_t value_for_key_as_uint16(MMDB_entry_s *start, char *key);
LOCAL uint32_t value_for_key_as_uint32(MMDB_entry_s *start, char *key);
LOCAL uint64_t value_for_key_as_uint64(MMDB_entry_s *start, char *key);
LOCAL char *value_for_key_as_string(MMDB_entry_s *start, char *key);
LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                      MMDB_entry_s *metadata_start);
LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                        MMDB_entry_s *metadata_start);
LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses);
LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
                                      sa_family_t address_family,
                                      MMDB_lookup_result_s *result);
LOCAL record_info_s record_info_for_database(MMDB_s *mmdb);
LOCAL int find_ipv4_start_node(MMDB_s *mmdb);
LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
                          uint16_t netmask, MMDB_lookup_result_s *result);
LOCAL uint32_t get_left_28_bit_record(const uint8_t *record);
LOCAL uint32_t get_right_28_bit_record(const uint8_t *record);
LOCAL int path_length(va_list va_path);
LOCAL int lookup_path_in_array(const char *path_elem, MMDB_s *mmdb,
                               MMDB_entry_data_s *entry_data);
LOCAL int lookup_path_in_map(const char *path_elem, MMDB_s *mmdb,
                             MMDB_entry_data_s *entry_data);
LOCAL int skip_map_or_array(MMDB_s *mmdb, MMDB_entry_data_s *entry_data);
LOCAL int decode_one_follow(MMDB_s *mmdb, uint32_t offset,
                            MMDB_entry_data_s *entry_data);
LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
                     MMDB_entry_data_s *entry_data);
LOCAL int get_ext_type(int raw_ext_type);
LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr,
                            int ptr_size);
LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
                              MMDB_entry_data_list_s *const entry_data_list);
LOCAL float get_ieee754_float(const uint8_t *restrict p);
LOCAL double get_ieee754_double(const uint8_t *restrict p);
LOCAL uint32_t get_uint32(const uint8_t *p);
LOCAL uint32_t get_uint24(const uint8_t *p);
LOCAL uint32_t get_uint16(const uint8_t *p);
LOCAL uint64_t get_uintX(const uint8_t *p, int length);
LOCAL int32_t get_sintX(const uint8_t *p, int length);
LOCAL MMDB_entry_data_list_s *new_entry_data_list(void);
LOCAL void free_mmdb_struct(MMDB_s *const mmdb);
LOCAL void free_languages_metadata(MMDB_s *mmdb);
LOCAL void free_descriptions_metadata(MMDB_s *mmdb);
LOCAL MMDB_entry_data_list_s *dump_entry_data_list(
    FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent,
    int *status);
LOCAL void print_indentation(FILE *stream, int i);
LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size);
/* --prototypes end - don't remove this comment-- */
/* *INDENT-ON* */

#define CHECKED_DECODE_ONE(mmdb, offset, entry_data)       \
    do {                                                   \
        int status = decode_one(mmdb, offset, entry_data); \
        if (MMDB_SUCCESS != status) {                      \
            return status;                                 \
        }                                                  \
    } while (0)

#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data)       \
    do {                                                          \
        int status = decode_one_follow(mmdb, offset, entry_data); \
        if (MMDB_SUCCESS != status) {                             \
            return status;                                        \
        }                                                         \
    } while (0)

#define FREE_AND_SET_NULL(p) { free((void *)(p)); (p) = NULL; }

int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
{
    int status = MMDB_SUCCESS;

    mmdb->file_content = NULL;
    mmdb->data_section = NULL;
    mmdb->metadata.database_type = NULL;
    mmdb->metadata.languages.count = 0;
    mmdb->metadata.description.count = 0;

    mmdb->filename = mmdb_strdup(filename);
    if (NULL == mmdb->filename) {
        status = MMDB_OUT_OF_MEMORY_ERROR;
        goto cleanup;
    }

    if ((flags & MMDB_MODE_MASK) == 0) {
        flags |= MMDB_MODE_MMAP;
    }
    mmdb->flags = flags;

    if (MMDB_SUCCESS != (status = map_file(mmdb)) ) {
        goto cleanup;
    }

#ifdef _WIN32
    WSADATA wsa;
    WSAStartup(MAKEWORD(2, 2), &wsa);
#endif

    uint32_t metadata_size = 0;
    const uint8_t *metadata = find_metadata(mmdb->file_content, mmdb->file_size,
                                            &metadata_size);
    if (NULL == metadata) {
        status = MMDB_INVALID_METADATA_ERROR;
        goto cleanup;
    }

    mmdb->metadata_section = metadata;
    mmdb->metadata_section_size = metadata_size;

    status = read_metadata(mmdb);
    if (MMDB_SUCCESS != status) {
        goto cleanup;
    }

    if (mmdb->metadata.binary_format_major_version != 2) {
        status = MMDB_UNKNOWN_DATABASE_FORMAT_ERROR;
        goto cleanup;
    }

    uint32_t search_tree_size = mmdb->metadata.node_count *
                                mmdb->full_record_byte_size;

    mmdb->data_section = mmdb->file_content + search_tree_size;
    if (search_tree_size > mmdb->file_size) {
        status = MMDB_INVALID_METADATA_ERROR;
        goto cleanup;
    }
    mmdb->data_section_size = mmdb->file_size - search_tree_size;
    mmdb->metadata_section = metadata;
    mmdb->ipv4_start_node.node_value = 0;
    mmdb->ipv4_start_node.netmask = 0;

 cleanup:
    if (MMDB_SUCCESS != status) {
        int saved_errno = errno;
        free_mmdb_struct(mmdb);
        errno = saved_errno;
    }
    return status;
}

#ifdef _WIN32

LOCAL int map_file(MMDB_s *const mmdb)
{
    ssize_t size;
    int status = MMDB_SUCCESS;
    HANDLE mmh = NULL;
    HANDLE fd = CreateFileA(mmdb->filename, GENERIC_READ, FILE_SHARE_READ, NULL,
                            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
    if (fd == INVALID_HANDLE_VALUE) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }
    size = GetFileSize(fd, NULL);
    if (size == INVALID_FILE_SIZE) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }
    mmh = CreateFileMappingA(fd, NULL, PAGE_READONLY, 0, size, NULL);
    /* Microsoft documentation for CreateFileMapping indicates this returns
        NULL not INVALID_HANDLE_VALUE on error */
    if (NULL == mmh) {
        status = MMDB_IO_ERROR;
        goto cleanup;
    }
    uint8_t *file_content =
        (uint8_t *)MapViewOfFile(mmh, FILE_MAP_READ, 0, 0, 0);
    if (file_content == NULL) {
        status = MMDB_IO_ERROR;
        goto cleanup;
    }

    mmdb->file_size = size;
    mmdb->file_content = file_content;

 cleanup:;
    int saved_errno = errno;
    if (INVALID_HANDLE_VALUE != fd) {
        CloseHandle(fd);
    }
    if (NULL != mmh) {
        CloseHandle(mmh);
    }
    errno = saved_errno;

    return status;
}

#else

LOCAL int map_file(MMDB_s *const mmdb)
{
    ssize_t size;
    int status = MMDB_SUCCESS;

    int fd = open(mmdb->filename, O_RDONLY);
    struct stat s;
    if (fd < 0 || fstat(fd, &s)) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }

    size = s.st_size;

    uint8_t *file_content =
        (uint8_t *)mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
    if (MAP_FAILED == file_content) {
        if (ENOMEM == errno) {
            status = MMDB_OUT_OF_MEMORY_ERROR;
        } else {
            status = MMDB_IO_ERROR;
        }
        goto cleanup;
    }

    mmdb->file_size = size;
    mmdb->file_content = file_content;

 cleanup:;
    int saved_errno = errno;
    if (fd >= 0) {
        close(fd);
    }
    errno = saved_errno;

    return status;
}

#endif

LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
                                   ssize_t file_size, uint32_t *metadata_size)
{
    ssize_t max_size = file_size >
                       METADATA_BLOCK_MAX_SIZE ? METADATA_BLOCK_MAX_SIZE :
                       file_size;

    uint8_t *search_area = (uint8_t *)(file_content + (file_size - max_size));
    uint8_t *tmp;
    do {
        tmp = mmdb_memmem(search_area, max_size,
                          METADATA_MARKER, strlen(METADATA_MARKER));

        if (NULL != tmp) {
            max_size -= tmp - search_area;
            search_area = tmp;
        }
    } while (NULL != tmp && tmp != search_area);

    const uint8_t *metadata_start = search_area + strlen(METADATA_MARKER);
    *metadata_size = file_size - (search_area - file_content);

    return metadata_start;
}

LOCAL int read_metadata(MMDB_s *mmdb)
{
    /* We need to create a fake MMDB_s struct in order to decode values from
       the metadata. The metadata is basically just like the data section, so we
       want to use the same functions we use for the data section to get metadata
       values. */
    MMDB_s metadata_db = make_fake_metadata_db(mmdb);

    MMDB_entry_s metadata_start = {
        .mmdb   = &metadata_db,
        .offset = 0
    };

    mmdb->metadata.node_count =
        value_for_key_as_uint32(&metadata_start, "node_count");
    if (!mmdb->metadata.node_count) {
        DEBUG_MSG("could not find node_count value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    mmdb->metadata.record_size =
        value_for_key_as_uint16(&metadata_start, "record_size");
    if (!mmdb->metadata.record_size) {
        DEBUG_MSG("could not find record_size value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    if (mmdb->metadata.record_size != 24 && mmdb->metadata.record_size != 28
        && mmdb->metadata.record_size != 32) {
        DEBUG_MSGF("bad record size in metadata: %i",
                   mmdb->metadata.record_size);
        return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR;
    }

    mmdb->metadata.ip_version =
        value_for_key_as_uint16(&metadata_start, "ip_version");
    if (!mmdb->metadata.ip_version) {
        DEBUG_MSG("could not find ip_version value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }
    if (!(mmdb->metadata.ip_version == 4 || mmdb->metadata.ip_version == 6)) {
        DEBUG_MSGF("ip_version value in metadata is not 4 or 6 - it was %i",
                   mmdb->metadata.ip_version);
        return MMDB_INVALID_METADATA_ERROR;
    }

    mmdb->metadata.database_type =
        value_for_key_as_string(&metadata_start, "database_type");
    if (NULL == mmdb->metadata.database_type) {
        DEBUG_MSG("could not find database_type value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    int status =
        populate_languages_metadata(mmdb, &metadata_db, &metadata_start);
    if (MMDB_SUCCESS != status) {
        DEBUG_MSG("could not populate languages from metadata");
        return status;
    }

    mmdb->metadata.binary_format_major_version =
        value_for_key_as_uint16(&metadata_start, "binary_format_major_version");
    if (!mmdb->metadata.binary_format_major_version) {
        DEBUG_MSG(
            "could not find binary_format_major_version value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    mmdb->metadata.binary_format_minor_version =
        value_for_key_as_uint16(&metadata_start, "binary_format_minor_version");

    mmdb->metadata.build_epoch =
        value_for_key_as_uint64(&metadata_start, "build_epoch");
    if (!mmdb->metadata.build_epoch) {
        DEBUG_MSG("could not find build_epoch value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    status = populate_description_metadata(mmdb, &metadata_db, &metadata_start);
    if (MMDB_SUCCESS != status) {
        DEBUG_MSG("could not populate description from metadata");
        return status;
    }

    mmdb->full_record_byte_size = mmdb->metadata.record_size * 2 / 8U;

    mmdb->depth = mmdb->metadata.ip_version == 4 ? 32 : 128;

    return MMDB_SUCCESS;
}

LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb)
{
    MMDB_s fake_metadata_db = {
        .data_section      = mmdb->metadata_section,
        .data_section_size = mmdb->metadata_section_size
    };

    return fake_metadata_db;
}

LOCAL uint16_t value_for_key_as_uint16(MMDB_entry_s *start, char *key)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    MMDB_aget_value(start, &entry_data, path);
    return entry_data.uint16;
}

LOCAL uint32_t value_for_key_as_uint32(MMDB_entry_s *start, char *key)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    MMDB_aget_value(start, &entry_data, path);
    return entry_data.uint32;
}

LOCAL uint64_t value_for_key_as_uint64(MMDB_entry_s *start, char *key)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    MMDB_aget_value(start, &entry_data, path);
    return entry_data.uint64;
}

LOCAL char *value_for_key_as_string(MMDB_entry_s *start, char *key)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    MMDB_aget_value(start, &entry_data, path);
    if (MMDB_DATA_TYPE_UTF8_STRING != entry_data.type) {
        return NULL;
    }
    return mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size);
}

LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                      MMDB_entry_s *metadata_start)
{
    MMDB_entry_data_s entry_data;

    const char *path[] = { "languages", NULL };
    MMDB_aget_value(metadata_start, &entry_data, path);

    if (MMDB_DATA_TYPE_ARRAY != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }

    MMDB_entry_s array_start = {
        .mmdb   = metadata_db,
        .offset = entry_data.offset
    };

    MMDB_entry_data_list_s *member;
    MMDB_get_entry_data_list(&array_start, &member);<commit_msg>Check for more errors while decoding metadata<commit_after>#if HAVE_CONFIG_H
#include <config.h>
#endif
#include "maxminddb.h"
#include "maxminddb-compat-util.h"
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>

#ifdef _WIN32
#include <windows.h>
#include <ws2ipdef.h>
#else
#include <arpa/inet.h>
#include <sys/mman.h>
#include <unistd.h>
#endif

#define MMDB_DATA_SECTION_SEPARATOR (16)
#define MAXIMUM_DATA_STRUCTURE_DEPTH (512)

#ifdef MMDB_DEBUG
#define LOCAL
#define NO_PROTO
#define DEBUG_FUNC
#define DEBUG_MSG(msg) fprintf(stderr, msg "\n")
#define DEBUG_MSGF(fmt, ...) fprintf(stderr, fmt "\n", __VA_ARGS__)
#define DEBUG_BINARY(fmt, byte)                                 \
    do {                                                        \
        char *binary = byte_to_binary(byte);                    \
        if (NULL == binary) {                                   \
            fprintf(stderr, "Malloc failed in DEBUG_BINARY\n"); \
            abort();                                            \
        }                                                       \
        fprintf(stderr, fmt "\n", binary);                      \
        free(binary);                                           \
    } while (0)
#define DEBUG_NL fprintf(stderr, "\n")
#else
#define LOCAL static
#define NO_PROTO static
#define DEBUG_MSG(...)
#define DEBUG_MSGF(...)
#define DEBUG_BINARY(...)
#define DEBUG_NL
#endif

#ifdef MMDB_DEBUG
DEBUG_FUNC char *byte_to_binary(uint8_t byte)
{
    char *bits = malloc(sizeof(char) * 9);
    if (NULL == bits) {
        return bits;
    }

    for (uint8_t i = 0; i < 8; i++) {
        bits[i] = byte & (128 >> i) ? '1' : '0';
    }
    bits[8] = '\0';

    return bits;
}

DEBUG_FUNC char *type_num_to_name(uint8_t num)
{
    switch (num) {
    case 0:
        return "extended";
    case 1:
        return "pointer";
    case 2:
        return "utf8_string";
    case 3:
        return "double";
    case 4:
        return "bytes";
    case 5:
        return "uint16";
    case 6:
        return "uint32";
    case 7:
        return "map";
    case 8:
        return "int32";
    case 9:
        return "uint64";
    case 10:
        return "uint128";
    case 11:
        return "array";
    case 12:
        return "container";
    case 13:
        return "end_marker";
    case 14:
        return "boolean";
    case 15:
        return "float";
    default:
        return "unknown type";
    }
}
#endif

typedef struct record_info_s {
    uint16_t record_length;
    uint32_t (*left_record_getter)(const uint8_t *);
    uint32_t (*right_record_getter)(const uint8_t *);
    uint8_t right_record_offset;
} record_info_s;

#define METADATA_MARKER "\xab\xcd\xefMaxMind.com"
/* This is 128kb */
#define METADATA_BLOCK_MAX_SIZE 131072

/* *INDENT-OFF* */
/* --prototypes automatically generated by dev-bin/regen-prototypes.pl - don't remove this comment */
LOCAL int map_file(MMDB_s *const mmdb);
LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
                                   ssize_t file_size, uint32_t *metadata_size);
LOCAL int read_metadata(MMDB_s *mmdb);
LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb);
LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key,
                                  uint16_t *value);
LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key,
                                  uint32_t *value);
LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key,
                                  uint64_t *value);
LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key,
                                  char const **value);
LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                      MMDB_entry_s *metadata_start);
LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                        MMDB_entry_s *metadata_start);
LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses);
LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
                                      sa_family_t address_family,
                                      MMDB_lookup_result_s *result);
LOCAL record_info_s record_info_for_database(MMDB_s *mmdb);
LOCAL int find_ipv4_start_node(MMDB_s *mmdb);
LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
                          uint16_t netmask, MMDB_lookup_result_s *result);
LOCAL uint32_t get_left_28_bit_record(const uint8_t *record);
LOCAL uint32_t get_right_28_bit_record(const uint8_t *record);
LOCAL int path_length(va_list va_path);
LOCAL int lookup_path_in_array(const char *path_elem, MMDB_s *mmdb,
                               MMDB_entry_data_s *entry_data);
LOCAL int lookup_path_in_map(const char *path_elem, MMDB_s *mmdb,
                             MMDB_entry_data_s *entry_data);
LOCAL int skip_map_or_array(MMDB_s *mmdb, MMDB_entry_data_s *entry_data);
LOCAL int decode_one_follow(MMDB_s *mmdb, uint32_t offset,
                            MMDB_entry_data_s *entry_data);
LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
                     MMDB_entry_data_s *entry_data);
LOCAL int get_ext_type(int raw_ext_type);
LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr,
                            int ptr_size);
LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
                              MMDB_entry_data_list_s *const entry_data_list);
LOCAL float get_ieee754_float(const uint8_t *restrict p);
LOCAL double get_ieee754_double(const uint8_t *restrict p);
LOCAL uint32_t get_uint32(const uint8_t *p);
LOCAL uint32_t get_uint24(const uint8_t *p);
LOCAL uint32_t get_uint16(const uint8_t *p);
LOCAL uint64_t get_uintX(const uint8_t *p, int length);
LOCAL int32_t get_sintX(const uint8_t *p, int length);
LOCAL MMDB_entry_data_list_s *new_entry_data_list(void);
LOCAL void free_mmdb_struct(MMDB_s *const mmdb);
LOCAL void free_languages_metadata(MMDB_s *mmdb);
LOCAL void free_descriptions_metadata(MMDB_s *mmdb);
LOCAL MMDB_entry_data_list_s *dump_entry_data_list(
    FILE *stream, MMDB_entry_data_list_s *entry_data_list, int indent,
    int *status);
LOCAL void print_indentation(FILE *stream, int i);
LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size);
/* --prototypes end - don't remove this comment-- */
/* *INDENT-ON* */

#define CHECKED_DECODE_ONE(mmdb, offset, entry_data)       \
    do {                                                   \
        int status = decode_one(mmdb, offset, entry_data); \
        if (MMDB_SUCCESS != status) {                      \
            return status;                                 \
        }                                                  \
    } while (0)

#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data)       \
    do {                                                          \
        int status = decode_one_follow(mmdb, offset, entry_data); \
        if (MMDB_SUCCESS != status) {                             \
            return status;                                        \
        }                                                         \
    } while (0)

#define FREE_AND_SET_NULL(p) { free((void *)(p)); (p) = NULL; }

int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
{
    int status = MMDB_SUCCESS;

    mmdb->file_content = NULL;
    mmdb->data_section = NULL;
    mmdb->metadata.database_type = NULL;
    mmdb->metadata.languages.count = 0;
    mmdb->metadata.description.count = 0;

    mmdb->filename = mmdb_strdup(filename);
    if (NULL == mmdb->filename) {
        status = MMDB_OUT_OF_MEMORY_ERROR;
        goto cleanup;
    }

    if ((flags & MMDB_MODE_MASK) == 0) {
        flags |= MMDB_MODE_MMAP;
    }
    mmdb->flags = flags;

    if (MMDB_SUCCESS != (status = map_file(mmdb)) ) {
        goto cleanup;
    }

#ifdef _WIN32
    WSADATA wsa;
    WSAStartup(MAKEWORD(2, 2), &wsa);
#endif

    uint32_t metadata_size = 0;
    const uint8_t *metadata = find_metadata(mmdb->file_content, mmdb->file_size,
                                            &metadata_size);
    if (NULL == metadata) {
        status = MMDB_INVALID_METADATA_ERROR;
        goto cleanup;
    }

    mmdb->metadata_section = metadata;
    mmdb->metadata_section_size = metadata_size;

    status = read_metadata(mmdb);
    if (MMDB_SUCCESS != status) {
        goto cleanup;
    }

    if (mmdb->metadata.binary_format_major_version != 2) {
        status = MMDB_UNKNOWN_DATABASE_FORMAT_ERROR;
        goto cleanup;
    }

    uint32_t search_tree_size = mmdb->metadata.node_count *
                                mmdb->full_record_byte_size;

    mmdb->data_section = mmdb->file_content + search_tree_size;
    if (search_tree_size > mmdb->file_size) {
        status = MMDB_INVALID_METADATA_ERROR;
        goto cleanup;
    }
    mmdb->data_section_size = mmdb->file_size - search_tree_size;
    mmdb->metadata_section = metadata;
    mmdb->ipv4_start_node.node_value = 0;
    mmdb->ipv4_start_node.netmask = 0;

 cleanup:
    if (MMDB_SUCCESS != status) {
        int saved_errno = errno;
        free_mmdb_struct(mmdb);
        errno = saved_errno;
    }
    return status;
}

#ifdef _WIN32

LOCAL int map_file(MMDB_s *const mmdb)
{
    ssize_t size;
    int status = MMDB_SUCCESS;
    HANDLE mmh = NULL;
    HANDLE fd = CreateFileA(mmdb->filename, GENERIC_READ, FILE_SHARE_READ, NULL,
                            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
    if (fd == INVALID_HANDLE_VALUE) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }
    size = GetFileSize(fd, NULL);
    if (size == INVALID_FILE_SIZE) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }
    mmh = CreateFileMappingA(fd, NULL, PAGE_READONLY, 0, size, NULL);
    /* Microsoft documentation for CreateFileMapping indicates this returns
        NULL not INVALID_HANDLE_VALUE on error */
    if (NULL == mmh) {
        status = MMDB_IO_ERROR;
        goto cleanup;
    }
    uint8_t *file_content =
        (uint8_t *)MapViewOfFile(mmh, FILE_MAP_READ, 0, 0, 0);
    if (file_content == NULL) {
        status = MMDB_IO_ERROR;
        goto cleanup;
    }

    mmdb->file_size = size;
    mmdb->file_content = file_content;

 cleanup:;
    int saved_errno = errno;
    if (INVALID_HANDLE_VALUE != fd) {
        CloseHandle(fd);
    }
    if (NULL != mmh) {
        CloseHandle(mmh);
    }
    errno = saved_errno;

    return status;
}

#else

LOCAL int map_file(MMDB_s *const mmdb)
{
    ssize_t size;
    int status = MMDB_SUCCESS;

    int fd = open(mmdb->filename, O_RDONLY);
    struct stat s;
    if (fd < 0 || fstat(fd, &s)) {
        status = MMDB_FILE_OPEN_ERROR;
        goto cleanup;
    }

    size = s.st_size;

    uint8_t *file_content =
        (uint8_t *)mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
    if (MAP_FAILED == file_content) {
        if (ENOMEM == errno) {
            status = MMDB_OUT_OF_MEMORY_ERROR;
        } else {
            status = MMDB_IO_ERROR;
        }
        goto cleanup;
    }

    mmdb->file_size = size;
    mmdb->file_content = file_content;

 cleanup:;
    int saved_errno = errno;
    if (fd >= 0) {
        close(fd);
    }
    errno = saved_errno;

    return status;
}

#endif

LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
                                   ssize_t file_size, uint32_t *metadata_size)
{
    ssize_t max_size = file_size >
                       METADATA_BLOCK_MAX_SIZE ? METADATA_BLOCK_MAX_SIZE :
                       file_size;

    uint8_t *search_area = (uint8_t *)(file_content + (file_size - max_size));
    uint8_t *tmp;
    do {
        tmp = mmdb_memmem(search_area, max_size,
                          METADATA_MARKER, strlen(METADATA_MARKER));

        if (NULL != tmp) {
            max_size -= tmp - search_area;
            search_area = tmp;
        }
    } while (NULL != tmp && tmp != search_area);

    const uint8_t *metadata_start = search_area + strlen(METADATA_MARKER);
    *metadata_size = file_size - (search_area - file_content);

    return metadata_start;
}

LOCAL int read_metadata(MMDB_s *mmdb)
{
    /* We need to create a fake MMDB_s struct in order to decode values from
       the metadata. The metadata is basically just like the data section, so we
       want to use the same functions we use for the data section to get metadata
       values. */
    MMDB_s metadata_db = make_fake_metadata_db(mmdb);

    MMDB_entry_s metadata_start = {
        .mmdb   = &metadata_db,
        .offset = 0
    };

    int status =
        value_for_key_as_uint32(&metadata_start, "node_count",
                                &mmdb->metadata.node_count);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (!mmdb->metadata.node_count) {
        DEBUG_MSG("could not find node_count value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    status = value_for_key_as_uint16(&metadata_start, "record_size",
                                     &mmdb->metadata.record_size);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (!mmdb->metadata.record_size) {
        DEBUG_MSG("could not find record_size value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    if (mmdb->metadata.record_size != 24 && mmdb->metadata.record_size != 28
        && mmdb->metadata.record_size != 32) {
        DEBUG_MSGF("bad record size in metadata: %i",
                   mmdb->metadata.record_size);
        return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR;
    }

    status = value_for_key_as_uint16(&metadata_start, "ip_version",
                                     &mmdb->metadata.ip_version);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (!mmdb->metadata.ip_version) {
        DEBUG_MSG("could not find ip_version value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }
    if (!(mmdb->metadata.ip_version == 4 || mmdb->metadata.ip_version == 6)) {
        DEBUG_MSGF("ip_version value in metadata is not 4 or 6 - it was %i",
                   mmdb->metadata.ip_version);
        return MMDB_INVALID_METADATA_ERROR;
    }

    status = value_for_key_as_string(&metadata_start, "database_type",
                                     &mmdb->metadata.database_type);
    if (MMDB_SUCCESS != status) {
        DEBUG_MSG("error finding database_type value in metadata");
        return status;
    }

    status =
        populate_languages_metadata(mmdb, &metadata_db, &metadata_start);
    if (MMDB_SUCCESS != status) {
        DEBUG_MSG("could not populate languages from metadata");
        return status;
    }

    status = value_for_key_as_uint16(
        &metadata_start, "binary_format_major_version",
        &mmdb->metadata.binary_format_major_version);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (!mmdb->metadata.binary_format_major_version) {
        DEBUG_MSG(
            "could not find binary_format_major_version value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    status = value_for_key_as_uint16(
        &metadata_start, "binary_format_minor_version",
        &mmdb->metadata.binary_format_minor_version);
    if (MMDB_SUCCESS != status) {
        return status;
    }

    status = value_for_key_as_uint64(&metadata_start, "build_epoch",
                                     &mmdb->metadata.build_epoch);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (!mmdb->metadata.build_epoch) {
        DEBUG_MSG("could not find build_epoch value in metadata");
        return MMDB_INVALID_METADATA_ERROR;
    }

    status = populate_description_metadata(mmdb, &metadata_db, &metadata_start);
    if (MMDB_SUCCESS != status) {
        DEBUG_MSG("could not populate description from metadata");
        return status;
    }

    mmdb->full_record_byte_size = mmdb->metadata.record_size * 2 / 8U;

    mmdb->depth = mmdb->metadata.ip_version == 4 ? 32 : 128;

    return MMDB_SUCCESS;
}

LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb)
{
    MMDB_s fake_metadata_db = {
        .data_section      = mmdb->metadata_section,
        .data_section_size = mmdb->metadata_section_size
    };

    return fake_metadata_db;
}

LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key,
                                  uint16_t *value)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    int status = MMDB_aget_value(start, &entry_data, path);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (MMDB_DATA_TYPE_UINT16 != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }
    *value = entry_data.uint16;
    return MMDB_SUCCESS;
}

LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key,
                                  uint32_t *value)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    int status = MMDB_aget_value(start, &entry_data, path);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (MMDB_DATA_TYPE_UINT32 != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }
    *value = entry_data.uint32;
    return MMDB_SUCCESS;
}

LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key,
                                  uint64_t *value)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    int status = MMDB_aget_value(start, &entry_data, path);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (MMDB_DATA_TYPE_UINT64 != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }
    *value = entry_data.uint64;
    return MMDB_SUCCESS;
}

LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key,
                                  char const **value)
{
    MMDB_entry_data_s entry_data;
    const char *path[] = { key, NULL };
    int status = MMDB_aget_value(start, &entry_data, path);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (MMDB_DATA_TYPE_UTF8_STRING != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }
    *value = mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size);
    return MMDB_SUCCESS;
}

LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
                                      MMDB_entry_s *metadata_start)
{
    MMDB_entry_data_s entry_data;

    const char *path[] = { "languages", NULL };
    int status = MMDB_aget_value(metadata_start, &entry_data, path);
    if (MMDB_SUCCESS != status) {
        return status;
    }
    if (MMDB_DATA_TYPE_ARRAY != entry_data.type) {
        return MMDB_INVALID_METADATA_ERROR;
    }

    MMDB_entry_s array_start = {
        .mmdb   = metadata_db,
        .offset = entry_data.offset
    };

    MMDB_entry_data_list_s *member;
    status = MMDB_get_entry_data_list(&array_start, &member);
    if (MMDB_SUCCESS != status) {
        return status;
    }
{'fraction_non_alphanumeric': 0.0964111334859728, 'fraction_numerical': 0.013627210283854238, 'mean_word_length': 2.397563261480787, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 148, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '298218', 'n_tokens_mistral': 12682, 'n_tokens_neox': 12093, 'n_words': 2613}