<commit_before>#define _XOPEN_SOURCE 500
/* need to define _XOPEN_SOURCE to get IOV_MAX */

#include "mcdb.h"
#include "mcdb_makefmt.h"
#include "mcdb_error.h"
#include "mcdb_uint32.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>   /* open(), O_RDONLY */
#include <stdbool.h> /* bool */
#include <stdio.h>   /* printf(), IOV_MAX */
#include <stdlib.h>  /* malloc(), free(), EXIT_SUCCESS */
#include <string.h>  /* strlen() */
#include <unistd.h>  /* STDIN_FILENO, STDOUT_FILENO */
#include <sys/uio.h> /* writev() */
#include <libgen.h>  /* basename() */
#include <limits.h>  /* SSIZE_MAX */

static bool
writev_loop(const int fd, struct iovec * restrict iov, int iovcnt)
{
    /* Note: unlike writev(), this routine might modify the iovecs */
    ssize_t len;
    while (iovcnt && (len = writev(fd, iov, iovcnt)) != -1) {
        while (len != 0) {
            if (len >= iov[0].iov_len) {
                len -= iov[0].iov_len;
                --iovcnt;
                ++iov;
            }
            else {
                iov[0].iov_len -= len;
                iov[0].iov_base = ((char *)iov[0].iov_base) + len;
            }
        }
    }
    return (iovcnt == 0);
}

/* read and dump data section of mcdb
 * start of hash tables is end of key/data section (eod)
 * subtract 7 for maximal added alignment between data and hash table
 * (each record key,len,keystr,datastr is at least 8 bytes) */
static int
mcdbctl_dump(struct mcdb * const restrict m)
{
    unsigned char * restrict p = m->map->ptr;
    uint32_t klen;
    uint32_t dlen;
    unsigned char * const eod =
      p + mcdb_uint32_unpack_bigendian_aligned_macro(p) - 7;
    int    iovcnt = 0;
    size_t iovlen = 0;
    size_t buflen = 0;
    enum { MCDB_IOVNUM = IOV_MAX };/* assert(IOV_MAX >= 7) */
    struct iovec iov[MCDB_IOVNUM]; /* each db entry uses 7 iovecs */
    char buf[(MCDB_IOVNUM * 3)];   /* each db entry might use (2) * 10 chars */
      /* oversized buffer since all num strings must add up to less than max */

    for (p += MCDB_HEADER_SZ; p < eod; p += 8+klen+dlen) {

        klen = mcdb_uint32_unpack_bigendian_macro(p);
        dlen = mcdb_uint32_unpack_bigendian_macro(p+4);

        /* avoid printf("%.*s\n",...) due to mcdb arbitrary binary data */
        /* klen, dlen each limited to (2GB - 8); space for extra tokens exists*/
        if (iovlen + klen + 5 > SSIZE_MAX || iovcnt + 7 > MCDB_IOVNUM) {
            if (!writev_loop(STDOUT_FILENO, iov, iovcnt))
                return MCDB_ERROR_WRITE;
            iovcnt = 0;
            iovlen = 0;
            buflen = 0;
        }

        iov[iovcnt].iov_base = "+";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = buf+buflen;
        buflen += iov[iovcnt].iov_len = snprintf(buf+buflen, 11, "%u", klen);
        ++iovcnt;

        iov[iovcnt].iov_base = ",";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = buf+buflen;
        buflen += iov[iovcnt].iov_len = snprintf(buf+buflen, 11, "%u", dlen);
        ++iovcnt;

        iov[iovcnt].iov_base = ":";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = p+8;
        iov[iovcnt].iov_len  = klen;
        ++iovcnt;

        iov[iovcnt].iov_base = "->";
        iov[iovcnt].iov_len  = 2;
        ++iovcnt;

        iovlen += klen + 5;

        if (iovlen + dlen + 1 > SSIZE_MAX) {
            if (!writev_loop(STDOUT_FILENO, iov, iovcnt))
                return MCDB_ERROR_WRITE;
            iovcnt = 0;
            iovlen = 0;
            buflen = 0;
        }

        iov[iovcnt].iov_base = p+8+klen;
        iov[iovcnt].iov_len  = dlen;
        ++iovcnt;

        iov[iovcnt].iov_base = "\n";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iovlen += dlen + 1;

    }

    return (writev_loop(STDOUT_FILENO, iov, iovcnt)
            && write(STDOUT_FILENO, "\n", 1)) ? EXIT_SUCCESS : MCDB_ERROR_WRITE;
            /* append blank line ("\n") to indicate end of data */
}

/* Note: mcdbctl_stats() is equivalent test to pass/fail of djb cdbtest */
static int
mcdbctl_stats(struct mcdb * const restrict m)
{
    unsigned char * const map_ptr = m->map->ptr;
    unsigned char *p;
    uint32_t klen;
    uint32_t dlen;
    unsigned char * const eod =
      map_ptr + mcdb_uint32_unpack_bigendian_aligned_macro(map_ptr) - 7;
    unsigned long nrec = 0;
    unsigned long numd[11] = { 0,0,0,0,0,0,0,0,0,0,0 };
    unsigned int rv;
    bool rc;
    for (p = map_ptr+MCDB_HEADER_SZ; p < eod; p += klen+dlen) {
        klen = mcdb_uint32_unpack_bigendian_macro(p);
        dlen = mcdb_uint32_unpack_bigendian_macro(p+4);
        /* Search for key,data and track number of tries before found.
         * Technically, passing m (which contains m->map->ptr) and an
         * alias into the map (p+8) as key is in violation of C99 restrict
         * pointers, but is inconsequential since it is all read-only */
        p += 8;
        if ((rc = mcdb_findstart(m, (char *)p, klen))) {
            do { rc = mcdb_findnext(m, (char *)p, klen);
            } while (rc && map_ptr+m->dpos != p+klen);
        }
        if (!rc) return MCDB_ERROR_READFORMAT;
        ++nrec;
        ++numd[ ((m->loop < 11) ? m->loop - 1 : 10) ];
    }
    printf("records %lu\n", nrec);
    for (rv = 0; rv < 10; ++rv)
        printf("d%d      %lu\n", rv, numd[rv]);
    printf(">9      %lu\n", numd[10]);
    fflush(stdout);
    return EXIT_SUCCESS;
}

static int
mcdbctl_getseq(struct mcdb * const restrict m,
               const char * const restrict key, unsigned long seq)
{
    const size_t klen = strlen(key);
    struct iovec iov[2];
    if (mcdb_findstart(m, key, klen)) {
        bool rc;
        while ((rc = mcdb_findnext(m, key, klen)) && seq--)
            ;
        if (rc) {
            /* avoid printf("%.*s\n",...) due to mcdb arbitrary binary data */
            iov[0].iov_base = mcdb_dataptr(m);
            iov[0].iov_len  = mcdb_datalen(m);
            iov[0].iov_base = "\n";
            iov[0].iov_len  = 1;
            return
              writev_loop(STDOUT_FILENO, iov, sizeof(iov)/sizeof(struct iovec))
              ? EXIT_SUCCESS
              : MCDB_ERROR_WRITE;
        }
    }
    return EXIT_FAILURE;
}

static int
mcdbctl_query(const int argc, char ** restrict argv)
{
    struct mcdb m;
    struct mcdb_mmap map;
    int rv;
    int fd;
    unsigned long seq = 0;
    enum { MCDBCTL_BAD_QUERY_TYPE, MCDBCTL_GET, MCDBCTL_DUMP, MCDBCTL_STATS }
      query_type = MCDBCTL_BAD_QUERY_TYPE;

    /* validate args  (query type string == argv[1]) */
    if (0 == strcmp(argv[1], "get")) {
        if (argc == 5) {
            char *endptr;
            seq = strtoul(argv[4], &endptr, 10);
            if (seq != ULONG_MAX && argv[4] != endptr && *endptr == '\0')
                query_type = MCDBCTL_GET;<commit_msg>pass size to writev_loop() since we already calculated it fix some sloppy bugs<commit_after>#define _XOPEN_SOURCE 500
/* need to define _XOPEN_SOURCE to get IOV_MAX */

#include "mcdb.h"
#include "mcdb_makefmt.h"
#include "mcdb_error.h"
#include "mcdb_uint32.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>   /* open(), O_RDONLY */
#include <stdbool.h> /* bool */
#include <stdio.h>   /* printf(), IOV_MAX */
#include <stdlib.h>  /* malloc(), free(), EXIT_SUCCESS */
#include <string.h>  /* strlen() */
#include <unistd.h>  /* STDIN_FILENO, STDOUT_FILENO */
#include <sys/uio.h> /* writev() */
#include <libgen.h>  /* basename() */
#include <limits.h>  /* SSIZE_MAX */

static bool
writev_loop(const int fd, struct iovec * restrict iov, int iovcnt, ssize_t sz)
{
    /* Note: unlike writev(), this routine might modify the iovecs */
    ssize_t len;
    while (iovcnt && (len = writev(fd, iov, iovcnt)) != -1) {
        if ((sz -= len) == 0)
            return true;
        while (len != 0) {
            if (len >= iov->iov_len) {
                len -= iov->iov_len;
                --iovcnt;
                ++iov;
            }
            else {
                iov->iov_len -= len;
                iov->iov_base = ((char *)(iov->iov_base)) + len;
            }
        }
    }
    return (iovcnt == 0);
}

/* read and dump data section of mcdb
 * start of hash tables is end of key/data section (eod)
 * subtract 7 for maximal added alignment between data and hash table
 * (each record key,len,keystr,datastr is at least 8 bytes) */
static int
mcdbctl_dump(struct mcdb * const restrict m)
{
    unsigned char * restrict p = m->map->ptr;
    uint32_t klen;
    uint32_t dlen;
    unsigned char * const eod =
      p + mcdb_uint32_unpack_bigendian_aligned_macro(p) - 7;
    int    iovcnt = 0;
    size_t iovlen = 0;
    size_t buflen = 0;
    enum { MCDB_IOVNUM = IOV_MAX };/* assert(IOV_MAX >= 7) */
    struct iovec iov[MCDB_IOVNUM]; /* each db entry uses 7 iovecs */
    char buf[(MCDB_IOVNUM * 3)];   /* each db entry might use (2) * 10 chars */
      /* oversized buffer since all num strings must add up to less than max */

    for (p += MCDB_HEADER_SZ; p < eod; p += 8+klen+dlen) {

        klen = mcdb_uint32_unpack_bigendian_macro(p);
        dlen = mcdb_uint32_unpack_bigendian_macro(p+4);

        /* avoid printf("%.*s\n",...) due to mcdb arbitrary binary data */
        /* klen, dlen each limited to (2GB - 8); space for extra tokens exists*/
        if (iovlen + klen + 5 > SSIZE_MAX || iovcnt + 7 > MCDB_IOVNUM) {
            if (!writev_loop(STDOUT_FILENO, iov, iovcnt, (ssize_t)iovlen))
                return MCDB_ERROR_WRITE;
            iovcnt = 0;
            iovlen = 0;
            buflen = 0;
        }

        iov[iovcnt].iov_base = "+";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = buf+buflen;
        buflen += iov[iovcnt].iov_len = snprintf(buf+buflen, 11, "%u", klen);
        ++iovcnt;

        iov[iovcnt].iov_base = ",";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = buf+buflen;
        buflen += iov[iovcnt].iov_len = snprintf(buf+buflen, 11, "%u", dlen);
        ++iovcnt;

        iov[iovcnt].iov_base = ":";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iov[iovcnt].iov_base = p+8;
        iov[iovcnt].iov_len  = klen;
        ++iovcnt;

        iov[iovcnt].iov_base = "->";
        iov[iovcnt].iov_len  = 2;
        ++iovcnt;

        iovlen += klen + 5;

        if (iovlen + dlen + 1 > SSIZE_MAX) {
            if (!writev_loop(STDOUT_FILENO, iov, iovcnt, (ssize_t)iovlen))
                return MCDB_ERROR_WRITE;
            iovcnt = 0;
            iovlen = 0;
            buflen = 0;
        }

        iov[iovcnt].iov_base = p+8+klen;
        iov[iovcnt].iov_len  = dlen;
        ++iovcnt;

        iov[iovcnt].iov_base = "\n";
        iov[iovcnt].iov_len  = 1;
        ++iovcnt;

        iovlen += dlen + 1;

    }

    /* write out iovecs and append blank line ("\n") to indicate end of data */
    return (writev_loop(STDOUT_FILENO, iov, iovcnt, (ssize_t)iovlen)
            && write(STDOUT_FILENO, "\n", 1) == 1)
      ? EXIT_SUCCESS
      : MCDB_ERROR_WRITE;
}

/* Note: mcdbctl_stats() is equivalent test to pass/fail of djb cdbtest */
static int
mcdbctl_stats(struct mcdb * const restrict m)
{
    unsigned char * const map_ptr = m->map->ptr;
    unsigned char *p;
    uint32_t klen;
    uint32_t dlen;
    unsigned char * const eod =
      map_ptr + mcdb_uint32_unpack_bigendian_aligned_macro(map_ptr) - 7;
    unsigned long nrec = 0;
    unsigned long numd[11] = { 0,0,0,0,0,0,0,0,0,0,0 };
    unsigned int rv;
    bool rc;
    for (p = map_ptr+MCDB_HEADER_SZ; p < eod; p += klen+dlen) {
        klen = mcdb_uint32_unpack_bigendian_macro(p);
        dlen = mcdb_uint32_unpack_bigendian_macro(p+4);
        /* Search for key,data and track number of tries before found.
         * Technically, passing m (which contains m->map->ptr) and an
         * alias into the map (p+8) as key is in violation of C99 restrict
         * pointers, but is inconsequential since it is all read-only */
        p += 8;
        if ((rc = mcdb_findstart(m, (char *)p, klen))) {
            do { rc = mcdb_findnext(m, (char *)p, klen);
            } while (rc && map_ptr+m->dpos != p+klen);
        }
        if (!rc) return MCDB_ERROR_READFORMAT;
        ++nrec;
        ++numd[ ((m->loop < 11) ? m->loop - 1 : 10) ];
    }
    printf("records %lu\n", nrec);
    for (rv = 0; rv < 10; ++rv)
        printf("d%d      %lu\n", rv, numd[rv]);
    printf(">9      %lu\n", numd[10]);
    fflush(stdout);
    return EXIT_SUCCESS;
}

static int
mcdbctl_getseq(struct mcdb * const restrict m,
               const char * const restrict key, unsigned long seq)
{
    const size_t klen = strlen(key);
    struct iovec iov[2];
    if (mcdb_findstart(m, key, klen)) {
        bool rc;
        while ((rc = mcdb_findnext(m, key, klen)) && seq--)
            ;
        if (rc) {
            /* avoid printf("%.*s\n",...) due to mcdb arbitrary binary data */
            iov[0].iov_base = mcdb_dataptr(m);
            iov[0].iov_len  = mcdb_datalen(m);
            iov[1].iov_base = "\n";
            iov[1].iov_len  = 1;
            return writev_loop(STDOUT_FILENO,iov,2,(ssize_t)(iov[0].iov_len+1))
              ? EXIT_SUCCESS
              : MCDB_ERROR_WRITE;
        }
    }
    return EXIT_FAILURE;
}

static int
mcdbctl_query(const int argc, char ** restrict argv)
{
    struct mcdb m;
    struct mcdb_mmap map;
    int rv;
    int fd;
    unsigned long seq = 0;
    enum { MCDBCTL_BAD_QUERY_TYPE, MCDBCTL_GET, MCDBCTL_DUMP, MCDBCTL_STATS }
      query_type = MCDBCTL_BAD_QUERY_TYPE;

    /* validate args  (query type string == argv[1]) */
    if (0 == strcmp(argv[1], "get")) {
        if (argc == 5) {
            char *endptr;
            seq = strtoul(argv[4], &endptr, 10);
            if (seq != ULONG_MAX && argv[4] != endptr && *endptr == '\0')
                query_type = MCDBCTL_GET;
{'fraction_non_alphanumeric': 0.13872499640236005, 'fraction_numerical': 0.017124766153403366, 'mean_word_length': 2.1318161333934205, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1711018', 'n_tokens_mistral': 5335, 'n_tokens_neox': 5283, 'n_words': 1256}