<commit_before>/* Copyright 2010  Glue Logic LLC
 * License: GPLv3
 */

#define _POSIX_C_SOURCE 200112L
/* _GNU_SOURCE needed for mremap() */
#define _GNU_SOURCE
/* large file support needed for ftruncate() on cdb larger than 2 GB */
#define _FILE_OFFSET_BITS 64
/* GPS: TODO add large file support for other operating systems */

#include "mcdb_make.h"
#include "mcdb.h"
#include "mcdb_uint32.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>  /* memcpy() */
#include <stdint.h>  /* uint32_t */
#include <limits.h>  /* UINT_MAX, INT_MAX */

#define MCDB_HPLIST 1000

struct mcdb_hp { uint32_t h; uint32_t p; };

struct mcdb_hplist {
  struct mcdb_hp hp[MCDB_HPLIST];
  struct mcdb_hplist *next;
  uint32_t num;
};

static struct mcdb_hplist *  __attribute__((noinline))
mcdb_hplist_alloc(struct mcdb_make * const restrict m)
{
    struct mcdb_hplist * const restrict head =
      (struct mcdb_hplist *) m->fn_malloc(sizeof(struct mcdb_hplist));
    if (!head) return NULL;
    head->num  = 0;
    head->next = m->head;
    return (m->head = head);
}

/* (compilation with large file support enables off_t max > 2 GB in cast) */
static int  __attribute__((noinline))
mcdb_ftruncate_nointr(const int fd, const size_t sz)
{ int r; do{r=ftruncate(fd,(off_t)sz);}while(r != 0 && errno==EINTR);return r; }

static bool  __attribute__((noinline))
mcdb_mmap_resize(struct mcdb_make * const restrict m, const size_t sz)
{
    size_t fsz = (m->fsz < sz ? m->fsz : sz);
    void *x;

    /* assert(sz <= UINT_MAX); *//* caller must check or else infinite loop */

    while (fsz < sz)
        fsz = (fsz < (UINT_MAX>>2) ? (fsz << 2) : UINT_MAX);

    if (mcdb_ftruncate_nointr(m->fd, fsz) != 0) return false;

  #ifdef __GLIBC__

    x = mremap(m->map, m->fsz, fsz, MREMAP_MAYMOVE); /*fsz page-sized multiple*/
    if (x == MAP_FAILED) return false;

  #else

  #if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES-0 \
   && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO-0
    if (msync(m->map, m->pos, MS_SYNC) != 0) return false;
  #endif
    if (munmap(m->map, m->fsz) != 0) return false;
    x = mmap(0,fsz,PROT_WRITE,MAP_SHARED,fd,0);      /*fsz page-sized multiple*/
    if (x == MAP_FAILED) return false;
    posix_madvise(x, fsz, POSIX_MADV_SEQUENTIAL);

  #endif

    m->map = (char *)x;
    m->fsz = fsz;
    return true;
}

static bool  inline
mcdb_mmap_commit(struct mcdb_make * const restrict m)
{
    if (mcdb_ftruncate_nointr(m->fd, m->pos) != 0) return false;

  #if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES-0 \
   && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO-0
    if (msync(m->map, m->pos, MS_SYNC) != 0) return false;
  #endif<commit_msg>more incremental resource allocation when creating mcdb > 1 GB<commit_after>/* Copyright 2010  Glue Logic LLC
 * License: GPLv3
 */

#define _POSIX_C_SOURCE 200112L
/* _GNU_SOURCE needed for mremap() */
#define _GNU_SOURCE
/* large file support needed for ftruncate() on cdb larger than 2 GB */
#define _FILE_OFFSET_BITS 64
/* GPS: TODO add large file support for other operating systems */

#include "mcdb_make.h"
#include "mcdb.h"
#include "mcdb_uint32.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>  /* memcpy() */
#include <stdint.h>  /* uint32_t */
#include <limits.h>  /* UINT_MAX, INT_MAX */

#define MCDB_HPLIST 1000

struct mcdb_hp { uint32_t h; uint32_t p; };

struct mcdb_hplist {
  struct mcdb_hp hp[MCDB_HPLIST];
  struct mcdb_hplist *next;
  uint32_t num;
};

static struct mcdb_hplist *  __attribute__((noinline))
mcdb_hplist_alloc(struct mcdb_make * const restrict m)
{
    struct mcdb_hplist * const restrict head =
      (struct mcdb_hplist *) m->fn_malloc(sizeof(struct mcdb_hplist));
    if (!head) return NULL;
    head->num  = 0;
    head->next = m->head;
    return (m->head = head);
}

/* (compilation with large file support enables off_t max > 2 GB in cast) */
static int  __attribute__((noinline))
mcdb_ftruncate_nointr(const int fd, const size_t sz)
{ int r; do{r=ftruncate(fd,(off_t)sz);}while(r != 0 && errno==EINTR);return r; }

static bool  __attribute__((noinline))
mcdb_mmap_resize(struct mcdb_make * const restrict m, const size_t sz)
{
    size_t fsz = (m->fsz < sz ? m->fsz : sz);
    void *x;

    while (fsz < sz && fsz < (UINT_MAX>>2))
        fsz <<= 2;

    if (fsz < sz) { /* then sz >= (UINT_MAX>>2) */
        size_t incr = 1<<31;
        while (fsz > UINT_MAX-incr) incr >>= 1;
        do { fsz += incr; incr >>= 1; } while (fsz < sz && incr != 0);
        if (fsz < sz) { errno = EOVERFLOW; return false; }
    }

    if (mcdb_ftruncate_nointr(m->fd, fsz) != 0) return false;

  #ifdef __GLIBC__

    x = mremap(m->map, m->fsz, fsz, MREMAP_MAYMOVE); /*fsz page-sized multiple*/
    if (x == MAP_FAILED) return false;

  #else

  #if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES-0 \
   && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO-0
    if (msync(m->map, m->pos, MS_SYNC) != 0) return false;
  #endif
    if (munmap(m->map, m->fsz) != 0) return false;
    x = mmap(0,fsz,PROT_WRITE,MAP_SHARED,fd,0);      /*fsz page-sized multiple*/
    if (x == MAP_FAILED) return false;
    posix_madvise(x, fsz, POSIX_MADV_SEQUENTIAL);

  #endif

    m->map = (char *)x;
    m->fsz = fsz;
    return true;
}

static bool  inline
mcdb_mmap_commit(struct mcdb_make * const restrict m)
{
    if (m->fsz != m->pos
        && mcdb_ftruncate_nointr(m->fd, m->pos) != 0) return false;

  #if defined(_POSIX_MAPPED_FILES) && _POSIX_MAPPED_FILES-0 \
   && defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO-0
    if (msync(m->map, m->pos, MS_SYNC) != 0) return false;
  #endif
{'fraction_non_alphanumeric': 0.14322781525687597, 'fraction_numerical': 0.01660612350804359, 'mean_word_length': 3.681781376518219, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1139252', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2343, 'n_words': 567}