<commit_before>
/**
 * debug.c
 *
 * Debugging reports for Turbine
 * */

#include <assert.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "src/util/debug.h"

static bool  initialized = false;
static bool  enabled = true;
static char* buffer = NULL;
static const int buffer_size = 2*1024;

void
turbine_debug_init()
{
  if (initialized)
  {
    printf("turbine_debug: already initialized\n");
    return;
  }
  initialized = true;
  char* s = getenv("TURBINE_DEBUG");
  if (s != NULL)
    if (strcmp(s, "0") == 0)
    {
      enabled = false;
      return;
    }
  buffer = malloc(buffer_size);
}

/**
   Used only for snprintf checks in turbine_debug
*/
#define BUFFER_SIZE_CHECK(count, buffer_size)                           \
  if (count >= buffer_size)                                             \
    printf("turbine_debug: message exceeded buffer_size (%i)\n",        \
           buffer_size);

/**
   All turbine_debug messages may be disabled by setting
   DEBUG=0 (number 0) in the environment.
   We have to put everything into one string before we print it,
   otherwise mpiexec -l does not print the rank things [0]
   correctly.
*/
__attribute__ ((format (printf, 2, 3)))
void
turbine_debug(const char* token, const char* format, ...)
{
  if (!enabled)
    return;
  assert(initialized);

  va_list va;
  va_start(va, format);
  int count = 0;
  count += sprintf(buffer, "%s: ", token);
  count += vsnprintf(buffer+count, buffer_size-count, format, va);
  BUFFER_SIZE_CHECK(count, buffer_size);
  count += snprintf(buffer+count, buffer_size-count, "\n");
  BUFFER_SIZE_CHECK(count, buffer_size);
  printf("%s", buffer);
  fflush(stdout);
  va_end(va);
}

void
turbine_debug_finalize()
{
  if (buffer)
    free(buffer);
}<commit_msg>Better debug buffer size mgmt<commit_after>
/**
 * debug.c
 *
 * Debugging reports for Turbine
 * */

#include <assert.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <memory.h>
#include <tools.h>

#include "src/util/debug.h"

static bool  initialized = false;
static bool  enabled = true;
static char* buffer = NULL;
static int buffer_size;

void
turbine_debug_init()
{
  if (initialized)
  {
    printf("turbine_debug: already initialized\n");
    return;
  }
  initialized = true;
  char* s = getenv("TURBINE_DEBUG");
  if (s != NULL)
    if (strcmp(s, "0") == 0)
    {
      enabled = false;
      return;
    }
  buffer_size = 32*KB;
  buffer = malloc(buffer_size);
}

/**
   Used only for snprintf checks in turbine_debug
*/
#define BUFFER_SIZE_CHECK(count)                     \
  if (count >= buffer_size) {                        \
    printf("turbine_debug: "                         \
           "message exceeded buffer_size (%i/%i)\n", \
           count, buffer_size);               \
    buffer[buffer_size-1] = '\0';                    \
    printf("buffer: %s\n", buffer);  \
}

/**
   All turbine_debug messages may be disabled by setting
   DEBUG=0 (number 0) in the environment.
   We have to put everything into one string before we print it,
   otherwise mpiexec -l does not print the rank things [0]
   correctly.
*/
__attribute__ ((format (printf, 2, 3)))
void
turbine_debug(const char* token, const char* format, ...)
{
  if (!enabled)
    return;
  assert(initialized);

  va_list va;
  va_start(va, format);
  int count = 0;
  count += sprintf(buffer, "%s: ", token);
  count += vsnprintf(buffer+count, buffer_size-count, format, va);
  BUFFER_SIZE_CHECK(count);
  count += snprintf(buffer+count, buffer_size-count, "\n");
  BUFFER_SIZE_CHECK(count);
  printf("%s", buffer);
  fflush(stdout);
  va_end(va);
}

void
turbine_debug_finalize()
{
  if (buffer)
    free(buffer);
}
{'fraction_non_alphanumeric': 0.13442709711563905, 'fraction_numerical': 0.006615506747816882, 'mean_word_length': 2.921161825726141, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6304632', 'n_tokens_mistral': 1309, 'n_tokens_neox': 1281, 'n_words': 345}