<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

#include "sst.h"

#define test(fn) \
  fputs("\n\x1b[34m# " # fn "\x1b[0m\n", stderr); \
  fn();

#define t_ok(pred, msg) do {                       \
  fprintf(stderr, "  - \x1b[90m" msg "\x1b[0m\n"); \
  assert(pred && msg);                             \
} while(0)

#define t_equal_str(s1, s2, msg) t_ok(strcmp((s1), (s2)) == 0, msg)

char *strtoupper(char *s) {
  char *cp, *p;
  p = cp = strdup(s);
  while((*p = toupper(*p))) p++;
  return cp;
}

void upper_onwrite(sst_t* self, sst_chunk_t* chunk) {
  char *s;
  sst_chunk_t *chunk_out;

  s = (char*)chunk->data;

  chunk_out = sst_chunk_new(strtoupper(s), NULL);
   // TODO: on data should copy string instead
  self->emit(self, chunk_out);
}

static const int max_emit = 2;
static char *emit_data[max_emit];
static int  emit_count = 0;
static int  end_called = 0;

void setup() {
  int i;
  for (i = 0; i < max_emit; i++) {
    emit_data[i] = '\0';
  }
  emit_count = 0;
  end_called = 0;
}

void ondata(sst_t* self, sst_chunk_t* chunk) {
  emit_data[emit_count++] = chunk->data;
  sst_chunk_free(chunk);
}

void onend(sst_t* self) {
  end_called++;
  sst_free(self);<commit_msg>fixing to compile without errors on linux<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>

#include "sst.h"

#ifndef strdup
extern char *strdup(const char *str);
#endif

#define test(fn) \
  fputs("\n\x1b[34m# " # fn "\x1b[0m\n", stderr); \
  fn();

#define t_ok(pred, msg) do {                       \
  fprintf(stderr, "  - \x1b[90m" msg "\x1b[0m\n"); \
  assert(pred && msg);                             \
} while(0)

#define t_equal_str(s1, s2, msg) t_ok(strcmp((s1), (s2)) == 0, msg)

char *strtoupper(char *s) {
  char *cp, *p;
  p = cp = (char*)strdup(s);
  while((*p = toupper(*p))) p++;
  return cp;
}

void upper_onwrite(sst_t* self, sst_chunk_t* chunk) {
  char *s;
  sst_chunk_t *chunk_out;

  s = (char*)chunk->data;

  chunk_out = sst_chunk_new(strtoupper(s), NULL);
   // TODO: on data should copy string instead
  self->emit(self, chunk_out);
}

static const int max_emit = 2;
static char **emit_data;
static int  emit_count = 0;
static int  end_called = 0;

void setup() {
  int i;
  for (i = 0; i < max_emit; i++) {
    //free(emit_data[i]);
  }
  emit_count = 0;
  end_called = 0;
  emit_data = malloc(1000);
}

void ondata(sst_t* self, sst_chunk_t* chunk) {
  emit_data[emit_count++] = chunk->data;
  sst_chunk_free(chunk);
}

void onend(sst_t* self) {
  end_called++;
  sst_free(self);
{'fraction_non_alphanumeric': 0.17544539116963595, 'fraction_numerical': 0.018977536793183578, 'mean_word_length': 3.2205882352941178, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7409965', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1106, 'n_words': 250}