<commit_before>#define _GNU_SOURCE
#undef NDEBUG

#include <assert.h>
#include <ctype.h>
#include <malloc.h>
#include <regex.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/random.h>

struct string { char* buf; size_t len; size_t alloc; };
typedef struct string string;

#define MAXGROUP 9
struct match_t {
    // State for next_match
    const char *regexp;
    int cflags;

    bool result;
    regmatch_t matches[MAXGROUP + 1];
};
typedef struct match_t match_t;

static int lineNumber;
static int inputFileIndex;

//
// String functions
//

static void ensure_len_discard(string* s, size_t n)
{
    if (n > s->alloc) {
        free(s->buf);
        s->buf = malloc(n);
        if (!s->buf) abort();
        s->alloc = malloc_usable_size(s->buf);
        s->len = 0;
    }
}

static size_t grow(size_t prev, size_t min)
{
    const size_t plus10 = prev + prev / 10;
    // Grow 10% or directly to the given size
    return min > plus10 ? min : plus10;
}

static void ensure_len(string* s, size_t n)
{
    if (n > s->alloc) {
        n = grow(s->alloc, n);
        s->buf = realloc(s->buf, n);
        if (!s->buf) abort();
        s->alloc = malloc_usable_size(s->buf);
    }
}

static void free_string(string* s)
{
    free(s->buf);
    s->alloc = s->len = 0;
    s->buf = 0;
}

static void store_cstr(string* dst, const char* src, size_t n)
{
    ensure_len_discard(dst, n);
    memcpy(dst->buf, src, n);
    dst->len = n;
}

static void append_str(string* dst, const char *str, size_t n)
{
    ensure_len(dst, dst->len + n);
    memcpy(dst->buf + dst->len, str, n);
    dst->len += n;
}

static void copy(string* dst, string* src)
{
    ensure_len_discard(dst, src->len);
    memcpy(dst->buf, src->buf, src->len);
    dst->len = src->len;
}

static void concat_newline(string* dst, string* a, string* b)
{
    const size_t n = a->len + 1 + b->len;
    ensure_len_discard(dst, n);

    char *p = mempcpy(dst->buf, a->buf, a->len);
    *p++ = '\n';
    memcpy(p, b->buf, b->len);
    dst->len = n;
}

static void concat(string* dst, string* a, string* b)
{
    const size_t n = a->len + b->len;
    ensure_len_discard(dst, n);

    char *p = mempcpy(dst->buf, a->buf, a->len);
    memcpy(p, b->buf, b->len);
    dst->len = n;
}

static void concat_inplace(string* dst, string* b)
{
    const size_t n = dst->len + b->len;
    ensure_len(dst, n);
    memcpy(dst->buf + dst->len, b->buf, b->len);
    dst->len = n;
}

static void substring(string* dst, string* src, size_t i1, size_t i2)
{
    assert(i1 <= src->len && i2 <= src->len);
    assert(i1 <= i2);
    const size_t n = i2 - i1;
    ensure_len_discard(dst, n);
    memcpy(dst->buf, src->buf + i1, n);
    dst->len = n;
}

static void trans(string* dst, const char* from, const char* to, string* src)
{
    ensure_len_discard(dst, src->len);
    assert(strlen(to) >= strlen(from));
    for (size_t i = 0; i < src->len; i++) {
        char c = src->buf[i];
        const char *p = strchr(from, c);
        if (p) {
            c = to[p - from];
        }
        dst->buf[i] = c;
    }
    dst->len = src->len;
}

static void random_string(string* dst)
{
    uint8_t temp[16];
    getrandom(temp, sizeof(temp), 0);

    ensure_len_discard(dst, 2 * sizeof(temp) + 1);
    for (size_t i = 0; i < sizeof(temp); i++) {
        snprintf(dst->buf + 2 * i, 3, "%02x", temp[i]);
    }
    dst->len = 2 * sizeof(temp);
}

static void format_literal(string* dst, int width, string* s)
{
    int col = 0;
    dst->len = 0;
    for (size_t i = 0; i < s->len; i++) {
        char c = s->buf[i];
        size_t prev = dst->len;
        if (c == '\n') {
            append_str(dst, "$\n", 2);
            col = 0;
        } else if (c == '\\') {
            append_str(dst, "\\\\", 2);
            col += 2;
        } else if (!isprint(c)) {
            // one more here to have room for a NUL
            append_str(dst, "\\ooo", 5);
            dst->len--;
            snprintf(dst->buf + dst->len - 3, 4, "%03o", c);
            col += 4;
        } else {
            append_str(dst, &c, 1);
            col++;
        }
        // TODO Should be breaking before output? I think the haskell also gets
        // this wrong if e.g. an octal escape would straddle a line break.
        if (col >= width - 1) {
            append_str(dst, "\\\n", 2);
            col = 0;
        }
    }
    append_str(dst, "$", 1);
}

static void format_int(string* dst, int i)
{
    char temp[32];
    snprintf(temp, sizeof(temp), "%d", i);
    store_cstr(dst, temp, strlen(temp));
}

//
// Regex functions
//

static void match_regexp(match_t* m, string* s, const char* regexp, int cflags,
                         size_t offset)
{
    memset(m, 0, sizeof(*m));

    if (offset >= s->len) {
        return;
    }

    regex_t re;
    int res = regcomp(&re, regexp, cflags);
    if (res) {
        fprintf(stderr, "regcomp: error %d in %s\n", res, regexp);
        abort();
    }

    m->regexp = regexp;
    m->cflags = cflags;
    m->matches[0].rm_so = offset;
    m->matches[0].rm_eo = s->len;
    res = regexec(&re, s->buf, MAXGROUP, m->matches, REG_STARTEND);
    if (res != 0 && res != REG_NOMATCH) {
        fprintf(stderr, "regexec: error %d in %s\n", res, regexp);
        abort();
    }
    regfree(&re);
    m->result = (res == 0);
}

static void copy_match(match_t* dst, match_t* src)
{
    memcpy(dst, src, sizeof(match_t));<commit_msg>Partial fix for compiled dc.sed<commit_after>#define _GNU_SOURCE
#undef NDEBUG

#include <assert.h>
#include <ctype.h>
#include <malloc.h>
#include <regex.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/random.h>

struct string { char* buf; size_t len; size_t alloc; };
typedef struct string string;

#define MAXGROUP 9
struct match_t {
    // State for next_match
    const char *regexp;
    int cflags;

    bool result;
    regmatch_t matches[MAXGROUP + 1];
};
typedef struct match_t match_t;

static int lineNumber;
static int inputFileIndex;

//
// String functions
//

static void ensure_len_discard(string* s, size_t n)
{
    if (n > s->alloc) {
        free(s->buf);
        s->buf = malloc(n);
        if (!s->buf) abort();
        s->alloc = malloc_usable_size(s->buf);
        s->len = 0;
    }
}

static size_t grow(size_t prev, size_t min)
{
    const size_t plus10 = prev + prev / 10;
    // Grow 10% or directly to the given size
    return min > plus10 ? min : plus10;
}

static void ensure_len(string* s, size_t n)
{
    if (n > s->alloc) {
        n = grow(s->alloc, n);
        s->buf = realloc(s->buf, n);
        if (!s->buf) abort();
        s->alloc = malloc_usable_size(s->buf);
    }
}

static void free_string(string* s)
{
    free(s->buf);
    s->alloc = s->len = 0;
    s->buf = 0;
}

static void store_cstr(string* dst, const char* src, size_t n)
{
    ensure_len_discard(dst, n);
    memcpy(dst->buf, src, n);
    dst->len = n;
}

static void append_str(string* dst, const char *str, size_t n)
{
    ensure_len(dst, dst->len + n);
    memcpy(dst->buf + dst->len, str, n);
    dst->len += n;
}

static void copy(string* dst, string* src)
{
    ensure_len_discard(dst, src->len);
    memcpy(dst->buf, src->buf, src->len);
    dst->len = src->len;
}

static void concat_newline(string* dst, string* a, string* b)
{
    const size_t n = a->len + 1 + b->len;
    ensure_len_discard(dst, n);

    char *p = mempcpy(dst->buf, a->buf, a->len);
    *p++ = '\n';
    memcpy(p, b->buf, b->len);
    dst->len = n;
}

static void concat(string* dst, string* a, string* b)
{
    const size_t n = a->len + b->len;
    ensure_len_discard(dst, n);

    char *p = mempcpy(dst->buf, a->buf, a->len);
    memcpy(p, b->buf, b->len);
    dst->len = n;
}

static void concat_inplace(string* dst, string* b)
{
    const size_t n = dst->len + b->len;
    ensure_len(dst, n);
    memcpy(dst->buf + dst->len, b->buf, b->len);
    dst->len = n;
}

static void substring(string* dst, string* src, size_t i1, size_t i2)
{
    assert(i1 <= src->len && i2 <= src->len);
    assert(i1 <= i2);
    const size_t n = i2 - i1;
    ensure_len_discard(dst, n);
    memcpy(dst->buf, src->buf + i1, n);
    dst->len = n;
}

static void trans(string* dst, const char* from, const char* to, string* src)
{
    ensure_len_discard(dst, src->len);
    assert(strlen(to) >= strlen(from));
    for (size_t i = 0; i < src->len; i++) {
        char c = src->buf[i];
        const char *p = strchr(from, c);
        if (p) {
            c = to[p - from];
        }
        dst->buf[i] = c;
    }
    dst->len = src->len;
}

static void random_string(string* dst)
{
    uint8_t temp[16];
    getrandom(temp, sizeof(temp), 0);

    ensure_len_discard(dst, 2 * sizeof(temp) + 1);
    for (size_t i = 0; i < sizeof(temp); i++) {
        snprintf(dst->buf + 2 * i, 3, "%02x", temp[i]);
    }
    dst->len = 2 * sizeof(temp);
}

static void format_literal(string* dst, int width, string* s)
{
    int col = 0;
    dst->len = 0;
    for (size_t i = 0; i < s->len; i++) {
        char c = s->buf[i];
        size_t prev = dst->len;
        if (c == '\n') {
            append_str(dst, "$\n", 2);
            col = 0;
        } else if (c == '\\') {
            append_str(dst, "\\\\", 2);
            col += 2;
        } else if (!isprint(c)) {
            // one more here to have room for a NUL
            append_str(dst, "\\ooo", 5);
            dst->len--;
            snprintf(dst->buf + dst->len - 3, 4, "%03o", c);
            col += 4;
        } else {
            append_str(dst, &c, 1);
            col++;
        }
        // TODO Should be breaking before output? I think the haskell also gets
        // this wrong if e.g. an octal escape would straddle a line break.
        if (col >= width - 1) {
            append_str(dst, "\\\n", 2);
            col = 0;
        }
    }
    append_str(dst, "$", 1);
}

static void format_int(string* dst, int i)
{
    char temp[32];
    snprintf(temp, sizeof(temp), "%d", i);
    store_cstr(dst, temp, strlen(temp));
}

//
// Regex functions
//

static void match_regexp(match_t* m, string* s, const char* regexp, int cflags,
                         size_t offset)
{
    memset(m, 0, sizeof(*m));

    // Stop matching when we've consumed the whole string, but allow a zero
    // length match if it comes first?
    if (offset && offset >= s->len) {
        return;
    }

    regex_t re;
    int res = regcomp(&re, regexp, cflags);
    if (res) {
        fprintf(stderr, "regcomp: error %d in %s\n", res, regexp);
        abort();
    }

    m->regexp = regexp;
    m->cflags = cflags;
    m->matches[0].rm_so = offset;
    m->matches[0].rm_eo = s->len;
    res = regexec(&re, s->buf, MAXGROUP, m->matches, REG_STARTEND);
    if (res != 0 && res != REG_NOMATCH) {
        fprintf(stderr, "regexec: error %d in %s\n", res, regexp);
        abort();
    }
    regfree(&re);
    m->result = (res == 0);
}

static void copy_match(match_t* dst, match_t* src)
{
    memcpy(dst, src, sizeof(match_t));
{'fraction_non_alphanumeric': 0.17149562234858742, 'fraction_numerical': 0.011372867587327376, 'mean_word_length': 2.2839359810314166, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 201, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3539657', 'n_tokens_mistral': 4283, 'n_tokens_neox': 4153, 'n_words': 1142}