<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <util.h>

#ifndef UNUSED
#define UNUSED(x) (void)(x)
#endif

int main(int argc, char** argv) {
    UNUSED(argc);
    UNUSED(argv);

    return util_test();
}

typedef struct {
    char* input;
    char* expected;
} test;

int util_test() {
    test tests[] = {
        {"", ""},
        {"a", "a"},
        {"ab", "ab"},
        {"abcdefghijklmnopq", "abcdefghijklmnopq"},
    };
    int ntests = sizeof(tests) / sizeof(test);

    int i;
    test t;
    char* obtained;
    for (i=0; i<ntests; i++) {
        t = tests[i];
        obtained = strdup(t.input);
        if (obtained == t.input) {
            printf("util_test: test %d failed\n", i);
            printf("\tobtained and expected have the same mem address\n");<commit_msg>make the test generic, with test functions per functionality<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <util.h>

#ifndef UNUSED
#define UNUSED(x) (void)(x)
#endif

int test_util();

int main(int argc, char** argv) {
    UNUSED(argc);
    UNUSED(argv);

    return test_util();
}

typedef struct {
    char* input;
    char* expected;
} test;

int test_util() {
    test tests[] = {
        {"", ""},
        {"a", "a"},
        {"ab", "ab"},
        {"abcdefghijklmnopq", "abcdefghijklmnopq"},
    };
    int ntests = sizeof(tests) / sizeof(test);

    int i;
    test t;
    char* obtained;
    for (i=0; i<ntests; i++) {
        t = tests[i];
        obtained = strdup(t.input);
        if (obtained == t.input) {
            printf("util_test: test %d failed\n", i);
            printf("\tobtained and expected have the same mem address\n");
{'fraction_non_alphanumeric': 0.17028772753963595, 'fraction_numerical': 0.0011743981209630064, 'mean_word_length': 2.5352697095435683, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5642518', 'n_tokens_mistral': 625, 'n_tokens_neox': 606, 'n_words': 157}