<commit_before>#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "dir.h"

static int skip(char *dirname)
{
    return (dirname[0] == '.') &&
        ((dirname[1] == '\0') || (dirname[1] == '.' && dirname[2] == '\0'));
}

static void walk(char *name, int opts, void (*func)(char *, void *), void *ctx)
{
    DIR *dir = opendir(name);
	char* endp = name + strlen(name);
    struct dirent *d;

    if (!dir) return;

	*endp++ = '/';

    while ((d = readdir(dir)) != NULL) {
        if (skip(d->d_name)) continue;
        if (d->d_name[0] == '.' && !(opts & DW_HIDDEN)) continue;

        strcpy(endp, d->d_name);

		if (d->d_type == DT_DIR) {
           if (opts & DW_DIRECTORIES)
			   func(name, ctx);
           walk(name, opts, func, ctx);
		}
		else if (opts & DW_FILES) {
			func(name, ctx);
        }
    }
    closedir(dir);
}

void dirwalk(char *dirname, int opts, void (*func)(char *, void *), void *ctx)
{
	char* buf = malloc(FILENAME_MAX);
	strcpy(buf, dirname);
	walk(buf, opts, func, ctx);
}<commit_msg>Free allocated memory in dirwalk<commit_after>#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "dir.h"

static int skip(char *dirname)
{
    return (dirname[0] == '.') &&
        ((dirname[1] == '\0') || (dirname[1] == '.' && dirname[2] == '\0'));
}

static void walk(char *name, int opts, void (*func)(char *, void *), void *ctx)
{
    DIR *dir = opendir(name);
	char* endp = name + strlen(name);
    struct dirent *d;

    if (!dir) return;

	*endp++ = '/';

    while ((d = readdir(dir)) != NULL) {
        if (skip(d->d_name)) continue;
        if (d->d_name[0] == '.' && !(opts & DW_HIDDEN)) continue;

        strcpy(endp, d->d_name);

		if (d->d_type == DT_DIR) {
           if (opts & DW_DIRECTORIES)
			   func(name, ctx);
           walk(name, opts, func, ctx);
		}
		else if (opts & DW_FILES) {
			func(name, ctx);
        }
    }
    closedir(dir);
}

void dirwalk(char *dirname, int opts, void (*func)(char *, void *), void *ctx)
{
	char* buf = malloc(FILENAME_MAX);
	strcpy(buf, dirname);
	walk(buf, opts, func, ctx);
	free(buf);
}
{'fraction_non_alphanumeric': 0.20710612435717626, 'fraction_numerical': 0.0065451145395044414, 'mean_word_length': 2.8979963570127505, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4051241', 'n_tokens_mistral': 866, 'n_tokens_neox': 820, 'n_words': 201}