<commit_before>/*
 * apply filter file to all files in a newsgroup
 *
 * Written by <NAME> <<EMAIL>>.
 * Copyright 1999.
 *
 * Modified by <NAME> <<EMAIL>>
 * Copyright of modifications 2000 - 2007.
 *
 * See README for restrictions on the use of this software.
 */

#include "leafnode.h"
#include "critmem.h"
#include "mastring.h"
#include "ln_log.h"
#include "msgid.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <ctype.h>
#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <utime.h>
#include <fcntl.h>

#ifdef WITH_DMALLOC
#include <dmalloc.h>
#endif

extern char *optarg;
extern int optind, opterr, optopt;

static void
usage(void)
{
    fprintf(stderr,
	    "Usage:\n"
	    "applyfilter [options] object [...]\n"
	    "options are:\n"
	    GLOBALOPTLONGHELP
	    );
    fprintf(stderr,
	    "    -c             - check mode, print if filters match files on command line\n"
	    "    -n             - dry run, do not actually delete anything\n"
	    );
}

/* remove LF in LF+whitespace sequences, in place */
static void unfold(char *p)
{
    char *s, *q;

    q = p;
    s = p + strlen(p);
    while (p <= s) {
	if (p[0] != '\n' || p + 1 > s || !isspace((unsigned char)p[1])) {
	    *q++ = *p++;
	} else {
	    p++;
	}
    }
}

/* returns 0 for success */
static int applyfilter(const char *name, struct newsgroup *g,
	const struct filterlist *myfilter, int dryrun,
	unsigned long *kept, unsigned long *deleted)
{
    static size_t lsize = MAXHEADERSIZE + 1;
    static char *l;
    struct stat st;
    int score, fd;
    struct utimbuf u;
    unsigned long n;

    l = (char *)critmalloc(lsize, "Space for article");

    if (stat(name, &st)) {
	ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
		"cannot stat file \"%s\" in newsgroup %s: %m",
		g->name, name);
	return 0;
    }

    /* remove truncated files */
    if (st.st_size == 0)
	score = TRUE;

    if (!S_ISREG(st.st_mode)) {
	ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
		"not a regular file in newsgroup %s: %s",
		g->name, name);
	return 0;
    }

    if((fd = open(name, O_RDONLY)) >= 0)
    {
	int ret;

	/* read and unfold headers */
	ret = readheaders(fd, name, &l, &lsize, "\n\n");
	if (ret != -1)
	    unfold(l);

	switch (ret) {
	    case 0:
		score = killfilter(myfilter, l);
		break;
	    case -1:
		score = TRUE;
		break;
	    case -2: /* article has no body */
		if (delaybody_group(g->name))
		    score = killfilter(myfilter, l);
		else
		    score = TRUE;
		break;
	    default:
		/*@notreached@*/
		score = FALSE;
	}
	close(fd);

	if (score) {
	    if (!dryrun) {
		char *msgid = NULL;
		if (strlen(l)) {
		    msgid = mgetheader("Message-ID:", l);
		    if (msgid) {
			delete_article(msgid, "applyfilter", "filtered", 0);
		    } else {
			ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
				"%s: Article %s has no Message-ID header",
			       g->name, name);
			unlink(name);
		    }
		} else {
		    unlink(name);
		}
	    } else { /* !dryrun */
		if (verbose) {
		    printf("%s would be deleted\n", name);
		}
	    }
	    (*deleted)++;
	} else {
	    /* restore atime and mtime to keep texpire
	     * functionality intact */
	    u.actime = st.st_atime;
	    u.modtime = st.st_mtime;
	    utime(name, &u);
	    (*kept) ++;
	}

	n = strtoul(name, NULL, 10);
	if (n) {
	    if (n < g->first)
		g->first = n;
	    if (n > g->last)
		g->last = n;
	}
    } else /* if (fd = open) >= 0 */
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"could not open file \"%s\" in newsgroup %s\n",
		name, g->name);
    return 0;
}


static int
setupfilter(struct filterlist **myfilter, const char *ng) {
    if ((*myfilter = selectfilter(ng)) == NULL) {
	printf("Nothing to filter -- no filter for %s found.\n", ng);
	return 0;
    }
    return 1;
}<commit_msg>Plug memory leak in applyfilter().<commit_after>/*
 * apply filter file to all files in a newsgroup
 *
 * Written by <NAME> <<EMAIL>>.
 * Copyright 1999.
 *
 * Modified by <NAME> <<EMAIL>>
 * Copyright of modifications 2000 - 2007.
 *
 * See README for restrictions on the use of this software.
 */

#include "leafnode.h"
#include "critmem.h"
#include "mastring.h"
#include "ln_log.h"
#include "msgid.h"

#include <sys/stat.h>
#include <sys/types.h>
#include <ctype.h>
#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <utime.h>
#include <fcntl.h>

#ifdef WITH_DMALLOC
#include <dmalloc.h>
#endif

extern char *optarg;
extern int optind, opterr, optopt;

static void
usage(void)
{
    fprintf(stderr,
	    "Usage:\n"
	    "applyfilter [options] object [...]\n"
	    "options are:\n"
	    GLOBALOPTLONGHELP
	    );
    fprintf(stderr,
	    "    -c             - check mode, print if filters match files on command line\n"
	    "    -n             - dry run, do not actually delete anything\n"
	    );
}

/* remove LF in LF+whitespace sequences, in place */
static void unfold(char *p)
{
    char *s, *q;

    q = p;
    s = p + strlen(p);
    while (p <= s) {
	if (p[0] != '\n' || p + 1 > s || !isspace((unsigned char)p[1])) {
	    *q++ = *p++;
	} else {
	    p++;
	}
    }
}

/* returns 0 for success */
static int applyfilter(const char *name, struct newsgroup *g,
	const struct filterlist *myfilter, int dryrun,
	unsigned long *kept, unsigned long *deleted)
{
    static size_t lsize = MAXHEADERSIZE + 1;
    char *l;
    struct stat st;
    int score, fd;
    struct utimbuf u;
    unsigned long n;

    if (stat(name, &st)) {
	ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
		"cannot stat file \"%s\" in newsgroup %s: %m",
		g->name, name);
	return 0;
    }

    /* remove truncated files */
    if (st.st_size == 0)
	score = TRUE;

    if (!S_ISREG(st.st_mode)) {
	ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
		"not a regular file in newsgroup %s: %s",
		g->name, name);
	return 0;
    }

    l = (char *)critmalloc(lsize, "Space for article");

    if((fd = open(name, O_RDONLY)) >= 0)
    {
	int ret;

	/* read and unfold headers */
	ret = readheaders(fd, name, &l, &lsize, "\n\n");
	if (ret != -1)
	    unfold(l);

	switch (ret) {
	    case 0:
		score = killfilter(myfilter, l);
		break;
	    case -1:
		score = TRUE;
		break;
	    case -2: /* article has no body */
		if (delaybody_group(g->name))
		    score = killfilter(myfilter, l);
		else
		    score = TRUE;
		break;
	    default:
		/*@notreached@*/
		score = FALSE;
	}
	close(fd);

	if (score) {
	    if (!dryrun) {
		char *msgid = NULL;
		if (strlen(l)) {
		    msgid = mgetheader("Message-ID:", l);
		    if (msgid) {
			delete_article(msgid, "applyfilter", "filtered", 0);
		    } else {
			ln_log(LNLOG_SNOTICE, LNLOG_CARTICLE,
				"%s: Article %s has no Message-ID header",
			       g->name, name);
			unlink(name);
		    }
		} else {
		    unlink(name);
		}
	    } else { /* !dryrun */
		if (verbose) {
		    printf("%s would be deleted\n", name);
		}
	    }
	    (*deleted)++;
	} else {
	    /* restore atime and mtime to keep texpire
	     * functionality intact */
	    u.actime = st.st_atime;
	    u.modtime = st.st_mtime;
	    utime(name, &u);
	    (*kept) ++;
	}

	n = strtoul(name, NULL, 10);
	if (n) {
	    if (n < g->first)
		g->first = n;
	    if (n > g->last)
		g->last = n;
	}
    } else /* if (fd = open) >= 0 */
	ln_log(LNLOG_SERR, LNLOG_CARTICLE,
		"could not open file \"%s\" in newsgroup %s\n",
		name, g->name);

    free(l);

    return 0;
}


static int
setupfilter(struct filterlist **myfilter, const char *ng) {
    if ((*myfilter = selectfilter(ng)) == NULL) {
	printf("Nothing to filter -- no filter for %s found.\n", ng);
	return 0;
    }
    return 1;
}
{'fraction_non_alphanumeric': 0.16108980293611955, 'fraction_numerical': 0.008464488824229599, 'mean_word_length': 2.5535714285714284, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4387661', 'n_tokens_mistral': 3151, 'n_tokens_neox': 2870, 'n_words': 822}