<commit_before>/* **********************************************************
 * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
 * **********************************************************/

/* Dr. Memory: the memory debugger
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; 
 * version 2.1 of the License, and no later version.

 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Library General Public License for more details.

 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

/***************************************************************************
 * report.c: Dr. Memory error reporting
 */

#include "dr_api.h"
#include "drmemory.h"
#include "shadow.h"
#include "readwrite.h"
#include "syscall.h"
#include "alloc.h" 
#include "report.h"
#include "callstack.h"
#include "heap.h"
#include "alloc_drmem.h"
#ifdef LINUX
# include <errno.h>
#endif
#include <limits.h>

static uint error_id; /* errors + leaks */
static uint num_reported_errors;
static uint num_total_leaks;
static uint num_throttled_errors;
static uint num_throttled_leaks;
static uint num_leaks_ignored;
static size_t num_bytes_leaked;
static size_t num_bytes_possible_leaked;
static uint num_suppressions;
static uint num_suppressions_matched;
static uint num_suppressed_leaks;
static uint num_reachable_leaks;

static uint saved_throttled_leaks;
static uint saved_total_leaks;
static uint saved_leaks_ignored;
static uint saved_suppressed_leaks;
static uint saved_possible_leaks_total;
static uint saved_possible_leaks_unique;
static uint saved_reachable_leaks;
static uint saved_leaks_unique;
static uint saved_leaks_total;
static size_t saved_bytes_leaked;
static size_t saved_bytes_possible_leaked;

static uint64 timestamp_start;

/***************************************************************************/
/* Store all errors so we can eliminate duplicates (PR 484167) */

enum {
    ERROR_UNADDRESSABLE,
    ERROR_UNDEFINED,
    ERROR_INVALID_HEAP_ARG,
    ERROR_WARNING,
    ERROR_LEAK,
    ERROR_POSSIBLE_LEAK,
    ERROR_MAX_VAL,
};

static const char *const error_name[] = {
    "unaddressable access(es)",
    "uninitialized access(es)",
    "invalid heap argument(s)",
    "warning(s)",
    "leak(s)",    
    "possible leak(s)",    
};

static const char *const suppress_name[] = {
    "UNADDRESSABLE ACCESS",
    "UNINITIALIZED READ",
    "INVALID HEAP ARGUMENT",
    "WARNING",
    "LEAK",    
    "POSSIBLE LEAK",    
};

/* The error_lock protects these as well as error_table */
static uint num_unique[ERROR_MAX_VAL];
static uint num_total[ERROR_MAX_VAL];

/* Though any one instance of an address can have only one error
 * type, the same address could have multiple via different
 * executions.  Thus we must use a key combining the callstack and
 * the error type.
 */
typedef struct _stored_error_t {
    /* We can shrink some of these fields if memory is tight but we shouldn't
     * have more than a few thousand of these
     */
    uint id;
    uint errtype; /* from ERROR_ enum */
    uint count;
    bool suppressed;
    packed_callstack_t *pcs;
    /* We also keep a linked list so we can iterate in id order */
    struct _stored_error_t *next;
} stored_error_t;

#define ERROR_HASH_BITS 8
hashtable_t error_table;
/* We need an outer lock to synchronize stored_error_t data access.
 * Since we never remove from error_table we could instead have
 * a lock per stored_error_t but we save space, assuming errors
 * are rare enough to not be a bottleneck.
 */
static void *error_lock;
/* We also keep a linked list so we can iterate in id order, but composed
 * of hashtable payloads so no separate free is necessary.
 * Protected by error_lock.
 */
static stored_error_t *error_head;
static stored_error_t *error_tail;

/* Only initializes the errtype field */
stored_error_t *
stored_error_create(uint type)
{
    stored_error_t *err = global_alloc(sizeof(*err), HEAPSTAT_MISC);
    memset(err, 0, sizeof(*err));
    ASSERT(type < ERROR_MAX_VAL, "invalid error type");
    err->errtype = type;
    return err;
}

void
stored_error_free(stored_error_t *err)
{
    uint ref;
    ASSERT(err != NULL, "invalid arg");
    if (err->pcs != NULL) {
        ref = packed_callstack_free(err->pcs);
        ASSERT(ref == 0, "invalid ref count");
    }
    global_free(err, sizeof(*err), HEAPSTAT_MISC);
}

uint
stored_error_hash(stored_error_t *err)
{
    /* do NOT use id or count as they won't be filled out at lookup time */
    uint hash;
    ASSERT(err != NULL, "invalid arg");
    hash = packed_callstack_hash(err->pcs);
    hash ^= err->errtype;
    return hash;
}

bool
stored_error_cmp(stored_error_t *err1, stored_error_t *err2)
{
    /* do NOT use id or count as they won't be filled out at lookup time */
    ASSERT(err1 != NULL && err2 != NULL, "invalid arg");
    if (err1->errtype != err2->errtype)
        return false;
    return (packed_callstack_cmp(err1->pcs, err2->pcs));
}

/* A prefix for supplying additional info on a reported error beyond
 * the primary line, timestamp line, and callstack itself (from PR 535568)
 */
#define INFO_PFX IF_DRSYMS_ELSE("Note: ", "  info: ")

/***************************************************************************
 * suppression list
 */

/* For each error type, we have a list of callstacks, with each
 * callstack a variable-sized array of frames, each frame a string.
 */
typedef struct _suppress_spec_t {
    uint num_frames;
    char **frames; /* variable-sized array of strings */
    /* During initial reading it's easier to build a linked list rather than
     * add resizable array support.  We could convert to an array after
     * reading both suppress files, but we have pointers scattered all
     * over anyway so we leave it a list.
     */
    struct _suppress_spec_t *next;
} suppress_spec_t;

/* We suppress error type separately (PR 507837) */
static suppress_spec_t *supp_list[ERROR_MAX_VAL];
static uint supp_num[ERROR_MAX_VAL];

/* We are single-threaded when we read the suppression file */
#define BUFSIZE 4096
static char buf[BUFSIZE];

#ifdef USE_DRSYMS
static void *suppress_file_lock;
#endif

static int
get_suppress_type(char *line)
{
    int i;
    ASSERT(line != NULL, "invalid param");
    if (line[0] == '\0')
        return -1;
    /* Perf: we could stick the 6 names in a hashtable */
    for (i = 0; i < ERROR_MAX_VAL; i++) {
        if (strstr(line, suppress_name[i]) == line)
            return i;
    }
    return -1;
}

static void
report_malformed_suppression(int type, uint num_frames, const char **frames,
                             const char *message)
{
    char *supp = NULL;
    const char *type_string = suppress_name[type];
    int i, supp_size = 0, sofar = 0, len = 0;

    supp_size += strlen(type_string) + strlen(NL);
    for (i = 0; i < num_frames; i++)
        supp_size += strlen(frames[i]) + strlen(NL);
    supp_size++; /* +\0 */

    /* we can't use pt->errbuf since it happens pre-thread-init */
    supp = (char*)global_alloc(supp_size,  HEAPSTAT_MISC);
    BUFPRINT(supp, supp_size, sofar, len, "%s"NL, type_string);
    for (i = 0; i < num_frames; i++)
        BUFPRINT(supp, supp_size, sofar, len, "%s"NL, frames[i]);
    ASSERT(sofar + 1 == supp_size, "");
    NOTIFY("Malformed suppression:\n%s\n%s\n", supp, message);
    global_free(supp, supp_size, HEAPSTAT_MISC);
    usage_error("Malformed suppression. See the log file for the details", "");
}

static suppress_spec_t *
add_suppress_spec(int type, uint num_frames, char **frames)
{
    suppress_spec_t *spec;
    uint i;
    ASSERT(type >= 0 && type < ERROR_MAX_VAL, "internal error type error");
    ASSERT(num_frames > 0, "");
    if (strcmp(frames[num_frames-1], "...") == 0) {
        report_malformed_suppression(type, num_frames, (const char **) frames,
                                     "The given suppression ends with '...'");
        ASSERT(false, "should not reach here");
    }
    spec = (suppress_spec_t *) global_alloc(sizeof(*spec), HEAPSTAT_MISC);
    spec->num_frames = num_frames;
    spec->frames = (char **)
        global_alloc(num_frames*sizeof(char*), HEAPSTAT_MISC);
    LOG(2, "read suppression %d of type %s\n", num_suppressions,
        suppress_name[type]);
    for (i = 0; i < num_frames; i++) {
        LOG(2, "  frame %d: \"%s\"\n", i, frames[i]);
        spec->frames[i] = frames[i];
        frames[i] = NULL;
    }
    /* insert into list */
    spec->next = supp_list[type];
    supp_list[type] = spec;
    supp_num[type]++;
    num_suppressions++;
    return spec;
}

static void
read_suppression_file(file_t f)
{
    char *line, *newline = NULL;
    int bufread = 0, bufwant;
    int len;
    /* current callstack */
    int type, curtype = -1;
    bool has_symbolic_frames = false;
    /* even if a suppression callstack is longer than our max, we match
     * any prefix so we can truncate at the max.
     * the entries of this array are dynamically allocated and become
     * the final strings stored in supp_list.
     */
    char **frames = (char **)
        global_alloc(options.callstack_max_frames*sizeof(char*), HEAPSTAT_MISC);
    uint i, num_frames = 0;
    memset(frames, 0, options.callstack_max_frames*sizeof(char*));

    while (true) {
        if (newline == NULL) {
            bufwant = BUFSIZE-1;
            bufread = dr_read_file(f, buf, bufwant);
            ASSERT(bufread <= bufwant, "internal error reading suppression file");
            if (bufread <= 0)
                break;
            buf[bufread] = '\0';
            newline = strchr(buf, '\n');
            line = buf;
        } else {
            line = newline + 1;
            newline = strchr(line, '\n');
            if (newline == NULL) {
                /* shift 1st part of line to start of buf, then read in rest */
                /* the memory for the processed part can be reused  */
                bufwant = line - buf;
                ASSERT(bufwant <= bufread, "internal error reading suppression file");
                len = bufread - bufwant; /* what is left from last time */
                /* since strings may overlap, should use memmove, not strncpy */
                /* FIXME corner case: if len == 0, nothing to move */
                memmove(buf, line, len);
                bufread = dr_read_file(f, buf+len, bufwant);
                ASSERT(bufread <= bufwant, "internal error reading suppression file");
                if (bufread <= 0)
                    break;
                bufread += len; /* bufread is total in buf */
                buf[bufread] = '\0';
                newline = strchr(buf, '\n');
                line = buf;
            }
        }
        /* buffer is big enough to hold at least one line */
        ASSERT(newline != NULL, "internal error: suppression file malformed?");
        *newline = '\0';
        /* strip one or more carriage returns */
        len = 1;
        while (newline-len >= line && *(newline-len) == '\r') {
            *(newline-len) = '\0';
            len++;
        }
        LOG(3, "suppression file line: \"%s\"\n", line);
        /* Lines look like this:
         * UNINITIALIZED READ
         * <ADVAPI32.dll+0x3c0d>
         * # comment line; blank (newline) lines are allowed too
         * LEAK
         * <libc.so.6+0x2bc80>
         * <+0x2bc80>
         * <not in a module>
         *
         * Note: no leading white spaces.
         * Note: <+0x###> is only on esxi due to PR 363063;  it will go way once
         *       the bug is fixed
         *
         * For USE_DRSYMS, this client now also supports mod!func callstacks:
         * INVALID HEAP ARGUMENT
         * suppress.exe!invalid_free_test1
         * suppress.exe!test
         * suppress.exe!main
         */
        if (line[0] == '\0' || line[0] == '#')
            continue; /* Skip blank and comment lines. */
        type = get_suppress_type(line);
        if (type > -1) {
            if (curtype > -1) {
                /* the prior callstack completed successfully */
                if (IF_DRSYMS_ELSE(true, !has_symbolic_frames))
                    add_suppress_spec(curtype, num_frames, frames);
            }
            /* starting a new callstack */
            curtype = type;
            has_symbolic_frames = false;
            num_frames = 0;
        } else if (num_frames >= options.callstack_max_frames) {
            /* we truncate suppression callstacks to match requested max */
            LOG(1, "WARNING: requested max frames truncates suppression callstacks\n");
            /* just continue to next line */
            /* FIXME: should we abort? */
        } else {
            if (curtype == -1) {
                usage_error("malformed suppression: no error type on line ", line);
                ASSERT(false, "should not reach here");
            }

            if (/* Doesn't match mod!func or has '+' in the middle */
                !(strchr(line, '!') != NULL && strchr(line, '+') == NULL) &&
                /* Doesn't match <mod+off> or has '!' in the middle */
                !(line[0] == '<' && strchr(line, '+') != NULL &&
                  strchr(line, '>') != NULL && strchr(line, '!') == NULL) &&
                strcmp(line, "<not in a module>") != 0 &&
                strcmp(line, "...") != 0 &&
                strstr(line, "system call ") == NULL) {
                /* Found a malformed suppression frame.
                 * Add it to the frames[] and report error. */
                frames[num_frames++] = line;
                report_malformed_suppression(curtype, num_frames,
                                             (const char **) frames,
                            "The last frame is incorrect!"NL NL
                            "Frames should be one of the following:"NL
                            " module!function"NL
                            " <module+offset>"NL
                            " <not in a module>"NL
                            " system call Name"NL
                            " ...");
                ASSERT(false, "should not reach here");
            }
            if (strchr(line, '!') != NULL)
                has_symbolic_frames = true;

            if (frames[num_frames] != NULL) {
                global_free(frames[num_frames], strlen(frames[num_frames])+1,
                            HEAPSTAT_MISC);
            }
            frames[num_frames] = drmem_strdup(line, HEAPSTAT_MISC);
            num_frames++;
        }
    }
    if (curtype > -1) {
        /* the last callstack completed successfully */
        if (IF_DRSYMS_ELSE(true, !has_symbolic_frames))
            add_suppress_spec(curtype, num_frames, frames);
    }
    for (i = 0; i < options.callstack_max_frames; i++) {
        if (frames[i] != NULL)
            global_free(frames[i], strlen(frames[i])+1, HEAPSTAT_MISC);
    }
    global_free(frames, options.callstack_max_frames*sizeof(char*), HEAPSTAT_MISC);
}

static void
open_and_read_suppression_file(const char *fname, bool is_default)
{
    uint prev_suppressions = num_suppressions;
    const char *label = (is_default) ? "default" : "user";
    if (fname == NULL || fname[0] == '\0') {
        dr_fprintf(f_global, "No %s suppression file specified\n", label);
    } else {
        file_t f = dr_open_file(fname, DR_FILE_READ);
        if (f == INVALID_FILE) {
            NOTIFY_ERROR("Error opening %s suppression file %s\n", label, fname);
            dr_abort();
            return;
        }
        read_suppression_file(f);
        /* Don't print to stderr about default suppression file */
        NOTIFY_COND(!is_default, f_global, "Recorded %d suppression(s) from %s %s\n",
                    num_suppressions - prev_suppressions, label, fname);
#ifdef USE_DRSYMS
        ELOGF(0, f_results, "Recorded %d suppression(s) from %s %s"NL,
              num_suppressions - prev_suppressions, label, fname);
#endif
        dr_close_file(f);
    }
}

static const char *
next_error_frame(const char *error_stack)
{
    error_stack = strstr(error_stack, NL);
    ASSERT(error_stack != NULL, "malformed error stack");
    error_stack += strlen(NL);
#ifdef USE_DRSYMS
    /* skip file:line# or <syscall> line */
    error_stack = strstr(error_stack, NL);
    ASSERT(error_stack != NULL, "malformed error stack");
    error_stack += strlen(NL);
#endif
    return error_stack;
}

#ifdef USE_DRSYMS
/* up to caller to lock f_results file */
static void
write_suppress_pattern(uint type, const char *cstack, bool symbolic)
{
    const char *eframe, *epos, *end, *ques;
    ASSERT(type >= 0 && type < ERROR_MAX_VAL, "invalid error type");
    ASSERT(cstack != NULL, "invalid param");
    dr_fprintf(f_suppress, "%s"NL, suppress_name[type]);
    eframe = cstack;
    while (eframe != NULL && *eframe != '\0') {
        epos = strstr(eframe, "system call");
        if (epos != NULL) {
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
            dr_fprintf(f_suppress, "%.*s"NL, end - epos, epos);
        } else if (symbolic) {
            epos = strchr(eframe, '>');
            ASSERT(epos != NULL && *(epos+1) == ' ', "suppress generation error");
            epos += 2; /* skip '> ' */
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
            /* i#285: replace ? with * */
            ques = strchr(epos, '?');
            if (ques != NULL && ques < end) {
                dr_fprintf(f_suppress, "%.*s*", ques - epos, epos);
                epos = ques + 1;
            }
            dr_fprintf(f_suppress, "%.*s"NL, end - epos, epos);
        } else {
            epos = strchr(eframe, '<');
            ASSERT(epos != NULL, "suppress generation error");
            end = strchr(epos, '>');
            ASSERT(end != NULL, "suppress generation error");
            dr_fprintf(f_suppress, "%.*s>"NL, end - epos, epos);
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
        }
        eframe = next_error_frame(eframe);
    }
}
#endif

static bool text_matches_pattern(const char *text, int text_length,
                                 const char *pattern)
{
    /* Match text[0...text_length) with pattern and return the result.
     * The pattern may contain '*' and '?' wildcards.
     */
    const char *cur_text = text,
               *text_last_asterisk = NULL,
               *pattern_last_asterisk = NULL;
    while (*cur_text != '\0' && (cur_text < text + text_length)) {
        if (*pattern == '*') {
            while (*++pattern == '*') {
                /* Skip consecutive '*'s */
            }
            if (*pattern == '\0') {
                /* the pattern ends with a series of '*' */
                return true;
            }
            text_last_asterisk = cur_text;
            pattern_last_asterisk = pattern;
        } else if ((*cur_text == *pattern) || (*pattern == '?')) {
            ++cur_text;
            ++pattern;
        } else if (text_last_asterisk != NULL) {
            /* No match. But we have seen at least one '*', so go back
             * and try at the next position.
             */
            pattern = pattern_last_asterisk;
            cur_text = text_last_asterisk++;
        } else {
            return false;
        }
    }
    while (*pattern == '*')
        ++pattern;
    return *pattern == '\0';
}

static bool
top_frame_matches_suppression_frame(const char *error_stack,
                                    const char *supp_frame)
{
    /* error_stack looks like
     * "0x00123456 <some.dll+0xabcd> some.dll!some_function"NL
     * <other frames>
     */
    const char *epos,
               *eol = strstr(error_stack, NL);
    ASSERT(eol != NULL, "malformed error stack - newline missing");

    epos = strstr(error_stack, "error end");
    if (epos != NULL && epos < eol)
        return false;

    epos = strstr(error_stack, "system call");
    if (epos != NULL && epos < eol)
        return text_matches_pattern(epos, eol - epos, supp_frame);

    if (*supp_frame == '<') {
        /* "<mod+off>" suppression frame */
        const char *end_of_mod_off;
        epos = strchr(error_stack, '<');
        ASSERT(epos != NULL && epos < eol, "malformed frame");
        end_of_mod_off = strchr(epos, '>');
        ASSERT(end_of_mod_off != NULL && end_of_mod_off < eol,
               "malformed frame");
        end_of_mod_off++; /* skip '>' */<commit_msg>fixes issue 308 support "not in a module" during suppression matching<commit_after>/* **********************************************************
 * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
 * **********************************************************/

/* Dr. Memory: the memory debugger
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; 
 * version 2.1 of the License, and no later version.

 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Library General Public License for more details.

 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */

/***************************************************************************
 * report.c: Dr. Memory error reporting
 */

#include "dr_api.h"
#include "drmemory.h"
#include "shadow.h"
#include "readwrite.h"
#include "syscall.h"
#include "alloc.h" 
#include "report.h"
#include "callstack.h"
#include "heap.h"
#include "alloc_drmem.h"
#ifdef LINUX
# include <errno.h>
#endif
#include <limits.h>

static uint error_id; /* errors + leaks */
static uint num_reported_errors;
static uint num_total_leaks;
static uint num_throttled_errors;
static uint num_throttled_leaks;
static uint num_leaks_ignored;
static size_t num_bytes_leaked;
static size_t num_bytes_possible_leaked;
static uint num_suppressions;
static uint num_suppressions_matched;
static uint num_suppressed_leaks;
static uint num_reachable_leaks;

static uint saved_throttled_leaks;
static uint saved_total_leaks;
static uint saved_leaks_ignored;
static uint saved_suppressed_leaks;
static uint saved_possible_leaks_total;
static uint saved_possible_leaks_unique;
static uint saved_reachable_leaks;
static uint saved_leaks_unique;
static uint saved_leaks_total;
static size_t saved_bytes_leaked;
static size_t saved_bytes_possible_leaked;

static uint64 timestamp_start;

/***************************************************************************/
/* Store all errors so we can eliminate duplicates (PR 484167) */

enum {
    ERROR_UNADDRESSABLE,
    ERROR_UNDEFINED,
    ERROR_INVALID_HEAP_ARG,
    ERROR_WARNING,
    ERROR_LEAK,
    ERROR_POSSIBLE_LEAK,
    ERROR_MAX_VAL,
};

static const char *const error_name[] = {
    "unaddressable access(es)",
    "uninitialized access(es)",
    "invalid heap argument(s)",
    "warning(s)",
    "leak(s)",    
    "possible leak(s)",    
};

static const char *const suppress_name[] = {
    "UNADDRESSABLE ACCESS",
    "UNINITIALIZED READ",
    "INVALID HEAP ARGUMENT",
    "WARNING",
    "LEAK",    
    "POSSIBLE LEAK",    
};

/* The error_lock protects these as well as error_table */
static uint num_unique[ERROR_MAX_VAL];
static uint num_total[ERROR_MAX_VAL];

/* Though any one instance of an address can have only one error
 * type, the same address could have multiple via different
 * executions.  Thus we must use a key combining the callstack and
 * the error type.
 */
typedef struct _stored_error_t {
    /* We can shrink some of these fields if memory is tight but we shouldn't
     * have more than a few thousand of these
     */
    uint id;
    uint errtype; /* from ERROR_ enum */
    uint count;
    bool suppressed;
    packed_callstack_t *pcs;
    /* We also keep a linked list so we can iterate in id order */
    struct _stored_error_t *next;
} stored_error_t;

#define ERROR_HASH_BITS 8
hashtable_t error_table;
/* We need an outer lock to synchronize stored_error_t data access.
 * Since we never remove from error_table we could instead have
 * a lock per stored_error_t but we save space, assuming errors
 * are rare enough to not be a bottleneck.
 */
static void *error_lock;
/* We also keep a linked list so we can iterate in id order, but composed
 * of hashtable payloads so no separate free is necessary.
 * Protected by error_lock.
 */
static stored_error_t *error_head;
static stored_error_t *error_tail;

/* Only initializes the errtype field */
stored_error_t *
stored_error_create(uint type)
{
    stored_error_t *err = global_alloc(sizeof(*err), HEAPSTAT_MISC);
    memset(err, 0, sizeof(*err));
    ASSERT(type < ERROR_MAX_VAL, "invalid error type");
    err->errtype = type;
    return err;
}

void
stored_error_free(stored_error_t *err)
{
    uint ref;
    ASSERT(err != NULL, "invalid arg");
    if (err->pcs != NULL) {
        ref = packed_callstack_free(err->pcs);
        ASSERT(ref == 0, "invalid ref count");
    }
    global_free(err, sizeof(*err), HEAPSTAT_MISC);
}

uint
stored_error_hash(stored_error_t *err)
{
    /* do NOT use id or count as they won't be filled out at lookup time */
    uint hash;
    ASSERT(err != NULL, "invalid arg");
    hash = packed_callstack_hash(err->pcs);
    hash ^= err->errtype;
    return hash;
}

bool
stored_error_cmp(stored_error_t *err1, stored_error_t *err2)
{
    /* do NOT use id or count as they won't be filled out at lookup time */
    ASSERT(err1 != NULL && err2 != NULL, "invalid arg");
    if (err1->errtype != err2->errtype)
        return false;
    return (packed_callstack_cmp(err1->pcs, err2->pcs));
}

/* A prefix for supplying additional info on a reported error beyond
 * the primary line, timestamp line, and callstack itself (from PR 535568)
 */
#define INFO_PFX IF_DRSYMS_ELSE("Note: ", "  info: ")

/***************************************************************************
 * suppression list
 */

/* For each error type, we have a list of callstacks, with each
 * callstack a variable-sized array of frames, each frame a string.
 */
typedef struct _suppress_spec_t {
    uint num_frames;
    char **frames; /* variable-sized array of strings */
    /* During initial reading it's easier to build a linked list rather than
     * add resizable array support.  We could convert to an array after
     * reading both suppress files, but we have pointers scattered all
     * over anyway so we leave it a list.
     */
    struct _suppress_spec_t *next;
} suppress_spec_t;

/* We suppress error type separately (PR 507837) */
static suppress_spec_t *supp_list[ERROR_MAX_VAL];
static uint supp_num[ERROR_MAX_VAL];

/* We are single-threaded when we read the suppression file */
#define BUFSIZE 4096
static char buf[BUFSIZE];

#ifdef USE_DRSYMS
static void *suppress_file_lock;
#endif

static int
get_suppress_type(char *line)
{
    int i;
    ASSERT(line != NULL, "invalid param");
    if (line[0] == '\0')
        return -1;
    /* Perf: we could stick the 6 names in a hashtable */
    for (i = 0; i < ERROR_MAX_VAL; i++) {
        if (strstr(line, suppress_name[i]) == line)
            return i;
    }
    return -1;
}

static void
report_malformed_suppression(int type, uint num_frames, const char **frames,
                             const char *message)
{
    char *supp = NULL;
    const char *type_string = suppress_name[type];
    int i, supp_size = 0, sofar = 0, len = 0;

    supp_size += strlen(type_string) + strlen(NL);
    for (i = 0; i < num_frames; i++)
        supp_size += strlen(frames[i]) + strlen(NL);
    supp_size++; /* +\0 */

    /* we can't use pt->errbuf since it happens pre-thread-init */
    supp = (char*)global_alloc(supp_size,  HEAPSTAT_MISC);
    BUFPRINT(supp, supp_size, sofar, len, "%s"NL, type_string);
    for (i = 0; i < num_frames; i++)
        BUFPRINT(supp, supp_size, sofar, len, "%s"NL, frames[i]);
    ASSERT(sofar + 1 == supp_size, "");
    NOTIFY("Malformed suppression:\n%s\n%s\n", supp, message);
    global_free(supp, supp_size, HEAPSTAT_MISC);
    usage_error("Malformed suppression. See the log file for the details", "");
}

static suppress_spec_t *
add_suppress_spec(int type, uint num_frames, char **frames)
{
    suppress_spec_t *spec;
    uint i;
    ASSERT(type >= 0 && type < ERROR_MAX_VAL, "internal error type error");
    ASSERT(num_frames > 0, "");
    if (strcmp(frames[num_frames-1], "...") == 0) {
        report_malformed_suppression(type, num_frames, (const char **) frames,
                                     "The given suppression ends with '...'");
        ASSERT(false, "should not reach here");
    }
    spec = (suppress_spec_t *) global_alloc(sizeof(*spec), HEAPSTAT_MISC);
    spec->num_frames = num_frames;
    spec->frames = (char **)
        global_alloc(num_frames*sizeof(char*), HEAPSTAT_MISC);
    LOG(2, "read suppression %d of type %s\n", num_suppressions,
        suppress_name[type]);
    for (i = 0; i < num_frames; i++) {
        LOG(2, "  frame %d: \"%s\"\n", i, frames[i]);
        spec->frames[i] = frames[i];
        frames[i] = NULL;
    }
    /* insert into list */
    spec->next = supp_list[type];
    supp_list[type] = spec;
    supp_num[type]++;
    num_suppressions++;
    return spec;
}

static void
read_suppression_file(file_t f)
{
    char *line, *newline = NULL;
    int bufread = 0, bufwant;
    int len;
    /* current callstack */
    int type, curtype = -1;
    bool has_symbolic_frames = false;
    /* even if a suppression callstack is longer than our max, we match
     * any prefix so we can truncate at the max.
     * the entries of this array are dynamically allocated and become
     * the final strings stored in supp_list.
     */
    char **frames = (char **)
        global_alloc(options.callstack_max_frames*sizeof(char*), HEAPSTAT_MISC);
    uint i, num_frames = 0;
    memset(frames, 0, options.callstack_max_frames*sizeof(char*));

    while (true) {
        if (newline == NULL) {
            bufwant = BUFSIZE-1;
            bufread = dr_read_file(f, buf, bufwant);
            ASSERT(bufread <= bufwant, "internal error reading suppression file");
            if (bufread <= 0)
                break;
            buf[bufread] = '\0';
            newline = strchr(buf, '\n');
            line = buf;
        } else {
            line = newline + 1;
            newline = strchr(line, '\n');
            if (newline == NULL) {
                /* shift 1st part of line to start of buf, then read in rest */
                /* the memory for the processed part can be reused  */
                bufwant = line - buf;
                ASSERT(bufwant <= bufread, "internal error reading suppression file");
                len = bufread - bufwant; /* what is left from last time */
                /* since strings may overlap, should use memmove, not strncpy */
                /* FIXME corner case: if len == 0, nothing to move */
                memmove(buf, line, len);
                bufread = dr_read_file(f, buf+len, bufwant);
                ASSERT(bufread <= bufwant, "internal error reading suppression file");
                if (bufread <= 0)
                    break;
                bufread += len; /* bufread is total in buf */
                buf[bufread] = '\0';
                newline = strchr(buf, '\n');
                line = buf;
            }
        }
        /* buffer is big enough to hold at least one line */
        ASSERT(newline != NULL, "internal error: suppression file malformed?");
        *newline = '\0';
        /* strip one or more carriage returns */
        len = 1;
        while (newline-len >= line && *(newline-len) == '\r') {
            *(newline-len) = '\0';
            len++;
        }
        LOG(3, "suppression file line: \"%s\"\n", line);
        /* Lines look like this:
         * UNINITIALIZED READ
         * <ADVAPI32.dll+0x3c0d>
         * # comment line; blank (newline) lines are allowed too
         * LEAK
         * <libc.so.6+0x2bc80>
         * <+0x2bc80>
         * <not in a module>
         *
         * Note: no leading white spaces.
         * Note: <+0x###> is only on esxi due to PR 363063;  it will go way once
         *       the bug is fixed
         *
         * For USE_DRSYMS, this client now also supports mod!func callstacks:
         * INVALID HEAP ARGUMENT
         * suppress.exe!invalid_free_test1
         * suppress.exe!test
         * suppress.exe!main
         */
        if (line[0] == '\0' || line[0] == '#')
            continue; /* Skip blank and comment lines. */
        type = get_suppress_type(line);
        if (type > -1) {
            if (curtype > -1) {
                /* the prior callstack completed successfully */
                if (IF_DRSYMS_ELSE(true, !has_symbolic_frames))
                    add_suppress_spec(curtype, num_frames, frames);
            }
            /* starting a new callstack */
            curtype = type;
            has_symbolic_frames = false;
            num_frames = 0;
        } else if (num_frames >= options.callstack_max_frames) {
            /* we truncate suppression callstacks to match requested max */
            LOG(1, "WARNING: requested max frames truncates suppression callstacks\n");
            /* just continue to next line */
            /* FIXME: should we abort? */
        } else {
            if (curtype == -1) {
                usage_error("malformed suppression: no error type on line ", line);
                ASSERT(false, "should not reach here");
            }

            if (/* Doesn't match mod!func or has '+' in the middle */
                !(strchr(line, '!') != NULL && strchr(line, '+') == NULL) &&
                /* Doesn't match <mod+off> or has '!' in the middle */
                !(line[0] == '<' && strchr(line, '+') != NULL &&
                  strchr(line, '>') != NULL && strchr(line, '!') == NULL) &&
                strcmp(line, "<not in a module>") != 0 &&
                strcmp(line, "...") != 0 &&
                strstr(line, "system call ") == NULL) {
                /* Found a malformed suppression frame.
                 * Add it to the frames[] and report error. */
                frames[num_frames++] = line;
                report_malformed_suppression(curtype, num_frames,
                                             (const char **) frames,
                            "The last frame is incorrect!"NL NL
                            "Frames should be one of the following:"NL
                            " module!function"NL
                            " <module+offset>"NL
                            " <not in a module>"NL
                            " system call Name"NL
                            " ...");
                ASSERT(false, "should not reach here");
            }
            if (strchr(line, '!') != NULL)
                has_symbolic_frames = true;

            if (frames[num_frames] != NULL) {
                global_free(frames[num_frames], strlen(frames[num_frames])+1,
                            HEAPSTAT_MISC);
            }
            frames[num_frames] = drmem_strdup(line, HEAPSTAT_MISC);
            num_frames++;
        }
    }
    if (curtype > -1) {
        /* the last callstack completed successfully */
        if (IF_DRSYMS_ELSE(true, !has_symbolic_frames))
            add_suppress_spec(curtype, num_frames, frames);
    }
    for (i = 0; i < options.callstack_max_frames; i++) {
        if (frames[i] != NULL)
            global_free(frames[i], strlen(frames[i])+1, HEAPSTAT_MISC);
    }
    global_free(frames, options.callstack_max_frames*sizeof(char*), HEAPSTAT_MISC);
}

static void
open_and_read_suppression_file(const char *fname, bool is_default)
{
    uint prev_suppressions = num_suppressions;
    const char *label = (is_default) ? "default" : "user";
    if (fname == NULL || fname[0] == '\0') {
        dr_fprintf(f_global, "No %s suppression file specified\n", label);
    } else {
        file_t f = dr_open_file(fname, DR_FILE_READ);
        if (f == INVALID_FILE) {
            NOTIFY_ERROR("Error opening %s suppression file %s\n", label, fname);
            dr_abort();
            return;
        }
        read_suppression_file(f);
        /* Don't print to stderr about default suppression file */
        NOTIFY_COND(!is_default, f_global, "Recorded %d suppression(s) from %s %s\n",
                    num_suppressions - prev_suppressions, label, fname);
#ifdef USE_DRSYMS
        ELOGF(0, f_results, "Recorded %d suppression(s) from %s %s"NL,
              num_suppressions - prev_suppressions, label, fname);
#endif
        dr_close_file(f);
    }
}

static const char *
next_error_frame(const char *error_stack)
{
    error_stack = strstr(error_stack, NL);
    ASSERT(error_stack != NULL, "malformed error stack");
    error_stack += strlen(NL);
#ifdef USE_DRSYMS
    /* skip file:line# or <syscall> line */
    error_stack = strstr(error_stack, NL);
    ASSERT(error_stack != NULL, "malformed error stack");
    error_stack += strlen(NL);
#endif
    return error_stack;
}

#ifdef USE_DRSYMS
/* up to caller to lock f_results file */
static void
write_suppress_pattern(uint type, const char *cstack, bool symbolic)
{
    const char *eframe, *epos, *end, *ques;
    ASSERT(type >= 0 && type < ERROR_MAX_VAL, "invalid error type");
    ASSERT(cstack != NULL, "invalid param");
    dr_fprintf(f_suppress, "%s"NL, suppress_name[type]);
    eframe = cstack;
    while (eframe != NULL && *eframe != '\0') {
        epos = strstr(eframe, "system call");
        if (epos != NULL) {
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
            dr_fprintf(f_suppress, "%.*s"NL, end - epos, epos);
        } else if (symbolic) {
            epos = strchr(eframe, '>');
            ASSERT(epos != NULL && *(epos+1) == ' ', "suppress generation error");
            epos += 2; /* skip '> ' */
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
            /* i#285: replace ? with * */
            ques = strchr(epos, '?');
            if (ques != NULL && ques < end) {
                dr_fprintf(f_suppress, "%.*s*", ques - epos, epos);
                epos = ques + 1;
            }
            dr_fprintf(f_suppress, "%.*s"NL, end - epos, epos);
        } else {
            epos = strchr(eframe, '<');
            ASSERT(epos != NULL, "suppress generation error");
            end = strchr(epos, '>');
            ASSERT(end != NULL, "suppress generation error");
            dr_fprintf(f_suppress, "%.*s>"NL, end - epos, epos);
            end = strchr(epos, '\n');
            ASSERT(end != NULL, "suppress generation error");
        }
        eframe = next_error_frame(eframe);
    }
}
#endif

static bool text_matches_pattern(const char *text, int text_length,
                                 const char *pattern)
{
    /* Match text[0...text_length) with pattern and return the result.
     * The pattern may contain '*' and '?' wildcards.
     */
    const char *cur_text = text,
               *text_last_asterisk = NULL,
               *pattern_last_asterisk = NULL;
    while (*cur_text != '\0' && (cur_text < text + text_length)) {
        if (*pattern == '*') {
            while (*++pattern == '*') {
                /* Skip consecutive '*'s */
            }
            if (*pattern == '\0') {
                /* the pattern ends with a series of '*' */
                return true;
            }
            text_last_asterisk = cur_text;
            pattern_last_asterisk = pattern;
        } else if ((*cur_text == *pattern) || (*pattern == '?')) {
            ++cur_text;
            ++pattern;
        } else if (text_last_asterisk != NULL) {
            /* No match. But we have seen at least one '*', so go back
             * and try at the next position.
             */
            pattern = pattern_last_asterisk;
            cur_text = text_last_asterisk++;
        } else {
            return false;
        }
    }
    while (*pattern == '*')
        ++pattern;
    return *pattern == '\0';
}

static bool
top_frame_matches_suppression_frame(const char *error_stack,
                                    const char *supp_frame)
{
    /* error_stack looks like
     * "0x00123456 <some.dll+0xabcd> some.dll!some_function"NL
     * <other frames>
     */
    const char *epos,
               *eol = strstr(error_stack, NL);
    ASSERT(eol != NULL, "malformed error stack - newline missing");

    epos = strstr(error_stack, "error end");
    if (epos != NULL && epos < eol)
        return false;

    epos = strstr(error_stack, "system call");
    if (epos != NULL && epos < eol)
        return text_matches_pattern(epos, eol - epos, supp_frame);

    epos = strstr(error_stack, "not in a module");
    if (epos != NULL && epos < eol)
        return text_matches_pattern(epos, eol - epos, supp_frame);

    if (*supp_frame == '<') {
        /* "<mod+off>" suppression frame */
        const char *end_of_mod_off;
        epos = strchr(error_stack, '<');
        ASSERT(epos != NULL && epos < eol, "malformed frame");
        end_of_mod_off = strchr(epos, '>');
        ASSERT(end_of_mod_off != NULL && end_of_mod_off < eol,
               "malformed frame");
        end_of_mod_off++; /* skip '>' */
{'fraction_non_alphanumeric': 0.13187966696089892, 'fraction_numerical': 0.007419424100007157, 'mean_word_length': 2.366907630522088, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3345931', 'n_tokens_mistral': 13046, 'n_tokens_neox': 12582, 'n_words': 3810}