<commit_before>/*-
 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Berkeley Software Design Inc's name may not be used to endorse or
 *    promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
 *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
 */

/*
 * Machine independent bits of mutex implementation.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include "opt_adaptive_mutexes.h"
#include "opt_ddb.h"
#include "opt_mprof.h"
#include "opt_mutex_wake_all.h"
#include "opt_sched.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <sys/vmmeter.h>

#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/clock.h>
#include <machine/cpu.h>

#include <ddb/ddb.h>

#include <vm/vm.h>
#include <vm/vm_extern.h>

/* 
 * Force MUTEX_WAKE_ALL for now.
 * single thread wakeup needs fixes to avoid race conditions with 
 * priority inheritance.
 */
#ifndef MUTEX_WAKE_ALL
#define MUTEX_WAKE_ALL
#endif

/*
 * Internal utility macros.
 */
#define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)

#define mtx_owner(m)	(mtx_unowned((m)) ? NULL \
	: (struct thread *)((m)->mtx_lock & MTX_FLAGMASK))

/*
 * Lock classes for sleep and spin mutexes.
 */
struct lock_class lock_class_mtx_sleep = {
	"sleep mutex",
	LC_SLEEPLOCK | LC_RECURSABLE
};
struct lock_class lock_class_mtx_spin = {
	"spin mutex",
	LC_SPINLOCK | LC_RECURSABLE
};

/*
 * System-wide mutexes
 */
struct mtx sched_lock;
struct mtx Giant;

#ifdef MUTEX_PROFILING
SYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging");
SYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling");
static int mutex_prof_enable = 0;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW,
    &mutex_prof_enable, 0, "Enable tracing of mutex holdtime");

struct mutex_prof {
	const char	*name;
	const char	*file;
	int		line;
	uintmax_t	cnt_max;
	uintmax_t	cnt_tot;
	uintmax_t	cnt_cur;
	uintmax_t	cnt_contest_holding;
	uintmax_t	cnt_contest_locking;
	struct mutex_prof *next;
};

/*
 * mprof_buf is a static pool of profiling records to avoid possible
 * reentrance of the memory allocation functions.
 *
 * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE.
 */
#ifdef MPROF_BUFFERS
#define NUM_MPROF_BUFFERS	MPROF_BUFFERS
#else
#define	NUM_MPROF_BUFFERS	1000
#endif
static struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS];
static int first_free_mprof_buf;
#ifndef MPROF_HASH_SIZE
#define	MPROF_HASH_SIZE		1009
#endif
#if NUM_MPROF_BUFFERS >= MPROF_HASH_SIZE
#error MPROF_BUFFERS must be larger than MPROF_HASH_SIZE
#endif
static struct mutex_prof *mprof_hash[MPROF_HASH_SIZE];
/* SWAG: sbuf size = avg stat. line size * number of locks */
#define MPROF_SBUF_SIZE		256 * 400

static int mutex_prof_acquisitions;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD,
    &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded");
static int mutex_prof_records;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD,
    &mutex_prof_records, 0, "Number of profiling records");
static int mutex_prof_maxrecords = NUM_MPROF_BUFFERS;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD,
    &mutex_prof_maxrecords, 0, "Maximum number of profiling records");
static int mutex_prof_rejected;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD,
    &mutex_prof_rejected, 0, "Number of rejected profiling records");
static int mutex_prof_hashsize = MPROF_HASH_SIZE;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD,
    &mutex_prof_hashsize, 0, "Hash size");
static int mutex_prof_collisions = 0;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD,
    &mutex_prof_collisions, 0, "Number of hash collisions");

/*
 * mprof_mtx protects the profiling buffers and the hash.
 */
static struct mtx mprof_mtx;
MTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET);

static u_int64_t
nanoseconds(void)
{
	struct timespec tv;

	nanotime(&tv);
	return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec);
}

static int
dump_mutex_prof_stats(SYSCTL_HANDLER_ARGS)
{
	struct sbuf *sb;
	int error, i;
	static int multiplier = 1;

	if (first_free_mprof_buf == 0)
		return (SYSCTL_OUT(req, "No locking recorded",
		    sizeof("No locking recorded")));

retry_sbufops:
	sb = sbuf_new(NULL, NULL, MPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN);
	sbuf_printf(sb, "%6s %12s %11s %5s %12s %12s %s\n",
	    "max", "total", "count", "avg", "cnt_hold", "cnt_lock", "name");
	/*
	 * XXX this spinlock seems to be by far the largest perpetrator
	 * of spinlock latency (1.6 msec on an Athlon1600 was recorded
	 * even before I pessimized it further by moving the average
	 * computation here).
	 */
	mtx_lock_spin(&mprof_mtx);
	for (i = 0; i < first_free_mprof_buf; ++i) {
		sbuf_printf(sb, "%6ju %12ju %11ju %5ju %12ju %12ju %s:%d (%s)\n",
		    mprof_buf[i].cnt_max / 1000,
		    mprof_buf[i].cnt_tot / 1000,
		    mprof_buf[i].cnt_cur,
		    mprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 :
			mprof_buf[i].cnt_tot / (mprof_buf[i].cnt_cur * 1000),
		    mprof_buf[i].cnt_contest_holding,
		    mprof_buf[i].cnt_contest_locking,
		    mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name);
		if (sbuf_overflowed(sb)) {
			mtx_unlock_spin(&mprof_mtx);
			sbuf_delete(sb);
			multiplier++;
			goto retry_sbufops;
		}
	}
	mtx_unlock_spin(&mprof_mtx);
	sbuf_finish(sb);
	error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
	sbuf_delete(sb);
	return (error);<commit_msg>Add additional newline to debug.mutex.prof.stats header, so that column names are printed exactly above the columns.<commit_after>/*-
 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Berkeley Software Design Inc's name may not be used to endorse or
 *    promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $
 *	and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $
 */

/*
 * Machine independent bits of mutex implementation.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include "opt_adaptive_mutexes.h"
#include "opt_ddb.h"
#include "opt_mprof.h"
#include "opt_mutex_wake_all.h"
#include "opt_sched.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <sys/vmmeter.h>

#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/clock.h>
#include <machine/cpu.h>

#include <ddb/ddb.h>

#include <vm/vm.h>
#include <vm/vm_extern.h>

/* 
 * Force MUTEX_WAKE_ALL for now.
 * single thread wakeup needs fixes to avoid race conditions with 
 * priority inheritance.
 */
#ifndef MUTEX_WAKE_ALL
#define MUTEX_WAKE_ALL
#endif

/*
 * Internal utility macros.
 */
#define mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)

#define mtx_owner(m)	(mtx_unowned((m)) ? NULL \
	: (struct thread *)((m)->mtx_lock & MTX_FLAGMASK))

/*
 * Lock classes for sleep and spin mutexes.
 */
struct lock_class lock_class_mtx_sleep = {
	"sleep mutex",
	LC_SLEEPLOCK | LC_RECURSABLE
};
struct lock_class lock_class_mtx_spin = {
	"spin mutex",
	LC_SPINLOCK | LC_RECURSABLE
};

/*
 * System-wide mutexes
 */
struct mtx sched_lock;
struct mtx Giant;

#ifdef MUTEX_PROFILING
SYSCTL_NODE(_debug, OID_AUTO, mutex, CTLFLAG_RD, NULL, "mutex debugging");
SYSCTL_NODE(_debug_mutex, OID_AUTO, prof, CTLFLAG_RD, NULL, "mutex profiling");
static int mutex_prof_enable = 0;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, enable, CTLFLAG_RW,
    &mutex_prof_enable, 0, "Enable tracing of mutex holdtime");

struct mutex_prof {
	const char	*name;
	const char	*file;
	int		line;
	uintmax_t	cnt_max;
	uintmax_t	cnt_tot;
	uintmax_t	cnt_cur;
	uintmax_t	cnt_contest_holding;
	uintmax_t	cnt_contest_locking;
	struct mutex_prof *next;
};

/*
 * mprof_buf is a static pool of profiling records to avoid possible
 * reentrance of the memory allocation functions.
 *
 * Note: NUM_MPROF_BUFFERS must be smaller than MPROF_HASH_SIZE.
 */
#ifdef MPROF_BUFFERS
#define NUM_MPROF_BUFFERS	MPROF_BUFFERS
#else
#define	NUM_MPROF_BUFFERS	1000
#endif
static struct mutex_prof mprof_buf[NUM_MPROF_BUFFERS];
static int first_free_mprof_buf;
#ifndef MPROF_HASH_SIZE
#define	MPROF_HASH_SIZE		1009
#endif
#if NUM_MPROF_BUFFERS >= MPROF_HASH_SIZE
#error MPROF_BUFFERS must be larger than MPROF_HASH_SIZE
#endif
static struct mutex_prof *mprof_hash[MPROF_HASH_SIZE];
/* SWAG: sbuf size = avg stat. line size * number of locks */
#define MPROF_SBUF_SIZE		256 * 400

static int mutex_prof_acquisitions;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, acquisitions, CTLFLAG_RD,
    &mutex_prof_acquisitions, 0, "Number of mutex acquistions recorded");
static int mutex_prof_records;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, records, CTLFLAG_RD,
    &mutex_prof_records, 0, "Number of profiling records");
static int mutex_prof_maxrecords = NUM_MPROF_BUFFERS;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, maxrecords, CTLFLAG_RD,
    &mutex_prof_maxrecords, 0, "Maximum number of profiling records");
static int mutex_prof_rejected;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, rejected, CTLFLAG_RD,
    &mutex_prof_rejected, 0, "Number of rejected profiling records");
static int mutex_prof_hashsize = MPROF_HASH_SIZE;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, hashsize, CTLFLAG_RD,
    &mutex_prof_hashsize, 0, "Hash size");
static int mutex_prof_collisions = 0;
SYSCTL_INT(_debug_mutex_prof, OID_AUTO, collisions, CTLFLAG_RD,
    &mutex_prof_collisions, 0, "Number of hash collisions");

/*
 * mprof_mtx protects the profiling buffers and the hash.
 */
static struct mtx mprof_mtx;
MTX_SYSINIT(mprof, &mprof_mtx, "mutex profiling lock", MTX_SPIN | MTX_QUIET);

static u_int64_t
nanoseconds(void)
{
	struct timespec tv;

	nanotime(&tv);
	return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec);
}

static int
dump_mutex_prof_stats(SYSCTL_HANDLER_ARGS)
{
	struct sbuf *sb;
	int error, i;
	static int multiplier = 1;

	if (first_free_mprof_buf == 0)
		return (SYSCTL_OUT(req, "No locking recorded",
		    sizeof("No locking recorded")));

retry_sbufops:
	sb = sbuf_new(NULL, NULL, MPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN);
	sbuf_printf(sb, "\n%6s %12s %11s %5s %12s %12s %s\n",
	    "max", "total", "count", "avg", "cnt_hold", "cnt_lock", "name");
	/*
	 * XXX this spinlock seems to be by far the largest perpetrator
	 * of spinlock latency (1.6 msec on an Athlon1600 was recorded
	 * even before I pessimized it further by moving the average
	 * computation here).
	 */
	mtx_lock_spin(&mprof_mtx);
	for (i = 0; i < first_free_mprof_buf; ++i) {
		sbuf_printf(sb, "%6ju %12ju %11ju %5ju %12ju %12ju %s:%d (%s)\n",
		    mprof_buf[i].cnt_max / 1000,
		    mprof_buf[i].cnt_tot / 1000,
		    mprof_buf[i].cnt_cur,
		    mprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 :
			mprof_buf[i].cnt_tot / (mprof_buf[i].cnt_cur * 1000),
		    mprof_buf[i].cnt_contest_holding,
		    mprof_buf[i].cnt_contest_locking,
		    mprof_buf[i].file, mprof_buf[i].line, mprof_buf[i].name);
		if (sbuf_overflowed(sb)) {
			mtx_unlock_spin(&mprof_mtx);
			sbuf_delete(sb);
			multiplier++;
			goto retry_sbufops;
		}
	}
	mtx_unlock_spin(&mprof_mtx);
	sbuf_finish(sb);
	error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
	sbuf_delete(sb);
	return (error);
{'fraction_non_alphanumeric': 0.10488594859072428, 'fraction_numerical': 0.017366136034732273, 'mean_word_length': 4.920848633210935, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6680295', 'n_tokens_mistral': 5706, 'n_tokens_neox': 5297, 'n_words': 1486}