<commit_before>/*-
 * Copyright (c)2006,2007,2008,2009 <NAME>,
 * Copyright (c) 2013 EMC Corp.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * From:
 *	$NetBSD: vmem_impl.h,v 1.2 2013/01/29 21:26:24 para Exp $
 *	$NetBSD: subr_vmem.c,v 1.83 2013/03/06 11:20:10 yamt Exp $
 */

/*
 * reference:
 * -	Magazines and Vmem: Extending the Slab Allocator
 *	to Many CPUs and Arbitrary Resources
 *	http://www.usenix.org/event/usenix01/bonwick.html
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/callout.h>
#include <sys/hash.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/condvar.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/vmem.h>

#include "opt_vm.h"

#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/vm_param.h>
#include <vm/vm_pageout.h>

#define	VMEM_OPTORDER		5
#define	VMEM_OPTVALUE		(1 << VMEM_OPTORDER)
#define	VMEM_MAXORDER						\
    (VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)

#define	VMEM_HASHSIZE_MIN	16
#define	VMEM_HASHSIZE_MAX	131072

#define	VMEM_QCACHE_IDX_MAX	16

#define	VMEM_FITMASK	(M_BESTFIT | M_FIRSTFIT)

#define	VMEM_FLAGS						\
    (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | M_FIRSTFIT)

#define	BT_FLAGS	(M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM)

#define	QC_NAME_MAX	16

/*
 * Data structures private to vmem.
 */
MALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures");

typedef struct vmem_btag bt_t;

TAILQ_HEAD(vmem_seglist, vmem_btag);
LIST_HEAD(vmem_freelist, vmem_btag);
LIST_HEAD(vmem_hashlist, vmem_btag);

struct qcache {
	uma_zone_t	qc_cache;
	vmem_t 		*qc_vmem;
	vmem_size_t	qc_size;
	char		qc_name[QC_NAME_MAX];
};
typedef struct qcache qcache_t;
#define	QC_POOL_TO_QCACHE(pool)	((qcache_t *)(pool->pr_qcache))

#define	VMEM_NAME_MAX	16

/* vmem arena */
struct vmem {
	struct mtx_padalign	vm_lock;
	struct cv		vm_cv;
	char			vm_name[VMEM_NAME_MAX+1];
	LIST_ENTRY(vmem)	vm_alllist;
	struct vmem_hashlist	vm_hash0[VMEM_HASHSIZE_MIN];
	struct vmem_freelist	vm_freelist[VMEM_MAXORDER];
	struct vmem_seglist	vm_seglist;
	struct vmem_hashlist	*vm_hashlist;
	vmem_size_t		vm_hashsize;

	/* Constant after init */
	vmem_size_t		vm_qcache_max;
	vmem_size_t		vm_quantum_mask;
	vmem_size_t		vm_import_quantum;
	int			vm_quantum_shift;

	/* Written on alloc/free */
	LIST_HEAD(, vmem_btag)	vm_freetags;
	int			vm_nfreetags;
	int			vm_nbusytag;
	vmem_size_t		vm_inuse;
	vmem_size_t		vm_size;

	/* Used on import. */
	vmem_import_t		*vm_importfn;
	vmem_release_t		*vm_releasefn;
	void			*vm_arg;

	/* Space exhaustion callback. */
	vmem_reclaim_t		*vm_reclaimfn;

	/* quantum cache */
	qcache_t		vm_qcache[VMEM_QCACHE_IDX_MAX];
};

/* boundary tag */
struct vmem_btag {
	TAILQ_ENTRY(vmem_btag) bt_seglist;
	union {
		LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */
		LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */
	} bt_u;
#define	bt_hashlist	bt_u.u_hashlist
#define	bt_freelist	bt_u.u_freelist
	vmem_addr_t	bt_start;
	vmem_size_t	bt_size;
	int		bt_type;
};

#define	BT_TYPE_SPAN		1	/* Allocated from importfn */
#define	BT_TYPE_SPAN_STATIC	2	/* vmem_add() or create. */
#define	BT_TYPE_FREE		3	/* Available space. */
#define	BT_TYPE_BUSY		4	/* Used space. */
#define	BT_ISSPAN_P(bt)	((bt)->bt_type <= BT_TYPE_SPAN_STATIC)

#define	BT_END(bt)	((bt)->bt_start + (bt)->bt_size - 1)

#if defined(DIAGNOSTIC)
static int enable_vmem_check = 1;
SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RW,
    &enable_vmem_check, 0, "Enable vmem check");
static void vmem_check(vmem_t *);
#endif

static struct callout	vmem_periodic_ch;
static int		vmem_periodic_interval;
static struct task	vmem_periodic_wk;

static struct mtx_padalign vmem_list_lock;
static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list);

/* ---- misc */
#define	VMEM_CONDVAR_INIT(vm, wchan)	cv_init(&vm->vm_cv, wchan)
#define	VMEM_CONDVAR_DESTROY(vm)	cv_destroy(&vm->vm_cv)
#define	VMEM_CONDVAR_WAIT(vm)		cv_wait(&vm->vm_cv, &vm->vm_lock)<commit_msg>MFC r280797: Make debug.vmem_check a tunable.  It is useful to set it early.<commit_after>/*-
 * Copyright (c)2006,2007,2008,2009 <NAME>,
 * Copyright (c) 2013 EMC Corp.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * From:
 *	$NetBSD: vmem_impl.h,v 1.2 2013/01/29 21:26:24 para Exp $
 *	$NetBSD: subr_vmem.c,v 1.83 2013/03/06 11:20:10 yamt Exp $
 */

/*
 * reference:
 * -	Magazines and Vmem: Extending the Slab Allocator
 *	to Many CPUs and Arbitrary Resources
 *	http://www.usenix.org/event/usenix01/bonwick.html
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/callout.h>
#include <sys/hash.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/condvar.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/vmem.h>

#include "opt_vm.h"

#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/vm_param.h>
#include <vm/vm_pageout.h>

#define	VMEM_OPTORDER		5
#define	VMEM_OPTVALUE		(1 << VMEM_OPTORDER)
#define	VMEM_MAXORDER						\
    (VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)

#define	VMEM_HASHSIZE_MIN	16
#define	VMEM_HASHSIZE_MAX	131072

#define	VMEM_QCACHE_IDX_MAX	16

#define	VMEM_FITMASK	(M_BESTFIT | M_FIRSTFIT)

#define	VMEM_FLAGS						\
    (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | M_BESTFIT | M_FIRSTFIT)

#define	BT_FLAGS	(M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM)

#define	QC_NAME_MAX	16

/*
 * Data structures private to vmem.
 */
MALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures");

typedef struct vmem_btag bt_t;

TAILQ_HEAD(vmem_seglist, vmem_btag);
LIST_HEAD(vmem_freelist, vmem_btag);
LIST_HEAD(vmem_hashlist, vmem_btag);

struct qcache {
	uma_zone_t	qc_cache;
	vmem_t 		*qc_vmem;
	vmem_size_t	qc_size;
	char		qc_name[QC_NAME_MAX];
};
typedef struct qcache qcache_t;
#define	QC_POOL_TO_QCACHE(pool)	((qcache_t *)(pool->pr_qcache))

#define	VMEM_NAME_MAX	16

/* vmem arena */
struct vmem {
	struct mtx_padalign	vm_lock;
	struct cv		vm_cv;
	char			vm_name[VMEM_NAME_MAX+1];
	LIST_ENTRY(vmem)	vm_alllist;
	struct vmem_hashlist	vm_hash0[VMEM_HASHSIZE_MIN];
	struct vmem_freelist	vm_freelist[VMEM_MAXORDER];
	struct vmem_seglist	vm_seglist;
	struct vmem_hashlist	*vm_hashlist;
	vmem_size_t		vm_hashsize;

	/* Constant after init */
	vmem_size_t		vm_qcache_max;
	vmem_size_t		vm_quantum_mask;
	vmem_size_t		vm_import_quantum;
	int			vm_quantum_shift;

	/* Written on alloc/free */
	LIST_HEAD(, vmem_btag)	vm_freetags;
	int			vm_nfreetags;
	int			vm_nbusytag;
	vmem_size_t		vm_inuse;
	vmem_size_t		vm_size;

	/* Used on import. */
	vmem_import_t		*vm_importfn;
	vmem_release_t		*vm_releasefn;
	void			*vm_arg;

	/* Space exhaustion callback. */
	vmem_reclaim_t		*vm_reclaimfn;

	/* quantum cache */
	qcache_t		vm_qcache[VMEM_QCACHE_IDX_MAX];
};

/* boundary tag */
struct vmem_btag {
	TAILQ_ENTRY(vmem_btag) bt_seglist;
	union {
		LIST_ENTRY(vmem_btag) u_freelist; /* BT_TYPE_FREE */
		LIST_ENTRY(vmem_btag) u_hashlist; /* BT_TYPE_BUSY */
	} bt_u;
#define	bt_hashlist	bt_u.u_hashlist
#define	bt_freelist	bt_u.u_freelist
	vmem_addr_t	bt_start;
	vmem_size_t	bt_size;
	int		bt_type;
};

#define	BT_TYPE_SPAN		1	/* Allocated from importfn */
#define	BT_TYPE_SPAN_STATIC	2	/* vmem_add() or create. */
#define	BT_TYPE_FREE		3	/* Available space. */
#define	BT_TYPE_BUSY		4	/* Used space. */
#define	BT_ISSPAN_P(bt)	((bt)->bt_type <= BT_TYPE_SPAN_STATIC)

#define	BT_END(bt)	((bt)->bt_start + (bt)->bt_size - 1)

#if defined(DIAGNOSTIC)
static int enable_vmem_check = 1;
SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RWTUN,
    &enable_vmem_check, 0, "Enable vmem check");
static void vmem_check(vmem_t *);
#endif

static struct callout	vmem_periodic_ch;
static int		vmem_periodic_interval;
static struct task	vmem_periodic_wk;

static struct mtx_padalign vmem_list_lock;
static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list);

/* ---- misc */
#define	VMEM_CONDVAR_INIT(vm, wchan)	cv_init(&vm->vm_cv, wchan)
#define	VMEM_CONDVAR_DESTROY(vm)	cv_destroy(&vm->vm_cv)
#define	VMEM_CONDVAR_WAIT(vm)		cv_wait(&vm->vm_cv, &vm->vm_lock)
{'fraction_non_alphanumeric': 0.10554755043227666, 'fraction_numerical': 0.015489913544668587, 'mean_word_length': 4.756868843960602, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4361645', 'n_tokens_mistral': 4777, 'n_tokens_neox': 4457, 'n_words': 1097}