pvpage = TAILQ_FIRST(&pv_freepages); if (pvpage != NULL) {
if (pv_freepages.tqh_first != NULL) { pvpage = pv_freepages.tqh_first;
static struct pv_entry	*pmap_remove_pv __P((struct pv_head *, struct pmap *,					     vaddr_t));static void		 pmap_do_remove __P((struct pmap *, vaddr_t,						vaddr_t, int));static boolean_t	 pmap_remove_pte __P((struct pmap *, struct vm_page *,					      pt_entry_t *, vaddr_t, int));static void		 pmap_remove_ptes __P((struct pmap *,					       struct pmap_remove_record *,					       struct vm_page *, vaddr_t,					       vaddr_t, vaddr_t, int));#define PMAP_REMOVE_ALL		0	/* remove all mappings */#define PMAP_REMOVE_SKIPWIRED	1	/* skip wired mappings */static vaddr_t		 pmap_tmpmap_pa __P((paddr_t));static pt_entry_t	*pmap_tmpmap_pvepte __P((struct pv_entry *));static void		 pmap_tmpunmap_pa __P((void));static void		 pmap_tmpunmap_pvepte __P((struct pv_entry *));static void		pmap_unmap_ptes __P((struct pmap *));/* * p m a p   i n l i n e   h e l p e r   f u n c t i o n s *//* * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]? *		of course the kernel is always loaded */__inline static boolean_tpmap_is_curpmap(pmap)	struct pmap *pmap;{	return((pmap == pmap_kernel()) ||	       (pmap->pm_pdirpa == (paddr_t) rcr3()));}/* * pmap_tmpmap_pa: map a page in for tmp usage * * => returns with pmap_tmpptp_lock held */__inline static vaddr_tpmap_tmpmap_pa(pa)	paddr_t pa;{	simple_lock(&pmap_tmpptp_lock);#if defined(DIAGNOSTIC)	if (*ptp_pte)		panic("pmap_tmpmap_pa: ptp_pte in use?");#endif	*ptp_pte = PG_V | PG_RW | pa;		/* always a new mapping */	return((vaddr_t)ptpp);}/* * pmap_tmpunmap_pa: unmap a tmp use page (undoes pmap_tmpmap_pa) * * => we release pmap_tmpptp_lock */__inline static voidpmap_tmpunmap_pa(){#if defined(DIAGNOSTIC)	if (!pmap_valid_entry(*ptp_pte))		panic("pmap_tmpunmap_pa: our pte invalid?");#endif	*ptp_pte = 0;		/* zap! */	pmap_update_pg((vaddr_t)ptpp);	simple_unlock(&pmap_tmpptp_lock);}/* * pmap_tmpmap_pvepte: get a quick mapping of a PTE for a pv_entry * * => do NOT use this on kernel mappings [why?  because pv_ptp may be NULL] * => we may grab pmap_tmpptp_lock and return with it held */__inline static pt_entry_t *pmap_tmpmap_pvepte(pve)	struct pv_entry *pve;{#ifdef DIAGNOSTIC	if (pve->pv_pmap == pmap_kernel())		panic("pmap_tmpmap_pvepte: attempt to map kernel");#endif	/* is it current pmap?  use direct mapping... */	if (pmap_is_curpmap(pve->pv_pmap))		return(vtopte(pve->pv_va));	return(((pt_entry_t *)pmap_tmpmap_pa(VM_PAGE_TO_PHYS(pve->pv_ptp)))	       + ptei((unsigned)pve->pv_va));}/* * pmap_tmpunmap_pvepte: release a mapping obtained with pmap_tmpmap_pvepte * * => we will release pmap_tmpptp_lock if we hold it */__inline static voidpmap_tmpunmap_pvepte(pve)	struct pv_entry *pve;{	/* was it current pmap?   if so, return */	if (pmap_is_curpmap(pve->pv_pmap))		return;	pmap_tmpunmap_pa();}/* * pmap_map_ptes: map a pmap's PTEs into KVM and lock them in * * => we lock enough pmaps to keep things locked in * => must be undone with pmap_unmap_ptes before returning */__inline static pt_entry_t *pmap_map_ptes(pmap)	struct pmap *pmap;{	pd_entry_t opde;	/* the kernel's pmap is always accessible */	if (pmap == pmap_kernel()) {		return(PTE_BASE);	}	/* if curpmap then we are always mapped */	if (pmap_is_curpmap(pmap)) {		simple_lock(&pmap->pm_obj.vmobjlock);		return(PTE_BASE);	}	/* need to lock both curpmap and pmap: use ordered locking */	if ((unsigned) pmap < (unsigned) curpcb->pcb_pmap) {		simple_lock(&pmap->pm_obj.vmobjlock);		simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);	} else {		simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);		simple_lock(&pmap->pm_obj.vmobjlock);	}	/* need to load a new alternate pt space into curpmap? */	opde = *APDP_PDE;	if (!pmap_valid_entry(opde) || (opde & PG_FRAME) != pmap->pm_pdirpa) {		*APDP_PDE = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V);		if (pmap_valid_entry(opde))			tlbflush();	}	return(APTE_BASE);}/* * pmap_unmap_ptes: unlock the PTE mapping of "pmap" */__inline static voidpmap_unmap_ptes(pmap)	struct pmap *pmap;{	if (pmap == pmap_kernel()) {		return;	}	if (pmap_is_curpmap(pmap)) {		simple_unlock(&pmap->pm_obj.vmobjlock);	} else {		simple_unlock(&pmap->pm_obj.vmobjlock);		simple_unlock(&curpcb->pcb_pmap->pm_obj.vmobjlock);	}}/* * p m a p   k e n t e r   f u n c t i o n s * * functions to quickly enter/remove pages from the kernel address * space.   pmap_kremove is exported to MI kernel.  we make use of * the recursive PTE mappings. *//* * pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking * * => no need to lock anything, assume va is already allocated * => should be faster than normal pmap enter function */voidpmap_kenter_pa(va, pa, prot)	vaddr_t va;	paddr_t pa;	vm_prot_t prot;{	pt_entry_t *pte, opte;	if (va < VM_MIN_KERNEL_ADDRESS)		pte = vtopte(va);	else		pte = kvtopte(va);	opte = *pte;#ifdef LARGEPAGES	/* XXX For now... */	if (opte & PG_PS)		panic("pmap_kenter_pa: PG_PS");#endif	*pte = pa | ((prot & VM_PROT_WRITE)? PG_RW : PG_RO) |		PG_V | pmap_pg_g;	/* zap! */	if (pmap_valid_entry(opte))		pmap_update_pg(va);}/* * pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking * * => no need to lock anything * => caller must dispose of any vm_page mapped in the va range * => note: not an inline function * => we assume the va is page aligned and the len is a multiple of PAGE_SIZE * => we assume kernel only unmaps valid addresses and thus don't bother *    checking the valid bit before doing TLB flushing */voidpmap_kremove(va, len)	vaddr_t va;	vsize_t len;{	pt_entry_t *pte;	len >>= PAGE_SHIFT;	for ( /* null */ ; len ; len--, va += PAGE_SIZE) {		if (va < VM_MIN_KERNEL_ADDRESS)			pte = vtopte(va);		else			pte = kvtopte(va);#ifdef LARGEPAGES		/* XXX For now... */		if (*pte & PG_PS)			panic("pmap_kremove: PG_PS");#endif#ifdef DIAGNOSTIC		if (*pte & PG_PVLIST)			panic("pmap_kremove: PG_PVLIST mapping for 0x%lx\n",			      va);#endif		*pte = 0;		/* zap! */#if defined(I386_CPU)		if (cpu_class != CPUCLASS_386)#endif			pmap_update_pg(va);	}#if defined(I386_CPU)	if (cpu_class == CPUCLASS_386)		tlbflush();#endif}/* * p m a p   i n i t   f u n c t i o n s * * pmap_bootstrap and pmap_init are called during system startup * to init the pmap module.   pmap_bootstrap() does a low level * init just to get things rolling.   pmap_init() finishes the job. *//* * pmap_bootstrap: get the system in a state where it can run with VM *	properly enabled (called before main()).   the VM system is *      fully init'd later... * * => on i386, locore.s has already enabled the MMU by allocating *	a PDP for the kernel, and nkpde PTP's for the kernel. * => kva_start is the first free virtual address in kernel space */voidpmap_bootstrap(kva_start)	vaddr_t kva_start;{	struct pmap *kpm;	vaddr_t kva;	pt_entry_t *pte;	/*	 * set the page size (default value is 4K which is ok)	 */	uvm_setpagesize();	/*	 * a quick sanity check	 */	if (PAGE_SIZE != NBPG)		panic("pmap_bootstrap: PAGE_SIZE != NBPG");	/*	 * use the very last page of physical memory for the message buffer	 */	avail_end -= i386_round_page(MSGBUFSIZE);	/*	 * The arguments passed in from /boot needs space too.	 */	avail_end -= i386_round_page(bootargc);	/*	 * set up our local static global vars that keep track of the	 * usage of KVM before kernel_map is set up	 */	virtual_avail = kva_start;		/* first free KVA */	virtual_end = VM_MAX_KERNEL_ADDRESS;	/* last KVA */	/*	 * set up protection_codes: we need to be able to convert from	 * a MI protection code (some combo of VM_PROT...) to something	 * we can jam into a i386 PTE.	 */	protection_codes[VM_PROT_NONE] = 0;  			/* --- */	protection_codes[VM_PROT_EXECUTE] = PG_RO;		/* --x */	protection_codes[VM_PROT_READ] = PG_RO;			/* -r- */	protection_codes[VM_PROT_READ|VM_PROT_EXECUTE] = PG_RO;	/* -rx */	protection_codes[VM_PROT_WRITE] = PG_RW;		/* w-- */	protection_codes[VM_PROT_WRITE|VM_PROT_EXECUTE] = PG_RW;/* w-x */	protection_codes[VM_PROT_WRITE|VM_PROT_READ] = PG_RW;	/* wr- */	protection_codes[VM_PROT_ALL] = PG_RW;			/* wrx */	/*	 * now we init the kernel's pmap	 *	 * the kernel pmap's pm_obj is not used for much.   however, in	 * user pmaps the pm_obj contains the list of active PTPs.	 * the pm_obj currently does not have a pager.   it might be possible	 * to add a pager that would allow a process to read-only mmap its	 * own page tables (fast user level vtophys?).   this may or may not	 * be useful.	 */	kpm = pmap_kernel();	simple_lock_init(&kpm->pm_obj.vmobjlock);	kpm->pm_obj.pgops = NULL;	TAILQ_INIT(&kpm->pm_obj.memq);	kpm->pm_obj.uo_npages = 0;	kpm->pm_obj.uo_refs = 1;	bzero(&kpm->pm_list, sizeof(kpm->pm_list));  /* pm_list not used */	kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE);	kpm->pm_pdirpa = (u_int32_t) proc0.p_addr->u_pcb.pcb_cr3;	kpm->pm_stats.wired_count = kpm->pm_stats.resident_count =		i386_btop(kva_start - VM_MIN_KERNEL_ADDRESS);	/*	 * the above is just a rough estimate and not critical to the proper	 * operation of the system.	 */	curpcb->pcb_pmap = kpm;	/* proc0's pcb */	/*	 * enable global TLB entries if they are supported	 */	if (cpu_feature & CPUID_PGE) {		lcr4(rcr4() | CR4_PGE);	/* enable hardware (via %cr4) */		pmap_pg_g = PG_G;		/* enable software */		/* add PG_G attribute to already mapped kernel pages */		for (kva = VM_MIN_KERNEL_ADDRESS ; kva < virtual_avail ;		     kva += PAGE_SIZE)			if (pmap_valid_entry(PTE_BASE[i386_btop(kva)]))				PTE_BASE[i386_btop(kva)] |= PG_G;	}#ifdef LARGEPAGES	/*	 * enable large pages of they are supported.	 */	if (cpu_feature & CPUID_PSE) {		paddr_t pa;		vaddr_t kva_end;		pd_entry_t *pde;		extern char _etext;		lcr4(rcr4() | CR4_PSE);	/* enable hardware (via %cr4) */		pmap_largepages = 1;	/* enable software */		/*		 * the TLB must be flushed after enabling large pages		 * on Pentium CPUs, according to section 3.6.2.2 of		 * "Intel Architecture Software Developer's Manual,		 * Volume 3: System Programming".		 */		tlbflush();		/*		 * now, remap the kernel text using large pages.  we		 * assume that the linker has properly aligned the		 * .data segment to a 4MB boundary.		 */		kva_end = roundup((vaddr_t)&_etext, NBPD);		for (pa = 0, kva = KERNBASE; kva < kva_end;		     kva += NBPD, pa += NBPD) {			pde = &kpm->pm_pdir[pdei(kva)];			*pde = pa | pmap_pg_g | PG_PS |			    PG_KR | PG_V;	/* zap! */			tlbflush();		}	}#endif /* LARGEPAGES */	/*	 * now we allocate the "special" VAs which are used for tmp mappings	 * by the pmap (and other modules).    we allocate the VAs by advancing	 * virtual_avail (note that there are no pages mapped at these VAs).	 * we find the PTE that maps the allocated VA via the linear PTE	 * mapping.	 */	pte = PTE_BASE + i386_btop(virtual_avail);	csrcp = (caddr_t) virtual_avail;  csrc_pte = pte;	/* allocate */	virtual_avail += PAGE_SIZE; pte++;			/* advance */	cdstp = (caddr_t) virtual_avail;  cdst_pte = pte;	virtual_avail += PAGE_SIZE; pte++;	zerop = (caddr_t) virtual_avail;  zero_pte = pte;	virtual_avail += PAGE_SIZE; pte++;	ptpp = (caddr_t) virtual_avail;  ptp_pte = pte;	virtual_avail += PAGE_SIZE; pte++;	/* XXX: vmmap used by mem.c... should be uvm_map_reserve */	vmmap = (char *)virtual_avail;			/* don't need pte */	virtual_avail += PAGE_SIZE; pte++;#ifdef __NetBSD	msgbuf_vaddr = virtual_avail;			/* don't need pte */#endif#ifdef __OpenBSD__	msgbufp = (struct msgbuf *)virtual_avail;	/* don't need pte */#endif	virtual_avail += round_page(MSGBUFSIZE); pte++;#ifdef __NetBSD__	idt_vaddr = virtual_avail;			/* don't need pte */	virtual_avail += PAGE_SIZE; pte++;	idt_paddr = avail_start;			/* steal a page */	avail_start += PAGE_SIZE;#if defined(I586_CPU)	/* pentium f00f bug stuff */	pentium_idt_vaddr = virtual_avail;		/* don't need pte */	virtual_avail += PAGE_SIZE; pte++;#endif#endif#ifdef __OpenBSD__	bootargp = (bootarg_t *)virtual_avail;	virtual_avail += round_page(bootargc); pte++;#endif	/*	 * now we reserve some VM for mapping pages when doing a crash dump	 */	virtual_avail = reserve_dumppages(virtual_avail);	/*	 * init the static-global locks and global lists.	 */	spinlockinit(&pmap_main_lock, "pmaplk", 0);	simple_lock_init(&pvalloc_lock);	simple_lock_init(&pmaps_lock);	simple_lock_init(&pmap_copy_page_lock);	simple_lock_init(&pmap_zero_page_lock);	simple_lock_init(&pmap_tmpptp_lock);	LIST_INIT(&pmaps);	TAILQ_INIT(&pv_freepages);	TAILQ_INIT(&pv_unusedpgs);	/*	 * initialize the pmap pool.	 */	pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",		  0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP);  	/*	 * initialize the PDE pool and cache.	 */	pool_init(&pmap_pdp_pool, PAGE_SIZE, 0, 0, 0, "pdppl",		  0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP);	pool_cache_init(&pmap_pdp_cache, &pmap_pdp_pool,			pmap_pdp_ctor, NULL, NULL);	/*	 * ensure the TLB is sync'd with reality by flushing it...	 */	tlbflush();}/* * pmap_init: called from uvm_init, our job is to get the pmap * system ready to manage mappings... this mainly means initing * the pv_entry stuff. */voidpmap_init(){	int npages, lcv, i;	vaddr_t addr;	vsize_t s;	/*	 * compute the number of pages we have and then allocate RAM	 * for each pages' pv_head and saved attributes.	 */	npages = 0;	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)		npages += (vm_physmem[lcv].end - vm_physmem[lcv].start);	s = (vsize_t) (sizeof(struct pv_head) * npages +		       sizeof(char) * npages);	s = round_page(s); /* round up */	addr = (vaddr_t) uvm_km_zalloc(kernel_map, s);	if (addr == 0)		panic("pmap_init: unable to allocate pv_heads");	/*	 * init all pv_head's and attrs in one bzero	 */	/* allocate pv_head stuff first */	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {		vm_physmem[lcv].pmseg.pvhead = (struct pv_head *) addr;		addr = (vaddr_t)(vm_physmem[lcv].pmseg.pvhead +				 (vm_physmem[lcv].end - vm_physmem[lcv].start));		for (i = 0;		     i < (vm_physmem[lcv].end - vm_physmem[lcv].start); i++) {			simple_lock_init(			    &vm_physmem[lcv].pmseg.pvhead[i].pvh_lock);		}	}	/* now allocate attrs */	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {		vm_physmem[lcv].pmseg.attrs = (char *) addr;		addr = (vaddr_t)(vm_physmem[lcv].pmseg.attrs +				 (vm_physmem[lcv].end - vm_physmem[lcv].start));	}	/*	 * now we need to free enough pv_entry structures to allow us to get	 * the kmem_map/kmem_object allocated and inited (done after this	 * function is finished).  to do this we allocate one bootstrap page out	 * of kernel_map and use it to provide an initial pool of pv_entry	 * structures.   we never free this page.	 */	pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE);	if (pv_initpage == NULL)		panic("pmap_init: pv_initpage");	pv_cachedva = 0;   /* a VA we have allocated but not used yet */	pv_nfpvents = 0;	(void) pmap_add_pvpage(pv_initpage, FALSE);	/*	 * done: pmap module is up (and ready for business)	 */	pmap_initialized = TRUE;}/* * p v _ e n t r y   f u n c t i o n s *//* * pv_entry allocation functions: *   the main pv_entry allocation functions are: *     pmap_alloc_pv: allocate a pv_entry structure *     pmap_free_pv: free one pv_entry *     pmap_free_pvs: free a list of pv_entrys * * the rest are helper functions *//* * pmap_alloc_pv: inline function to allocate a pv_entry structure * => we lock pvalloc_lock * => if we fail, we call out to pmap_alloc_pvpage * => 3 modes: *    ALLOCPV_NEED   = we really need a pv_entry, even if we have to steal it *    ALLOCPV_TRY    = we want a pv_entry, but not enough to steal *    ALLOCPV_NONEED = we are trying to grow our free list, don't really need *			one now * * "try" is for optional functions like pmap_copy(). */__inline static struct pv_entry *pmap_alloc_pv(pmap, mode)	struct pmap *pmap;	int mode;{	struct pv_page *pvpage;	struct pv_entry *pv;	simple_lock(&pvalloc_lock);	pvpage = TAILQ_FIRST(&pv_freepages);	if (pvpage != NULL) {		pvpage->pvinfo.pvpi_nfree--;		if (pvpage->pvinfo.pvpi_nfree == 0) {			/* nothing left in this one? */			TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list);		}		pv = pvpage->pvinfo.pvpi_pvfree;#ifdef DIAGNOSTIC		if (pv == NULL)			panic("pmap_alloc_pv: pvpi_nfree off");#endif		pvpage->pvinfo.pvpi_pvfree = pv->pv_next;		pv_nfpvents--;  /* took one from pool */	} else {		pv = NULL;		/* need more of them */	}	/*	 * if below low water mark or we didn't get a pv_entry we try and	 * create more pv_entrys ...	 */	if (pv_nfpvents < PVE_LOWAT || pv == NULL) {		if (pv == NULL)			pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ?					       mode : ALLOCPV_NEED);		else			(void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED);	}	simple_unlock(&pvalloc_lock);	return(pv);}/* * pmap_alloc_pvpage: maybe allocate a new pvpage * * if need_entry is false: try and allocate a new pv_page * if need_entry is true: try and allocate a new pv_page and return a *	new pv_entry from it.   if we are unable to allocate a pv_page *	we make a last ditch effort to steal a pv_page from some other *	mapping.    if that fails, we panic... * * => we assume that the caller holds pvalloc_lock */static struct pv_entry *pmap_alloc_pvpage(pmap, mode)	struct pmap *pmap;	int mode;{	struct vm_page *pg;	struct pv_page *pvpage;	struct pv_entry *pv;	int s;	/*	 * if we need_entry and we've got unused pv_pages, allocate from there	 */	pvpage = TAILQ_FIRST(&pv_unusedpgs);	if (mode != ALLOCPV_NONEED && pvpage != NULL) {		/* move it to pv_freepages list */		TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list);		TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list);		/* allocate a pv_entry */		pvpage->pvinfo.pvpi_nfree--;	/* can't go to zero */		pv = pvpage->pvinfo.pvpi_pvfree;#ifdef DIAGNOSTIC		if (pv == NULL)			panic("pmap_alloc_pvpage: pvpi_nfree off");#endif		pvpage->pvinfo.pvpi_pvfree = pv->pv_next;		pv_nfpvents--;  /* took one from pool */		return(pv);	}	/*	 *  see if we've got a cached unmapped VA that we can map a page in.	 * if not, try to allocate one.	 */	s = splvm();   /* must protect kmem_map/kmem_object with splvm! */	if (pv_cachedva == 0) {		pv_cachedva = uvm_km_kmemalloc(kmem_map, uvmexp.kmem_object,		    PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC);		if (pv_cachedva == 0) {			splx(s);			return (NULL);		}	}	/*	 * we have a VA, now let's try and allocate a page in the object	 * note: we are still holding splvm to protect kmem_object	 */	if (!simple_lock_try(&uvmexp.kmem_object->vmobjlock)) {		splx(s);		return (NULL);	}	pg = uvm_pagealloc(uvmexp.kmem_object, pv_cachedva -			   vm_map_min(kernel_map),			   NULL, UVM_PGA_USERESERVE);	if (pg)		pg->flags &= ~PG_BUSY;	/* never busy */	simple_unlock(&uvmexp.kmem_object->vmobjlock);	splx(s);	/* splvm now dropped */	if (pg == NULL)		return (NULL);	/*	 * add a mapping for our new pv_page and free its entrys (save one!)	 *	 * NOTE: If we are allocating a PV page for the kernel pmap, the	 * pmap is already locked!  (...but entering the mapping is safe...)	 */	pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg),	    VM_PROT_READ|VM_PROT_WRITE);	pmap_update(pmap_kernel());	pvpage = (struct pv_page *)pv_cachedva;	pv_cachedva = 0;	return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED));}/* * pmap_add_pvpage: add a pv_page's pv_entrys to the free list * * => caller must hold pvalloc_lock * => if need_entry is true, we allocate and return one pv_entry */static struct pv_entry *pmap_add_pvpage(pvp, need_entry)	struct pv_page *pvp;	boolean_t need_entry;{	int tofree, lcv;	/* do we need to return one? */	tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE;	pvp->pvinfo.pvpi_pvfree = NULL;	pvp->pvinfo.pvpi_nfree = tofree;	for (lcv = 0 ; lcv < tofree ; lcv++) {		pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree;		pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv];	}	if (need_entry)		TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list);	else		TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list);	pv_nfpvents += tofree;	return((need_entry) ? &pvp->pvents[lcv] : NULL);}/* * pmap_free_pv_doit: actually free a pv_entry * * => do not call this directly!  instead use either *    1. pmap_free_pv ==> free a single pv_entry *    2. pmap_free_pvs => free a list of pv_entrys * => we must be holding pvalloc_lock */__inline static voidpmap_free_pv_doit(pv)	struct pv_entry *pv;{	struct pv_page *pvp;	pvp = (struct pv_page *) i386_trunc_page(pv);	pv_nfpvents++;	pvp->pvinfo.pvpi_nfree++;	/* nfree == 1 => fully allocated page just became partly allocated */	if (pvp->pvinfo.pvpi_nfree == 1) {		TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list);	}	/* free it */	pv->pv_next = pvp->pvinfo.pvpi_pvfree;	pvp->pvinfo.pvpi_pvfree = pv;	/*	 * are all pv_page's pv_entry's free?  move it to unused queue.	 */	if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) {		TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list);		TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list);	}}/* * pmap_free_pv: free a single pv_entry * * => we gain the pvalloc_lock */__inline static voidpmap_free_pv(pmap, pv)	struct pmap *pmap;	struct pv_entry *pv;{	simple_lock(&pvalloc_lock);	pmap_free_pv_doit(pv);	/*	 * Can't free the PV page if the PV entries were associated with	 * the kernel pmap; the pmap is already locked.	 */	if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL &&	    pmap != pmap_kernel())		pmap_free_pvpage();	simple_unlock(&pvalloc_lock);}/* * pmap_free_pvs: free a list of pv_entrys * * => we gain the pvalloc_lock */__inline static voidpmap_free_pvs(pmap, pvs)	struct pmap *pmap;	struct pv_entry *pvs;{	struct pv_entry *nextpv;	simple_lock(&pvalloc_lock);	for ( /* null */ ; pvs != NULL ; pvs = nextpv) {		nextpv = pvs->pv_next;		pmap_free_pv_doit(pvs);	}	/*	 * Can't free the PV page if the PV entries were associated with	 * the kernel pmap; the pmap is already locked.	 */	if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL &&	    pmap != pmap_kernel())		pmap_free_pvpage();	simple_unlock(&pvalloc_lock);}/* * pmap_free_pvpage: try and free an unused pv_page structure * * => assume caller is holding the pvalloc_lock and that *	there is a page on the pv_unusedpgs list * => if we can't get a lock on the kmem_map we try again later * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows *	that if we can lock the kmem_map then we are not already *	holding kmem_object's lock. */static voidpmap_free_pvpage(){	int s;	struct vm_map *map;	struct vm_map_entry *dead_entries;	struct pv_page *pvp;	s = splvm(); /* protect kmem_map */	pvp = TAILQ_FIRST(&pv_unusedpgs);	/*	 * note: watch out for pv_initpage which is allocated out of	 * kernel_map rather than kmem_map.	 */	if (pvp == pv_initpage)		map = kernel_map;	else		map = kmem_map;	if (vm_map_lock_try(map)) {		/* remove pvp from pv_unusedpgs */		TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list);		/* unmap the page */		dead_entries = NULL;		uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE,		    &dead_entries);		vm_map_unlock(map);		if (dead_entries != NULL)			uvm_unmap_detach(dead_entries, 0);		pv_nfpvents -= PVE_PER_PVPAGE;  /* update free count */	}	if (pvp == pv_initpage)		/* no more initpage, we've freed it */		pv_initpage = NULL;	splx(s);}/* * main pv_entry manipulation functions: *   pmap_enter_pv: enter a mapping onto a pv_head list *   pmap_remove_pv: remove a mappiing from a pv_head list * * NOTE: pmap_enter_pv expects to lock the pvh itself *       pmap_remove_pv expects te caller to lock the pvh before calling *//* * pmap_enter_pv: enter a mapping onto a pv_head lst * * => caller should hold the proper lock on pmap_main_lock * => caller should have pmap locked * => we will gain the lock on the pv_head and allocate the new pv_entry * => caller should adjust ptp's wire_count before calling */__inline static voidpmap_enter_pv(pvh, pve, pmap, va, ptp)	struct pv_head *pvh;	struct pv_entry *pve;	/* preallocated pve for us to use */	struct pmap *pmap;	vaddr_t va;	struct vm_page *ptp;	/* PTP in pmap that maps this VA */{	pve->pv_pmap = pmap;	pve->pv_va = va;	pve->pv_ptp = ptp;			/* NULL for kernel pmap */	simple_lock(&pvh->pvh_lock);		/* lock pv_head */	pve->pv_next = pvh->pvh_list;		/* add to ... */	pvh->pvh_list = pve;			/* ... locked list */	simple_unlock(&pvh->pvh_lock);		/* unlock, done! */}/* * pmap_remove_pv: try to remove a mapping from a pv_list * * => caller should hold proper lock on pmap_main_lock * => pmap should be locked * => caller should hold lock on pv_head [so that attrs can be adjusted] * => caller should adjust ptp's wire_count and free PTP if needed * => we return the removed pve */__inline static struct pv_entry *pmap_remove_pv(pvh, pmap, va)	struct pv_head *pvh;	struct pmap *pmap;	vaddr_t va;{	struct pv_entry *pve, **prevptr;	prevptr = &pvh->pvh_list;		/* previous pv_entry pointer */	pve = *prevptr;	while (pve) {		if (pve->pv_pmap == pmap && pve->pv_va == va) {	/* match? */			*prevptr = pve->pv_next;		/* remove it! */			break;		}		prevptr = &pve->pv_next;		/* previous pointer */		pve = pve->pv_next;			/* advance */	}	return(pve);				/* return removed pve */}