if (pv_cachedva == NULL) {
if (pv_cachedva == 0) {
static struct vm_page	*pmap_get_ptp __P((struct pmap *, int, boolean_t));static boolean_t	 pmap_is_curpmap __P((struct pmap *));static pt_entry_t	*pmap_map_ptes __P((struct pmap *));static struct pv_entry	*pmap_remove_pv __P((struct pv_head *, struct pmap *,					     vaddr_t));static boolean_t	 pmap_remove_pte __P((struct pmap *, struct vm_page *,					      pt_entry_t *, vaddr_t));static void		 pmap_remove_ptes __P((struct pmap *,					       struct pmap_remove_record *,					       struct vm_page *, vaddr_t,					       vaddr_t, vaddr_t));static struct vm_page	*pmap_steal_ptp __P((struct uvm_object *,					     vaddr_t));static vaddr_t		 pmap_tmpmap_pa __P((paddr_t));static pt_entry_t	*pmap_tmpmap_pvepte __P((struct pv_entry *));static void		 pmap_tmpunmap_pa __P((void));static void		 pmap_tmpunmap_pvepte __P((struct pv_entry *));static boolean_t	 pmap_transfer_ptes __P((struct pmap *,					 struct pmap_transfer_location *,					 struct pmap *,					 struct pmap_transfer_location *,					 int, boolean_t));static boolean_t	 pmap_try_steal_pv __P((struct pv_head *,						struct pv_entry *,						struct pv_entry *));static void		pmap_unmap_ptes __P((struct pmap *));void			pmap_pinit __P((pmap_t));void			pmap_release __P((pmap_t));/* * p m a p   i n l i n e   h e l p e r   f u n c t i o n s *//* * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]? *		of course the kernel is always loaded */__inline static boolean_tpmap_is_curpmap(pmap)	struct pmap *pmap;{	return((pmap == pmap_kernel()) ||	       (pmap->pm_pdirpa == (paddr_t) rcr3()));}/* * pmap_tmpmap_pa: map a page in for tmp usage * * => returns with pmap_tmpptp_lock held */__inline static vaddr_tpmap_tmpmap_pa(pa)	paddr_t pa;{	simple_lock(&pmap_tmpptp_lock);#if defined(DIAGNOSTIC)	if (*ptp_pte)		panic("pmap_tmpmap_pa: ptp_pte in use?");#endif	*ptp_pte = PG_V | PG_RW | pa;		/* always a new mapping */	return((vaddr_t)ptpp);}/* * pmap_tmpunmap_pa: unmap a tmp use page (undoes pmap_tmpmap_pa) * * => we release pmap_tmpptp_lock */__inline static voidpmap_tmpunmap_pa(){#if defined(DIAGNOSTIC)	if (!pmap_valid_entry(*ptp_pte))		panic("pmap_tmpunmap_pa: our pte invalid?");#endif	*ptp_pte = 0;		/* zap! */	pmap_update_pg((vaddr_t)ptpp);	simple_unlock(&pmap_tmpptp_lock);}/* * pmap_tmpmap_pvepte: get a quick mapping of a PTE for a pv_entry * * => do NOT use this on kernel mappings [why?  because pv_ptp may be NULL] * => we may grab pmap_tmpptp_lock and return with it held */__inline static pt_entry_t *pmap_tmpmap_pvepte(pve)	struct pv_entry *pve;{#ifdef DIAGNOSTIC	if (pve->pv_pmap == pmap_kernel())		panic("pmap_tmpmap_pvepte: attempt to map kernel");#endif	/* is it current pmap?  use direct mapping... */	if (pmap_is_curpmap(pve->pv_pmap))		return(vtopte(pve->pv_va));	return(((pt_entry_t *)pmap_tmpmap_pa(VM_PAGE_TO_PHYS(pve->pv_ptp)))	       + ptei((unsigned)pve->pv_va));}/* * pmap_tmpunmap_pvepte: release a mapping obtained with pmap_tmpmap_pvepte * * => we will release pmap_tmpptp_lock if we hold it */__inline static voidpmap_tmpunmap_pvepte(pve)	struct pv_entry *pve;{	/* was it current pmap?   if so, return */	if (pmap_is_curpmap(pve->pv_pmap))		return;	pmap_tmpunmap_pa();}/* * pmap_map_ptes: map a pmap's PTEs into KVM and lock them in * * => we lock enough pmaps to keep things locked in * => must be undone with pmap_unmap_ptes before returning */__inline static pt_entry_t *pmap_map_ptes(pmap)	struct pmap *pmap;{	pd_entry_t opde;	/* the kernel's pmap is always accessible */	if (pmap == pmap_kernel()) {		return(PTE_BASE);	}	/* if curpmap then we are always mapped */	if (pmap_is_curpmap(pmap)) {		simple_lock(&pmap->pm_obj.vmobjlock);		return(PTE_BASE);	}	/* need to lock both curpmap and pmap: use ordered locking */	if ((unsigned) pmap < (unsigned) curpcb->pcb_pmap) {		simple_lock(&pmap->pm_obj.vmobjlock);		simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);	} else {		simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);		simple_lock(&pmap->pm_obj.vmobjlock);	}	/* need to load a new alternate pt space into curpmap? */	opde = *APDP_PDE;	if (!pmap_valid_entry(opde) || (opde & PG_FRAME) != pmap->pm_pdirpa) {		*APDP_PDE = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V);		if (pmap_valid_entry(opde))			pmap_update();	}	return(APTE_BASE);}/* * pmap_unmap_ptes: unlock the PTE mapping of "pmap" */__inline static voidpmap_unmap_ptes(pmap)	struct pmap *pmap;{	if (pmap == pmap_kernel()) {		return;	}	if (pmap_is_curpmap(pmap)) {		simple_unlock(&pmap->pm_obj.vmobjlock);	} else {		simple_unlock(&pmap->pm_obj.vmobjlock);		simple_unlock(&curpcb->pcb_pmap->pm_obj.vmobjlock);	}}/* * p m a p   k e n t e r   f u n c t i o n s * * functions to quickly enter/remove pages from the kernel address * space.   pmap_kremove/pmap_kenter_pgs are exported to MI kernel. * we make use of the recursive PTE mappings. *//* * pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking * * => no need to lock anything, assume va is already allocated * => should be faster than normal pmap enter function */voidpmap_kenter_pa(va, pa, prot)	vaddr_t va;	paddr_t pa;	vm_prot_t prot;{	pt_entry_t *pte, opte;	pte = vtopte(va);	opte = *pte;	*pte = pa | ((prot & VM_PROT_WRITE)? PG_RW : PG_RO) |		PG_V | pmap_pg_g;	/* zap! */	if (pmap_valid_entry(opte))		pmap_update_pg(va);}/* * pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking * * => no need to lock anything * => caller must dispose of any vm_page mapped in the va range * => note: not an inline function * => we assume the va is page aligned and the len is a multiple of NBPG * => we assume kernel only unmaps valid addresses and thus don't bother *    checking the valid bit before doing TLB flushing */voidpmap_kremove(va, len)	vaddr_t va;	vsize_t len;{	pt_entry_t *pte;	len >>= PAGE_SHIFT;	for ( /* null */ ; len ; len--, va += NBPG) {		pte = vtopte(va);#ifdef DIAGNOSTIC		if (*pte & PG_PVLIST)			panic("pmap_kremove: PG_PVLIST mapping for 0x%lx\n",			      va);#endif		*pte = 0;		/* zap! */#if defined(I386_CPU)		if (cpu_class != CPUCLASS_386)#endif			pmap_update_pg(va);	}#if defined(I386_CPU)	if (cpu_class == CPUCLASS_386)		pmap_update();#endif}/* * pmap_kenter_pgs: enter in a number of vm_pages */voidpmap_kenter_pgs(va, pgs, npgs)	vaddr_t va;	struct vm_page **pgs;	int npgs;{	pt_entry_t *pte, opte;	int lcv;	vaddr_t tva;#if defined(I386_CPU)	boolean_t need_update = FALSE;#endif	for (lcv = 0 ; lcv < npgs ; lcv++) {		tva = va + lcv * NBPG;		pte = vtopte(tva);		opte = *pte;		*pte = VM_PAGE_TO_PHYS(pgs[lcv]) | PG_RW | PG_V | pmap_pg_g;#if defined(I386_CPU)		if (cpu_class == CPUCLASS_386) {			if (pmap_valid_entry(opte))				need_update = TRUE;			continue;		}#endif		if (pmap_valid_entry(opte))			pmap_update_pg(tva);	}#if defined(I386_CPU)	if (need_update && cpu_class == CPUCLASS_386)		pmap_update();#endif}/* * p m a p   i n i t   f u n c t i o n s * * pmap_bootstrap and pmap_init are called during system startup * to init the pmap module.   pmap_bootstrap() does a low level * init just to get things rolling.   pmap_init() finishes the job. *//* * pmap_bootstrap: get the system in a state where it can run with VM *	properly enabled (called before main()).   the VM system is *      fully init'd later... * * => on i386, locore.s has already enabled the MMU by allocating *	a PDP for the kernel, and nkpde PTP's for the kernel. * => kva_start is the first free virtual address in kernel space * => we make use of the global vars from machdep.c: *	avail_start, avail_end, hole_start, hole_end */voidpmap_bootstrap(kva_start)	vaddr_t kva_start;{	struct pmap *kpm;	vaddr_t kva;	pt_entry_t *pte;#ifdef __NetBSD__	int first16q;#endif	/*	 * set the page size (default value is 4K which is ok)	 */	uvm_setpagesize();	/*	 * a quick sanity check	 */	if (PAGE_SIZE != NBPG)		panic("pmap_bootstrap: PAGE_SIZE != NBPG");	/*	 * use the very last page of physical memory for the message buffer	 */	avail_end -= i386_round_page(MSGBUFSIZE);#ifdef __NetBSD__	msgbuf_paddr = avail_end;#endif#ifdef __OpenBSD__	/*	 * The arguments passed in from /boot needs space too.	 */	avail_end -= i386_round_page(bootargc);#endif	/*	 * set up our local static global vars that keep track of the	 * usage of KVM before kernel_map is set up	 */	virtual_avail = kva_start;		/* first free KVA */	virtual_end = VM_MAX_KERNEL_ADDRESS;	/* last KVA */	/*	 * set up protection_codes: we need to be able to convert from	 * a MI protection code (some combo of VM_PROT...) to something	 * we can jam into a i386 PTE.	 */	protection_codes[VM_PROT_NONE] = 0;  			/* --- */	protection_codes[VM_PROT_EXECUTE] = PG_RO;		/* --x */	protection_codes[VM_PROT_READ] = PG_RO;			/* -r- */	protection_codes[VM_PROT_READ|VM_PROT_EXECUTE] = PG_RO;	/* -rx */	protection_codes[VM_PROT_WRITE] = PG_RW;		/* w-- */	protection_codes[VM_PROT_WRITE|VM_PROT_EXECUTE] = PG_RW;/* w-x */	protection_codes[VM_PROT_WRITE|VM_PROT_READ] = PG_RW;	/* wr- */	protection_codes[VM_PROT_ALL] = PG_RW;			/* wrx */	/*	 * now we init the kernel's pmap	 *	 * the kernel pmap's pm_obj is not used for much.   however, in	 * user pmaps the pm_obj contains the list of active PTPs.	 * the pm_obj currently does not have a pager.   it might be possible	 * to add a pager that would allow a process to read-only mmap its	 * own page tables (fast user level vtophys?).   this may or may not	 * be useful.	 */	kpm = pmap_kernel();	simple_lock_init(&kpm->pm_obj.vmobjlock);	kpm->pm_obj.pgops = NULL;	TAILQ_INIT(&kpm->pm_obj.memq);	kpm->pm_obj.uo_npages = 0;	kpm->pm_obj.uo_refs = 1;	bzero(&kpm->pm_list, sizeof(kpm->pm_list));  /* pm_list not used */	kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE);	kpm->pm_pdirpa = (u_int32_t) proc0.p_addr->u_pcb.pcb_cr3;	kpm->pm_stats.wired_count = kpm->pm_stats.resident_count =		i386_btop(kva_start - VM_MIN_KERNEL_ADDRESS);	/*	 * the above is just a rough estimate and not critical to the proper	 * operation of the system.	 */	curpcb->pcb_pmap = kpm;	/* proc0's pcb */	/*	 * enable global TLB entries if they are supported	 */	if (cpu_feature & CPUID_PGE) {		lcr4(rcr4() | CR4_PGE);	/* enable hardware (via %cr4) */		pmap_pg_g = PG_G;		/* enable software */		/* add PG_G attribute to already mapped kernel pages */		for (kva = VM_MIN_KERNEL_ADDRESS ; kva < virtual_avail ;		     kva += NBPG)			if (pmap_valid_entry(PTE_BASE[i386_btop(kva)]))				PTE_BASE[i386_btop(kva)] |= PG_G;	}	/*	 * now we allocate the "special" VAs which are used for tmp mappings	 * by the pmap (and other modules).    we allocate the VAs by advancing	 * virtual_avail (note that there are no pages mapped at these VAs).	 * we find the PTE that maps the allocated VA via the linear PTE	 * mapping.	 */	pte = PTE_BASE + i386_btop(virtual_avail);	csrcp = (caddr_t) virtual_avail;  csrc_pte = pte;  /* allocate */	virtual_avail += NBPG; pte++;			     /* advance */	cdstp = (caddr_t) virtual_avail;  cdst_pte = pte;	virtual_avail += NBPG; pte++;	zerop = (caddr_t) virtual_avail;  zero_pte = pte;	virtual_avail += NBPG; pte++;	ptpp = (caddr_t) virtual_avail;  ptp_pte = pte;	virtual_avail += NBPG; pte++;	/* XXX: vmmap used by mem.c... should be uvm_map_reserve */	vmmap = (char *)virtual_avail;			/* don't need pte */	virtual_avail += NBPG; pte++;#ifdef __NetBSD	msgbuf_vaddr = virtual_avail;			/* don't need pte */#endif#ifdef __OpenBSD__	msgbufp = (struct msgbuf *)virtual_avail;	/* don't need pte */#endif	virtual_avail += round_page(MSGBUFSIZE); pte++;#ifdef __NetBSD__	idt_vaddr = virtual_avail;			/* don't need pte */	virtual_avail += NBPG; pte++;	avail_end -= NBPG;	idt_paddr = avail_end;#if defined(I586_CPU)	/* pentium f00f bug stuff */	pentium_idt_vaddr = virtual_avail;		/* don't need pte */	virtual_avail += NBPG; pte++;#endif#endif#ifdef __OpenBSD__	bootargp = (bootarg_t *)virtual_avail;	virtual_avail += round_page(bootargc); pte++;#endif	/*	 * now we reserve some VM for mapping pages when doing a crash dump	 */	virtual_avail = reserve_dumppages(virtual_avail);	/*	 * init the static-global locks and global lists.	 */#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)	spinlockinit(&pmap_main_lock, "pmaplk", 0);	simple_lock_init(&pvalloc_lock);	simple_lock_init(&pmaps_lock);	simple_lock_init(&pmap_copy_page_lock);	simple_lock_init(&pmap_zero_page_lock);	simple_lock_init(&pmap_tmpptp_lock);#endif	LIST_INIT(&pmaps);	TAILQ_INIT(&pv_freepages);	TAILQ_INIT(&pv_unusedpgs);	/*	 * initialize the pmap pool.	 */	pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",		  0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP);#ifdef __NetBSD__	/*	 * we must call uvm_page_physload() after we are done playing with	 * virtual_avail but before we call pmap_steal_memory.  [i.e. here]	 * this call tells the VM system how much physical memory it	 * controls.  If we have 16M of RAM or less, just put it all on	 * the default free list.  Otherwise, put the first 16M of RAM	 * on a lower priority free list (so that all of the ISA DMA'able	 * memory won't be eaten up first-off).	 */	if (avail_end <= (16 * 1024 * 1024))		first16q = VM_FREELIST_DEFAULT;	else		first16q = VM_FREELIST_FIRST16;	if (avail_start < hole_start)   /* any free memory before the hole? */		uvm_page_physload(atop(avail_start), atop(hole_start),				  atop(avail_start), atop(hole_start),				  first16q);	if (first16q != VM_FREELIST_DEFAULT &&	    hole_end < 16 * 1024 * 1024) {		uvm_page_physload(atop(hole_end), atop(16 * 1024 * 1024),				  atop(hole_end), atop(16 * 1024 * 1024),				  first16q);		uvm_page_physload(atop(16 * 1024 * 1024), atop(avail_end),				  atop(16 * 1024 * 1024), atop(avail_end),				  VM_FREELIST_DEFAULT);	} else {		uvm_page_physload(atop(hole_end), atop(avail_end),				  atop(hole_end), atop(avail_end),				  VM_FREELIST_DEFAULT);	}#endif	/*	 * ensure the TLB is sync'd with reality by flushing it...	 */	pmap_update();}/* * pmap_init: called from uvm_init, our job is to get the pmap * system ready to manage mappings... this mainly means initing * the pv_entry stuff. */voidpmap_init(){	int npages, lcv;	vaddr_t addr;	vsize_t s;	/*	 * compute the number of pages we have and then allocate RAM	 * for each pages' pv_head and saved attributes.	 */	npages = 0;	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)		npages += (vm_physmem[lcv].end - vm_physmem[lcv].start);	s = (vsize_t) (sizeof(struct pv_head) * npages +		       sizeof(char) * npages);	s = round_page(s); /* round up */	addr = (vaddr_t) uvm_km_zalloc(kernel_map, s);	if (addr == NULL)		panic("pmap_init: unable to allocate pv_heads");	/*	 * init all pv_head's and attrs in one bzero	 */	/* allocate pv_head stuff first */	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {		vm_physmem[lcv].pmseg.pvhead = (struct pv_head *) addr;		addr = (vaddr_t)(vm_physmem[lcv].pmseg.pvhead +				 (vm_physmem[lcv].end - vm_physmem[lcv].start));	}	/* now allocate attrs */	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {		vm_physmem[lcv].pmseg.attrs = (char *) addr;		addr = (vaddr_t)(vm_physmem[lcv].pmseg.attrs +				 (vm_physmem[lcv].end - vm_physmem[lcv].start));	}	/*	 * now we need to free enough pv_entry structures to allow us to get	 * the kmem_map/kmem_object allocated and inited (done after this	 * function is finished).  to do this we allocate one bootstrap page out	 * of kernel_map and use it to provide an initial pool of pv_entry	 * structures.   we never free this page.	 */	pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, NBPG);	if (pv_initpage == NULL)		panic("pmap_init: pv_initpage");	pv_cachedva = NULL;   /* a VA we have allocated but not used yet */	pv_nfpvents = 0;	(void) pmap_add_pvpage(pv_initpage, FALSE);	/*	 * done: pmap module is up (and ready for business)	 */	pmap_initialized = TRUE;}/* * p v _ e n t r y   f u n c t i o n s *//* * pv_entry allocation functions: *   the main pv_entry allocation functions are: *     pmap_alloc_pv: allocate a pv_entry structure *     pmap_free_pv: free one pv_entry *     pmap_free_pvs: free a list of pv_entrys * * the rest are helper functions *//* * pmap_alloc_pv: inline function to allocate a pv_entry structure * => we lock pvalloc_lock * => if we fail, we call out to pmap_alloc_pvpage * => 3 modes: *    ALLOCPV_NEED   = we really need a pv_entry, even if we have to steal it *    ALLOCPV_TRY    = we want a pv_entry, but not enough to steal *    ALLOCPV_NONEED = we are trying to grow our free list, don't really need *			one now * * "try" is for optional functions like pmap_copy(). */__inline static struct pv_entry *pmap_alloc_pv(pmap, mode)	struct pmap *pmap;	int mode;{	struct pv_page *pvpage;	struct pv_entry *pv;	simple_lock(&pvalloc_lock);	if (pv_freepages.tqh_first != NULL) {		pvpage = pv_freepages.tqh_first;		pvpage->pvinfo.pvpi_nfree--;		if (pvpage->pvinfo.pvpi_nfree == 0) {			/* nothing left in this one? */			TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list);		}		pv = pvpage->pvinfo.pvpi_pvfree;#ifdef DIAGNOSTIC		if (pv == NULL)			panic("pmap_alloc_pv: pvpi_nfree off");#endif		pvpage->pvinfo.pvpi_pvfree = pv->pv_next;		pv_nfpvents--;  /* took one from pool */	} else {		pv = NULL;		/* need more of them */	}	/*	 * if below low water mark or we didn't get a pv_entry we try and	 * create more pv_entrys ...	 */	if (pv_nfpvents < PVE_LOWAT || pv == NULL) {		if (pv == NULL)			pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ?					       mode : ALLOCPV_NEED);		else			(void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED);	}	simple_unlock(&pvalloc_lock);	return(pv);}/* * pmap_alloc_pvpage: maybe allocate a new pvpage * * if need_entry is false: try and allocate a new pv_page * if need_entry is true: try and allocate a new pv_page and return a *	new pv_entry from it.   if we are unable to allocate a pv_page *	we make a last ditch effort to steal a pv_page from some other *	mapping.    if that fails, we panic... * * => we assume that the caller holds pvalloc_lock */static struct pv_entry *pmap_alloc_pvpage(pmap, mode)	struct pmap *pmap;	int mode;{	struct vm_page *pg;	struct pv_page *pvpage;	int lcv, idx, npg, s;	struct pv_entry *pv, *cpv, *prevpv;	/*	 * if we need_entry and we've got unused pv_pages, allocate from there	 */	if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) {		/* move it to pv_freepages list */		pvpage = pv_unusedpgs.tqh_first;		TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list);		TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list);		/* allocate a pv_entry */		pvpage->pvinfo.pvpi_nfree--;	/* can't go to zero */		pv = pvpage->pvinfo.pvpi_pvfree;#ifdef DIAGNOSTIC		if (pv == NULL)			panic("pmap_alloc_pvpage: pvpi_nfree off");#endif		pvpage->pvinfo.pvpi_pvfree = pv->pv_next;		pv_nfpvents--;  /* took one from pool */		return(pv);	}	/*	 *  see if we've got a cached unmapped VA that we can map a page in.	 * if not, try to allocate one.	 */	s = splimp();   /* must protect kmem_map/kmem_object with splimp! */	if (pv_cachedva == NULL) {		pv_cachedva = uvm_km_kmemalloc(kmem_map, uvmexp.kmem_object,		    NBPG, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC);		if (pv_cachedva == NULL) {			splx(s);			goto steal_one;		}	}	/*	 * we have a VA, now let's try and allocate a page in the object	 * note: we are still holding splimp to protect kmem_object	 */	if (!simple_lock_try(&uvmexp.kmem_object->vmobjlock)) {		splx(s);		goto steal_one;	}	pg = uvm_pagealloc(uvmexp.kmem_object, pv_cachedva -			   vm_map_min(kernel_map),			   NULL, UVM_PGA_USERESERVE);	if (pg)		pg->flags &= ~PG_BUSY;	/* never busy */	simple_unlock(&uvmexp.kmem_object->vmobjlock);	splx(s);	/* splimp now dropped */	if (pg == NULL)		goto steal_one;	/*	 * add a mapping for our new pv_page and free its entrys (save one!)	 *	 * NOTE: If we are allocating a PV page for the kernel pmap, the	 * pmap is already locked!  (...but entering the mapping is safe...)	 */	pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL);	pvpage = (struct pv_page *) pv_cachedva;	pv_cachedva = NULL;	return(pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED));steal_one:	/*	 * if we don't really need a pv_entry right now, we can just return.	 */	if (mode != ALLOCPV_NEED)		return(NULL);	/*	 * last ditch effort!   we couldn't allocate a free page to make	 * more pv_entrys so we try and steal one from someone else.	 */	pv = NULL;	for (lcv = 0 ; pv == NULL && lcv < vm_nphysseg ; lcv++) {		npg = vm_physmem[lcv].end - vm_physmem[lcv].start;		for (idx = 0 ; idx < npg ; idx++) {			struct pv_head *pvhead = vm_physmem[lcv].pmseg.pvhead;			if (pvhead->pvh_list == NULL)				continue;	/* spot check */			if (!simple_lock_try(&pvhead->pvh_lock))				continue;			cpv = prevpv = pvhead->pvh_list;			while (cpv) {				if (pmap_try_steal_pv(pvhead, cpv, prevpv))					break;				prevpv = cpv;				cpv = cpv->pv_next;			}			simple_unlock(&pvhead->pvh_lock);			/* got one?  break out of the loop! */			if (cpv) {				pv = cpv;				break;			}		}	}	return(pv);}/* * pmap_try_steal_pv: try and steal a pv_entry from a pmap * * => return true if we did it! */static boolean_tpmap_try_steal_pv(pvh, cpv, prevpv)	struct pv_head *pvh;	struct pv_entry *cpv, *prevpv;{	pt_entry_t *ptep;	/* pointer to a PTE */	/*	 * we never steal kernel mappings or mappings from pmaps we can't lock	 */	if (cpv->pv_pmap == pmap_kernel() ||	    !simple_lock_try(&cpv->pv_pmap->pm_obj.vmobjlock))		return(FALSE);	/*	 * yes, we can try and steal it.   first we need to remove the	 * mapping from the pmap.	 */	ptep = pmap_tmpmap_pvepte(cpv);	if (*ptep & PG_W) {		ptep = NULL;	/* wired page, avoid stealing this one */	} else {		*ptep = 0;		/* zap! */		if (pmap_is_curpmap(cpv->pv_pmap))			pmap_update_pg(cpv->pv_va);		pmap_tmpunmap_pvepte(cpv);	}	if (ptep == NULL) {		simple_unlock(&cpv->pv_pmap->pm_obj.vmobjlock);		return(FALSE);	/* wired page, abort! */	}	cpv->pv_pmap->pm_stats.resident_count--;	if (cpv->pv_ptp && cpv->pv_ptp->wire_count)		/* drop PTP's wired count */		cpv->pv_ptp->wire_count--;	/*	 * XXX: if wire_count goes to one the PTP could be freed, however,	 * we'd have to lock the page queues (etc.) to do that and it could	 * cause deadlock headaches.   besides, the pmap we just stole from	 * may want the mapping back anyway, so leave the PTP around.	 */	/*	 * now we need to remove the entry from the pvlist	 */	if (cpv == pvh->pvh_list)		pvh->pvh_list = cpv->pv_next;	else		prevpv->pv_next = cpv->pv_next;	return(TRUE);}/* * pmap_add_pvpage: add a pv_page's pv_entrys to the free list * * => caller must hold pvalloc_lock * => if need_entry is true, we allocate and return one pv_entry */static struct pv_entry *pmap_add_pvpage(pvp, need_entry)	struct pv_page *pvp;	boolean_t need_entry;{	int tofree, lcv;	/* do we need to return one? */	tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE;	pvp->pvinfo.pvpi_pvfree = NULL;	pvp->pvinfo.pvpi_nfree = tofree;	for (lcv = 0 ; lcv < tofree ; lcv++) {		pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree;		pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv];	}	if (need_entry)		TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list);	else		TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list);	pv_nfpvents += tofree;	return((need_entry) ? &pvp->pvents[lcv] : NULL);}/* * pmap_free_pv_doit: actually free a pv_entry * * => do not call this directly!  instead use either *    1. pmap_free_pv ==> free a single pv_entry *    2. pmap_free_pvs => free a list of pv_entrys * => we must be holding pvalloc_lock */__inline static voidpmap_free_pv_doit(pv)	struct pv_entry *pv;{	struct pv_page *pvp;	pvp = (struct pv_page *) i386_trunc_page(pv);	pv_nfpvents++;	pvp->pvinfo.pvpi_nfree++;	/* nfree == 1 => fully allocated page just became partly allocated */	if (pvp->pvinfo.pvpi_nfree == 1) {		TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list);	}	/* free it */	pv->pv_next = pvp->pvinfo.pvpi_pvfree;	pvp->pvinfo.pvpi_pvfree = pv;	/*	 * are all pv_page's pv_entry's free?  move it to unused queue.	 */	if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) {		TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list);		TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list);	}}/* * pmap_free_pv: free a single pv_entry * * => we gain the pvalloc_lock */__inline static voidpmap_free_pv(pmap, pv)	struct pmap *pmap;	struct pv_entry *pv;{	simple_lock(&pvalloc_lock);	pmap_free_pv_doit(pv);	/*	 * Can't free the PV page if the PV entries were associated with	 * the kernel pmap; the pmap is already locked.	 */	if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL &&	    pmap != pmap_kernel())		pmap_free_pvpage();	simple_unlock(&pvalloc_lock);}/* * pmap_free_pvs: free a list of pv_entrys * * => we gain the pvalloc_lock */__inline static voidpmap_free_pvs(pmap, pvs)	struct pmap *pmap;	struct pv_entry *pvs;{	struct pv_entry *nextpv;	simple_lock(&pvalloc_lock);	for ( /* null */ ; pvs != NULL ; pvs = nextpv) {		nextpv = pvs->pv_next;		pmap_free_pv_doit(pvs);	}	/*	 * Can't free the PV page if the PV entries were associated with	 * the kernel pmap; the pmap is already locked.	 */	if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL &&	    pmap != pmap_kernel())		pmap_free_pvpage();	simple_unlock(&pvalloc_lock);}/* * pmap_free_pvpage: try and free an unused pv_page structure * * => assume caller is holding the pvalloc_lock and that *	there is a page on the pv_unusedpgs list * => if we can't get a lock on the kmem_map we try again later * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows *	that if we can lock the kmem_map then we are not already *	holding kmem_object's lock. */static voidpmap_free_pvpage(){	int s;	struct vm_map *map;	vm_map_entry_t dead_entries;	struct pv_page *pvp;	s = splimp(); /* protect kmem_map */	pvp = pv_unusedpgs.tqh_first;	/*	 * note: watch out for pv_initpage which is allocated out of	 * kernel_map rather than kmem_map.	 */	if (pvp == pv_initpage)		map = kernel_map;	else		map = kmem_map;	if (vm_map_lock_try(map)) {		/* remove pvp from pv_unusedpgs */		TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list);		/* unmap the page */		dead_entries = NULL;		(void)uvm_unmap_remove(map, (vaddr_t) pvp,				       ((vaddr_t) pvp) + NBPG, &dead_entries);		vm_map_unlock(map);		if (dead_entries != NULL)			uvm_unmap_detach(dead_entries, 0);		pv_nfpvents -= PVE_PER_PVPAGE;  /* update free count */	}	if (pvp == pv_initpage)		/* no more initpage, we've freed it */		pv_initpage = NULL;	splx(s);}/* * main pv_entry manipulation functions: *   pmap_enter_pv: enter a mapping onto a pv_head list *   pmap_remove_pv: remove a mappiing from a pv_head list * * NOTE: pmap_enter_pv expects to lock the pvh itself *       pmap_remove_pv expects te caller to lock the pvh before calling *//* * pmap_enter_pv: enter a mapping onto a pv_head lst * * => caller should hold the proper lock on pmap_main_lock * => caller should have pmap locked * => we will gain the lock on the pv_head and allocate the new pv_entry * => caller should adjust ptp's wire_count before calling */__inline static voidpmap_enter_pv(pvh, pve, pmap, va, ptp)	struct pv_head *pvh;	struct pv_entry *pve;	/* preallocated pve for us to use */	struct pmap *pmap;	vaddr_t va;	struct vm_page *ptp;	/* PTP in pmap that maps this VA */{	pve->pv_pmap = pmap;	pve->pv_va = va;	pve->pv_ptp = ptp;			/* NULL for kernel pmap */	simple_lock(&pvh->pvh_lock);		/* lock pv_head */	pve->pv_next = pvh->pvh_list;		/* add to ... */	pvh->pvh_list = pve;			/* ... locked list */	simple_unlock(&pvh->pvh_lock);		/* unlock, done! */}/* * pmap_remove_pv: try to remove a mapping from a pv_list * * => caller should hold proper lock on pmap_main_lock * => pmap should be locked * => caller should hold lock on pv_head [so that attrs can be adjusted] * => caller should adjust ptp's wire_count and free PTP if needed * => we return the removed pve */__inline static struct pv_entry *pmap_remove_pv(pvh, pmap, va)	struct pv_head *pvh;	struct pmap *pmap;	vaddr_t va;{	struct pv_entry *pve, **prevptr;	prevptr = &pvh->pvh_list;		/* previous pv_entry pointer */	pve = *prevptr;	while (pve) {		if (pve->pv_pmap == pmap && pve->pv_va == va) {	/* match? */			*prevptr = pve->pv_next;		/* remove it! */			break;		}		prevptr = &pve->pv_next;		/* previous pointer */		pve = pve->pv_next;			/* advance */	}	return(pve);				/* return removed pve */}/* * p t p   f u n c t i o n s *//* * pmap_alloc_ptp: allocate a PTP for a PMAP * * => pmap should already be locked by caller * => we use the ptp's wire_count to count the number of active mappings *	in the PTP (we start it at one to prevent any chance this PTP *	will ever leak onto the active/inactive queues) * => we should not be holding any pv_head locks (in case we are forced *	to call pmap_steal_ptp()) * => we may need to lock pv_head's if we have to steal a PTP * => just_try: true if we want a PTP, but not enough to steal one * 	from another pmap (e.g. during optional functions like pmap_copy) */__inline static struct vm_page *pmap_alloc_ptp(pmap, pde_index, just_try)	struct pmap *pmap;	int pde_index;	boolean_t just_try;{	struct vm_page *ptp;	ptp = uvm_pagealloc(&pmap->pm_obj, ptp_i2o(pde_index), NULL,			    UVM_PGA_USERESERVE);	if (ptp == NULL) {		if (just_try)			return(NULL);		ptp = pmap_steal_ptp(&pmap->pm_obj, ptp_i2o(pde_index));		if (ptp == NULL) {			return (NULL);		}	}	/* got one! */	ptp->flags &= ~PG_BUSY;	/* never busy */	ptp->wire_count = 1;	/* no mappings yet */	pmap_zero_page(VM_PAGE_TO_PHYS(ptp));	pmap->pm_pdir[pde_index] =		(pd_entry_t) (VM_PAGE_TO_PHYS(ptp) | PG_u | PG_RW | PG_V);	pmap->pm_stats.resident_count++;	/* count PTP as resident */	pmap->pm_ptphint = ptp;	return(ptp);}/* * pmap_steal_ptp: steal a PTP from any pmap that we can access * * => obj is locked by caller. * => we can throw away mappings at this level (except in the kernel's pmap) * => stolen PTP is placed in <obj,offset> pmap * => we lock pv_head's * => hopefully, this function will be seldom used [much better to have *	enough free pages around for us to allocate off the free page list] */static struct vm_page *pmap_steal_ptp(obj, offset)	struct uvm_object *obj;	vaddr_t offset;{	struct vm_page *ptp = NULL;	struct pmap *firstpmap;	struct uvm_object *curobj;	pt_entry_t *ptes;	int idx, lcv;	boolean_t caller_locked, we_locked;	simple_lock(&pmaps_lock);	if (pmaps_hand == NULL)		pmaps_hand = LIST_FIRST(&pmaps);	firstpmap = pmaps_hand;	do { /* while we haven't looped back around to firstpmap */		curobj = &pmaps_hand->pm_obj;		we_locked = FALSE;		caller_locked = (curobj == obj);		if (!caller_locked) {			we_locked = simple_lock_try(&curobj->vmobjlock);		}		if (caller_locked || we_locked) {			ptp = curobj->memq.tqh_first;			for (/*null*/; ptp != NULL; ptp = ptp->listq.tqe_next) {				/*				 * might have found a PTP we can steal				 * (unless it has wired pages).				 */				idx = ptp_o2i(ptp->offset);#ifdef DIAGNOSTIC				if (VM_PAGE_TO_PHYS(ptp) !=				    (pmaps_hand->pm_pdir[idx] & PG_FRAME))					panic("pmap_steal_ptp: PTP mismatch!");#endif				ptes = (pt_entry_t *)					pmap_tmpmap_pa(VM_PAGE_TO_PHYS(ptp));				for (lcv = 0 ; lcv < PTES_PER_PTP ; lcv++)					if ((ptes[lcv] & (PG_V|PG_W)) ==					    (PG_V|PG_W))						break;				if (lcv == PTES_PER_PTP)					pmap_remove_ptes(pmaps_hand, NULL, ptp,							 (vaddr_t)ptes,							 ptp_i2v(idx),							 ptp_i2v(idx+1));				pmap_tmpunmap_pa();				if (lcv != PTES_PER_PTP)					/* wired, try next PTP */					continue;				/*				 * got it!!!				 */				pmaps_hand->pm_pdir[idx] = 0;	/* zap! */				pmaps_hand->pm_stats.resident_count--;				if (pmap_is_curpmap(pmaps_hand))					pmap_update();				else if (pmap_valid_entry(*APDP_PDE) &&					 (*APDP_PDE & PG_FRAME) ==					 pmaps_hand->pm_pdirpa) {					pmap_update_pg(((vaddr_t)APTE_BASE) +						       ptp->offset);				}				/* put it in our pmap! */				uvm_pagerealloc(ptp, obj, offset);				break;	/* break out of "for" loop */			}			if (we_locked) {				simple_unlock(&curobj->vmobjlock);			}		}		/* advance the pmaps_hand */		pmaps_hand = LIST_NEXT(pmaps_hand, pm_list);		if (pmaps_hand == NULL) {			pmaps_hand = LIST_FIRST(&pmaps);		}	} while (ptp == NULL && pmaps_hand != firstpmap);	simple_unlock(&pmaps_lock);	return(ptp);}/* * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) * * => pmap should NOT be pmap_kernel() * => pmap should be locked */static struct vm_page *pmap_get_ptp(pmap, pde_index, just_try)	struct pmap *pmap;	int pde_index;	boolean_t just_try;{	struct vm_page *ptp;	if (pmap_valid_entry(pmap->pm_pdir[pde_index])) {		/* valid... check hint (saves us a PA->PG lookup) */		if (pmap->pm_ptphint &&		    (pmap->pm_pdir[pde_index] & PG_FRAME) ==		    VM_PAGE_TO_PHYS(pmap->pm_ptphint))			return(pmap->pm_ptphint);		ptp = uvm_pagelookup(&pmap->pm_obj, ptp_i2o(pde_index));#ifdef DIAGNOSTIC		if (ptp == NULL)			panic("pmap_get_ptp: unmanaged user PTP");#endif		pmap->pm_ptphint = ptp;		return(ptp);	}	/* allocate a new PTP (updates ptphint) */	return(pmap_alloc_ptp(pmap, pde_index, just_try));}