return EACCES;
return (KERN_PROTECTION_FAILURE); }    if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { UVMHIST_LOG(maphist, "<- map %p not pageable", ufi.map, 0, 0, 0); uvmfault_unlockmaps(&ufi, FALSE); return (KERN_FAILURE);
uvm_fault(orig_map, vaddr, fault_type, access_type)	struct vm_map *orig_map;	vaddr_t vaddr;	vm_fault_t fault_type;	vm_prot_t access_type;{	struct uvm_faultinfo ufi;	vm_prot_t enter_prot;	boolean_t wired, narrow, promote, locked, shadowed;	int npages, nback, nforw, centeridx, error, lcv, gotpages;	vaddr_t startva, objaddr, currva, offset, uoff;	paddr_t pa;	struct vm_amap *amap;	struct uvm_object *uobj;	struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;	struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;	UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);	UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)",	      orig_map, vaddr, fault_type, access_type);	anon = NULL;	pg = NULL;	uvmexp.faults++;	/* XXX: locking? */	/*	 * init the IN parameters in the ufi	 */	ufi.orig_map = orig_map;	ufi.orig_rvaddr = trunc_page(vaddr);	ufi.orig_size = PAGE_SIZE;	/* can't get any smaller than this */	if (fault_type == VM_FAULT_WIRE)		narrow = TRUE;		/* don't look for neighborhood					 * pages on wire */	else		narrow = FALSE;		/* normal fault */	/*	 * "goto ReFault" means restart the page fault from ground zero.	 */ReFault:	/*	 * lookup and lock the maps	 */	if (uvmfault_lookup(&ufi, FALSE) == FALSE) {		UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);		return (EFAULT);	}	/* locked: maps(read) */#ifdef DIAGNOSTIC	if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) {		printf("Page fault on non-pageable map:\n");		printf("ufi.map = %p\n", ufi.map);		printf("ufi.orig_map = %p\n", ufi.orig_map);		printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr);		panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0");	}#endif	/*	 * check protection	 */	if ((ufi.entry->protection & access_type) != access_type) {		UVMHIST_LOG(maphist,		    "<- protection failure (prot=0x%x, access=0x%x)",		    ufi.entry->protection, access_type, 0, 0);		uvmfault_unlockmaps(&ufi, FALSE);		return EACCES;	}	/*	 * "enter_prot" is the protection we want to enter the page in at.	 * for certain pages (e.g. copy-on-write pages) this protection can	 * be more strict than ufi.entry->protection.  "wired" means either	 * the entry is wired or we are fault-wiring the pg.	 */	enter_prot = ufi.entry->protection;	wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE);	if (wired)		access_type = enter_prot; /* full access for wired */	/*	 * handle "needs_copy" case.   if we need to copy the amap we will	 * have to drop our readlock and relock it with a write lock.  (we	 * need a write lock to change anything in a map entry [e.g.	 * needs_copy]).	 */	if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {		if ((access_type & VM_PROT_WRITE) ||		    (ufi.entry->object.uvm_obj == NULL)) {			/* need to clear */			UVMHIST_LOG(maphist,			    "  need to clear needs_copy and refault",0,0,0,0);			uvmfault_unlockmaps(&ufi, FALSE);			uvmfault_amapcopy(&ufi);			uvmexp.fltamcopy++;			goto ReFault;		} else {			/*			 * ensure that we pmap_enter page R/O since			 * needs_copy is still true			 */			enter_prot &= ~VM_PROT_WRITE;		}	}	/*	 * identify the players	 */	amap = ufi.entry->aref.ar_amap;		/* top layer */	uobj = ufi.entry->object.uvm_obj;	/* bottom layer */	/*	 * check for a case 0 fault.  if nothing backing the entry then	 * error now.	 */	if (amap == NULL && uobj == NULL) {		uvmfault_unlockmaps(&ufi, FALSE);		UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);		return (EFAULT);	}	/*	 * establish range of interest based on advice from mapper	 * and then clip to fit map entry.   note that we only want	 * to do this the first time through the fault.   if we	 * ReFault we will disable this by setting "narrow" to true.	 */	if (narrow == FALSE) {		/* wide fault (!narrow) */		KASSERT(uvmadvice[ufi.entry->advice].advice ==			 ufi.entry->advice);		nback = min(uvmadvice[ufi.entry->advice].nback,			    (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);		startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);		nforw = min(uvmadvice[ufi.entry->advice].nforw,			    ((ufi.entry->end - ufi.orig_rvaddr) >>			     PAGE_SHIFT) - 1);		/*		 * note: "-1" because we don't want to count the		 * faulting page as forw		 */		npages = nback + nforw + 1;		centeridx = nback;		narrow = TRUE;	/* ensure only once per-fault */	} else {		/* narrow fault! */		nback = nforw = 0;		startva = ufi.orig_rvaddr;		npages = 1;		centeridx = 0;	}	/* locked: maps(read) */	UVMHIST_LOG(maphist, "  narrow=%d, back=%d, forw=%d, startva=0x%x",		    narrow, nback, nforw, startva);	UVMHIST_LOG(maphist, "  entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry,		    amap, uobj, 0);	/*	 * if we've got an amap, lock it and extract current anons.	 */	if (amap) {		amap_lock(amap);		anons = anons_store;		amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,		    anons, npages);	} else {		anons = NULL;	/* to be safe */	}	/* locked: maps(read), amap(if there) */	/*	 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages	 * now and then forget about them (for the rest of the fault).	 */	if (ufi.entry->advice == MADV_SEQUENTIAL) {		UVMHIST_LOG(maphist, "  MADV_SEQUENTIAL: flushing backpages",		    0,0,0,0);		/* flush back-page anons? */		if (amap)			uvmfault_anonflush(anons, nback);		/* flush object? */		if (uobj) {			objaddr =			    (startva - ufi.entry->start) + ufi.entry->offset;			simple_lock(&uobj->vmobjlock);			(void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr +				    (nback << PAGE_SHIFT), PGO_DEACTIVATE);			simple_unlock(&uobj->vmobjlock);		}		/* now forget about the backpages */		if (amap)			anons += nback;		startva += (nback << PAGE_SHIFT);		npages -= nback;		nback = centeridx = 0;	}	/* locked: maps(read), amap(if there) */	/*	 * map in the backpages and frontpages we found in the amap in hopes	 * of preventing future faults.    we also init the pages[] array as	 * we go.	 */	currva = startva;	shadowed = FALSE;	for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {		/*		 * dont play with VAs that are already mapped		 * except for center)		 */		if (lcv != centeridx &&		    pmap_extract(ufi.orig_map->pmap, currva, &pa)) {			pages[lcv] = PGO_DONTCARE;			continue;		}		/*		 * unmapped or center page.   check if any anon at this level.		 */		if (amap == NULL || anons[lcv] == NULL) {			pages[lcv] = NULL;			continue;		}		/*		 * check for present page and map if possible.   re-activate it.		 */		pages[lcv] = PGO_DONTCARE;		if (lcv == centeridx) {		/* save center for later! */			shadowed = TRUE;			continue;		}		anon = anons[lcv];		simple_lock(&anon->an_lock);		/* ignore loaned pages */		if (anon->u.an_page && anon->u.an_page->loan_count == 0 &&			(anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) {			uvm_lock_pageq();			uvm_pageactivate(anon->u.an_page);	/* reactivate */			uvm_unlock_pageq();			UVMHIST_LOG(maphist,			    "  MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",			    ufi.orig_map->pmap, currva, anon->u.an_page, 0);			uvmexp.fltnamap++;			/*			 * Since this isn't the page that's actually faulting,			 * ignore pmap_enter() failures; it's not critical			 * that we enter these right now.			 */			(void) pmap_enter(ufi.orig_map->pmap, currva,			    VM_PAGE_TO_PHYS(anon->u.an_page),			    (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :			    enter_prot,			    PMAP_CANFAIL |			     (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));		}		simple_unlock(&anon->an_lock);		pmap_update(ufi.orig_map->pmap);	}	/* locked: maps(read), amap(if there) */	/* (shadowed == TRUE) if there is an anon at the faulting address */	UVMHIST_LOG(maphist, "  shadowed=%d, will_get=%d", shadowed,	    (uobj && shadowed == FALSE),0,0);	/*	 * note that if we are really short of RAM we could sleep in the above	 * call to pmap_enter with everything locked.   bad?	 *	 * XXX Actually, that is bad; pmap_enter() should just fail in that	 * XXX case.  --thorpej	 */	/*	 * if the desired page is not shadowed by the amap and we have a	 * backing object, then we check to see if the backing object would	 * prefer to handle the fault itself (rather than letting us do it	 * with the usual pgo_get hook).  the backing object signals this by	 * providing a pgo_fault routine.	 */	if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) {		simple_lock(&uobj->vmobjlock);		/* locked: maps(read), amap (if there), uobj */		error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,		    centeridx, fault_type, access_type, PGO_LOCKED|PGO_SYNCIO);		/* locked: nothing, pgo_fault has unlocked everything */		if (error == ERESTART)			goto ReFault;		/* try again! */		/*		 * object fault routine responsible for pmap_update().		 */		return error;	}	/*	 * now, if the desired page is not shadowed by the amap and we have	 * a backing object that does not have a special fault routine, then	 * we ask (with pgo_get) the object for resident pages that we care	 * about and attempt to map them in.  we do not let pgo_get block	 * (PGO_LOCKED).	 *	 * ("get" has the option of doing a pmap_enter for us)	 */	if (uobj && shadowed == FALSE) {		simple_lock(&uobj->vmobjlock);		/* locked (!shadowed): maps(read), amap (if there), uobj */		/*		 * the following call to pgo_get does _not_ change locking state		 */		uvmexp.fltlget++;		gotpages = npages;		(void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +				(startva - ufi.entry->start),				pages, &gotpages, centeridx,				access_type & MASK(ufi.entry),				ufi.entry->advice, PGO_LOCKED);		/*		 * check for pages to map, if we got any		 */		uobjpage = NULL;		if (gotpages) {			currva = startva;			for (lcv = 0 ; lcv < npages ;			    lcv++, currva += PAGE_SIZE) {				if (pages[lcv] == NULL ||				    pages[lcv] == PGO_DONTCARE)					continue;				KASSERT((pages[lcv]->flags & PG_RELEASED) == 0);				/*				 * if center page is resident and not				 * PG_BUSY|PG_RELEASED then pgo_get				 * made it PG_BUSY for us and gave				 * us a handle to it.   remember this				 * page as "uobjpage." (for later use).				 */				if (lcv == centeridx) {					uobjpage = pages[lcv];					UVMHIST_LOG(maphist, "  got uobjpage "					    "(0x%x) with locked get",					    uobjpage, 0,0,0);					continue;				}				/*				 * note: calling pgo_get with locked data				 * structures returns us pages which are				 * neither busy nor released, so we don't				 * need to check for this.   we can just				 * directly enter the page (after moving it				 * to the head of the active queue [useful?]).				 */				uvm_lock_pageq();				uvm_pageactivate(pages[lcv]);	/* reactivate */				uvm_unlock_pageq();				UVMHIST_LOG(maphist,				  "  MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",				  ufi.orig_map->pmap, currva, pages[lcv], 0);				uvmexp.fltnomap++;				/*				 * Since this page isn't the page that's				 * actually fauling, ignore pmap_enter()				 * failures; it's not critical that we				 * enter these right now.				 */				(void) pmap_enter(ufi.orig_map->pmap, currva,				    VM_PAGE_TO_PHYS(pages[lcv]),				    pages[lcv]->flags & PG_RDONLY ?				    VM_PROT_READ : enter_prot & MASK(ufi.entry),				    PMAP_CANFAIL |				     (wired ? PMAP_WIRED : 0));				/*				 * NOTE: page can't be PG_WANTED or PG_RELEASED				 * because we've held the lock the whole time				 * we've had the handle.				 */				pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */				UVM_PAGE_OWN(pages[lcv], NULL);			}	/* for "lcv" loop */			pmap_update(ufi.orig_map->pmap);		}   /* "gotpages" != 0 */		/* note: object still _locked_ */	} else {		uobjpage = NULL;	}	/* locked (shadowed): maps(read), amap */	/* locked (!shadowed): maps(read), amap(if there),		 uobj(if !null), uobjpage(if !null) */	/*	 * note that at this point we are done with any front or back pages.	 * we are now going to focus on the center page (i.e. the one we've	 * faulted on).  if we have faulted on the top (anon) layer	 * [i.e. case 1], then the anon we want is anons[centeridx] (we have	 * not touched it yet).  if we have faulted on the bottom (uobj)	 * layer [i.e. case 2] and the page was both present and available,	 * then we've got a pointer to it as "uobjpage" and we've already	 * made it BUSY.	 */	/*	 * there are four possible cases we must address: 1A, 1B, 2A, and 2B	 */	/*	 * redirect case 2: if we are not shadowed, go to case 2.	 */	if (shadowed == FALSE)		goto Case2;	/* locked: maps(read), amap */	/*	 * handle case 1: fault on an anon in our amap	 */	anon = anons[centeridx];	UVMHIST_LOG(maphist, "  case 1 fault: anon=0x%x", anon, 0,0,0);	simple_lock(&anon->an_lock);	/* locked: maps(read), amap, anon */	/*	 * no matter if we have case 1A or case 1B we are going to need to	 * have the anon's memory resident.   ensure that now.	 */	/*	 * let uvmfault_anonget do the dirty work.	 * if it fails (!OK) it will unlock everything for us.	 * if it succeeds, locks are still valid and locked.	 * also, if it is OK, then the anon's page is on the queues.	 * if the page is on loan from a uvm_object, then anonget will	 * lock that object for us if it does not fail.	 */	error = uvmfault_anonget(&ufi, amap, anon);	switch (error) {	case 0:		break;	case ERESTART:		goto ReFault;	case EAGAIN:		tsleep(&lbolt, PVM, "fltagain1", 0);		goto ReFault;	default:		return error;	}	/*	 * uobj is non null if the page is on loan from an object (i.e. uobj)	 */	uobj = anon->u.an_page->uobject;	/* locked by anonget if !NULL */	/* locked: maps(read), amap, anon, uobj(if one) */	/*	 * special handling for loaned pages	 */	if (anon->u.an_page->loan_count) {		if ((access_type & VM_PROT_WRITE) == 0) {			/*			 * for read faults on loaned pages we just cap the			 * protection at read-only.			 */			enter_prot = enter_prot & ~VM_PROT_WRITE;		} else {			/*			 * note that we can't allow writes into a loaned page!			 *			 * if we have a write fault on a loaned page in an			 * anon then we need to look at the anon's ref count.			 * if it is greater than one then we are going to do			 * a normal copy-on-write fault into a new anon (this			 * is not a problem).  however, if the reference count			 * is one (a case where we would normally allow a			 * write directly to the page) then we need to kill			 * the loan before we continue.			 */			/* >1 case is already ok */			if (anon->an_ref == 1) {				/* get new un-owned replacement page */				pg = uvm_pagealloc(NULL, 0, NULL, 0);				if (pg == NULL) {					uvmfault_unlockall(&ufi, amap, uobj,					    anon);					uvm_wait("flt_noram2");					goto ReFault;				}				/*				 * copy data, kill loan, and drop uobj lock				 * (if any)				 */				/* copy old -> new */				uvm_pagecopy(anon->u.an_page, pg);				/* force reload */				pmap_page_protect(anon->u.an_page,						  VM_PROT_NONE);				uvm_lock_pageq();	  /* KILL loan */				if (uobj)					/* if we were loaning */					anon->u.an_page->loan_count--;				anon->u.an_page->uanon = NULL;				/* in case we owned */				anon->u.an_page->pqflags &= ~PQ_ANON;				uvm_unlock_pageq();				if (uobj) {					simple_unlock(&uobj->vmobjlock);					uobj = NULL;				}				/* install new page in anon */				anon->u.an_page = pg;				pg->uanon = anon;				pg->pqflags |= PQ_ANON;				pg->flags &= ~(PG_BUSY|PG_FAKE);				UVM_PAGE_OWN(pg, NULL);				/* done! */			}     /* ref == 1 */		}       /* write fault */	}         /* loan count */	/*	 * if we are case 1B then we will need to allocate a new blank	 * anon to transfer the data into.   note that we have a lock	 * on anon, so no one can busy or release the page until we are done.	 * also note that the ref count can't drop to zero here because	 * it is > 1 and we are only dropping one ref.	 *	 * in the (hopefully very rare) case that we are out of RAM we	 * will unlock, wait for more RAM, and refault.	 *	 * if we are out of anon VM we kill the process (XXX: could wait?).	 */	if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) {		UVMHIST_LOG(maphist, "  case 1B: COW fault",0,0,0,0);		uvmexp.flt_acow++;		oanon = anon;		/* oanon = old, locked anon */		anon = uvm_analloc();		if (anon) {			/* new anon is locked! */			pg = uvm_pagealloc(NULL, 0, anon, 0);		}		/* check for out of RAM */		if (anon == NULL || pg == NULL) {			if (anon) {				anon->an_ref--;				simple_unlock(&anon->an_lock);				uvm_anfree(anon);			}			uvmfault_unlockall(&ufi, amap, uobj, oanon);			KASSERT(uvmexp.swpgonly <= uvmexp.swpages);			if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {				UVMHIST_LOG(maphist,				    "<- failed.  out of VM",0,0,0,0);				uvmexp.fltnoanon++;				return ENOMEM;			}			uvmexp.fltnoram++;			uvm_wait("flt_noram3");	/* out of RAM, wait for more */			goto ReFault;		}		/* got all resources, replace anon with nanon */		uvm_pagecopy(oanon->u.an_page, pg);	/* pg now !PG_CLEAN */		pg->flags &= ~(PG_BUSY|PG_FAKE);	/* un-busy! new page */		UVM_PAGE_OWN(pg, NULL);		amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,		    anon, 1);		/* deref: can not drop to zero here by defn! */		oanon->an_ref--;		/*		 * note: oanon is still locked, as is the new anon.  we		 * need to check for this later when we unlock oanon; if		 * oanon != anon, we'll have to unlock anon, too.		 */	} else {		uvmexp.flt_anon++;		oanon = anon;		/* old, locked anon is same as anon */		pg = anon->u.an_page;		if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */			enter_prot = enter_prot & ~VM_PROT_WRITE;	}	/* locked: maps(read), amap, oanon, anon (if different from oanon) */	/*	 * now map the page in ...	 * XXX: old fault unlocks object before pmap_enter.  this seems	 * suspect since some other thread could blast the page out from	 * under us between the unlock and the pmap_enter.	 */	UVMHIST_LOG(maphist, "  MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x",	    ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);	if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),	    enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))	    != 0) {		/*		 * No need to undo what we did; we can simply think of		 * this as the pmap throwing away the mapping information.		 *		 * We do, however, have to go through the ReFault path,		 * as the map may change while we're asleep.		 */		if (anon != oanon)			simple_unlock(&anon->an_lock);		uvmfault_unlockall(&ufi, amap, uobj, oanon);		KASSERT(uvmexp.swpgonly <= uvmexp.swpages);		if (uvmexp.swpgonly == uvmexp.swpages) {			UVMHIST_LOG(maphist,			    "<- failed.  out of VM",0,0,0,0);			/* XXX instrumentation */			return ENOMEM;		}		/* XXX instrumentation */		uvm_wait("flt_pmfail1");		goto ReFault;	}	/*	 * ... update the page queues.	 */	uvm_lock_pageq();	if (fault_type == VM_FAULT_WIRE) {		uvm_pagewire(pg);		/*		 * since the now-wired page cannot be paged out,		 * release its swap resources for others to use.		 * since an anon with no swap cannot be PG_CLEAN,		 * clear its clean flag now.		 */		pg->flags &= ~(PG_CLEAN);		uvm_anon_dropswap(anon);	} else {		/* activate it */		uvm_pageactivate(pg);	}	uvm_unlock_pageq();	/*	 * done case 1!  finish up by unlocking everything and returning success	 */	if (anon != oanon)		simple_unlock(&anon->an_lock);	uvmfault_unlockall(&ufi, amap, uobj, oanon);	pmap_update(ufi.orig_map->pmap);	return 0;Case2:	/*	 * handle case 2: faulting on backing object or zero fill	 */	/*	 * locked:	 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)	 */	/*	 * note that uobjpage can not be PGO_DONTCARE at this point.  we now	 * set uobjpage to PGO_DONTCARE if we are doing a zero fill.  if we	 * have a backing object, check and see if we are going to promote	 * the data up to an anon during the fault.	 */	if (uobj == NULL) {		uobjpage = PGO_DONTCARE;		promote = TRUE;		/* always need anon here */	} else {		KASSERT(uobjpage != PGO_DONTCARE);		promote = (access_type & VM_PROT_WRITE) &&		     UVM_ET_ISCOPYONWRITE(ufi.entry);	}	UVMHIST_LOG(maphist, "  case 2 fault: promote=%d, zfill=%d",	    promote, (uobj == NULL), 0,0);	/*	 * if uobjpage is not null then we do not need to do I/O to get the	 * uobjpage.	 *	 * if uobjpage is null, then we need to unlock and ask the pager to	 * get the data for us.   once we have the data, we need to reverify	 * the state the world.   we are currently not holding any resources.	 */	if (uobjpage) {		/* update rusage counters */		curproc->p_addr->u_stats.p_ru.ru_minflt++;	} else {		/* update rusage counters */		curproc->p_addr->u_stats.p_ru.ru_majflt++;		/* locked: maps(read), amap(if there), uobj */		uvmfault_unlockall(&ufi, amap, NULL, NULL);		/* locked: uobj */		uvmexp.fltget++;		gotpages = 1;		uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;		error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,		    0, access_type & MASK(ufi.entry), ufi.entry->advice,		    PGO_SYNCIO);		/* locked: uobjpage(if no error) */		/*		 * recover from I/O		 */		if (error) {			if (error == EAGAIN) {				UVMHIST_LOG(maphist,				    "  pgo_get says TRY AGAIN!",0,0,0,0);				tsleep(&lbolt, PVM, "fltagain2", 0);				goto ReFault;			}			UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",			    error, 0,0,0);			return error;		}		/* locked: uobjpage */		/*		 * re-verify the state of the world by first trying to relock		 * the maps.  always relock the object.		 */		locked = uvmfault_relock(&ufi);		if (locked && amap)			amap_lock(amap);		simple_lock(&uobj->vmobjlock);		/* locked(locked): maps(read), amap(if !null), uobj, uobjpage */		/* locked(!locked): uobj, uobjpage */		/*		 * verify that the page has not be released and re-verify		 * that amap slot is still free.   if there is a problem,		 * we unlock and clean up.		 */		if ((uobjpage->flags & PG_RELEASED) != 0 ||		    (locked && amap &&		    amap_lookup(&ufi.entry->aref,		      ufi.orig_rvaddr - ufi.entry->start))) {			if (locked)				uvmfault_unlockall(&ufi, amap, NULL, NULL);			locked = FALSE;		}		/*		 * didn't get the lock?   release the page and retry.		 */		if (locked == FALSE) {			UVMHIST_LOG(maphist,			    "  wasn't able to relock after fault: retry",			    0,0,0,0);			if (uobjpage->flags & PG_WANTED)				/* still holding object lock */				wakeup(uobjpage);			if (uobjpage->flags & PG_RELEASED) {				uvmexp.fltpgrele++;				KASSERT(uobj->pgops->pgo_releasepg != NULL);				/* frees page */				if (uobj->pgops->pgo_releasepg(uobjpage,NULL))					/* unlock if still alive */					simple_unlock(&uobj->vmobjlock);				goto ReFault;			}			uvm_lock_pageq();			/* make sure it is in queues */			uvm_pageactivate(uobjpage);			uvm_unlock_pageq();			uobjpage->flags &= ~(PG_BUSY|PG_WANTED);			UVM_PAGE_OWN(uobjpage, NULL);			simple_unlock(&uobj->vmobjlock);			goto ReFault;		}		/*		 * we have the data in uobjpage which is PG_BUSY and		 * !PG_RELEASED.  we are holding object lock (so the page		 * can't be released on us).		 */		/* locked: maps(read), amap(if !null), uobj, uobjpage */	}	/*	 * locked:	 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)	 */	/*	 * notes:	 *  - at this point uobjpage can not be NULL	 *  - at this point uobjpage can not be PG_RELEASED (since we checked	 *  for it above)	 *  - at this point uobjpage could be PG_WANTED (handle later)	 */	if (promote == FALSE) {		/*		 * we are not promoting.   if the mapping is COW ensure that we		 * don't give more access than we should (e.g. when doing a read		 * fault on a COPYONWRITE mapping we want to map the COW page in		 * R/O even though the entry protection could be R/W).		 *		 * set "pg" to the page we want to map in (uobjpage, usually)		 */		/* no anon in this case. */		anon = NULL;		uvmexp.flt_obj++;		if (UVM_ET_ISCOPYONWRITE(ufi.entry))			enter_prot &= ~VM_PROT_WRITE;		pg = uobjpage;		/* map in the actual object */		/* assert(uobjpage != PGO_DONTCARE) */		/*		 * we are faulting directly on the page.   be careful		 * about writing to loaned pages...		 */		if (uobjpage->loan_count) {			if ((access_type & VM_PROT_WRITE) == 0) {				/* read fault: cap the protection at readonly */				/* cap! */				enter_prot = enter_prot & ~VM_PROT_WRITE;			} else {				/* write fault: must break the loan here */				/* alloc new un-owned page */				pg = uvm_pagealloc(NULL, 0, NULL, 0);				if (pg == NULL) {					/*					 * drop ownership of page, it can't					 * be released					 */					if (uobjpage->flags & PG_WANTED)						wakeup(uobjpage);					uobjpage->flags &= ~(PG_BUSY|PG_WANTED);					UVM_PAGE_OWN(uobjpage, NULL);					uvm_lock_pageq();					/* activate: we will need it later */					uvm_pageactivate(uobjpage);					uvm_unlock_pageq();					uvmfault_unlockall(&ufi, amap, uobj,					  NULL);					UVMHIST_LOG(maphist,					  "  out of RAM breaking loan, waiting",					  0,0,0,0);					uvmexp.fltnoram++;					uvm_wait("flt_noram4");					goto ReFault;				}				/*				 * copy the data from the old page to the new				 * one and clear the fake/clean flags on the				 * new page (keep it busy).  force a reload				 * of the old page by clearing it from all				 * pmaps.  then lock the page queues to				 * rename the pages.				 */				uvm_pagecopy(uobjpage, pg);	/* old -> new */				pg->flags &= ~(PG_FAKE|PG_CLEAN);				pmap_page_protect(uobjpage, VM_PROT_NONE);				if (uobjpage->flags & PG_WANTED)					wakeup(uobjpage);				/* uobj still locked */				uobjpage->flags &= ~(PG_WANTED|PG_BUSY);				UVM_PAGE_OWN(uobjpage, NULL);				uvm_lock_pageq();				offset = uobjpage->offset;				/* remove old page */				uvm_pagerealloc(uobjpage, NULL, 0);				/*				 * at this point we have absolutely no				 * control over uobjpage				 */				/* install new page */				uvm_pagerealloc(pg, uobj, offset);				uvm_unlock_pageq();				/*				 * done!  loan is broken and "pg" is				 * PG_BUSY.   it can now replace uobjpage.				 */				uobjpage = pg;			}		/* write fault case */		}		/* if loan_count */	} else {		/*		 * if we are going to promote the data to an anon we		 * allocate a blank anon here and plug it into our amap.		 */#if DIAGNOSTIC		if (amap == NULL)			panic("uvm_fault: want to promote data, but no anon");#endif		anon = uvm_analloc();		if (anon) {			/*			 * The new anon is locked.			 *			 * In `Fill in data...' below, if			 * uobjpage == PGO_DONTCARE, we want			 * a zero'd, dirty page, so have			 * uvm_pagealloc() do that for us.			 */			pg = uvm_pagealloc(NULL, 0, anon,			    (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);		}		/*		 * out of memory resources?		 */		if (anon == NULL || pg == NULL) {			if (anon != NULL) {				anon->an_ref--;				simple_unlock(&anon->an_lock);				uvm_anfree(anon);			}			/*			 * arg!  must unbusy our page and fail or sleep.			 */			if (uobjpage != PGO_DONTCARE) {				if (uobjpage->flags & PG_WANTED)					/* still holding object lock */					wakeup(uobjpage);				uvm_lock_pageq();				uvm_pageactivate(uobjpage);				uvm_unlock_pageq();				uobjpage->flags &= ~(PG_BUSY|PG_WANTED);				UVM_PAGE_OWN(uobjpage, NULL);			}			/* unlock and fail ... */			uvmfault_unlockall(&ufi, amap, uobj, NULL);			KASSERT(uvmexp.swpgonly <= uvmexp.swpages);			if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {				UVMHIST_LOG(maphist, "  promote: out of VM",				    0,0,0,0);				uvmexp.fltnoanon++;				return ENOMEM;			}			UVMHIST_LOG(maphist, "  out of RAM, waiting for more",			    0,0,0,0);			uvmexp.fltnoram++;			uvm_wait("flt_noram5");			goto ReFault;		}		/*		 * fill in the data		 */		if (uobjpage != PGO_DONTCARE) {			uvmexp.flt_prcopy++;			/* copy page [pg now dirty] */			uvm_pagecopy(uobjpage, pg);			/*			 * promote to shared amap?  make sure all sharing			 * procs see it			 */			if ((amap_flags(amap) & AMAP_SHARED) != 0) {				pmap_page_protect(uobjpage, VM_PROT_NONE);				/*				 * XXX: PAGE MIGHT BE WIRED!				 */			}			/*			 * dispose of uobjpage.  it can't be PG_RELEASED			 * since we still hold the object lock.			 * drop handle to uobj as well.			 */			if (uobjpage->flags & PG_WANTED)				/* still have the obj lock */				wakeup(uobjpage);			uobjpage->flags &= ~(PG_BUSY|PG_WANTED);			UVM_PAGE_OWN(uobjpage, NULL);			uvm_lock_pageq();			uvm_pageactivate(uobjpage);			uvm_unlock_pageq();			simple_unlock(&uobj->vmobjlock);			uobj = NULL;			UVMHIST_LOG(maphist,			    "  promote uobjpage 0x%x to anon/page 0x%x/0x%x",			    uobjpage, anon, pg, 0);		} else {			uvmexp.flt_przero++;			/*			 * Page is zero'd and marked dirty by uvm_pagealloc()			 * above.			 */			UVMHIST_LOG(maphist,"  zero fill anon/page 0x%x/0%x",			    anon, pg, 0, 0);		}		amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,		    anon, 0);	}	/*	 * locked:	 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),	 *   anon(if !null), pg(if anon)	 *	 * note: pg is either the uobjpage or the new page in the new anon	 */	/*	 * all resources are present.   we can now map it in and free our	 * resources.	 */	UVMHIST_LOG(maphist,	    "  MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",	    ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);	KASSERT(access_type == VM_PROT_READ || (pg->flags & PG_RDONLY) == 0);	if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),	    pg->flags & PG_RDONLY ? VM_PROT_READ : enter_prot,	    access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) {		/*		 * No need to undo what we did; we can simply think of		 * this as the pmap throwing away the mapping information.		 *		 * We do, however, have to go through the ReFault path,		 * as the map may change while we're asleep.		 */		if (pg->flags & PG_WANTED)			wakeup(pg);		/* lock still held */		/*		 * note that pg can't be PG_RELEASED since we did not drop		 * the object lock since the last time we checked.		 */		pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);		UVM_PAGE_OWN(pg, NULL);		uvmfault_unlockall(&ufi, amap, uobj, anon);		KASSERT(uvmexp.swpgonly <= uvmexp.swpages);		if (uvmexp.swpgonly == uvmexp.swpages) {			UVMHIST_LOG(maphist,			    "<- failed.  out of VM",0,0,0,0);			/* XXX instrumentation */			return ENOMEM;		}		/* XXX instrumentation */		uvm_wait("flt_pmfail2");		goto ReFault;	}	uvm_lock_pageq();	if (fault_type == VM_FAULT_WIRE) {		uvm_pagewire(pg);		if (pg->pqflags & PQ_AOBJ) {			/*			 * since the now-wired page cannot be paged out,			 * release its swap resources for others to use.			 * since an aobj page with no swap cannot be PG_CLEAN,			 * clear its clean flag now.			 */			pg->flags &= ~(PG_CLEAN);			uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);		}	} else {		/* activate it */		uvm_pageactivate(pg);	}	uvm_unlock_pageq();	if (pg->flags & PG_WANTED)		wakeup(pg);		/* lock still held */	/*	 * note that pg can't be PG_RELEASED since we did not drop the object	 * lock since the last time we checked.	 */	pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);	UVM_PAGE_OWN(pg, NULL);	uvmfault_unlockall(&ufi, amap, uobj, anon);	pmap_update(ufi.orig_map->pmap);	UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);	return 0;}