} #endif
static struct pte_ovfl *poalloc __P((void));static void pofree __P((struct pte_ovfl *, int));static u_int usedsr[NPMAPS / sizeof(u_int) / 8];static int pmap_initialized;/* * These small routines may have to be replaced, * if/when we support processors other that the 604. */static inline voidtlbie(ea)	caddr_t ea;{	asm volatile ("tlbie %0" :: "r"(ea));}static inline voidtlbsync(){	asm volatile ("sync; tlbsync; sync");}static voidtlbia(){	caddr_t i;		asm volatile ("sync");	for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000)		tlbie(i);	tlbsync();}static inline intptesr(sr, addr)	sr_t *sr;	vm_offset_t addr;{	return sr[(u_int)addr >> ADDR_SR_SHFT];}static inline intpteidx(sr, addr)	sr_t sr;	vm_offset_t addr;{	int hash;		hash = (sr & SR_VSID) ^ (((u_int)addr & ADDR_PIDX) >> ADDR_PIDX_SHFT);	return hash & ptab_mask;}static inline intptematch(ptp, sr, va, which)	pte_t *ptp;	sr_t sr;	vm_offset_t va;	int which;{	return ptp->pte_hi		== (((sr & SR_VSID) << PTE_VSID_SHFT)		    | (((u_int)va >> ADDR_API_SHFT) & PTE_API)		    | which);}/* * Try to insert page table entry *pt into the ptable at idx. * * Note: *pt mustn't have PTE_VALID set. * This is done here as required by Book III, 4.12. */static intpte_insert(idx, pt)	int idx;	pte_t *pt;{	pte_t *ptp;	int i;		/*	 * First try primary hash.	 */	for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++)		if (!(ptp->pte_hi & PTE_VALID)) {			*ptp = *pt;			ptp->pte_hi &= ~PTE_HID;			asm volatile ("sync");			ptp->pte_hi |= PTE_VALID;			return 1;		}	idx ^= ptab_mask;	for (ptp = ptable + idx * 8, i = 8; --i >= 0; ptp++)		if (!(ptp->pte_hi & PTE_VALID)) {			*ptp = *pt;			ptp->pte_hi |= PTE_HID;			asm volatile ("sync");			ptp->pte_hi |= PTE_VALID;			return 1;		}	return 0;}/* * Spill handler. * * Tries to spill a page table entry from the overflow area. * Note that this routine runs in real mode on a separate stack, * with interrupts disabled. */intpte_spill(addr)	vm_offset_t addr;{	int idx, i;	sr_t sr;	struct pte_ovfl *po;	pte_t ps;	pte_t *pt;	asm ("mfsrin %0,%1" : "=r"(sr) : "r"(addr));	idx = pteidx(sr, addr);	for (po = potable[idx].lh_first; po; po = po->po_list.le_next)		if (ptematch(&po->po_pte, sr, addr, 0)) {			/*			 * Now found an entry to be spilled into the real ptable.			 */			if (pte_insert(idx, &po->po_pte)) {				LIST_REMOVE(po, po_list);				pofree(po, 0);				return 1;			}			/*			 * Have to substitute some entry. Use the primary hash for this.			 *			 * Use low bits of timebase as random generator			 */			asm ("mftb %0" : "=r"(i));			pt = ptable + idx * 8 + (i & 7);			pt->pte_hi &= ~PTE_VALID;			ps = *pt;			asm volatile ("sync");			tlbie(addr);			tlbsync();			*pt = po->po_pte;			asm volatile ("sync");			pt->pte_hi |= PTE_VALID;			po->po_pte = ps;			if (ps.pte_hi & PTE_HID) {				/*				 * We took an entry that was on the alternate hash				 * chain, so move it to it's original chain.				 */				po->po_pte.pte_hi &= ~PTE_HID;				LIST_REMOVE(po, po_list);				LIST_INSERT_HEAD(potable + (idx ^ ptab_mask),						 po, po_list);			}			return 1;		}	return 0;}int avail_start;int avail_end;/* * This is called during initppc, before the system is really initialized. */voidpmap_bootstrap(kernelstart, kernelend)	u_int kernelstart, kernelend;{	struct mem_region *mp, *mp1;	int cnt, i;	u_int s, sz;	/*	 * Get memory.	 */	(fw->mem_regions)(&mem, &avail);#if 0	/* work around repeated entries bug */	for (mp = mem; mp->size; mp++) {		if (mp[1].start == mp[0].start) {			int i;			i = 0;			do {				mp[0+i].size = mp[1+i].size;				mp[0+i].start = mp[1+i].start;			} while ( mp[0+(i++)].size != 0);		}	}#endif	physmem = 0;	for (mp = mem; mp->size; mp++) {		physmem += btoc(mp->size);	}	/*	 * Count the number of available entries.	 */	for (cnt = 0, mp = avail; mp->size; mp++)		cnt++;	/*	 * Page align all regions.	 * Non-page memory isn't very interesting to us.	 * Also, sort the entries for ascending addresses.	 */	kernelstart &= ~PGOFSET;	kernelend = (kernelend + PGOFSET) & ~PGOFSET;	/* make certain that each section is page aligned for base and size */	for (mp = avail; mp->size; mp++) {		u_int32_t end;		s = mp->start - round_page(mp->start);		if (s != 0) {			mp->start = round_page(mp->start);			end = trunc_page(mp->size + mp->start);			mp->size = end - mp->start;		}		mp->size = trunc_page(mp->size);	}	for (mp = avail; mp->size; mp++) {		/*		 * Check whether this region holds all of the kernel.		 */		s = mp->start + mp->size;		if (mp->start < kernelstart && s > kernelend) {			avail[cnt].start = kernelend;			avail[cnt++].size = s - kernelend;			mp->size = kernelstart - mp->start;		}		/*		 * Look whether this regions starts within the kernel.		 */		if (mp->start >= kernelstart && mp->start < kernelend) {			s = kernelend - mp->start;			if (mp->size > s)				mp->size -= s;			else				mp->size = 0;			mp->start = kernelend;		}		/*		 * Now look whether this region ends within the kernel.		 */		s = mp->start + mp->size;		if (s > kernelstart && s < kernelend)			mp->size -= s - kernelstart;		/*		 * Now page align the start of the region.		 */		s = mp->start % NBPG;		if (mp->size >= s) {			mp->size -= s;			mp->start += s;		}		/*		 * And now align the size of the region.		 */		mp->size -= mp->size % NBPG;		/*		 * Check whether some memory is left here.		 */		if (mp->size == 0) {			bcopy(mp + 1, mp,			      (cnt - (mp - avail)) * sizeof *mp);			cnt--;			mp--;			continue;		}		s = mp->start;		sz = mp->size;		npgs += btoc(sz);		for (mp1 = avail; mp1 < mp; mp1++)			if (s < mp1->start)				break;		if (mp1 < mp) {			bcopy(mp1, mp1 + 1, (void *)mp - (void *)mp1);			mp1->start = s;			mp1->size = sz;		}	}#if 0avail_start = 0;avail_end = npgs * NBPG;#endif#ifdef  HTABENTS	ptab_cnt = HTABENTS;#else /* HTABENTS */	ptab_cnt = 1024;	#if 0	while ((HTABSIZE << 7) < ctob(physmem)) {		ptab_cnt <<= 1;	}	#endif#endif /* HTABENTS */	/*	 * Find suitably aligned memory for HTAB.	 */	for (mp = avail; mp->size; mp++) {		s = mp->start % HTABSIZE;		if (mp->start % HTABSIZE == 0) {			s = 0;		} else {			s = HTABSIZE - (mp->start % HTABSIZE) ;		}		if (mp->size < s + HTABSIZE)			continue;		ptable = (pte_t *)(mp->start + s);		if (mp->size == s + HTABSIZE) {			if (s)				mp->size = s;			else {				bcopy(mp + 1, mp,				      (cnt - (mp - avail)) * sizeof *mp);				mp = avail;			}			break;		}		if (s != 0) {			bcopy(mp, mp + 1,			      (cnt - (mp - avail)) * sizeof *mp);			mp++->size = s;		}		mp->start += s + HTABSIZE;		mp->size -= s + HTABSIZE;		break;	}	if (!mp->size)		panic("not enough memory?");	bzero((void *)ptable, HTABSIZE);	ptab_mask = ptab_cnt - 1;		/*	 * We cannot do vm_bootstrap_steal_memory here,	 * since we don't run with translation enabled yet.	 */	s = sizeof(struct pte_ovtab) * ptab_cnt;	sz = round_page(s);	for (mp = avail; mp->size; mp++)		if (mp->size >= sz)			break;	if (!mp->size)		panic("not enough memory?");	potable = (struct pte_ovtab *)mp->start;	mp->size -= sz;	mp->start += sz;	if (mp->size == 0)		bcopy(mp + 1, mp, (cnt - (mp - avail)) * sizeof *mp);	for (i = 0; i < ptab_cnt; i++)		LIST_INIT(potable + i);	LIST_INIT(&pv_page_freelist);	#ifdef UVM	for (mp = avail; mp->size; mp++) {		uvm_page_physload(atop(mp->start), atop(mp->start + mp->size),			atop(mp->start), atop(mp->start + mp->size),			VM_FREELIST_DEFAULT);	}#endif	/*	 * Initialize kernel pmap and hardware.	 */#if NPMAPS >= KERNEL_SEGMENT / 16	usedsr[KERNEL_SEGMENT / 16 / (sizeof usedsr[0] * 8)]		|= 1 << ((KERNEL_SEGMENT / 16) % (sizeof usedsr[0] * 8));#endif	for (i = 0; i < 16; i++) {		pmap_kernel()->pm_sr[i] = EMPTY_SEGMENT;		asm volatile ("mtsrin %0,%1"			      :: "r"(EMPTY_SEGMENT), "r"(i << ADDR_SR_SHFT) );	}	pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT;	asm volatile ("mtsr %0,%1"		      :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT));	asm volatile ("sync; mtsdr1 %0; isync"		      :: "r"((u_int)ptable | (ptab_mask >> 10)));	tlbia();	nextavail = avail->start;}/* * Restrict given range to physical memory */voidpmap_real_memory(start, size)	vm_offset_t *start;	vm_size_t *size;{	struct mem_region *mp;		for (mp = mem; mp->size; mp++) {		if (*start + *size > mp->start		    && *start < mp->start + mp->size) {			if (*start < mp->start) {				*size -= mp->start - *start;				*start = mp->start;			}			if (*start + *size > mp->start + mp->size)				*size = mp->start + mp->size - *start;			return;		}	}	*size = 0;}/* * Initialize anything else for pmap handling. * Called during vm_init(). */voidpmap_init(){	struct pv_entry *pv;	vsize_t sz;	vaddr_t addr;	int i, s;#ifdef UVM	int bank;	char *attr;#endif		sz = (vm_size_t)((sizeof(struct pv_entry) + 1) * npgs);	sz = round_page(sz);#ifdef UVM	addr = uvm_km_zalloc(kernel_map, sz);#else	addr = kmem_alloc(kernel_map, sz);#endif	s = splimp();	pv = pv_table = (struct pv_entry *)addr;	for (i = npgs; --i >= 0;)		pv++->pv_idx = -1;	LIST_INIT(&pv_page_freelist);	pmap_attrib = (char *)pv;	bzero(pv, npgs);#ifdef UVM	pv = pv_table;	attr = pmap_attrib;	for (bank = 0; bank < vm_nphysseg; bank++) {		sz = vm_physmem[bank].end - vm_physmem[bank].start;		vm_physmem[bank].pmseg.pvent = pv;		vm_physmem[bank].pmseg.attrs = attr;		pv += sz;		attr += sz;	}#endif	pmap_initialized = 1;	splx(s);}/* * Return the index of the given page in terms of pmap_next_page() calls. */intpmap_page_index(pa)	vm_offset_t pa;{	struct mem_region *mp;	vm_size_t pre;		pa &= ~PGOFSET;	for (pre = 0, mp = avail; mp->size; mp++) {		if (pa >= mp->start		    && pa < mp->start + mp->size)			return btoc(pre + (pa - mp->start));		pre += mp->size;	}	return -1;}vm_offset_t ppc_kvm_size = VM_KERN_ADDR_SIZE_DEF;/* * How much virtual space is available to the kernel? */voidpmap_virtual_space(start, end)	vm_offset_t *start, *end;{	/*	 * Reserve one segment for kernel virtual memory	 */	*start = (vm_offset_t)(KERNEL_SR << ADDR_SR_SHFT);	*end = *start + VM_KERN_ADDRESS_SIZE;}/* * Return the number of possible page indices returned * from pmap_page_index for any page provided by pmap_next_page. */u_intpmap_free_pages(){	return npgs;}/* * If there are still physical pages available, put the address of * the next available one at paddr and return TRUE.  Otherwise, * return FALSE to indicate that there are no more free pages. */intpmap_next_page(paddr)	vm_offset_t *paddr;{	static int lastidx = -1;		if (lastidx < 0	    || nextavail >= avail[lastidx].start + avail[lastidx].size) {		if (avail[++lastidx].size == 0)			return FALSE;		nextavail = avail[lastidx].start;	}	*paddr = nextavail;	nextavail += NBPG;	return TRUE;}/* * Create and return a physical map. */struct pmap *pmap_create(size)	vm_size_t size;{	struct pmap *pm;		pm = (struct pmap *)malloc(sizeof *pm, M_VMPMAP, M_WAITOK);	bzero((caddr_t)pm, sizeof *pm);	pmap_pinit(pm);	return pm;}/* * Initialize a preallocated and zeroed pmap structure. */voidpmap_pinit(pm)	struct pmap *pm;{	int i, j;	int s;		/*	 * Allocate some segment registers for this pmap.	 */	s = splimp();	pm->pm_refs = 1;	for (i = 0; i < sizeof usedsr / sizeof usedsr[0]; i++)		if (usedsr[i] != 0xffffffff) {			j = ffs(~usedsr[i]) - 1;			usedsr[i] |= 1 << j;			pm->pm_sr[0] = (i * sizeof usedsr[0] * 8 + j) * 16;			for (i = 1; i < 16; i++)				pm->pm_sr[i] = pm->pm_sr[i - 1] + 1;			splx(s);			return;		}	splx(s);	panic("out of segments");}/* * Add a reference to the given pmap. */voidpmap_reference(pm)	struct pmap *pm;{	pm->pm_refs++;}/* * Retire the given pmap from service. * Should only be called if the map contains no valid mappings. */voidpmap_destroy(pm)	struct pmap *pm;{	if (--pm->pm_refs == 0) {		pmap_release(pm);		free((caddr_t)pm, M_VMPMAP);	}}/* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. */voidpmap_release(pm)	struct pmap *pm;{	int i, j;		if (!pm->pm_sr[0])		panic("pmap_release");	i = pm->pm_sr[0] / 16;	j = i % (sizeof usedsr[0] * 8);	i /= sizeof usedsr[0] * 8;	usedsr[i] &= ~(1 << j);}/* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */voidpmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)	struct pmap *dst_pmap, *src_pmap;	vm_offset_t dst_addr, src_addr;	vm_size_t len;{}/* * Require that all active physical maps contain no * incorrect entries NOW. */voidpmap_update(){}/* * Garbage collects the physical map system for * pages which are no longer used. * Success need not be guaranteed -- that is, there * may well be pages which are not referenced, but * others may be collected. * Called by the pageout daemon when pages are scarce. */voidpmap_collect(pm)	struct pmap *pm;{}/* * Make the specified pages pageable or not as requested. * * This routine is merely advisory. */voidpmap_pageable(pm, start, end, pageable)	struct pmap *pm;	vm_offset_t start, end;	int pageable;{}/* * Fill the given physical page with zeroes. */voidpmap_zero_page(pa)	vm_offset_t pa;{	bzero((caddr_t)pa, NBPG);}/* * Copy the given physical source page to its destination. */voidpmap_copy_page(src, dst)	vm_offset_t src, dst;{	bcopy((caddr_t)src, (caddr_t)dst, NBPG);}static struct pv_entry *pmap_alloc_pv(){	struct pv_page *pvp;	struct pv_entry *pv;	int i;		if (pv_nfree == 0) {#ifdef UVM		if (!(pvp = (struct pv_page *)uvm_km_zalloc(kernel_map, NBPG)))			panic("pmap_alloc_pv: uvm_km_zalloc() failed");#else		if (!(pvp = (struct pv_page *)kmem_alloc(kernel_map, NBPG)))			panic("pmap_alloc_pv: kmem_alloc() failed");#endif		pv_pcnt++;		pvp->pvp_pgi.pgi_freelist = pv = &pvp->pvp_pv[1];		for (i = NPVPPG - 2; --i >= 0; pv++)			pv->pv_next = pv + 1;		pv->pv_next = 0;		pv_nfree += pvp->pvp_pgi.pgi_nfree = NPVPPG - 1;		LIST_INSERT_HEAD(&pv_page_freelist, pvp, pvp_pgi.pgi_list);		pv = pvp->pvp_pv;	} else {		pv_nfree--;		pvp = pv_page_freelist.lh_first;		if (--pvp->pvp_pgi.pgi_nfree <= 0)			LIST_REMOVE(pvp, pvp_pgi.pgi_list);		pv = pvp->pvp_pgi.pgi_freelist;		pvp->pvp_pgi.pgi_freelist = pv->pv_next;	}	return pv;}static voidpmap_free_pv(pv)	struct pv_entry *pv;{	struct pv_page *pvp;		pvp = (struct pv_page *)trunc_page(pv);	switch (++pvp->pvp_pgi.pgi_nfree) {	case 1:		LIST_INSERT_HEAD(&pv_page_freelist, pvp, pvp_pgi.pgi_list);	default:		pv->pv_next = pvp->pvp_pgi.pgi_freelist;		pvp->pvp_pgi.pgi_freelist = pv;		pv_nfree++;		break;	case NPVPPG:		pv_nfree -= NPVPPG - 1;		pv_pcnt--;		LIST_REMOVE(pvp, pvp_pgi.pgi_list);#ifdef UVM		uvm_km_free(kernel_map, (vaddr_t)pvp, NBPG);#else		kmem_free(kernel_map, (vm_offset_t)pvp, NBPG);#endif		break;	}}/* * We really hope that we don't need overflow entries * before the VM system is initialized!							XXX */static struct pte_ovfl *poalloc(){	struct po_page *pop;	struct pte_ovfl *po;	vm_page_t mem;	int i;		if (!pmap_initialized)		panic("poalloc");		if (po_nfree == 0) {		/*		 * Since we cannot use maps for potable allocation,		 * we have to steal some memory from the VM system.			XXX		 */#ifdef UVM		mem = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);#else		mem = vm_page_alloc(NULL, NULL);#endif		po_pcnt++;		pop = (struct po_page *)VM_PAGE_TO_PHYS(mem);		pop->pop_pgi.pgi_page = mem;		LIST_INIT(&pop->pop_pgi.pgi_freelist);		for (i = NPOPPG - 1, po = pop->pop_po + 1; --i >= 0; po++)			LIST_INSERT_HEAD(&pop->pop_pgi.pgi_freelist, po, po_list);		po_nfree += pop->pop_pgi.pgi_nfree = NPOPPG - 1;		LIST_INSERT_HEAD(&po_page_freelist, pop, pop_pgi.pgi_list);		po = pop->pop_po;	} else {		po_nfree--;		pop = po_page_freelist.lh_first;		if (--pop->pop_pgi.pgi_nfree <= 0)			LIST_REMOVE(pop, pop_pgi.pgi_list);		po = pop->pop_pgi.pgi_freelist.lh_first;		LIST_REMOVE(po, po_list);	}	return po;}