FM / asm

/usr/src/linux-headers-5.15.0-190/arch/sparc/include/asm/

/usr/src/linux-headers-5.15.0-190/arch/sparc/include/asm

Name	Size	Mode	Actions
adi.h	138	0644	edit dl rm
adi_64.h	812	0644	edit dl rm
agp.h	434	0644	edit dl rm
apb.h	1082	0644	edit dl rm
asm-offsets.h	35	0644	edit dl rm
asm-prototypes.h	729	0644	edit dl rm
asm.h	1101	0644	edit dl rm
asmmacro.h	1183	0644	edit dl rm
atomic.h	219	0644	edit dl rm
atomic_32.h	1760	0644	edit dl rm
atomic_64.h	1849	0644	edit dl rm
auxio.h	310	0644	edit dl rm
auxio_32.h	2615	0644	edit dl rm
auxio_64.h	3253	0644	edit dl rm
backoff.h	2756	0644	edit dl rm
barrier.h	223	0644	edit dl rm
barrier_32.h	160	0644	edit dl rm
barrier_64.h	2004	0644	edit dl rm
bbc.h	9996	0644	edit dl rm
bitext.h	631	0644	edit dl rm
bitops.h	219	0644	edit dl rm
bitops_32.h	2853	0644	edit dl rm
bitops_64.h	1675	0644	edit dl rm
btext.h	145	0644	edit dl rm
bug.h	588	0644	edit dl rm
cache.h	630	0644	edit dl rm
cacheflush.h	373	0644	edit dl rm
cacheflush_32.h	1951	0644	edit dl rm
cacheflush_64.h	2557	0644	edit dl rm
cachetlb_32.h	882	0644	edit dl rm
chafsr.h	9710	0644	edit dl rm
checksum.h	331	0644	edit dl rm
checksum_32.h	5791	0644	edit dl rm
checksum_64.h	3855	0644	edit dl rm
chmctrl.h	8096	0644	edit dl rm
clock.h	231	0644	edit dl rm
clocksource.h	407	0644	edit dl rm
cmpxchg.h	223	0644	edit dl rm
cmpxchg_32.h	2483	0644	edit dl rm
cmpxchg_64.h	5288	0644	edit dl rm
compat.h	3890	0644	edit dl rm
compat_signal.h	565	0644	edit dl rm
contregs.h	1941	0644	edit dl rm
cpudata.h	378	0644	edit dl rm
cpudata_32.h	729	0644	edit dl rm
cpudata_64.h	1153	0644	edit dl rm
cpu_type.h	579	0644	edit dl rm
current.h	991	0644	edit dl rm
dcr.h	728	0644	edit dl rm
dcu.h	1520	0644	edit dl rm
delay.h	215	0644	edit dl rm
delay_32.h	907	0644	edit dl rm
delay_64.h	403	0644	edit dl rm
device.h	565	0644	edit dl rm
dma-mapping.h	336	0644	edit dl rm
dma.h	4989	0644	edit dl rm
ebus_dma.h	1098	0644	edit dl rm
ecc.h	4448	0644	edit dl rm
eeprom.h	254	0644	edit dl rm
elf.h	207	0644	edit dl rm
elf_32.h	3267	0644	edit dl rm
elf_64.h	6848	0644	edit dl rm
estate.h	2287	0644	edit dl rm
extable.h	723	0644	edit dl rm
fb.h	680	0644	edit dl rm
fbio.h	2315	0644	edit dl rm
fhc.h	4533	0644	edit dl rm
floppy.h	219	0644	edit dl rm
floppy_32.h	9796	0644	edit dl rm
floppy_64.h	19392	0644	edit dl rm
fpumacro.h	710	0644	edit dl rm
ftrace.h	801	0644	edit dl rm
futex.h	215	0644	edit dl rm
futex_32.h	82	0644	edit dl rm
futex_64.h	2160	0644	edit dl rm
hardirq.h	223	0644	edit dl rm
hardirq_32.h	334	0644	edit dl rm
hardirq_64.h	412	0644	edit dl rm
head.h	211	0644	edit dl rm
head_32.h	2617	0644	edit dl rm
head_64.h	2184	0644	edit dl rm
hibernate.h	421	0644	edit dl rm
highmem.h	1966	0644	edit dl rm
hugetlb.h	1592	0644	edit dl rm
hvtramp.h	782	0644	edit dl rm
hw_irq.h	88	0644	edit dl rm
hypervisor.h	117649	0644	edit dl rm
ide.h	2222	0644	edit dl rm
idprom.h	656	0644	edit dl rm
intr_queue.h	794	0644	edit dl rm
io-unit.h	2469	0644	edit dl rm
io.h	620	0644	edit dl rm
ioctls.h	358	0644	edit dl rm
iommu-common.h	1445	0644	edit dl rm
iommu.h	215	0644	edit dl rm
iommu_32.h	5865	0644	edit dl rm
iommu_64.h	2489	0644	edit dl rm
io_32.h	3430	0644	edit dl rm
io_64.h	11076	0644	edit dl rm
irq.h	207	0644	edit dl rm
irqflags.h	227	0644	edit dl rm
irqflags_32.h	1058	0644	edit dl rm
irqflags_64.h	1952	0644	edit dl rm
irq_32.h	526	0644	edit dl rm
irq_64.h	3105	0644	edit dl rm
jump_label.h	1036	0644	edit dl rm
Kbuild	177	0644	edit dl rm
kdebug.h	219	0644	edit dl rm
kdebug_32.h	2041	0644	edit dl rm
kdebug_64.h	393	0644	edit dl rm
kgdb.h	1014	0644	edit dl rm
kprobes.h	1410	0644	edit dl rm
ldc.h	4475	0644	edit dl rm
leon.h	7844	0644	edit dl rm
leon_amba.h	8280	0644	edit dl rm
leon_pci.h	512	0644	edit dl rm
lsu.h	1062	0644	edit dl rm
machines.h	1540	0644	edit dl rm
mbus.h	2996	0644	edit dl rm
mc146818rtc.h	298	0644	edit dl rm
mc146818rtc_32.h	699	0644	edit dl rm
mc146818rtc_64.h	689	0644	edit dl rm
mdesc.h	3066	0644	edit dl rm
memctrl.h	311	0644	edit dl rm
mman.h	2633	0644	edit dl rm
mmu.h	207	0644	edit dl rm
mmu_32.h	209	0644	edit dl rm
mmu_64.h	3967	0644	edit dl rm
mmu_context.h	239	0644	edit dl rm
mmu_context_32.h	1082	0644	edit dl rm
mmu_context_64.h	5481	0644	edit dl rm
mmzone.h	363	0644	edit dl rm
mxcc.h	4429	0644	edit dl rm
nmi.h	354	0644	edit dl rm
ns87303.h	3299	0644	edit dl rm
obio.h	6410	0644	edit dl rm
openprom.h	7475	0644	edit dl rm
oplib.h	215	0644	edit dl rm
oplib_32.h	6062	0644	edit dl rm
oplib_64.h	8357	0644	edit dl rm
page.h	274	0644	edit dl rm
page_32.h	3826	0644	edit dl rm
page_64.h	4753	0644	edit dl rm
parport.h	5819	0644	edit dl rm
pbm.h	1505	0644	edit dl rm
pci.h	1367	0644	edit dl rm
pcic.h	5904	0644	edit dl rm
pcr.h	1893	0644	edit dl rm
percpu.h	219	0644	edit dl rm
percpu_32.h	168	0644	edit dl rm
percpu_64.h	541	0644	edit dl rm
perf_event.h	802	0644	edit dl rm
pgalloc.h	223	0644	edit dl rm
pgalloc_32.h	1823	0644	edit dl rm
pgalloc_64.h	2764	0644	edit dl rm
pgtable.h	223	0644	edit dl rm
pgtable_32.h	11021	0644	edit dl rm
pgtable_64.h	32369	0644	edit dl rm
pgtsrmmu.h	4879	0644	edit dl rm
pil.h	1109	0644	edit dl rm
processor.h	231	0644	edit dl rm
processor_32.h	2836	0644	edit dl rm
processor_64.h	7577	0644	edit dl rm
prom.h	1776	0644	edit dl rm
psr.h	1414	0644	edit dl rm
ptrace.h	4294	0644	edit dl rm
qrwlock.h	205	0644	edit dl rm
qspinlock.h	215	0644	edit dl rm
ross.h	5655	0644	edit dl rm
sbi.h	3425	0644	edit dl rm
scratchpad.h	547	0644	edit dl rm
seccomp.h	225	0644	edit dl rm
sections.h	289	0644	edit dl rm
setup.h	1552	0644	edit dl rm
sfafsr.h	3216	0644	edit dl rm
sfp-machine.h	239	0644	edit dl rm
sfp-machine_32.h	6954	0644	edit dl rm
sfp-machine_64.h	3178	0644	edit dl rm
shmparam.h	227	0644	edit dl rm
shmparam_32.h	253	0644	edit dl rm
shmparam_64.h	306	0644	edit dl rm
sigcontext.h	2616	0644	edit dl rm
signal.h	351	0644	edit dl rm
smp.h	207	0644	edit dl rm
smp_32.h	3368	0644	edit dl rm
smp_64.h	1791	0644	edit dl rm
sparsemem.h	295	0644	edit dl rm
spinlock.h	227	0644	edit dl rm
spinlock_32.h	4324	0644	edit dl rm
spinlock_64.h	409	0644	edit dl rm
spinlock_types.h	549	0644	edit dl rm
spitfire.h	9967	0644	edit dl rm
stacktrace.h	166	0644	edit dl rm
starfire.h	418	0644	edit dl rm
string.h	1254	0644	edit dl rm
string_32.h	405	0644	edit dl rm
string_64.h	409	0644	edit dl rm
sunbpp.h	3346	0644	edit dl rm
swift.h	3147	0644	edit dl rm
switch_to.h	231	0644	edit dl rm
switch_to_32.h	3616	0644	edit dl rm
switch_to_64.h	2640	0644	edit dl rm
syscall.h	3464	0644	edit dl rm
syscalls.h	299	0644	edit dl rm
termbits.h	198	0644	edit dl rm
termios.h	5063	0644	edit dl rm
thread_info.h	239	0644	edit dl rm
thread_info_32.h	3773	0644	edit dl rm
thread_info_64.h	8195	0644	edit dl rm
timer.h	215	0644	edit dl rm
timer_32.h	1211	0644	edit dl rm
timer_64.h	2448	0644	edit dl rm
timex.h	215	0644	edit dl rm
timex_32.h	266	0644	edit dl rm
timex_64.h	423	0644	edit dl rm
tlb.h	207	0644	edit dl rm
tlbflush.h	227	0644	edit dl rm
tlbflush_32.h	621	0644	edit dl rm
tlbflush_64.h	1776	0644	edit dl rm
tlb_32.h	138	0644	edit dl rm
tlb_64.h	1056	0644	edit dl rm
topology.h	227	0644	edit dl rm
topology_32.h	170	0644	edit dl rm
topology_64.h	1546	0644	edit dl rm
traps.h	577	0644	edit dl rm
trap_block.h	6786	0644	edit dl rm
tsb.h	12457	0644	edit dl rm
tsunami.h	1890	0644	edit dl rm
ttable.h	20827	0644	edit dl rm
turbosparc.h	3866	0644	edit dl rm
uaccess.h	390	0644	edit dl rm
uaccess_32.h	7032	0644	edit dl rm
uaccess_64.h	6142	0644	edit dl rm
unistd.h	1797	0644	edit dl rm
upa.h	3808	0644	edit dl rm
uprobes.h	1304	0644	edit dl rm
user.h	102	0644	edit dl rm
vaddrs.h	2066	0644	edit dl rm
vdso.h	491	0644	edit dl rm
vga.h	964	0644	edit dl rm
viking.h	8341	0644	edit dl rm
vio.h	12097	0644	edit dl rm
visasm.h	1544	0644	edit dl rm
vmalloc.h	93	0644	edit dl rm
vvar.h	1577	0644	edit dl rm
winmacro.h	4767	0644	edit dl rm
xor.h	207	0644	edit dl rm
xor_32.h	7086	0644	edit dl rm
xor_64.h	2156	0644	edit dl rm

Edit: /usr/src/linux-headers-5.15.0-190/arch/sparc/include/asm/tsb.h (12457B)

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _SPARC64_TSB_H
#define _SPARC64_TSB_H

/* The sparc64 TSB is similar to the powerpc hashtables.  It's a
 * power-of-2 sized table of TAG/PTE pairs.  The cpu precomputes
 * pointers into this table for 8K and 64K page sizes, and also a
 * comparison TAG based upon the virtual address and context which
 * faults.
 *
 * TLB miss trap handler software does the actual lookup via something
 * of the form:
 *
 * 	ldxa		[%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
 * 	ldxa		[%g0] ASI_{D,I}MMU, %g6
 *	sllx		%g6, 22, %g6
 *	srlx		%g6, 22, %g6
 * 	ldda		[%g1] ASI_NUCLEUS_QUAD_LDD, %g4
 * 	cmp		%g4, %g6
 * 	bne,pn	%xcc, tsb_miss_{d,i}tlb
 * 	 mov		FAULT_CODE_{D,I}TLB, %g3
 * 	stxa		%g5, [%g0] ASI_{D,I}TLB_DATA_IN
 * 	retry
 *
 *
 * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
 * PTE.  The TAG is of the same layout as the TLB TAG TARGET mmu
 * register which is:
 *
 * -------------------------------------------------
 * |  -  |  CONTEXT |  -  |    VADDR bits 63:22    |
 * -------------------------------------------------
 *  63 61 60      48 47 42 41                     0
 *
 * But actually, since we use per-mm TSB's, we zero out the CONTEXT
 * field.
 *
 * Like the powerpc hashtables we need to use locking in order to
 * synchronize while we update the entries.  PTE updates need locking
 * as well.
 *
 * We need to carefully choose a lock bits for the TSB entry.  We
 * choose to use bit 47 in the tag.  Also, since we never map anything
 * at page zero in context zero, we use zero as an invalid tag entry.
 * When the lock bit is set, this forces a tag comparison failure.
 */

#define TSB_TAG_LOCK_BIT	47
#define TSB_TAG_LOCK_HIGH	(1 << (TSB_TAG_LOCK_BIT - 32))

#define TSB_TAG_INVALID_BIT	46
#define TSB_TAG_INVALID_HIGH	(1 << (TSB_TAG_INVALID_BIT - 32))

/* Some cpus support physical address quad loads.  We want to use
 * those if possible so we don't need to hard-lock the TSB mapping
 * into the TLB.  We encode some instruction patching in order to
 * support this.
 *
 * The kernel TSB is locked into the TLB by virtue of being in the
 * kernel image, so we don't play these games for swapper_tsb access.
 */
#ifndef __ASSEMBLY__
struct tsb_ldquad_phys_patch_entry {
	unsigned int	addr;
	unsigned int	sun4u_insn;
	unsigned int	sun4v_insn;
};
extern struct tsb_ldquad_phys_patch_entry __tsb_ldquad_phys_patch,
	__tsb_ldquad_phys_patch_end;

struct tsb_phys_patch_entry {
	unsigned int	addr;
	unsigned int	insn;
};
extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
#endif
#define TSB_LOAD_QUAD(TSB, REG)	\
661:	ldda		[TSB] ASI_NUCLEUS_QUAD_LDD, REG; \
	.section	.tsb_ldquad_phys_patch, "ax"; \
	.word		661b; \
	ldda		[TSB] ASI_QUAD_LDD_PHYS, REG; \
	ldda		[TSB] ASI_QUAD_LDD_PHYS_4V, REG; \
	.previous

#define TSB_LOAD_TAG_HIGH(TSB, REG) \
661:	lduwa		[TSB] ASI_N, REG; \
	.section	.tsb_phys_patch, "ax"; \
	.word		661b; \
	lduwa		[TSB] ASI_PHYS_USE_EC, REG; \
	.previous

#define TSB_LOAD_TAG(TSB, REG) \
661:	ldxa		[TSB] ASI_N, REG; \
	.section	.tsb_phys_patch, "ax"; \
	.word		661b; \
	ldxa		[TSB] ASI_PHYS_USE_EC, REG; \
	.previous

#define TSB_CAS_TAG_HIGH(TSB, REG1, REG2) \
661:	casa		[TSB] ASI_N, REG1, REG2; \
	.section	.tsb_phys_patch, "ax"; \
	.word		661b; \
	casa		[TSB] ASI_PHYS_USE_EC, REG1, REG2; \
	.previous

#define TSB_CAS_TAG(TSB, REG1, REG2) \
661:	casxa		[TSB] ASI_N, REG1, REG2; \
	.section	.tsb_phys_patch, "ax"; \
	.word		661b; \
	casxa		[TSB] ASI_PHYS_USE_EC, REG1, REG2; \
	.previous

#define TSB_STORE(ADDR, VAL) \
661:	stxa		VAL, [ADDR] ASI_N; \
	.section	.tsb_phys_patch, "ax"; \
	.word		661b; \
	stxa		VAL, [ADDR] ASI_PHYS_USE_EC; \
	.previous

#define TSB_LOCK_TAG(TSB, REG1, REG2)	\
99:	TSB_LOAD_TAG_HIGH(TSB, REG1);	\
	sethi	%hi(TSB_TAG_LOCK_HIGH), REG2;\
	andcc	REG1, REG2, %g0;	\
	bne,pn	%icc, 99b;		\
	 nop;				\
	TSB_CAS_TAG_HIGH(TSB, REG1, REG2);	\
	cmp	REG1, REG2;		\
	bne,pn	%icc, 99b;		\
	 nop;				\

#define TSB_WRITE(TSB, TTE, TAG) \
	add	TSB, 0x8, TSB;   \
	TSB_STORE(TSB, TTE);     \
	sub	TSB, 0x8, TSB;   \
	TSB_STORE(TSB, TAG);

/* Do a kernel page table walk.  Leaves valid PTE value in
	 * REG1.  Jumps to FAIL_LABEL on early page table walk
	 * termination.  VADDR will not be clobbered, but REG2 will.
	 *
	 * There are two masks we must apply to propagate bits from
	 * the virtual address into the PTE physical address field
	 * when dealing with huge pages.  This is because the page
	 * table boundaries do not match the huge page size(s) the
	 * hardware supports.
	 *
	 * In these cases we propagate the bits that are below the
	 * page table level where we saw the huge page mapping, but
	 * are still within the relevant physical bits for the huge
	 * page size in question.  So for PMD mappings (which fall on
	 * bit 23, for 8MB per PMD) we must propagate bit 22 for a
	 * 4MB huge page.  For huge PUDs (which fall on bit 33, for
	 * 8GB per PUD), we have to accommodate 256MB and 2GB huge
	 * pages.  So for those we propagate bits 32 to 28.
	 */
#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL)	\
	sethi		%hi(swapper_pg_dir), REG1; \
	or		REG1, %lo(swapper_pg_dir), REG1; \
	sllx		VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldx		[REG1 + REG2], REG1; \
	brz,pn		REG1, FAIL_LABEL; \
	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
	brz,pn		REG1, FAIL_LABEL; \
	sethi		%uhi(_PAGE_PUD_HUGE), REG2; \
	brz,pn		REG1, FAIL_LABEL; \
	 sllx		REG2, 32, REG2; \
	andcc		REG1, REG2, %g0; \
	sethi		%hi(0xf8000000), REG2; \
	bne,pt		%xcc, 697f; \
	 sllx		REG2, 1, REG2; \
	sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
	sethi		%uhi(_PAGE_PMD_HUGE), REG2; \
	brz,pn		REG1, FAIL_LABEL; \
	 sllx		REG2, 32, REG2; \
	andcc		REG1, REG2, %g0; \
	be,pn		%xcc, 698f; \
	 sethi		%hi(0x400000), REG2; \
697:	brgez,pn	REG1, FAIL_LABEL; \
	 andn		REG1, REG2, REG1; \
	and		VADDR, REG2, REG2; \
	ba,pt		%xcc, 699f; \
	 or		REG1, REG2, REG1; \
698:	sllx		VADDR, 64 - PMD_SHIFT, REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
	brgez,pn	REG1, FAIL_LABEL; \
	 nop; \
699:

/* PUD has been loaded into REG1, interpret the value, seeing
	 * if it is a HUGE PUD or a normal one.  If it is not valid
	 * then jump to FAIL_LABEL.  If it is a HUGE PUD, and it
	 * translates to a valid PTE, branch to PTE_LABEL.
	 *
	 * We have to propagate bits [32:22] from the virtual address
	 * to resolve at 4M granularity.
	 */
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
700:	ba 700f;					\
	 nop;						\
	.section	.pud_huge_patch, "ax";		\
	.word		700b;				\
	nop;						\
	.previous;					\
	brz,pn		REG1, FAIL_LABEL;		\
	 sethi		%uhi(_PAGE_PUD_HUGE), REG2;	\
	sllx		REG2, 32, REG2;			\
	andcc		REG1, REG2, %g0;		\
	be,pt		%xcc, 700f;			\
	 sethi		%hi(0xffe00000), REG2;		\
	sllx		REG2, 1, REG2;			\
	brgez,pn	REG1, FAIL_LABEL;		\
	 andn		REG1, REG2, REG1;		\
	and		VADDR, REG2, REG2;		\
	brlz,pt		REG1, PTE_LABEL;		\
	 or		REG1, REG2, REG1;		\
700:
#else
#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
	brz,pn		REG1, FAIL_LABEL; \
	 nop;
#endif

/* PMD has been loaded into REG1, interpret the value, seeing
	 * if it is a HUGE PMD or a normal one.  If it is not valid
	 * then jump to FAIL_LABEL.  If it is a HUGE PMD, and it
	 * translates to a valid PTE, branch to PTE_LABEL.
	 *
	 * We have to propagate the 4MB bit of the virtual address
	 * because we are fabricating 8MB pages using 4MB hw pages.
	 */
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
	brz,pn		REG1, FAIL_LABEL;		\
	 sethi		%uhi(_PAGE_PMD_HUGE), REG2;	\
	sllx		REG2, 32, REG2;			\
	andcc		REG1, REG2, %g0;		\
	be,pt		%xcc, 700f;			\
	 sethi		%hi(4 * 1024 * 1024), REG2;	\
	brgez,pn	REG1, FAIL_LABEL;		\
	 andn		REG1, REG2, REG1;		\
	and		VADDR, REG2, REG2;		\
	brlz,pt		REG1, PTE_LABEL;		\
	 or		REG1, REG2, REG1;		\
700:
#else
#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
	brz,pn		REG1, FAIL_LABEL; \
	 nop;
#endif

/* Do a user page table walk in MMU globals.  Leaves final,
	 * valid, PTE value in REG1.  Jumps to FAIL_LABEL on early
	 * page table walk termination or if the PTE is not valid.
	 *
	 * Physical base of page tables is in PHYS_PGD which will not
	 * be modified.
	 *
	 * VADDR will not be clobbered, but REG1 and REG2 will.
	 */
#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL)	\
	sllx		VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
	brz,pn		REG1, FAIL_LABEL; \
	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
	USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
	brz,pn		REG1, FAIL_LABEL; \
	 sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
	USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
	sllx		VADDR, 64 - PMD_SHIFT, REG2; \
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
	andn		REG2, 0x7, REG2; \
	add		REG1, REG2, REG1; \
	ldxa		[REG1] ASI_PHYS_USE_EC, REG1; \
	brgez,pn	REG1, FAIL_LABEL; \
	 nop; \
800:

/* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
 * If no entry is found, FAIL_LABEL will be branched to.  On success
 * the resulting PTE value will be left in REG1.  VADDR is preserved
 * by this routine.
 */
#define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
	sethi		%hi(prom_trans), REG1; \
	or		REG1, %lo(prom_trans), REG1; \
97:	ldx		[REG1 + 0x00], REG2; \
	brz,pn		REG2, FAIL_LABEL; \
	 nop; \
	ldx		[REG1 + 0x08], REG3; \
	add		REG2, REG3, REG3; \
	cmp		REG2, VADDR; \
	bgu,pt		%xcc, 98f; \
	 cmp		VADDR, REG3; \
	bgeu,pt		%xcc, 98f; \
	 ldx		[REG1 + 0x10], REG3; \
	sub		VADDR, REG2, REG2; \
	ba,pt		%xcc, 99f; \
	 add		REG3, REG2, REG1; \
98:	ba,pt		%xcc, 97b; \
	 add		REG1, (3 * 8), REG1; \
99:

/* We use a 32K TSB for the whole kernel, this allows to
	 * handle about 16MB of modules and vmalloc mappings without
	 * incurring many hash conflicts.
	 */
#define KERNEL_TSB_SIZE_BYTES	(32 * 1024)
#define KERNEL_TSB_NENTRIES	\
	(KERNEL_TSB_SIZE_BYTES / 16)
#define KERNEL_TSB4M_NENTRIES	4096

/* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
	 * on TSB hit.  REG1, REG2, REG3, and REG4 are used as temporaries
	 * and the found TTE will be left in REG1.  REG3 and REG4 must
	 * be an even/odd pair of registers.
	 *
	 * VADDR and TAG will be preserved and not clobbered by this macro.
	 */
#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
661:	sethi		%uhi(swapper_tsb), REG1; \
	sethi		%hi(swapper_tsb), REG2; \
	or		REG1, %ulo(swapper_tsb), REG1; \
	or		REG2, %lo(swapper_tsb), REG2; \
	.section	.swapper_tsb_phys_patch, "ax"; \
	.word		661b; \
	.previous; \
	sllx		REG1, 32, REG1; \
	or		REG1, REG2, REG1; \
	srlx		VADDR, PAGE_SHIFT, REG2; \
	and		REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
	sllx		REG2, 4, REG2; \
	add		REG1, REG2, REG2; \
	TSB_LOAD_QUAD(REG2, REG3); \
	cmp		REG3, TAG; \
	be,a,pt		%xcc, OK_LABEL; \
	 mov		REG4, REG1;

#ifndef CONFIG_DEBUG_PAGEALLOC
	/* This version uses a trick, the TAG is already (VADDR >> 22) so
	 * we can make use of that for the index computation.
	 */
#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
661:	sethi		%uhi(swapper_4m_tsb), REG1; \
	sethi		%hi(swapper_4m_tsb), REG2; \
	or		REG1, %ulo(swapper_4m_tsb), REG1; \
	or		REG2, %lo(swapper_4m_tsb), REG2; \
	.section	.swapper_4m_tsb_phys_patch, "ax"; \
	.word		661b; \
	.previous; \
	sllx		REG1, 32, REG1; \
	or		REG1, REG2, REG1; \
	and		TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
	sllx		REG2, 4, REG2; \
	add		REG1, REG2, REG2; \
	TSB_LOAD_QUAD(REG2, REG3); \
	cmp		REG3, TAG; \
	be,a,pt		%xcc, OK_LABEL; \
	 mov		REG4, REG1;
#endif

#endif /* !(_SPARC64_TSB_H) */

cmd: