/usr/src/linux-headers-5.15.0-190/arch/x86/include/asm
NameSizeModeActions
e820/-0755rm
fpu/-0755rm
numachip/-0755rm
trace/-0755rm
uv/-0755rm
vdso/-0755rm
xen/-0755rm
acenv.h14500644editdlrm
acpi.h48040644editdlrm
acrn.h18340644editdlrm
agp.h10730644editdlrm
alternative.h135740644editdlrm
amd-ibs.h42470644editdlrm
amd_nb.h31360644editdlrm
apic.h136460644editdlrm
apicdef.h115630644editdlrm
apm.h18420644editdlrm
archrandom.h23470644editdlrm
arch_hweight.h10830644editdlrm
asm-offsets.h350644editdlrm
asm-prototypes.h4310644editdlrm
asm.h60600644editdlrm
atomic.h67660644editdlrm
atomic64_32.h83670644editdlrm
atomic64_64.h64100644editdlrm
audit.h1700644editdlrm
barrier.h23680644editdlrm
bios_ebda.h9140644editdlrm
bitops.h101490644editdlrm
boot.h23970644editdlrm
bootparam_utils.h28660644editdlrm
bug.h23920644editdlrm
bugs.h3440644editdlrm
cache.h6220644editdlrm
cacheflush.h3290644editdlrm
cacheinfo.h2630644editdlrm
ce4100.h1210644editdlrm
checksum.h2400644editdlrm
checksum_32.h46820644editdlrm
checksum_64.h51640644editdlrm
clocksource.h4800644editdlrm
cmdline.h3020644editdlrm
cmpxchg.h79370644editdlrm
cmpxchg_32.h32430644editdlrm
cmpxchg_64.h5610644editdlrm
compat.h41020644editdlrm
cpu.h20990644editdlrm
cpufeature.h84280644editdlrm
cpufeatures.h333830644editdlrm
cpuidle_haltpoll.h1890644editdlrm
cpumask.h8290644editdlrm
cpu_device_id.h109450644editdlrm
cpu_entry_area.h41960644editdlrm
crash.h3000644editdlrm
current.h4430644editdlrm
debugreg.h36410644editdlrm
delay.h2750644editdlrm
desc.h115240644editdlrm
desc_defs.h33960644editdlrm
device.h1760644editdlrm
disabled-features.h31730644editdlrm
div64.h23350644editdlrm
dma-mapping.h5190644editdlrm
dma.h98110644editdlrm
dmi.h5560644editdlrm
doublefault.h2710644editdlrm
dwarf2.h13250644editdlrm
edac.h4740644editdlrm
efi.h123080644editdlrm
elf.h126230644editdlrm
elfcore-compat.h8900644editdlrm
emergency-restart.h2020644editdlrm
emulate_prefix.h4840644editdlrm
enclu.h1810644editdlrm
entry-common.h33220644editdlrm
espfix.h4260644editdlrm
exec.h370644editdlrm
extable.h18710644editdlrm
extable_fixup_types.h18750644editdlrm
fb.h5400644editdlrm
fixmap.h60940644editdlrm
floppy.h68600644editdlrm
frame.h25550644editdlrm
fsgsbase.h20720644editdlrm
ftrace.h32340644editdlrm
futex.h25530644editdlrm
gart.h27040644editdlrm
GEN-for-each-reg.h3450644editdlrm
genapic.h220644editdlrm
geode.h6930644editdlrm
hardirq.h23420644editdlrm
highmem.h24660644editdlrm
hpet.h30020644editdlrm
hugetlb.h2430644editdlrm
hw_breakpoint.h21120644editdlrm
hw_irq.h30700644editdlrm
hyperv-tlfs.h175550644editdlrm
hypervisor.h23600644editdlrm
i8259.h20610644editdlrm
ia32.h17910644editdlrm
ia32_unistd.h3130644editdlrm
idtentry.h241970644editdlrm
imr.h16840644editdlrm
inat.h60920644editdlrm
inat_types.h3410644editdlrm
init.h6320644editdlrm
insn-eval.h13790644editdlrm
insn.h78670644editdlrm
inst.h20660644editdlrm
intel-family.h57750644editdlrm
intel-mid.h11320644editdlrm
intel_ds.h8490644editdlrm
intel_pconfig.h15450644editdlrm
intel_pt.h12350644editdlrm
intel_punit_ipc.h46660644editdlrm
intel_scu_ipc.h22340644editdlrm
intel_telemetry.h37230644editdlrm
invpcid.h14870644editdlrm
io.h126900644editdlrm
iomap.h5360644editdlrm
iommu.h7800644editdlrm
iommu_table.h39100644editdlrm
iosf_mbi.h73840644editdlrm
io_apic.h52040644editdlrm
io_bitmap.h13700644editdlrm
irq.h12800644editdlrm
irqdomain.h19150644editdlrm
irqflags.h29760644editdlrm
irq_remapping.h20610644editdlrm
irq_stack.h75360644editdlrm
irq_vectors.h42960644editdlrm
irq_work.h3580644editdlrm
ist.h2940644editdlrm
jailhouse_para.h4490644editdlrm
jump_label.h14060644editdlrm
kasan.h12910644editdlrm
kaslr.h3980644editdlrm
kbdleds.h4540644editdlrm
Kbuild3000644editdlrm
kdebug.h10870644editdlrm
kexec-bzimage64.h1950644editdlrm
kexec.h60110644editdlrm
kfence.h21400644editdlrm
kgdb.h21430644editdlrm
kprobes.h34630644editdlrm
kvm-x86-ops.h39210644editdlrm
kvmclock.h5060644editdlrm
kvm_host.h600660644editdlrm
kvm_page_track.h25040644editdlrm
kvm_para.h34670644editdlrm
kvm_types.h1780644editdlrm
kvm_vcpu_regs.h6060644editdlrm
linkage.h11390644editdlrm
livepatch.h4720644editdlrm
local.h39250644editdlrm
mach_timer.h15900644editdlrm
mach_traps.h10130644editdlrm
math_emu.h3950644editdlrm
mc146818rtc.h28350644editdlrm
mce.h131250644editdlrm
memtype.h8990644editdlrm
mem_encrypt.h34270644editdlrm
microcode.h38650644editdlrm
microcode_amd.h15730644editdlrm
microcode_intel.h25210644editdlrm
misc.h1430644editdlrm
mmconfig.h3740644editdlrm
mmu.h17400644editdlrm
mmu_context.h60740644editdlrm
mmx.h3370644editdlrm
mmzone.h1290644editdlrm
mmzone_32.h3430644editdlrm
mmzone_64.h4300644editdlrm
module.h3240644editdlrm
mpspec.h36520644editdlrm
mpspec_def.h42560644editdlrm
mshyperv.h56040644editdlrm
msi.h15490644editdlrm
msr-index.h402300644editdlrm
msr-trace.h13870644editdlrm
msr.h110610644editdlrm
mtrr.h47530644editdlrm
mwait.h44570644editdlrm
nmi.h14440644editdlrm
nops.h19050644editdlrm
nospec-branch.h130730644editdlrm
numa.h21930644editdlrm
numa_32.h2560644editdlrm
olpc.h23820644editdlrm
olpc_ofw.h11280644editdlrm
orc_lookup.h10650644editdlrm
orc_types.h20250644editdlrm
page.h25100644editdlrm
page_32.h10430644editdlrm
page_32_types.h23990644editdlrm
page_64.h27370644editdlrm
page_64_types.h31390644editdlrm
page_types.h22620644editdlrm
paravirt.h188130644editdlrm
paravirt_types.h196710644editdlrm
parport.h3140644editdlrm
pc-conf-reg.h7230644editdlrm
pci-direct.h8500644editdlrm
pci-functions.h6540644editdlrm
pci.h31910644editdlrm
pci_x86.h64770644editdlrm
percpu.h169920644editdlrm
perf_event.h160790644editdlrm
perf_event_p4.h267310644editdlrm
pgalloc.h53060644editdlrm
pgtable-2level.h28150644editdlrm
pgtable-2level_types.h9210644editdlrm
pgtable-3level.h89630644editdlrm
pgtable-3level_types.h10300644editdlrm
pgtable-invert.h10910644editdlrm
pgtable.h334640644editdlrm
pgtable_32.h23460644editdlrm
pgtable_32_areas.h15820644editdlrm
pgtable_32_types.h6340644editdlrm
pgtable_64.h72570644editdlrm
pgtable_64_types.h44680644editdlrm
pgtable_areas.h6230644editdlrm
pgtable_types.h175800644editdlrm
pkeys.h35130644editdlrm
pkru.h13460644editdlrm
platform_sst_audio.h31010644editdlrm
pm-trace.h6110644editdlrm
posix_types.h1440644editdlrm
preempt.h40510644editdlrm
probe_roms.h2730644editdlrm
processor-cyrix.h3830644editdlrm
processor-flags.h17500644editdlrm
processor.h224710644editdlrm
prom.h8210644editdlrm
proto.h9520644editdlrm
pti.h3720644editdlrm
ptrace.h104910644editdlrm
purgatory.h2350644editdlrm
pvclock-abi.h15300644editdlrm
pvclock.h27020644editdlrm
qrwlock.h1990644editdlrm
qspinlock.h31320644editdlrm
qspinlock_paravirt.h19080644editdlrm
realmode.h20990644editdlrm
reboot.h9480644editdlrm
reboot_fixups.h1830644editdlrm
required-features.h28390644editdlrm
resctrl.h28470644editdlrm
rmwcc.h22770644editdlrm
seccomp.h12050644editdlrm
sections.h8950644editdlrm
segment.h106130644editdlrm
serial.h11370644editdlrm
setup.h36330644editdlrm
setup_arch.h770644editdlrm
set_memory.h53880644editdlrm
sev-common.h30530644editdlrm
sev.h24430644editdlrm
sgx.h113890644editdlrm
shmparam.h1930644editdlrm
sigcontext.h2610644editdlrm
sigframe.h21410644editdlrm
sighandling.h5260644editdlrm
signal.h24280644editdlrm
simd.h2870644editdlrm
smap.h20380644editdlrm
smp.h50670644editdlrm
softirq_stack.h2160644editdlrm
sparsemem.h11020644editdlrm
spec-ctrl.h28750644editdlrm
special_insns.h71700644editdlrm
spinlock.h12140644editdlrm
spinlock_types.h2530644editdlrm
sta2x11.h3520644editdlrm
stackprotector.h30950644editdlrm
stacktrace.h28500644editdlrm
static_call.h28680644editdlrm
string.h1290644editdlrm
string_32.h61110644editdlrm
string_64.h28520644editdlrm
suspend.h4960644editdlrm
suspend_32.h8760644editdlrm
suspend_64.h18330644editdlrm
svm.h107910644editdlrm
swiotlb.h6450644editdlrm
switch_to.h23300644editdlrm
sync_bitops.h33360644editdlrm
sync_core.h33150644editdlrm
syscall.h43100644editdlrm
syscalls.h3740644editdlrm
syscall_wrapper.h99980644editdlrm
text-patching.h56460644editdlrm
thermal.h4280644editdlrm
thread_info.h79950644editdlrm
time.h3310644editdlrm
timer.h10270644editdlrm
timex.h5460644editdlrm
tlb.h12000644editdlrm
tlbbatch.h3320644editdlrm
tlbflush.h77100644editdlrm
topology.h64820644editdlrm
trace_clock.h4060644editdlrm
trapnr.h13160644editdlrm
traps.h12600644editdlrm
trap_pf.h7090644editdlrm
tsc.h20160644editdlrm
uaccess.h222100644editdlrm
uaccess_32.h10060644editdlrm
uaccess_64.h24400644editdlrm
umip.h3170644editdlrm
unistd.h16300644editdlrm
unwind.h32200644editdlrm
unwind_hints.h14610644editdlrm
uprobes.h10170644editdlrm
user.h22570644editdlrm
user32.h21560644editdlrm
user_32.h50420644editdlrm
user_64.h53390644editdlrm
vdso.h13910644editdlrm
vermagic.h19930644editdlrm
vga.h7400644editdlrm
vgtod.h5180644editdlrm
virtext.h34600644editdlrm
vm86.h21820644editdlrm
vmalloc.h5600644editdlrm
vmware.h19420644editdlrm
vmx.h261410644editdlrm
vmxfeatures.h60490644editdlrm
vsyscall.h9880644editdlrm
vvar.h15910644editdlrm
word-at-a-time.h25960644editdlrm
x86_init.h101760644editdlrm
xor.h101070644editdlrm
xor_32.h143470644editdlrm
xor_64.h7160644editdlrm
xor_avx.h43440644editdlrm
Edit: /usr/src/linux-headers-5.15.0-190/arch/x86/include/asm/paravirt.h (18813B)
/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_PARAVIRT_H #define _ASM_X86_PARAVIRT_H /* Various instructions on x86 need to be replaced for * para-virtualization: those hooks are defined here. */ #ifdef CONFIG_PARAVIRT #include #include #include #include #ifndef __ASSEMBLY__ #include #include #include #include #include u64 dummy_steal_clock(int cpu); u64 dummy_sched_clock(void); DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)); static inline u64 paravirt_sched_clock(void) { return static_call(pv_sched_clock)(); } struct static_key; extern struct static_key paravirt_steal_enabled; extern struct static_key paravirt_steal_rq_enabled; __visible void __native_queued_spin_unlock(struct qspinlock *lock); bool pv_is_native_spin_unlock(void); __visible bool __native_vcpu_is_preempted(long cpu); bool pv_is_native_vcpu_is_preempted(void); static inline u64 paravirt_steal_clock(int cpu) { return static_call(pv_steal_clock)(cpu); } #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init paravirt_set_cap(void); #endif /* The paravirtualized I/O functions */ static inline void slow_down_io(void) { pv_ops.cpu.io_delay(); #ifdef REALLY_SLOW_IO pv_ops.cpu.io_delay(); pv_ops.cpu.io_delay(); pv_ops.cpu.io_delay(); #endif } void native_flush_tlb_local(void); void native_flush_tlb_global(void); void native_flush_tlb_one_user(unsigned long addr); void native_flush_tlb_multi(const struct cpumask *cpumask, const struct flush_tlb_info *info); static inline void __flush_tlb_local(void) { PVOP_VCALL0(mmu.flush_tlb_user); } static inline void __flush_tlb_global(void) { PVOP_VCALL0(mmu.flush_tlb_kernel); } static inline void __flush_tlb_one_user(unsigned long addr) { PVOP_VCALL1(mmu.flush_tlb_one_user, addr); } static inline void __flush_tlb_multi(const struct cpumask *cpumask, const struct flush_tlb_info *info) { PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info); } static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) { PVOP_VCALL2(mmu.tlb_remove_table, tlb, table); } static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) { PVOP_VCALL1(mmu.exit_mmap, mm); } #ifdef CONFIG_PARAVIRT_XXL static inline void load_sp0(unsigned long sp0) { PVOP_VCALL1(cpu.load_sp0, sp0); } /* The paravirtualized CPUID instruction. */ static inline void __cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx); } /* * These special macros can be used to get or set a debugging register */ static inline unsigned long paravirt_get_debugreg(int reg) { return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg); } #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) static inline void set_debugreg(unsigned long val, int reg) { PVOP_VCALL2(cpu.set_debugreg, reg, val); } static inline unsigned long read_cr0(void) { return PVOP_CALL0(unsigned long, cpu.read_cr0); } static inline void write_cr0(unsigned long x) { PVOP_VCALL1(cpu.write_cr0, x); } static inline unsigned long read_cr2(void) { return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2, "mov %%cr2, %%rax;", ALT_NOT(X86_FEATURE_XENPV)); } static inline void write_cr2(unsigned long x) { PVOP_VCALL1(mmu.write_cr2, x); } static inline unsigned long __read_cr3(void) { return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3, "mov %%cr3, %%rax;", ALT_NOT(X86_FEATURE_XENPV)); } static inline void write_cr3(unsigned long x) { PVOP_ALT_VCALL1(mmu.write_cr3, x, "mov %%rdi, %%cr3", ALT_NOT(X86_FEATURE_XENPV)); } static inline void __write_cr4(unsigned long x) { PVOP_VCALL1(cpu.write_cr4, x); } static inline void arch_safe_halt(void) { PVOP_VCALL0(irq.safe_halt); } static inline void halt(void) { PVOP_VCALL0(irq.halt); } static inline void wbinvd(void) { PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT(X86_FEATURE_XENPV)); } static inline u64 paravirt_read_msr(unsigned msr) { return PVOP_CALL1(u64, cpu.read_msr, msr); } static inline void paravirt_write_msr(unsigned msr, unsigned low, unsigned high) { PVOP_VCALL3(cpu.write_msr, msr, low, high); } static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) { return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err); } static inline int paravirt_write_msr_safe(unsigned msr, unsigned low, unsigned high) { return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high); } #define rdmsr(msr, val1, val2) \ do { \ u64 _l = paravirt_read_msr(msr); \ val1 = (u32)_l; \ val2 = _l >> 32; \ } while (0) #define wrmsr(msr, val1, val2) \ do { \ paravirt_write_msr(msr, val1, val2); \ } while (0) #define rdmsrl(msr, val) \ do { \ val = paravirt_read_msr(msr); \ } while (0) static inline void wrmsrl(unsigned msr, u64 val) { wrmsr(msr, (u32)val, (u32)(val>>32)); } #define wrmsr_safe(msr, a, b) paravirt_write_msr_safe(msr, a, b) /* rdmsr with exception handling */ #define rdmsr_safe(msr, a, b) \ ({ \ int _err; \ u64 _l = paravirt_read_msr_safe(msr, &_err); \ (*a) = (u32)_l; \ (*b) = _l >> 32; \ _err; \ }) static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) { int err; *p = paravirt_read_msr_safe(msr, &err); return err; } static inline unsigned long long paravirt_read_pmc(int counter) { return PVOP_CALL1(u64, cpu.read_pmc, counter); } #define rdpmc(counter, low, high) \ do { \ u64 _l = paravirt_read_pmc(counter); \ low = (u32)_l; \ high = _l >> 32; \ } while (0) #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) { PVOP_VCALL2(cpu.alloc_ldt, ldt, entries); } static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) { PVOP_VCALL2(cpu.free_ldt, ldt, entries); } static inline void load_TR_desc(void) { PVOP_VCALL0(cpu.load_tr_desc); } static inline void load_gdt(const struct desc_ptr *dtr) { PVOP_VCALL1(cpu.load_gdt, dtr); } static inline void load_idt(const struct desc_ptr *dtr) { PVOP_VCALL1(cpu.load_idt, dtr); } static inline void set_ldt(const void *addr, unsigned entries) { PVOP_VCALL2(cpu.set_ldt, addr, entries); } static inline unsigned long paravirt_store_tr(void) { return PVOP_CALL0(unsigned long, cpu.store_tr); } #define store_tr(tr) ((tr) = paravirt_store_tr()) static inline void load_TLS(struct thread_struct *t, unsigned cpu) { PVOP_VCALL2(cpu.load_tls, t, cpu); } static inline void load_gs_index(unsigned int gs) { PVOP_VCALL1(cpu.load_gs_index, gs); } static inline void write_ldt_entry(struct desc_struct *dt, int entry, const void *desc) { PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc); } static inline void write_gdt_entry(struct desc_struct *dt, int entry, void *desc, int type) { PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type); } static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) { PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g); } #ifdef CONFIG_X86_IOPL_IOPERM static inline void tss_invalidate_io_bitmap(void) { PVOP_VCALL0(cpu.invalidate_io_bitmap); } static inline void tss_update_io_bitmap(void) { PVOP_VCALL0(cpu.update_io_bitmap); } #endif static inline void paravirt_activate_mm(struct mm_struct *prev, struct mm_struct *next) { PVOP_VCALL2(mmu.activate_mm, prev, next); } static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { PVOP_VCALL2(mmu.dup_mmap, oldmm, mm); } static inline int paravirt_pgd_alloc(struct mm_struct *mm) { return PVOP_CALL1(int, mmu.pgd_alloc, mm); } static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) { PVOP_VCALL2(mmu.pgd_free, mm, pgd); } static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) { PVOP_VCALL2(mmu.alloc_pte, mm, pfn); } static inline void paravirt_release_pte(unsigned long pfn) { PVOP_VCALL1(mmu.release_pte, pfn); } static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) { PVOP_VCALL2(mmu.alloc_pmd, mm, pfn); } static inline void paravirt_release_pmd(unsigned long pfn) { PVOP_VCALL1(mmu.release_pmd, pfn); } static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) { PVOP_VCALL2(mmu.alloc_pud, mm, pfn); } static inline void paravirt_release_pud(unsigned long pfn) { PVOP_VCALL1(mmu.release_pud, pfn); } static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) { PVOP_VCALL2(mmu.alloc_p4d, mm, pfn); } static inline void paravirt_release_p4d(unsigned long pfn) { PVOP_VCALL1(mmu.release_p4d, pfn); } static inline pte_t __pte(pteval_t val) { return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)) }; } static inline pteval_t pte_val(pte_t pte) { return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); } static inline pgd_t __pgd(pgdval_t val) { return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)) }; } static inline pgdval_t pgd_val(pgd_t pgd) { return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); } #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { pteval_t ret; ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, vma, addr, ptep); return (pte_t) { .pte = ret }; } static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t old_pte, pte_t pte) { PVOP_VCALL4(mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte); } static inline void set_pte(pte_t *ptep, pte_t pte) { PVOP_VCALL2(mmu.set_pte, ptep, pte.pte); } static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { PVOP_VCALL2(mmu.set_pmd, pmdp, native_pmd_val(pmd)); } static inline pmd_t __pmd(pmdval_t val) { return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)) }; } static inline pmdval_t pmd_val(pmd_t pmd) { return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); } static inline void set_pud(pud_t *pudp, pud_t pud) { PVOP_VCALL2(mmu.set_pud, pudp, native_pud_val(pud)); } static inline pud_t __pud(pudval_t val) { pudval_t ret; ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); return (pud_t) { ret }; } static inline pudval_t pud_val(pud_t pud) { return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); } static inline void pud_clear(pud_t *pudp) { set_pud(pudp, native_make_pud(0)); } static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { p4dval_t val = native_p4d_val(p4d); PVOP_VCALL2(mmu.set_p4d, p4dp, val); } #if CONFIG_PGTABLE_LEVELS >= 5 static inline p4d_t __p4d(p4dval_t val) { p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); return (p4d_t) { ret }; } static inline p4dval_t p4d_val(p4d_t p4d) { return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d, "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); } static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) { PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); } #define set_pgd(pgdp, pgdval) do { \ if (pgtable_l5_enabled()) \ __set_pgd(pgdp, pgdval); \ else \ set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \ } while (0) #define pgd_clear(pgdp) do { \ if (pgtable_l5_enabled()) \ set_pgd(pgdp, native_make_pgd(0)); \ } while (0) #endif /* CONFIG_PGTABLE_LEVELS == 5 */ static inline void p4d_clear(p4d_t *p4dp) { set_p4d(p4dp, native_make_p4d(0)); } static inline void set_pte_atomic(pte_t *ptep, pte_t pte) { set_pte(ptep, pte); } static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { set_pte(ptep, native_make_pte(0)); } static inline void pmd_clear(pmd_t *pmdp) { set_pmd(pmdp, native_make_pmd(0)); } #define __HAVE_ARCH_START_CONTEXT_SWITCH static inline void arch_start_context_switch(struct task_struct *prev) { PVOP_VCALL1(cpu.start_context_switch, prev); } static inline void arch_end_context_switch(struct task_struct *next) { PVOP_VCALL1(cpu.end_context_switch, next); } #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE static inline void arch_enter_lazy_mmu_mode(void) { PVOP_VCALL0(mmu.lazy_mode.enter); } static inline void arch_leave_lazy_mmu_mode(void) { PVOP_VCALL0(mmu.lazy_mode.leave); } static inline void arch_flush_lazy_mmu_mode(void) { PVOP_VCALL0(mmu.lazy_mode.flush); } static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, phys_addr_t phys, pgprot_t flags) { pv_ops.mmu.set_fixmap(idx, phys, flags); } #endif #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) { PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val); } static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) { PVOP_ALT_VCALLEE1(lock.queued_spin_unlock, lock, "movb $0, (%%" _ASM_ARG1 ");", ALT_NOT(X86_FEATURE_PVUNLOCK)); } static __always_inline void pv_wait(u8 *ptr, u8 val) { PVOP_VCALL2(lock.wait, ptr, val); } static __always_inline void pv_kick(int cpu) { PVOP_VCALL1(lock.kick, cpu); } static __always_inline bool pv_vcpu_is_preempted(long cpu) { return PVOP_ALT_CALLEE1(bool, lock.vcpu_is_preempted, cpu, "xor %%" _ASM_AX ", %%" _ASM_AX ";", ALT_NOT(X86_FEATURE_VCPUPREEMPT)); } void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); bool __raw_callee_save___native_vcpu_is_preempted(long cpu); #endif /* SMP && PARAVIRT_SPINLOCKS */ #ifdef CONFIG_X86_32 /* save and restore all caller-save registers, except return value */ #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" #else /* save and restore all caller-save registers, except return value */ #define PV_SAVE_ALL_CALLER_REGS \ "push %rcx;" \ "push %rdx;" \ "push %rsi;" \ "push %rdi;" \ "push %r8;" \ "push %r9;" \ "push %r10;" \ "push %r11;" #define PV_RESTORE_ALL_CALLER_REGS \ "pop %r11;" \ "pop %r10;" \ "pop %r9;" \ "pop %r8;" \ "pop %rdi;" \ "pop %rsi;" \ "pop %rdx;" \ "pop %rcx;" #endif /* * Generate a thunk around a function which saves all caller-save * registers except for the return value. This allows C functions to * be called from assembler code where fewer than normal registers are * available. It may also help code generation around calls from C * code if the common case doesn't use many registers. * * When a callee is wrapped in a thunk, the caller can assume that all * arg regs and all scratch registers are preserved across the * call. The return value in rax/eax will not be saved, even for void * functions. */ #define PV_THUNK_NAME(func) "__raw_callee_save_" #func #define PV_CALLEE_SAVE_REGS_THUNK(func) \ extern typeof(func) __raw_callee_save_##func; \ \ asm(".pushsection .text;" \ ".globl " PV_THUNK_NAME(func) ";" \ ".type " PV_THUNK_NAME(func) ", @function;" \ PV_THUNK_NAME(func) ":" \ FRAME_BEGIN \ PV_SAVE_ALL_CALLER_REGS \ "call " #func ";" \ PV_RESTORE_ALL_CALLER_REGS \ FRAME_END \ ASM_RET \ ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ ".popsection") /* Get a reference to a callee-save function */ #define PV_CALLEE_SAVE(func) \ ((struct paravirt_callee_save) { __raw_callee_save_##func }) /* Promise that "func" already uses the right calling convention */ #define __PV_IS_CALLEE_SAVE(func) \ ((struct paravirt_callee_save) { func }) #ifdef CONFIG_PARAVIRT_XXL static inline notrace unsigned long arch_local_save_flags(void) { return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;", ALT_NOT(X86_FEATURE_XENPV)); } static inline notrace void arch_local_irq_disable(void) { PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV)); } static inline notrace void arch_local_irq_enable(void) { PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV)); } static inline notrace unsigned long arch_local_irq_save(void) { unsigned long f; f = arch_local_save_flags(); arch_local_irq_disable(); return f; } #endif /* Make sure as little as possible of this mess escapes. */ #undef PARAVIRT_CALL #undef __PVOP_CALL #undef __PVOP_VCALL #undef PVOP_VCALL0 #undef PVOP_CALL0 #undef PVOP_VCALL1 #undef PVOP_CALL1 #undef PVOP_VCALL2 #undef PVOP_CALL2 #undef PVOP_VCALL3 #undef PVOP_CALL3 #undef PVOP_VCALL4 #undef PVOP_CALL4 extern void default_banner(void); #else /* __ASSEMBLY__ */ #define _PVSITE(ptype, ops, word, algn) \ 771:; \ ops; \ 772:; \ .pushsection .parainstructions,"a"; \ .align algn; \ word 771b; \ .byte ptype; \ .byte 772b-771b; \ .popsection #ifdef CONFIG_X86_64 #ifdef CONFIG_PARAVIRT_XXL #define PARA_PATCH(off) ((off) / 8) #define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8) #define PARA_INDIRECT(addr) *addr(%rip) #define INTERRUPT_RETURN \ ANNOTATE_RETPOLINE_SAFE; \ ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \ X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;") #ifdef CONFIG_DEBUG_ENTRY .macro PARA_IRQ_save_fl PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), ANNOTATE_RETPOLINE_SAFE; call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);) .endm #define SAVE_FLAGS ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \ ALT_NOT(X86_FEATURE_XENPV) #endif #endif /* CONFIG_PARAVIRT_XXL */ #endif /* CONFIG_X86_64 */ #endif /* __ASSEMBLY__ */ #else /* CONFIG_PARAVIRT */ # define default_banner x86_init_noop #endif /* !CONFIG_PARAVIRT */ #ifndef __ASSEMBLY__ #ifndef CONFIG_PARAVIRT_XXL static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { } #endif #ifndef CONFIG_PARAVIRT static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) { } #endif #ifndef CONFIG_PARAVIRT_SPINLOCKS static inline void paravirt_set_cap(void) { } #endif #endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_PARAVIRT_H */