mirror of
https://github.com/torvalds/linux.git
synced 2025-12-07 20:06:24 +00:00
51d90a15fe
Pull KVM updates from Paolo Bonzini:
"ARM:
- Support for userspace handling of synchronous external aborts
(SEAs), allowing the VMM to potentially handle the abort in a
non-fatal manner
- Large rework of the VGIC's list register handling with the goal of
supporting more active/pending IRQs than available list registers
in hardware. In addition, the VGIC now supports EOImode==1 style
deactivations for IRQs which may occur on a separate vCPU than the
one that acked the IRQ
- Support for FEAT_XNX (user / privileged execute permissions) and
FEAT_HAF (hardware update to the Access Flag) in the software page
table walkers and shadow MMU
- Allow page table destruction to reschedule, fixing long
need_resched latencies observed when destroying a large VM
- Minor fixes to KVM and selftests
Loongarch:
- Get VM PMU capability from HW GCFG register
- Add AVEC basic support
- Use 64-bit register definition for EIOINTC
- Add KVM timer test cases for tools/selftests
RISC/V:
- SBI message passing (MPXY) support for KVM guest
- Give a new, more specific error subcode for the case when in-kernel
AIA virtualization fails to allocate IMSIC VS-file
- Support KVM_DIRTY_LOG_INITIALLY_SET, enabling dirty log gradually
in small chunks
- Fix guest page fault within HLV* instructions
- Flush VS-stage TLB after VCPU migration for Andes cores
s390:
- Always allocate ESCA (Extended System Control Area), instead of
starting with the basic SCA and converting to ESCA with the
addition of the 65th vCPU. The price is increased number of exits
(and worse performance) on z10 and earlier processor; ESCA was
introduced by z114/z196 in 2010
- VIRT_XFER_TO_GUEST_WORK support
- Operation exception forwarding support
- Cleanups
x86:
- Skip the costly "zap all SPTEs" on an MMIO generation wrap if MMIO
SPTE caching is disabled, as there can't be any relevant SPTEs to
zap
- Relocate a misplaced export
- Fix an async #PF bug where KVM would clear the completion queue
when the guest transitioned in and out of paging mode, e.g. when
handling an SMI and then returning to paged mode via RSM
- Leave KVM's user-return notifier registered even when disabling
virtualization, as long as kvm.ko is loaded. On reboot/shutdown,
keeping the notifier registered is ok; the kernel does not use the
MSRs and the callback will run cleanly and restore host MSRs if the
CPU manages to return to userspace before the system goes down
- Use the checked version of {get,put}_user()
- Fix a long-lurking bug where KVM's lack of catch-up logic for
periodic APIC timers can result in a hard lockup in the host
- Revert the periodic kvmclock sync logic now that KVM doesn't use a
clocksource that's subject to NTP corrections
- Clean up KVM's handling of MMIO Stale Data and L1TF, and bury the
latter behind CONFIG_CPU_MITIGATIONS
- Context switch XCR0, XSS, and PKRU outside of the entry/exit fast
path; the only reason they were handled in the fast path was to
paper of a bug in the core #MC code, and that has long since been
fixed
- Add emulator support for AVX MOV instructions, to play nice with
emulated devices whose guest drivers like to access PCI BARs with
large multi-byte instructions
x86 (AMD):
- Fix a few missing "VMCB dirty" bugs
- Fix the worst of KVM's lack of EFER.LMSLE emulation
- Add AVIC support for addressing 4k vCPUs in x2AVIC mode
- Fix incorrect handling of selective CR0 writes when checking
intercepts during emulation of L2 instructions
- Fix a currently-benign bug where KVM would clobber SPEC_CTRL[63:32]
on VMRUN and #VMEXIT
- Fix a bug where KVM corrupt the guest code stream when re-injecting
a soft interrupt if the guest patched the underlying code after the
VM-Exit, e.g. when Linux patches code with a temporary INT3
- Add KVM_X86_SNP_POLICY_BITS to advertise supported SNP policy bits
to userspace, and extend KVM "support" to all policy bits that
don't require any actual support from KVM
x86 (Intel):
- Use the root role from kvm_mmu_page to construct EPTPs instead of
the current vCPU state, partly as worthwhile cleanup, but mostly to
pave the way for tracking per-root TLB flushes, and elide EPT
flushes on pCPU migration if the root is clean from a previous
flush
- Add a few missing nested consistency checks
- Rip out support for doing "early" consistency checks via hardware
as the functionality hasn't been used in years and is no longer
useful in general; replace it with an off-by-default module param
to WARN if hardware fails a check that KVM does not perform
- Fix a currently-benign bug where KVM would drop the guest's
SPEC_CTRL[63:32] on VM-Enter
- Misc cleanups
- Overhaul the TDX code to address systemic races where KVM (acting
on behalf of userspace) could inadvertantly trigger lock contention
in the TDX-Module; KVM was either working around these in weird,
ugly ways, or was simply oblivious to them (though even Yan's
devilish selftests could only break individual VMs, not the host
kernel)
- Fix a bug where KVM could corrupt a vCPU's cpu_list when freeing a
TDX vCPU, if creating said vCPU failed partway through
- Fix a few sparse warnings (bad annotation, 0 != NULL)
- Use struct_size() to simplify copying TDX capabilities to userspace
- Fix a bug where TDX would effectively corrupt user-return MSR
values if the TDX Module rejects VP.ENTER and thus doesn't clobber
host MSRs as expected
Selftests:
- Fix a math goof in mmu_stress_test when running on a single-CPU
system/VM
- Forcefully override ARCH from x86_64 to x86 to play nice with
specifying ARCH=x86_64 on the command line
- Extend a bunch of nested VMX to validate nested SVM as well
- Add support for LA57 in the core VM_MODE_xxx macro, and add a test
to verify KVM can save/restore nested VMX state when L1 is using
5-level paging, but L2 is not
- Clean up the guest paging code in anticipation of sharing the core
logic for nested EPT and nested NPT
guest_memfd:
- Add NUMA mempolicy support for guest_memfd, and clean up a variety
of rough edges in guest_memfd along the way
- Define a CLASS to automatically handle get+put when grabbing a
guest_memfd from a memslot to make it harder to leak references
- Enhance KVM selftests to make it easer to develop and debug
selftests like those added for guest_memfd NUMA support, e.g. where
test and/or KVM bugs often result in hard-to-debug SIGBUS errors
- Misc cleanups
Generic:
- Use the recently-added WQ_PERCPU when creating the per-CPU
workqueue for irqfd cleanup
- Fix a goof in the dirty ring documentation
- Fix choice of target for directed yield across different calls to
kvm_vcpu_on_spin(); the function was always starting from the first
vCPU instead of continuing the round-robin search"
* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (260 commits)
KVM: arm64: at: Update AF on software walk only if VM has FEAT_HAFDBS
KVM: arm64: at: Use correct HA bit in TCR_EL2 when regime is EL2
KVM: arm64: Document KVM_PGTABLE_PROT_{UX,PX}
KVM: arm64: Fix spelling mistake "Unexpeced" -> "Unexpected"
KVM: arm64: Add break to default case in kvm_pgtable_stage2_pte_prot()
KVM: arm64: Add endian casting to kvm_swap_s[12]_desc()
KVM: arm64: Fix compilation when CONFIG_ARM64_USE_LSE_ATOMICS=n
KVM: arm64: selftests: Add test for AT emulation
KVM: arm64: nv: Expose hardware access flag management to NV guests
KVM: arm64: nv: Implement HW access flag management in stage-2 SW PTW
KVM: arm64: Implement HW access flag management in stage-1 SW PTW
KVM: arm64: Propagate PTW errors up to AT emulation
KVM: arm64: Add helper for swapping guest descriptor
KVM: arm64: nv: Use pgtable definitions in stage-2 walk
KVM: arm64: Handle endianness in read helper for emulated PTW
KVM: arm64: nv: Stop passing vCPU through void ptr in S2 PTW
KVM: arm64: Call helper for reading descriptors directly
KVM: arm64: nv: Advertise support for FEAT_XNX
KVM: arm64: Teach ptdump about FEAT_XNX permissions
KVM: s390: Use generic VIRT_XFER_TO_GUEST_WORK functions
...
386 lines
12 KiB
C
386 lines
12 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2012,2013 - ARM Ltd
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
*/
|
|
|
|
#ifndef __ARM_KVM_ASM_H__
|
|
#define __ARM_KVM_ASM_H__
|
|
|
|
#include <asm/hyp_image.h>
|
|
#include <asm/insn.h>
|
|
#include <asm/virt.h>
|
|
#include <asm/sysreg.h>
|
|
|
|
#define ARM_EXIT_WITH_SERROR_BIT 31
|
|
#define ARM_EXCEPTION_CODE(x) ((x) & ~(1U << ARM_EXIT_WITH_SERROR_BIT))
|
|
#define ARM_EXCEPTION_IS_TRAP(x) (ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_TRAP)
|
|
#define ARM_SERROR_PENDING(x) !!((x) & (1U << ARM_EXIT_WITH_SERROR_BIT))
|
|
|
|
#define ARM_EXCEPTION_IRQ 0
|
|
#define ARM_EXCEPTION_EL1_SERROR 1
|
|
#define ARM_EXCEPTION_TRAP 2
|
|
#define ARM_EXCEPTION_IL 3
|
|
/* The hyp-stub will return this for any kvm_call_hyp() call */
|
|
#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR
|
|
|
|
#define kvm_arm_exception_type \
|
|
{ARM_EXCEPTION_IRQ, "IRQ" }, \
|
|
{ARM_EXCEPTION_EL1_SERROR, "SERROR" }, \
|
|
{ARM_EXCEPTION_TRAP, "TRAP" }, \
|
|
{ARM_EXCEPTION_HYP_GONE, "HYP_GONE" }
|
|
|
|
/*
|
|
* Size of the HYP vectors preamble. kvm_patch_vector_branch() generates code
|
|
* that jumps over this.
|
|
*/
|
|
#define KVM_VECTOR_PREAMBLE (2 * AARCH64_INSN_SIZE)
|
|
|
|
#define KVM_HOST_SMCCC_ID(id) \
|
|
ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
|
|
ARM_SMCCC_SMC_64, \
|
|
ARM_SMCCC_OWNER_VENDOR_HYP, \
|
|
(id))
|
|
|
|
#define KVM_HOST_SMCCC_FUNC(name) KVM_HOST_SMCCC_ID(__KVM_HOST_SMCCC_FUNC_##name)
|
|
|
|
#define __KVM_HOST_SMCCC_FUNC___kvm_hyp_init 0
|
|
|
|
#ifndef __ASSEMBLER__
|
|
|
|
#include <linux/mm.h>
|
|
|
|
enum __kvm_host_smccc_func {
|
|
/* Hypercalls available only prior to pKVM finalisation */
|
|
/* __KVM_HOST_SMCCC_FUNC___kvm_hyp_init */
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_init = __KVM_HOST_SMCCC_FUNC___kvm_hyp_init + 1,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs,
|
|
__KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs,
|
|
__KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize,
|
|
|
|
/* Hypercalls available after pKVM finalisation */
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_guest,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_guest,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_relax_perms_guest,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_wrprotect_guest,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_test_clear_young_guest,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_host_mkyoung_guest,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
|
|
__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
|
|
__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
|
|
__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_reserve_vm,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_init_vm,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put,
|
|
__KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid,
|
|
};
|
|
|
|
#define DECLARE_KVM_VHE_SYM(sym) extern char sym[]
|
|
#define DECLARE_KVM_NVHE_SYM(sym) extern char kvm_nvhe_sym(sym)[]
|
|
|
|
/*
|
|
* Define a pair of symbols sharing the same name but one defined in
|
|
* VHE and the other in nVHE hyp implementations.
|
|
*/
|
|
#define DECLARE_KVM_HYP_SYM(sym) \
|
|
DECLARE_KVM_VHE_SYM(sym); \
|
|
DECLARE_KVM_NVHE_SYM(sym)
|
|
|
|
#define DECLARE_KVM_VHE_PER_CPU(type, sym) \
|
|
DECLARE_PER_CPU(type, sym)
|
|
#define DECLARE_KVM_NVHE_PER_CPU(type, sym) \
|
|
DECLARE_PER_CPU(type, kvm_nvhe_sym(sym))
|
|
|
|
#define DECLARE_KVM_HYP_PER_CPU(type, sym) \
|
|
DECLARE_KVM_VHE_PER_CPU(type, sym); \
|
|
DECLARE_KVM_NVHE_PER_CPU(type, sym)
|
|
|
|
/*
|
|
* Compute pointer to a symbol defined in nVHE percpu region.
|
|
* Returns NULL if percpu memory has not been allocated yet.
|
|
*/
|
|
#define this_cpu_ptr_nvhe_sym(sym) per_cpu_ptr_nvhe_sym(sym, smp_processor_id())
|
|
#define per_cpu_ptr_nvhe_sym(sym, cpu) \
|
|
({ \
|
|
unsigned long base, off; \
|
|
base = kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; \
|
|
off = (unsigned long)&CHOOSE_NVHE_SYM(sym) - \
|
|
(unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start); \
|
|
base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL; \
|
|
})
|
|
|
|
#if defined(__KVM_NVHE_HYPERVISOR__)
|
|
|
|
#define CHOOSE_NVHE_SYM(sym) sym
|
|
#define CHOOSE_HYP_SYM(sym) CHOOSE_NVHE_SYM(sym)
|
|
|
|
/* The nVHE hypervisor shouldn't even try to access VHE symbols */
|
|
extern void *__nvhe_undefined_symbol;
|
|
#define CHOOSE_VHE_SYM(sym) __nvhe_undefined_symbol
|
|
#define this_cpu_ptr_hyp_sym(sym) (&__nvhe_undefined_symbol)
|
|
#define per_cpu_ptr_hyp_sym(sym, cpu) (&__nvhe_undefined_symbol)
|
|
|
|
#elif defined(__KVM_VHE_HYPERVISOR__)
|
|
|
|
#define CHOOSE_VHE_SYM(sym) sym
|
|
#define CHOOSE_HYP_SYM(sym) CHOOSE_VHE_SYM(sym)
|
|
|
|
/* The VHE hypervisor shouldn't even try to access nVHE symbols */
|
|
extern void *__vhe_undefined_symbol;
|
|
#define CHOOSE_NVHE_SYM(sym) __vhe_undefined_symbol
|
|
#define this_cpu_ptr_hyp_sym(sym) (&__vhe_undefined_symbol)
|
|
#define per_cpu_ptr_hyp_sym(sym, cpu) (&__vhe_undefined_symbol)
|
|
|
|
#else
|
|
|
|
/*
|
|
* BIG FAT WARNINGS:
|
|
*
|
|
* - Don't be tempted to change the following is_kernel_in_hyp_mode()
|
|
* to has_vhe(). has_vhe() is implemented as a *final* capability,
|
|
* while this is used early at boot time, when the capabilities are
|
|
* not final yet....
|
|
*
|
|
* - Don't let the nVHE hypervisor have access to this, as it will
|
|
* pick the *wrong* symbol (yes, it runs at EL2...).
|
|
*/
|
|
#define CHOOSE_HYP_SYM(sym) (is_kernel_in_hyp_mode() \
|
|
? CHOOSE_VHE_SYM(sym) \
|
|
: CHOOSE_NVHE_SYM(sym))
|
|
|
|
#define this_cpu_ptr_hyp_sym(sym) (is_kernel_in_hyp_mode() \
|
|
? this_cpu_ptr(&sym) \
|
|
: this_cpu_ptr_nvhe_sym(sym))
|
|
|
|
#define per_cpu_ptr_hyp_sym(sym, cpu) (is_kernel_in_hyp_mode() \
|
|
? per_cpu_ptr(&sym, cpu) \
|
|
: per_cpu_ptr_nvhe_sym(sym, cpu))
|
|
|
|
#define CHOOSE_VHE_SYM(sym) sym
|
|
#define CHOOSE_NVHE_SYM(sym) kvm_nvhe_sym(sym)
|
|
|
|
#endif
|
|
|
|
struct kvm_nvhe_init_params {
|
|
unsigned long mair_el2;
|
|
unsigned long tcr_el2;
|
|
unsigned long tpidr_el2;
|
|
unsigned long stack_hyp_va;
|
|
unsigned long stack_pa;
|
|
phys_addr_t pgd_pa;
|
|
unsigned long hcr_el2;
|
|
unsigned long vttbr;
|
|
unsigned long vtcr;
|
|
unsigned long tmp;
|
|
};
|
|
|
|
/*
|
|
* Used by the host in EL1 to dump the nVHE hypervisor backtrace on
|
|
* hyp_panic() in non-protected mode.
|
|
*
|
|
* @stack_base: hyp VA of the hyp_stack base.
|
|
* @overflow_stack_base: hyp VA of the hyp_overflow_stack base.
|
|
* @fp: hyp FP where the backtrace begins.
|
|
* @pc: hyp PC where the backtrace begins.
|
|
*/
|
|
struct kvm_nvhe_stacktrace_info {
|
|
unsigned long stack_base;
|
|
unsigned long overflow_stack_base;
|
|
unsigned long fp;
|
|
unsigned long pc;
|
|
};
|
|
|
|
/* Translate a kernel address @ptr into its equivalent linear mapping */
|
|
#define kvm_ksym_ref(ptr) \
|
|
({ \
|
|
void *val = (ptr); \
|
|
if (!is_kernel_in_hyp_mode()) \
|
|
val = lm_alias((ptr)); \
|
|
val; \
|
|
})
|
|
#define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym))
|
|
|
|
struct kvm;
|
|
struct kvm_vcpu;
|
|
struct kvm_s2_mmu;
|
|
|
|
DECLARE_KVM_NVHE_SYM(__kvm_hyp_init);
|
|
DECLARE_KVM_HYP_SYM(__kvm_hyp_vector);
|
|
#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
|
|
#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
|
|
|
|
extern unsigned long kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[];
|
|
DECLARE_KVM_NVHE_SYM(__per_cpu_start);
|
|
DECLARE_KVM_NVHE_SYM(__per_cpu_end);
|
|
|
|
DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs);
|
|
#define __bp_harden_hyp_vecs CHOOSE_HYP_SYM(__bp_harden_hyp_vecs)
|
|
|
|
extern void __kvm_flush_vm_context(void);
|
|
extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu);
|
|
extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa,
|
|
int level);
|
|
extern void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,
|
|
phys_addr_t ipa,
|
|
int level);
|
|
extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,
|
|
phys_addr_t start, unsigned long pages);
|
|
extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
|
|
|
|
extern int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding);
|
|
|
|
extern void __kvm_timer_set_cntvoff(u64 cntvoff);
|
|
extern int __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
|
|
extern int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
|
|
extern int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
|
|
|
|
extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
|
|
|
|
extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu);
|
|
|
|
extern u64 __vgic_v3_get_gic_config(void);
|
|
extern void __vgic_v3_init_lrs(void);
|
|
|
|
#define __KVM_EXTABLE(from, to) \
|
|
" .pushsection __kvm_ex_table, \"a\"\n" \
|
|
" .align 3\n" \
|
|
" .long (" #from " - .), (" #to " - .)\n" \
|
|
" .popsection\n"
|
|
|
|
|
|
#define __kvm_at(at_op, addr) \
|
|
( { \
|
|
int __kvm_at_err = 0; \
|
|
u64 spsr, elr; \
|
|
asm volatile( \
|
|
" mrs %1, spsr_el2\n" \
|
|
" mrs %2, elr_el2\n" \
|
|
"1: " __msr_s(at_op, "%3") "\n" \
|
|
" isb\n" \
|
|
" b 9f\n" \
|
|
"2: msr spsr_el2, %1\n" \
|
|
" msr elr_el2, %2\n" \
|
|
" mov %w0, %4\n" \
|
|
"9:\n" \
|
|
__KVM_EXTABLE(1b, 2b) \
|
|
: "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \
|
|
: "r" (addr), "i" (-EFAULT)); \
|
|
__kvm_at_err; \
|
|
} )
|
|
|
|
void __noreturn hyp_panic(void);
|
|
asmlinkage void kvm_unexpected_el2_exception(void);
|
|
asmlinkage void __noreturn hyp_panic(void);
|
|
asmlinkage void __noreturn hyp_panic_bad_stack(void);
|
|
asmlinkage void kvm_unexpected_el2_exception(void);
|
|
struct kvm_cpu_context;
|
|
void handle_trap(struct kvm_cpu_context *host_ctxt);
|
|
asmlinkage void __noreturn __kvm_host_psci_cpu_entry(bool is_cpu_on);
|
|
void __noreturn __pkvm_init_finalise(void);
|
|
void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc);
|
|
void kvm_patch_vector_branch(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
void kvm_get_kimage_voffset(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
void kvm_compute_final_ctr_el0(struct alt_instr *alt,
|
|
__le32 *origptr, __le32 *updptr, int nr_inst);
|
|
void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, u64 elr_virt,
|
|
u64 elr_phys, u64 par, uintptr_t vcpu, u64 far, u64 hpfar);
|
|
|
|
#else /* __ASSEMBLER__ */
|
|
|
|
.macro get_host_ctxt reg, tmp
|
|
adr_this_cpu \reg, kvm_host_data, \tmp
|
|
add \reg, \reg, #HOST_DATA_CONTEXT
|
|
.endm
|
|
|
|
.macro get_vcpu_ptr vcpu, ctxt
|
|
get_host_ctxt \ctxt, \vcpu
|
|
ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
|
|
.endm
|
|
|
|
.macro get_loaded_vcpu vcpu, ctxt
|
|
adr_this_cpu \ctxt, kvm_hyp_ctxt, \vcpu
|
|
ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
|
|
.endm
|
|
|
|
.macro set_loaded_vcpu vcpu, ctxt, tmp
|
|
adr_this_cpu \ctxt, kvm_hyp_ctxt, \tmp
|
|
str \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
|
|
.endm
|
|
|
|
/*
|
|
* KVM extable for unexpected exceptions.
|
|
* Create a struct kvm_exception_table_entry output to a section that can be
|
|
* mapped by EL2. The table is not sorted.
|
|
*
|
|
* The caller must ensure:
|
|
* x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
|
|
* code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
|
|
*/
|
|
.macro _kvm_extable, from, to
|
|
.pushsection __kvm_ex_table, "a"
|
|
.align 3
|
|
.long (\from - .), (\to - .)
|
|
.popsection
|
|
.endm
|
|
|
|
#define CPU_XREG_OFFSET(x) (CPU_USER_PT_REGS + 8*x)
|
|
#define CPU_LR_OFFSET CPU_XREG_OFFSET(30)
|
|
#define CPU_SP_EL0_OFFSET (CPU_LR_OFFSET + 8)
|
|
|
|
/*
|
|
* We treat x18 as callee-saved as the host may use it as a platform
|
|
* register (e.g. for shadow call stack).
|
|
*/
|
|
.macro save_callee_saved_regs ctxt
|
|
str x18, [\ctxt, #CPU_XREG_OFFSET(18)]
|
|
stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
|
|
stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
|
|
stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
|
|
stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
|
|
stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
|
|
stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
|
|
.endm
|
|
|
|
.macro restore_callee_saved_regs ctxt
|
|
// We require \ctxt is not x18-x28
|
|
ldr x18, [\ctxt, #CPU_XREG_OFFSET(18)]
|
|
ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
|
|
ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
|
|
ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
|
|
ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
|
|
ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
|
|
ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
|
|
.endm
|
|
|
|
.macro save_sp_el0 ctxt, tmp
|
|
mrs \tmp, sp_el0
|
|
str \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
|
|
.endm
|
|
|
|
.macro restore_sp_el0 ctxt, tmp
|
|
ldr \tmp, [\ctxt, #CPU_SP_EL0_OFFSET]
|
|
msr sp_el0, \tmp
|
|
.endm
|
|
|
|
#endif
|
|
|
|
#endif /* __ARM_KVM_ASM_H__ */
|