From 2da2d8d71610fec77780032c56f8e9fa20dd8068 Mon Sep 17 00:00:00 2001 From: hanliyang Date: Tue, 24 Sep 2024 15:16:39 +0800 Subject: [PATCH 1/8] KVM: SVM: CSV: Fix the vm_size even if CSV3 feature is unsupported on Hygon CPUs hygon inclusion category: bugfix CVE: NA --------------------------- Assume the userspace request CSV3's KVM ioctl interface on Hygon CPUs before C86-4G, the vm_size if as value sizeof(struct kvm_svm), and functions for the CSV3's KVM ioctl interface try to check whether the guest is a CSV3 guest by access the structure as below: struct kvm_csv_info { ...... bool csv3_active; ...... }; struct kvm_svm_csv { struct kvm_svm kvm_svm; struct kvm_csv_info csv_info; }; But the csv_info field of struct kvm_svm_csv will not be allocated, the functions for the CSV3's KVM ioctl interface will not get value of csv3_active field of struct kvm_csv_info. Always fix the vm_size in csv_init() to address the above issue. Fixes: 58ebba1dc7d6 ("KVM: SVM: CSV: Add KVM_CSV3_INIT command") Fixes: 51d21713cf36 ("KVM: SVM: CSV: Add KVM_CSV3_LAUNCH_ENCRYPT_DATA command") Fixes: 0ccf6765bd3b ("KVM: SVM: CSV: Add KVM_CSV3_LAUNCH_ENCRYPT_VMCB command") Fixes: a103ec4c986b ("KVM: SVM: CSV: Manage CSV3 guest's nested page table") Fixes: adc59986a1df ("KVM: SVM: CSV: Add KVM_CSV3_SEND_ENCRYPT_DATA command") Fixes: 2eec03fe05ae ("KVM: SVM: CSV: Add KVM_CSV3_SEND_ENCRYPT_CONTEXT command") Fixes: 943b908b70f6 ("KVM: SVM: CSV: Add KVM_CSV3_RECEIVE_ENCRYPT_DATA command") Fixes: 5032e75c2935 ("KVM: SVM: CSV: Add KVM_CSV3_RECEIVE_ENCRYPT_CONTEXT command") Fixes: 719354858bbe ("KVM: SVM: CSV: Add ioctl API to unpin shared pages of CSV3 guest") Signed-off-by: hanliyang --- arch/x86/kvm/svm/csv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index f0e2dda6468fa..13800c7e9c00f 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -2737,14 +2737,13 @@ void __init csv_init(struct kvm_x86_ops *ops) memcpy(&csv_x86_ops, ops, sizeof(struct kvm_x86_ops)); + ops->vm_size = sizeof(struct kvm_svm_csv); ops->mem_enc_ioctl = csv_mem_enc_ioctl; ops->vm_attestation = csv_vm_attestation; ops->control_pre_system_reset = csv_control_pre_system_reset; ops->control_post_system_reset = csv_control_post_system_reset; if (boot_cpu_has(X86_FEATURE_SEV_ES) && boot_cpu_has(X86_FEATURE_CSV3)) { - ops->vm_size = sizeof(struct kvm_svm_csv); - ops->vm_destroy = csv_vm_destroy; ops->handle_exit = csv_handle_exit; ops->guest_memory_reclaimed = csv_guest_memory_reclaimed; From d858beefa6822b511ac642391067791b88e3635f Mon Sep 17 00:00:00 2001 From: hanliyang Date: Mon, 7 Oct 2024 18:16:23 +0800 Subject: [PATCH 2/8] crypto: ccp: Get api version again when update Hygon CSV firmware at runtime hygon inclusion category: bugfix CVE: NA --------------------------- The commit 0aa9a168cca2 ("crypto: ccp: Implement CSV_DOWNLOAD_FIRMWARE ioctl command") support update Hygon CSV firmware at runtime, but it don't update API version info in the driver after issues the DOWNLOAD_FIRMWARE command. When we want use the new features in the updated firmware, the version check in this driver will fail. To address this problem, we should regain the api version when DOWNLOAD_FIRMWARE command returns. Fixes: 0aa9a168cca2 ("crypto: ccp: Implement CSV_DOWNLOAD_FIRMWARE ioctl command") Signed-off-by: hanliyang --- drivers/crypto/ccp/hygon/csv-dev.c | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/crypto/ccp/hygon/csv-dev.c b/drivers/crypto/ccp/hygon/csv-dev.c index 02ebcd8ef247f..16373f7dd4969 100644 --- a/drivers/crypto/ccp/hygon/csv-dev.c +++ b/drivers/crypto/ccp/hygon/csv-dev.c @@ -128,6 +128,46 @@ static int csv_ioctl_do_hgsc_import(struct sev_issue_cmd *argp) return ret; } +static int csv_get_api_version(void) +{ + struct sev_device *sev; + struct sev_user_data_status *status; + int error = 0, ret; + + if (!hygon_psp_hooks.sev_dev_hooks_installed) + return -ENODEV; + + if (!psp_master || !psp_master->sev_data) + return -ENODEV; + + sev = psp_master->sev_data; + + status = kzalloc(sizeof(*status), GFP_KERNEL); + if (!status) + return -ENOMEM; + + ret = hygon_psp_hooks.__sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, + status, &error); + if (ret) { + dev_err(sev->dev, + "CSV: failed to get status. Error: %#x\n", error); + ret = 1; + goto e_free_status; + } + + sev->api_major = status->api_major; + sev->api_minor = status->api_minor; + sev->build = status->build; + sev->state = status->state; + + csv_update_api_version(status); + + ret = 0; +e_free_status: + kfree(status); + return ret; +} + static int csv_ioctl_do_download_firmware(struct sev_issue_cmd *argp) { struct sev_data_download_firmware *data = NULL; @@ -190,6 +230,9 @@ static int csv_ioctl_do_download_firmware(struct sev_issue_cmd *argp) else pr_info("CSV firmware update successful\n"); + /* Sync api version status */ + csv_get_api_version(); + err_free_page: __free_pages(p, order); From ecfc428583ad0b9532eb2171900aee83b749183e Mon Sep 17 00:00:00 2001 From: hanliyang Date: Sat, 2 Nov 2024 18:56:25 +0800 Subject: [PATCH 3/8] KVM: SVM: CSV: Ensure all the GPRs and some non-GPRs are synced before LAUNCH_ENCRYPT_VMCB hygon inclusion category: feature CVE: NA --------------------------- Even though most of the GPRs is zero at reset state, we should explicitly set these before LAUNCH_ENCRYPT_VMCB. The DR6 register is not zero at reset state, we should explicitly set DR6 before LAUNCH_ENCRYPT_VMCB. The PKRU currently is unsupported on Hygon CPUs, this register is zero at reset state, nevertheless explicitly set PKRU before LAUNCH_ENCRYPT_VMCB. Signed-off-by: hanliyang --- arch/x86/kvm/svm/csv.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index 13800c7e9c00f..37adaddca5407 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -1295,14 +1295,43 @@ static int csv3_sync_vmsa(struct vcpu_svm *svm) if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) return -EINVAL; + /* + * CSV3 will use a VMSA that is pointed to by the VMCB, not + * the traditional VMSA that is part of the VMCB. Copy the + * traditional VMSA as it has been built so far (in prep + * for LAUNCH_ENCRYPT_VMCB) to be the initial CSV3 state. + */ memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save)); /* Sync registgers per spec. */ save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX]; + save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX]; + save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX]; + save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP]; + save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP]; + save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI]; + save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI]; +#ifdef CONFIG_X86_64 + save->r8 = svm->vcpu.arch.regs[VCPU_REGS_R8]; + save->r9 = svm->vcpu.arch.regs[VCPU_REGS_R9]; + save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10]; + save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11]; + save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12]; + save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13]; + save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14]; + save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15]; +#endif save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP]; + + /* Sync some non-GPR registers before encrypting */ save->xcr0 = svm->vcpu.arch.xcr0; + save->pkru = svm->vcpu.arch.pkru; save->xss = svm->vcpu.arch.ia32_xss; + save->dr6 = svm->vcpu.arch.dr6; + + pr_debug("Virtual Machine Save Area (VMSA):\n"); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false); return 0; } From 401bf1c4866c70bb1d7082862272e4ba6d41b6cf Mon Sep 17 00:00:00 2001 From: hanliyang Date: Wed, 25 Sep 2024 21:18:41 +0800 Subject: [PATCH 4/8] crypto: ccp: Provide csv_get_extension_info() to present extensions of newer CSV firmware hygon inclusion category: feature CVE: NA --------------------------- As more and more confidential computing features are provided, the hypervisor and userspace VMM should recognize the extended features. Provide csv_get_extension_info() to present the extended confidential computing features of the newer CSV firmware so that the hypervisor can utilize the extended features when launch and running a confidential guest. Signed-off-by: hanliyang --- drivers/crypto/ccp/hygon/csv-dev.c | 32 ++++++++++++++++++++++++++++++ include/linux/psp-hygon.h | 21 ++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/drivers/crypto/ccp/hygon/csv-dev.c b/drivers/crypto/ccp/hygon/csv-dev.c index 16373f7dd4969..64eb92f8625be 100644 --- a/drivers/crypto/ccp/hygon/csv-dev.c +++ b/drivers/crypto/ccp/hygon/csv-dev.c @@ -1222,6 +1222,38 @@ int vpsp_try_do_cmd(int cmd, phys_addr_t phy_addr, struct vpsp_ret *psp_ret) } EXPORT_SYMBOL_GPL(vpsp_try_do_cmd); +int csv_get_extension_info(void *buf, size_t *size) +{ + /* If @hygon_csv_build is 0, this means CSV firmware doesn't exist or + * the psp device doesn't exist. + */ + if (hygon_csv_build == 0) + return -ENODEV; + + /* The caller must provide valid @buf and the @buf must >= 4 bytes in + * size. + */ + if (!buf || !size || *size < sizeof(uint32_t)) { + if (size) + *size = sizeof(uint32_t); + + return -EINVAL; + } + + /* Since firmware with build id 2200, support: + * a. issue LAUNCH_ENCRYPT_DATA command more than once for a + * CSV3 guest. + * b. inject secret to a CSV3 guest. + */ + if (csv_version_greater_or_equal(2200)) { + *(uint32_t *)buf |= CSV_EXT_CSV3_MULT_LUP_DATA; + *(uint32_t *)buf |= CSV_EXT_CSV3_INJ_SECRET; + } + + return 0; +} +EXPORT_SYMBOL_GPL(csv_get_extension_info); + #ifdef CONFIG_HYGON_CSV int csv_platform_cmd_set_secure_memory_region(struct sev_device *sev, int *error) diff --git a/include/linux/psp-hygon.h b/include/linux/psp-hygon.h index 1888d9d725925..76ddf5cce089d 100644 --- a/include/linux/psp-hygon.h +++ b/include/linux/psp-hygon.h @@ -20,6 +20,11 @@ #define CSV_FW_MAX_SIZE 0x80000 /* 512KB */ +#define CSV_EXT_CSV3_MULT_LUP_DATA_BIT 0 +#define CSV_EXT_CSV3_MULT_LUP_DATA (1 << CSV_EXT_CSV3_MULT_LUP_DATA_BIT) +#define CSV_EXT_CSV3_INJ_SECRET_BIT 1 +#define CSV_EXT_CSV3_INJ_SECRET (1 << CSV_EXT_CSV3_INJ_SECRET_BIT) + /** * Guest/platform management commands for CSV */ @@ -508,6 +513,20 @@ int kvm_pv_psp_copy_forward_op(struct kvm_vpsp *vpsp, int cmd, gpa_t data_gpa, g int kvm_pv_psp_forward_op(struct kvm_vpsp *vpsp, uint32_t cmd, gpa_t data_gpa, uint32_t psp_ret); + +/** + * csv_get_extension_info - collect extension set of the firmware + * + * @buf: The buffer to save extension set + * @size: The size of the @buf + * + * Returns: + * 0 if @buf is filled with extension bitflags + * -%ENODEV if the CSV device is not available + * -%EINVAL if @buf is NULL or @size is too smaller + */ +int csv_get_extension_info(void *buf, size_t *size); + #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ static inline int psp_do_cmd(int cmd, void *data, int *psp_ret) { return -ENODEV; } @@ -546,6 +565,8 @@ static inline int kvm_pv_psp_forward_op(struct kvm_vpsp *vpsp, uint32_t cmd, gpa_t data_gpa, uint32_t psp_ret) { return -ENODEV; } +static inline int csv_get_extension_info(void *buf, size_t *size) { return -ENODEV; } + #endif /* CONFIG_CRYPTO_DEV_SP_PSP */ typedef int (*p2c_notifier_t)(uint32_t id, uint64_t data); From 7e32f18841656fe2930212f4a6dc183992adcfa8 Mon Sep 17 00:00:00 2001 From: hanliyang Date: Thu, 26 Sep 2024 14:59:25 +0800 Subject: [PATCH 5/8] KVM: SVM: CSV: Provide KVM_CAP_HYGON_COCO_EXT interface hygon inclusion category: feature CVE: NA --------------------------- The CSV1/2/3 firmware will provide more confidential features, it's recommended that the user space VMM (e.g. Qemu) inquiry about which features are supported by the system and decide to utilise some of these supported features. Provide KVM_CAP_HYGON_COCO_EXT ioctl interface so that the user space VMM, KVM, and firmware can negotiate how to interoperate with each other. The KVM_CAP_HYGON_COCO_EXT interface will address many compatibility issues when any one of the user space VMM, KVM, or firmware is not up-to-date. Signed-off-by: hanliyang --- arch/x86/include/asm/kvm-x86-ops.h | 2 + arch/x86/include/asm/kvm_host.h | 2 + arch/x86/kvm/svm/csv.c | 83 ++++++++++++++++++++++++++++++ arch/x86/kvm/x86.c | 23 +++++++++ include/uapi/linux/kvm.h | 7 +++ 5 files changed, 117 insertions(+) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index b54e72a0100b5..ca1c2a98656de 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -138,6 +138,8 @@ KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(vm_attestation) KVM_X86_OP_OPTIONAL(control_pre_system_reset) KVM_X86_OP_OPTIONAL(control_post_system_reset) +KVM_X86_OP_OPTIONAL(get_hygon_coco_extension) +KVM_X86_OP_OPTIONAL(enable_hygon_coco_extension) #undef KVM_X86_OP #undef KVM_X86_OP_OPTIONAL diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5f4963913d4ca..844e1abf8b48d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1758,6 +1758,8 @@ struct kvm_x86_ops { int (*vm_attestation)(struct kvm *kvm, unsigned long gpa, unsigned long len); int (*control_pre_system_reset)(struct kvm *kvm); int (*control_post_system_reset)(struct kvm *kvm); + int (*get_hygon_coco_extension)(struct kvm *kvm); + int (*enable_hygon_coco_extension)(struct kvm *kvm, u32 arg); }; struct kvm_x86_nested_ops { diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index 37adaddca5407..1ef26a0e5405d 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -879,6 +879,13 @@ struct kvm_csv_info { struct list_head smr_list; /* List of guest secure memory regions */ unsigned long nodemask; /* Nodemask where CSV3 guest's memory resides */ + + /* The following 5 fields record the extension status for current VM */ + bool fw_ext_valid; /* if @fw_ext field is valid */ + u32 fw_ext; /* extensions supported by current platform */ + bool kvm_ext_valid; /* if @kvm_ext field is valid */ + u32 kvm_ext; /* extensions supported by KVM */ + u32 inuse_ext; /* extensions inused by current VM */ }; struct kvm_svm_csv { @@ -2718,6 +2725,80 @@ static void csv_free_asid_userid_array(void) #endif /* CONFIG_KVM_SUPPORTS_CSV_REUSE_ASID */ +/** + * When userspace recognize these extensions, it's suggested that the userspace + * to enable these extensions through KVM_ENABLE_CAP, then both the userspace + * and KVM utilise these extensions. + */ +static int csv_get_hygon_coco_extension(struct kvm *kvm) +{ + struct kvm_csv_info *csv; + size_t len = sizeof(uint32_t); + int ret = 0; + + if (!kvm) + return 0; + + csv = &to_kvm_svm_csv(kvm)->csv_info; + + if (csv->fw_ext_valid == false) { + ret = csv_get_extension_info(&csv->fw_ext, &len); + + if (ret == -ENODEV) { + pr_err("Unable to interact with CSV firmware!\n"); + return 0; + } else if (ret == -EINVAL) { + pr_err("Need %ld bytes to record fw extension!\n", len); + return 0; + } + + csv->fw_ext_valid = true; + } + + /* The kvm_ext field of kvm_csv_info is filled in only if the fw_ext + * field of kvm_csv_info is valid. + */ + if (csv->kvm_ext_valid == false) { + /* Currently, KVM doesn't support any extensions, we don't need + * to fill in kvm_ext field of kvm_csv_info here. + */ + csv->kvm_ext_valid = true; + } + + /* Return extension info only if both fw_ext and kvm_ext fields of + * kvm_csv_info are valid. + */ + pr_debug("%s: fw_ext=%#x kvm_ext=%#x\n", + __func__, csv->fw_ext, csv->kvm_ext); + return (int)csv->kvm_ext; +} + +/** + * Return 0 means KVM accept the negotiation from userspace. Both the + * userspace and KVM should not utilise extensions if failed to negotiate. + */ +static int csv_enable_hygon_coco_extension(struct kvm *kvm, u32 arg) +{ + struct kvm_csv_info *csv; + + if (!kvm) + return -EINVAL; + + csv = &to_kvm_svm_csv(kvm)->csv_info; + + /* Negotiation is accepted only if both the fw_ext and kvm_ext fields + * of kvm_csv_info are valid and the virtual machine is a CSV3 guest. + */ + if (csv->fw_ext_valid && csv->kvm_ext_valid && csv3_guest(kvm)) { + csv->inuse_ext = csv->kvm_ext & arg; + pr_debug("%s: inuse_ext=%#x\n", __func__, csv->inuse_ext); + return csv->inuse_ext; + } + + /* Userspace should not utilise the extensions */ + return -EINVAL; +} + void __init csv_hardware_setup(unsigned int max_csv_asid) { unsigned int nr_asids = max_csv_asid + 1; @@ -2771,6 +2852,8 @@ void __init csv_init(struct kvm_x86_ops *ops) ops->vm_attestation = csv_vm_attestation; ops->control_pre_system_reset = csv_control_pre_system_reset; ops->control_post_system_reset = csv_control_post_system_reset; + ops->get_hygon_coco_extension = csv_get_hygon_coco_extension; + ops->enable_hygon_coco_extension = csv_enable_hygon_coco_extension; if (boot_cpu_has(X86_FEATURE_SEV_ES) && boot_cpu_has(X86_FEATURE_CSV3)) { ops->vm_destroy = csv_vm_destroy; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c6ef0c702b300..e57ed87ce64a6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4649,6 +4649,18 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_AMD64_SEV_ES_GHCB); break; + case KVM_CAP_HYGON_COCO_EXT: + r = 0; + + /* + * Before running a Hygon confidential guest, the userspace + * should find the advanced extensions of the Hygon CSV + * technology. If the userspace recognize the extensions, it's + * suggested that the userspace to utilise extensions. + */ + if (is_x86_vendor_hygon() && kvm_x86_ops.get_hygon_coco_extension) + r = static_call(kvm_x86_get_hygon_coco_extension)(kvm); + break; default: break; } @@ -6526,6 +6538,17 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, } mutex_unlock(&kvm->lock); break; + case KVM_CAP_HYGON_COCO_EXT: + r = -EINVAL; + + /* + * The userspace negotiate with KVM to utilise extensions of + * Hygon CSV technology. + */ + if (is_x86_vendor_hygon() && kvm_x86_ops.enable_hygon_coco_extension) + r = static_call(kvm_x86_enable_hygon_coco_extension)(kvm, + (u32)cap->args[0]); + break; default: r = -EINVAL; break; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index b2dc3c41a5d0b..c1c09a1b71ecb 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1202,6 +1202,13 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES 229 #define KVM_CAP_SEV_ES_GHCB 500 +#define KVM_CAP_HYGON_COCO_EXT 501 +/* support userspace to request firmware to build CSV3 guest's memory space */ +#define KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM (1 << 0) +/* support request to update CSV3 guest's memory region multiple times */ +#define KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA (1 << 1) +/* support request to inject secret to CSV3 guest */ +#define KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET (1 << 2) #ifdef KVM_CAP_IRQ_ROUTING From a3f2f9e6fe00b2ac6353308cd357f2f7c3252c92 Mon Sep 17 00:00:00 2001 From: hanliyang Date: Thu, 26 Sep 2024 16:49:28 +0800 Subject: [PATCH 6/8] KVM: SVM: CSV: Provide KVM_CSV3_SET_GUEST_PRIVATE_MEMORY ioctl interface hygon inclusion category: feature CVE: NA --------------------------- For newer CSV1/2/3 firmware, multiple LAUNCH_ENCRYPT_DATA commands are allowed to be issued. However, SET_GUEST_PRIVATE_MEMORY command can only be issued once. Provide a separate ioctl interface KVM_CSV3_SET_GUEST_PRIVATE_MEMORY here. The user space VMM can negotiate with KVM on whether to enable the capability KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM. When this capability is enabled, the user space VMM should explicitly request the KVM_CSV3_SET_GUEST_PRIVATE_MEMORY ioctl interface, and the KVM_CSV3_LAUNCH_ENCRYPT_DATA ioctl handler will skip the process of issuing the SET_GUEST_PRIVATE_MEMORY command, in addition, the user space VMM will have the chance to request KVM_CSV3_LAUNCH_ENCRYPT_DATA ioctl interface more than once. When this capability is disabled, the user space will not request the KVM_CSV3_SET_GUEST_PRIVATE_MEMORY ioctl interface, and the KVM_CSV3_LAUNCH_ENCRYPT_DATA ioctl handler will still issue the SET_GUEST_PRIVATE_MEMORY command. Signed-off-by: hanliyang --- arch/x86/kvm/svm/csv.c | 35 ++++++++++++++++++++++++----------- include/uapi/linux/kvm.h | 2 ++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index 1ef26a0e5405d..3d378bea1aa58 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -1052,7 +1052,7 @@ static bool csv3_is_mmio_pfn(kvm_pfn_t pfn) E820_TYPE_RAM); } -static int csv3_set_guest_private_memory(struct kvm *kvm) +static int csv3_set_guest_private_memory(struct kvm *kvm, struct kvm_sev_cmd *argp) { struct kvm_memslots *slots = kvm_memslots(kvm); struct kvm_memory_slot *memslot; @@ -1067,7 +1067,7 @@ static int csv3_set_guest_private_memory(struct kvm *kvm) LIST_HEAD(tmp_list); struct list_head *pos, *q; u32 i = 0, count = 0, remainder; - int ret = 0, error; + int ret = 0; u64 size = 0, nr_smr = 0, nr_pages = 0; u32 smr_entry_shift; int bkt; @@ -1079,6 +1079,10 @@ static int csv3_set_guest_private_memory(struct kvm *kvm) if (!csv3_guest(kvm)) return -ENOTTY; + /* The smr_list should be initialized only once */ + if (!list_empty(&csv->smr_list)) + return -EFAULT; + nodes_clear(nodemask); for_each_set_bit(i, &csv->nodemask, BITS_PER_LONG) if (i < MAX_NUMNODES) @@ -1161,7 +1165,7 @@ static int csv3_set_guest_private_memory(struct kvm *kvm) /* set secury memory region for launch enrypt data */ ret = hygon_kvm_hooks.sev_issue_cmd(kvm, CSV3_CMD_SET_GUEST_PRIVATE_MEMORY, - set_guest_private_memory, &error); + set_guest_private_memory, &argp->error); if (ret) goto e_free_smr; @@ -1221,10 +1225,17 @@ static int csv3_launch_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) goto exit; } - /* Allocate all the guest memory from CMA */ - ret = csv3_set_guest_private_memory(kvm); - if (ret) - goto exit; + /* + * If userspace request to invoke CSV3_CMD_SET_GUEST_PRIVATE_MEMORY + * explicitly, we should not calls to csv3_set_guest_private_memory() + * here. + */ + if (!(csv->inuse_ext & KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM)) { + /* Allocate all the guest memory from CMA */ + ret = csv3_set_guest_private_memory(kvm, argp); + if (ret) + goto exit; + } num_entries = params.len / PAGE_SIZE; num_entries_in_block = ARRAY_SIZE(blocks->entry); @@ -1694,7 +1705,7 @@ static int csv3_receive_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) if (unlikely(list_empty(&csv->smr_list))) { /* Allocate all the guest memory from CMA */ - ret = csv3_set_guest_private_memory(kvm); + ret = csv3_set_guest_private_memory(kvm, argp); if (ret) goto exit; } @@ -2463,6 +2474,9 @@ static int csv_mem_enc_ioctl(struct kvm *kvm, void __user *argp) case KVM_CSV3_HANDLE_MEMORY: r = csv3_handle_memory(kvm, &sev_cmd); break; + case KVM_CSV3_SET_GUEST_PRIVATE_MEMORY: + r = csv3_set_guest_private_memory(kvm, &sev_cmd); + break; default: /* * If the command is compatible between CSV and SEV, the @@ -2759,9 +2773,8 @@ static int csv_get_hygon_coco_extension(struct kvm *kvm) * field of kvm_csv_info is valid. */ if (csv->kvm_ext_valid == false) { - /* Currently, KVM doesn't support any extensions, we don't need - * to fill in kvm_ext field of kvm_csv_info here. - */ + if (csv3_guest(kvm)) + csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM; csv->kvm_ext_valid = true; } diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index c1c09a1b71ecb..5832689e142d2 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -2330,6 +2330,8 @@ enum csv3_cmd_id { KVM_CSV3_RECEIVE_ENCRYPT_CONTEXT, KVM_CSV3_HANDLE_MEMORY, + KVM_CSV3_SET_GUEST_PRIVATE_MEMORY = 0xc8, + KVM_CSV3_NR_MAX, }; From 5ae112d8ea9759dbf57812336fe711d9de1eefc3 Mon Sep 17 00:00:00 2001 From: hanliyang Date: Fri, 27 Sep 2024 15:12:20 +0800 Subject: [PATCH 7/8] KVM: SVM: CSV: Support issue non-4K aligned CSV3_CMD_LAUNCH_ENCRYPT_DATA and more than once hygon inclusion category: feature CVE: NA --------------------------- So far, the KVM_CSV3_LAUNCH_ENCRYPT_DATA handler only process 4K aligned data, this is insufficient because we need encrypt Non-4K aligned data to CSV3 guest's private memory in some cases. To address this, we provide new function csv3_launch_encrypt_data_alt_2 to process Non-4K aligned data. The new function will be called only when the cap KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA is enabled for current CSV3 guest. In addition, to simplify the KVM_CSV3_LAUNCH_ENCRYPT_DATA request from the user space, the function csv3_launch_encrypt_data_alt_2 allows issue CSV3_CMD_LAUNCH_ENCRYPT_DATA more than once if necessary. Signed-off-by: hanliyang --- arch/x86/kvm/svm/csv.c | 234 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index 3d378bea1aa58..3bc74af106574 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -1197,7 +1197,12 @@ static int csv3_set_guest_private_memory(struct kvm *kvm, struct kvm_sev_cmd *ar return ret; } -static int csv3_launch_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) +/** + * csv3_launch_encrypt_data_alt_1 - The legacy handler to encrypt CSV3 + * guest's memory before VMRUN. + */ +static int csv3_launch_encrypt_data_alt_1(struct kvm *kvm, + struct kvm_sev_cmd *argp) { struct kvm_csv_info *csv = &to_kvm_svm_csv(kvm)->csv_info; struct kvm_csv3_launch_encrypt_data params; @@ -1211,9 +1216,6 @@ static int csv3_launch_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long pfn, pfn_sme_mask; int ret = 0; - if (!csv3_guest(kvm)) - return -ENOTTY; - if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) { ret = -EFAULT; @@ -1305,6 +1307,225 @@ static int csv3_launch_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) return ret; } +#define MAX_ENTRIES_PER_BLOCK \ + (sizeof(((struct encrypt_data_block *)0)->entry) / \ + sizeof(((struct encrypt_data_block *)0)->entry[0])) +#define MAX_BLOCKS_PER_CSV3_LUP_DATA \ + (sizeof(((struct csv3_data_launch_encrypt_data *)0)->data_blocks) / \ + sizeof(((struct csv3_data_launch_encrypt_data *)0)->data_blocks[0])) +#define MAX_ENTRIES_PER_CSV3_LUP_DATA \ + (MAX_BLOCKS_PER_CSV3_LUP_DATA * MAX_ENTRIES_PER_BLOCK) + +/** + * __csv3_launch_encrypt_data - The helper for handler + * csv3_launch_encrypt_data_alt_2. + */ +static int __csv3_launch_encrypt_data(struct kvm *kvm, + struct kvm_sev_cmd *argp, + struct kvm_csv3_launch_encrypt_data *params, + void *src_buf, + unsigned int start_pgoff, + unsigned int end_pgoff) +{ + struct kvm_csv_info *csv = &to_kvm_svm_csv(kvm)->csv_info; + struct csv3_data_launch_encrypt_data *data = NULL; + struct encrypt_data_block *block = NULL; + struct page **pages = NULL; + unsigned long len, remain_len; + unsigned long pfn, pfn_sme_mask, last_pfn; + unsigned int pgoff = start_pgoff; + int i, j; + int ret = -ENOMEM; + + /* Alloc command buffer for CSV3_CMD_LAUNCH_ENCRYPT_DATA command */ + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + /* Alloc pages for data_blocks[] in the command buffer */ + len = ARRAY_SIZE(data->data_blocks) * sizeof(struct page *); + pages = kzalloc(len, GFP_KERNEL_ACCOUNT); + if (!pages) + goto e_free_data; + + for (i = 0; i < ARRAY_SIZE(data->data_blocks); i++) { + pages[i] = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!pages[i]) + goto e_free_pages; + } + + i = 0; + while (i < ARRAY_SIZE(data->data_blocks) && pgoff < end_pgoff) { + block = (struct encrypt_data_block *)page_to_virt(pages[i]); + + j = 0; + last_pfn = 0; + while (j < ARRAY_SIZE(block->entry) && pgoff < end_pgoff) { + pfn = vmalloc_to_pfn(src_buf + (pgoff << PAGE_SHIFT)); + pfn_sme_mask = __sme_set(pfn << PAGE_SHIFT) >> PAGE_SHIFT; + + /* + * One entry can record a number of contiguous physical + * pages. If the current page is not adjacent to the + * previous physical page, we should record the page to + * the next entry. If entries of current block is used + * up, we should try the next block. + */ + if (last_pfn && (last_pfn + 1 == pfn)) { + block->entry[j].npages++; + } else if (j < (ARRAY_SIZE(block->entry) - 1)) { + /* @last_pfn == 0 means fill in entry[0] */ + if (likely(last_pfn != 0)) + j++; + block->entry[j].pfn = pfn_sme_mask; + block->entry[j].npages = 1; + } else { + break; + } + + /* + * Succeed to record one page, increase the page offset. + * We also record the pfn of current page so that we can + * record the contiguous physical pages into one entry. + */ + last_pfn = pfn; + pgoff++; + } + + i++; + } + + if (pgoff < end_pgoff) { + pr_err("CSV3: Fail to fill in LAUNCH_ENCRYPT_DATA command!\n"); + goto e_free_pages; + } + + len = (end_pgoff - start_pgoff) << PAGE_SHIFT; + clflush_cache_range(src_buf + (start_pgoff << PAGE_SHIFT), len); + + /* Fill in command buffer */ + data->handle = csv->sev->handle; + + if (start_pgoff == 0) { + data->gpa = params->gpa; + len -= params->gpa & ~PAGE_MASK; + } else { + data->gpa = (params->gpa & PAGE_MASK) + (start_pgoff << PAGE_SHIFT); + } + remain_len = params->len - (data->gpa - params->gpa); + + data->length = (len <= remain_len) ? len : remain_len; + + for (j = 0; j < i; j++) + data->data_blocks[j] = __sme_set(page_to_phys(pages[j])); + + /* Issue command */ + ret = hygon_kvm_hooks.sev_issue_cmd(kvm, CSV3_CMD_LAUNCH_ENCRYPT_DATA, + data, &argp->error); + +e_free_pages: + for (i = 0; i < ARRAY_SIZE(data->data_blocks); i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); +e_free_data: + kfree(data); + + return ret; +} + +/** + * csv3_launch_encrypt_data_alt_2 - The handler to support encrypt CSV3 + * guest's memory before VMRUN. This handler support issue API command + * multiple times, both the GPA and length of the memory region are not + * required to be 4K-aligned. + */ +static int csv3_launch_encrypt_data_alt_2(struct kvm *kvm, + struct kvm_sev_cmd *argp) +{ + struct kvm_csv3_launch_encrypt_data params; + void *buffer = NULL; + unsigned long len; + unsigned int total_pages, start_pgoff, next_pgoff; + int ret = 0; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, + sizeof(params))) { + return -EFAULT; + } + + /* Both the GPA and length must be 16 Bytes aligned at least */ + if (!params.len || + !params.uaddr || + !IS_ALIGNED(params.len, 16) || + !IS_ALIGNED(params.gpa, 16)) { + return -EINVAL; + } + + /* + * Alloc buffer to save source data. When we copy source data from + * userspace to the buffer, the data in the first page of the buffer + * should keep the offset as params.gpa. + */ + len = PAGE_ALIGN((params.gpa & ~PAGE_MASK) + params.len); + total_pages = len >> PAGE_SHIFT; + next_pgoff = 0; + + buffer = vzalloc(len); + if (!buffer) + return -ENOMEM; + + if (copy_from_user(buffer + (params.gpa & ~PAGE_MASK), + (void __user *)params.uaddr, params.len)) { + ret = -EFAULT; + goto e_free_buffer; + } + + /* + * If the source data is too large, we should issue command more than + * once. The LAUNCH_ENCRYPT_DATA API updates not only the measurement + * of the data, but also the measurement of the metadata correspond to + * the data. The guest owner is obligated to verify the launch + * measurement, so guest owner must be aware of the launch measurement + * of each LAUNCH_ENCRYPT_DATA API command. If we processing pages more + * than MAX_ENTRIES_PER_CSV3_LUP_DATA in each API command, the guest + * owner could not able to calculate the correct measurement and fail + * to verify the launch measurement. For this reason, we limit the + * maximum number of pages processed by each API command to + * MAX_ENTRIES_PER_CSV3_LUP_DATA. + */ + while (next_pgoff < total_pages) { + start_pgoff = next_pgoff; + next_pgoff += MAX_ENTRIES_PER_CSV3_LUP_DATA; + + if (next_pgoff > total_pages) + next_pgoff = total_pages; + + ret = __csv3_launch_encrypt_data(kvm, argp, ¶ms, + buffer, start_pgoff, next_pgoff); + if (ret) + goto e_free_buffer; + } + +e_free_buffer: + vfree(buffer); + return ret; +} + +static int csv3_launch_encrypt_data(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_csv_info *csv = &to_kvm_svm_csv(kvm)->csv_info; + + if (!csv3_guest(kvm)) + return -ENOTTY; + + if (!(csv->inuse_ext & KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA)) + return csv3_launch_encrypt_data_alt_1(kvm, argp); + + return csv3_launch_encrypt_data_alt_2(kvm, argp); +} + static int csv3_sync_vmsa(struct vcpu_svm *svm) { struct sev_es_save_area *save = svm->sev_es.vmsa; @@ -2773,8 +2994,11 @@ static int csv_get_hygon_coco_extension(struct kvm *kvm) * field of kvm_csv_info is valid. */ if (csv->kvm_ext_valid == false) { - if (csv3_guest(kvm)) + if (csv3_guest(kvm)) { csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM; + if (csv->fw_ext & CSV_EXT_CSV3_MULT_LUP_DATA) + csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA; + } csv->kvm_ext_valid = true; } From 8e69def024ffb3dbe5613d12830010ceff2c4b3f Mon Sep 17 00:00:00 2001 From: hanliyang Date: Fri, 27 Sep 2024 21:07:06 +0800 Subject: [PATCH 8/8] KVM: SVM: CSV: Support inject secret to Hygon CSV3 guest hygon inclusion category: feature CVE: NA --------------------------- We should provide GPA in LAUNCH_SECRET API command buffer for CSV3 guest. We introduce a appropriate function csv_launch_secret to process user space KVM_SEV_LAUNCH_SECRET ioctl request, irrespective of whether it is a CSV, CSV2 or CSV3 guest. For CSV3 guest, the member guest_uaddr of the structure kvm_sev_launch_secret should be the value of GPA. Signed-off-by: hanliyang --- arch/x86/kvm/svm/csv.c | 95 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/arch/x86/kvm/svm/csv.c b/arch/x86/kvm/svm/csv.c index 3bc74af106574..25e86a8c4c5e8 100644 --- a/arch/x86/kvm/svm/csv.c +++ b/arch/x86/kvm/svm/csv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2626,6 +2627,95 @@ static int csv3_handle_memory(struct kvm *kvm, struct kvm_sev_cmd *argp) return r; }; +static int csv_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct kvm_csv_info *csv = &to_kvm_svm_csv(kvm)->csv_info; + struct sev_data_launch_secret data; + struct kvm_sev_launch_secret params; + struct page **pages; + void *blob, *hdr; + unsigned long n, i; + int ret, offset; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) + return -EFAULT; + + memset(&data, 0, sizeof(data)); + + if (!csv3_guest(kvm) || + !(csv->inuse_ext & KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET)) { + pages = hygon_kvm_hooks.sev_pin_memory(kvm, params.guest_uaddr, + params.guest_len, &n, 1); + if (IS_ERR(pages)) + return PTR_ERR(pages); + + /* + * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts + * pages in place; the cache may contain the data that was + * written unencrypted. + */ + hygon_kvm_hooks.sev_clflush_pages(pages, n); + + /* + * The secret must be copied into contiguous memory region, + * lets verify that userspace memory pages are contiguous + * before we issue command. + */ + if (hygon_kvm_hooks.get_num_contig_pages(0, pages, n) != n) { + ret = -EINVAL; + goto e_unpin_memory; + } + + offset = params.guest_uaddr & (PAGE_SIZE - 1); + data.guest_address = __sme_page_pa(pages[0]) + offset; + } else { + /* It's gpa for CSV3 guest */ + data.guest_address = params.guest_uaddr; + } + data.guest_len = params.guest_len; + + blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); + if (IS_ERR(blob)) { + ret = PTR_ERR(blob); + goto e_unpin_memory; + } + + data.trans_address = __psp_pa(blob); + data.trans_len = params.trans_len; + + hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); + if (IS_ERR(hdr)) { + ret = PTR_ERR(hdr); + goto e_free_blob; + } + data.hdr_address = __psp_pa(hdr); + data.hdr_len = params.hdr_len; + + data.handle = sev->handle; + ret = hygon_kvm_hooks.sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, + &data, &argp->error); + + kfree(hdr); + +e_free_blob: + kfree(blob); +e_unpin_memory: + if (!csv3_guest(kvm) || + !(csv->inuse_ext & KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET)) { + /* content of memory is updated, mark pages dirty */ + for (i = 0; i < n; i++) { + set_page_dirty_lock(pages[i]); + mark_page_accessed(pages[i]); + } + hygon_kvm_hooks.sev_unpin_memory(kvm, pages, n); + } + return ret; +} + static int csv_mem_enc_ioctl(struct kvm *kvm, void __user *argp) { struct kvm_sev_cmd sev_cmd; @@ -2649,6 +2739,9 @@ static int csv_mem_enc_ioctl(struct kvm *kvm, void __user *argp) r = csv_command_batch(kvm, &sev_cmd); mutex_unlock(&csv_cmd_batch_mutex); break; + case KVM_SEV_LAUNCH_SECRET: + r = csv_launch_secret(kvm, &sev_cmd); + break; case KVM_SEV_SEND_UPDATE_VMSA: /* * Hygon implement the specific interface, although @@ -2998,6 +3091,8 @@ static int csv_get_hygon_coco_extension(struct kvm *kvm) csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_SET_PRIV_MEM; if (csv->fw_ext & CSV_EXT_CSV3_MULT_LUP_DATA) csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_MULT_LUP_DATA; + if (csv->fw_ext & CSV_EXT_CSV3_INJ_SECRET) + csv->kvm_ext |= KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET; } csv->kvm_ext_valid = true; }