diff --git a/1001-Build-sample-code-with-PREBUILD_BINS-0.patch b/1001-Build-sample-code-with-PREBUILD_BINS-0.patch new file mode 100644 index 0000000000000000000000000000000000000000..f5e4fa18ad88423c2a0fcf5e8d01c092e9f359e5 --- /dev/null +++ b/1001-Build-sample-code-with-PREBUILD_BINS-0.patch @@ -0,0 +1,70 @@ +From 39905d130ed6e857692ec09db97814b5d6937d0a Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 9 Apr 2024 15:21:01 +0800 +Subject: [PATCH 01/28] Build sample code with PREBUILD_BINS != 0 + +Signed-off-by: Zelin Deng +--- + .../lookaside/access_layer/src/sample_code/Makefile | 6 +++--- + .../access_layer/src/sample_code/functional/common.mk | 7 ++++--- + .../access_layer/src/sample_code/performance/Makefile | 8 ++++---- + 3 files changed, 11 insertions(+), 10 deletions(-) + +diff --git a/quickassist/lookaside/access_layer/src/sample_code/Makefile b/quickassist/lookaside/access_layer/src/sample_code/Makefile +index c9803a8..65db858 100644 +--- a/quickassist/lookaside/access_layer/src/sample_code/Makefile ++++ b/quickassist/lookaside/access_layer/src/sample_code/Makefile +@@ -37,9 +37,9 @@ CC?=gcc + + PREBUILD_BINS = -1 + +-PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ +- | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ +- echo $$?) ++#PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ ++# | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ ++# echo $$?) + + #Set Upstream code based flags + ICP_BUILD_OUTPUT?=$(ICP_ROOT)/build +diff --git a/quickassist/lookaside/access_layer/src/sample_code/functional/common.mk b/quickassist/lookaside/access_layer/src/sample_code/functional/common.mk +index f4bef71..8b19f62 100755 +--- a/quickassist/lookaside/access_layer/src/sample_code/functional/common.mk ++++ b/quickassist/lookaside/access_layer/src/sample_code/functional/common.mk +@@ -37,9 +37,10 @@ + + #Set Upstream code based flags + +-PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ +- | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ +- echo $$?) ++#PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ ++# | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ ++# echo $$?) ++PREBUILD_BINS ?= -1 + + #QA API and SAL PATHS + MYPWD=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +diff --git a/quickassist/lookaside/access_layer/src/sample_code/performance/Makefile b/quickassist/lookaside/access_layer/src/sample_code/performance/Makefile +index bf8e42a..6b30616 100644 +--- a/quickassist/lookaside/access_layer/src/sample_code/performance/Makefile ++++ b/quickassist/lookaside/access_layer/src/sample_code/performance/Makefile +@@ -40,11 +40,11 @@ + + CC?=gcc + +-PREBUILD_BINS = -1 ++PREBUILD_BINS ?= -1 + +-PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ +- | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ +- echo $$?) ++#PREBUILD_BINS = $(shell echo -e "\#include \n void main () {}" \ ++# | $(CC) -lqat -lusdm -xc - -o /dev/null 2> /dev/null; \ ++# echo $$?) + MYPWD=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) + + INCLUDE_COMPRESSION=1 +-- +2.39.3 + diff --git a/1002-Just-regard-statistic-is-false-rather-than-error.patch b/1002-Just-regard-statistic-is-false-rather-than-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..6e07b6a00ef9e769aea2f3fcf586bba4c7010491 --- /dev/null +++ b/1002-Just-regard-statistic-is-false-rather-than-error.patch @@ -0,0 +1,52 @@ +From f0a6f180e20bf42d8aeed05ec16f1216cf2922d9 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Wed, 13 Sep 2023 15:44:21 +0800 +Subject: [PATCH 02/28] Just regard statistic is false rather than error + +If statistic is configured, SAL should also work. + +Signed-off-by: Zelin Deng +--- + .../access_layer/src/common/utils/sal_statistics.c | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/quickassist/lookaside/access_layer/src/common/utils/sal_statistics.c b/quickassist/lookaside/access_layer/src/common/utils/sal_statistics.c +index eb6a108..90ad511 100644 +--- a/quickassist/lookaside/access_layer/src/common/utils/sal_statistics.c ++++ b/quickassist/lookaside/access_layer/src/common/utils/sal_statistics.c +@@ -101,8 +101,10 @@ CpaStatus SalStatistics_GetStatEnabled(icp_accel_dev_t *device, + *pIsEnabled = CPA_FALSE; + return CPA_STATUS_SUCCESS; + } +- LAC_LOG_STRING_ERROR1("Failed to get %s from configuration file", +- statsName); ++ *pIsEnabled = CPA_FALSE; ++ /* If statistic is not configured, do not stop SAL initialized */ ++ LAC_LOG_DEBUG1("Failed to get %s from configuration file", ++ statsName); + return status; + } + +@@ -145,9 +147,8 @@ CpaStatus SalStatistics_InitStatisticsCollection(icp_accel_dev_t *device) + + status = SalStatistics_GetStatEnabled( + device, SAL_STATS_CFG_ENABLED, &pStatsCollection->bStatsEnabled); +- LAC_CHECK_STATUS(status); + +- if (CPA_FALSE == pStatsCollection->bStatsEnabled) ++ if (CPA_STATUS_SUCCESS != status || CPA_FALSE == pStatsCollection->bStatsEnabled) + { + pStatsCollection->bDcStatsEnabled = CPA_FALSE; + pStatsCollection->bDhStatsEnabled = CPA_FALSE; +@@ -159,7 +160,7 @@ CpaStatus SalStatistics_InitStatisticsCollection(icp_accel_dev_t *device) + pStatsCollection->bRsaStatsEnabled = CPA_FALSE; + pStatsCollection->bSymStatsEnabled = CPA_FALSE; + pStatsCollection->bMiscStatsEnabled = CPA_FALSE; +- return status; ++ return CPA_STATUS_SUCCESS; + } + + /* What services are enabled */ +-- +2.39.3 + diff --git a/1003-crypto-qat-adf_get_etr_base-helper.patch b/1003-crypto-qat-adf_get_etr_base-helper.patch new file mode 100644 index 0000000000000000000000000000000000000000..f5bb5855bc18951fb960d8ce2e4effcaa5512f23 --- /dev/null +++ b/1003-crypto-qat-adf_get_etr_base-helper.patch @@ -0,0 +1,97 @@ +From 1676256fdf618b621dd1f4fb60381c6763807e68 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 9 Apr 2024 16:50:16 +0800 +Subject: [PATCH 03/28] crypto: qat - adf_get_etr_base() helper + +derived from 1894cb1de656cfde345c3b2690e379be1eb9db96 + +Signed-off-by: Zelin Deng +--- + .../crypto/qat/qat_common/adf_common_drv.h | 29 +++++++++++++++++++ + .../crypto/qat/qat_common/adf_gen4_hw_data.c | 6 +--- + .../crypto/qat/qat_common/adf_transport.c | 4 +-- + 3 files changed, 31 insertions(+), 8 deletions(-) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h +index c26e114..e0ab348 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h +@@ -415,5 +415,34 @@ static inline void adf_flush_vf_wq(void) + } + + #endif ++static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_bar *pmisc; ++ ++ pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; ++ ++ return pmisc->virt_addr; ++} ++ ++static inline void __iomem *adf_get_etr_base(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_bar *etr; ++ ++ etr = &GET_BARS(accel_dev)[hw_data->get_etr_bar_id(hw_data)]; ++ ++ return etr->virt_addr; ++} ++ ++static inline void __iomem *adf_get_aram_base(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_bar *param; ++ ++ param = &GET_BARS(accel_dev)[hw_data->get_sram_bar_id(hw_data)]; ++ ++ return param->virt_addr; ++} + #endif + #endif +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +index 1567e64..1dac69a 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +@@ -238,16 +238,12 @@ int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, + u32 bank_number) + { + struct adf_hw_device_data *hw_data; +- struct adf_bar *etr_bar; +- void __iomem *csr; ++ void __iomem *csr = adf_get_etr_base(accel_dev); + + hw_data = accel_dev->hw_device; + if (bank_number >= hw_data->num_banks) + return -EINVAL; + +- etr_bar = &GET_BARS(accel_dev)[hw_data->get_etr_bar_id(hw_data)]; +- csr = etr_bar->virt_addr; +- + dev_dbg(&GET_DEV(accel_dev), "ring pair reset for bank:%d\n", + bank_number); + if (gen4_ring_pair_reset(csr, bank_number)) { +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport.c +index b27e2be..56eb104 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport.c +@@ -550,13 +550,11 @@ err_bank: + static int adf_init_banks(struct adf_accel_dev *accel_dev) + { + struct adf_etr_data *etr_data; +- struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr_addr; + u32 num_banks = 0; + int i, ret = 0; + +- i = hw_data->get_etr_bar_id(hw_data); +- csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; ++ csr_addr = adf_get_etr_base(accel_dev); + etr_data = accel_dev->transport; + num_banks = GET_MAX_BANKS(accel_dev); + +-- +2.39.3 + diff --git a/1003-src-fix-warning-when-compiling.patch b/1003-src-fix-warning-when-compiling.patch deleted file mode 100644 index 5227594ff51eecde2754766cec725abd595ab410..0000000000000000000000000000000000000000 --- a/1003-src-fix-warning-when-compiling.patch +++ /dev/null @@ -1,42 +0,0 @@ -From ee295f3a0d5a9b6f6a219d4d34adc84e6b9be170 Mon Sep 17 00:00:00 2001 -From: Xuchun Shang -Date: Tue, 29 Nov 2022 11:32:10 +0800 -Subject: [PATCH] src: fix warning when compiling - -The dev->deviceName is an array which will never be null. - -Signed-off-by: Xuchun Shang ---- - .../lookaside/access_layer/src/common/ctrl/sal_compression.c | 2 +- - quickassist/lookaside/access_layer/src/common/ctrl/sal_crypto.c | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/quickassist/lookaside/access_layer/src/common/ctrl/sal_compression.c b/quickassist/lookaside/access_layer/src/common/ctrl/sal_compression.c -index 88f4fdb..ed37178 100644 ---- a/quickassist/lookaside/access_layer/src/common/ctrl/sal_compression.c -+++ b/quickassist/lookaside/access_layer/src/common/ctrl/sal_compression.c -@@ -1671,7 +1671,7 @@ CpaStatus cpaDcInstanceGetInfo2(const CpaInstanceHandle instanceHandle, - pInstanceInfo2->isOffloaded = CPA_TRUE; - /* Get the instance name and part name from the config file */ - dev = icp_adf_getAccelDevByAccelId(pCompressionService->pkgID); -- if (NULL == dev || NULL == dev->deviceName) -+ if (NULL == dev) - { - LAC_LOG_ERROR("Can not find device for the instance\n"); - LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2)); -diff --git a/quickassist/lookaside/access_layer/src/common/ctrl/sal_crypto.c b/quickassist/lookaside/access_layer/src/common/ctrl/sal_crypto.c -index 64dea7b..1404595 100644 ---- a/quickassist/lookaside/access_layer/src/common/ctrl/sal_crypto.c -+++ b/quickassist/lookaside/access_layer/src/common/ctrl/sal_crypto.c -@@ -2480,7 +2480,7 @@ CpaStatus cpaCyInstanceGetInfo2(const CpaInstanceHandle instanceHandle_in, - - /* Get the instance name and part name*/ - dev = icp_adf_getAccelDevByAccelId(pCryptoService->pkgID); -- if (NULL == dev || NULL == dev->deviceName) -+ if (NULL == dev) - { - LAC_LOG_ERROR("Can not find device for the instance\n"); - LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2)); --- -2.37.3 - diff --git a/1004-crypto-qat-move-PFVF-compat-checker-to-a-function.patch b/1004-crypto-qat-move-PFVF-compat-checker-to-a-function.patch new file mode 100644 index 0000000000000000000000000000000000000000..f0a7f2905fa5b7c90a17613bc4b69233f7ad4ab7 --- /dev/null +++ b/1004-crypto-qat-move-PFVF-compat-checker-to-a-function.patch @@ -0,0 +1,36 @@ +From 1bfb2a3ee99a7424c510f4594ffefb7c00cd5e0b Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 9 Apr 2024 17:06:50 +0800 +Subject: [PATCH 04/28] crypto: qat - move PFVF compat checker to a function + +derived from 867e801005e9e76f7ae2d143fed0da440150c64d + +Signed-off-by: Zelin Deng +--- + .../drivers/crypto/qat/qat_common/adf_pf2vf_msg.h | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h +index 5ef82ab..1768665 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h +@@ -211,4 +211,16 @@ + #define ADF_CRC8_INIT_VALUE 0xFF + #define ADF_VF2PF_INT_MASK 0xFFFF + #define ADF_VF2PF_REG_MASK 0xFFFFFFFF ++ ++static inline u8 adf_vf_compat_checker(u8 vf_compat_ver) ++{ ++ if (vf_compat_ver == 0) ++ return ADF_PF2VF_VF_INCOMPATIBLE; ++ ++ if (vf_compat_ver <= ADF_PFVF_COMPATIBILITY_VERSION) ++ return ADF_PF2VF_VF_COMPATIBLE; ++ ++ return ADF_PF2VF_VF_COMPAT_UNKNOWN; ++} ++ + #endif /* ADF_IOV_MSG_H */ +-- +2.39.3 + diff --git a/1005-crypto-qat-add-bank-save-and-restore-flows.patch b/1005-crypto-qat-add-bank-save-and-restore-flows.patch new file mode 100644 index 0000000000000000000000000000000000000000..94ca092768708efa49f1f1129dc1048f48b3703f --- /dev/null +++ b/1005-crypto-qat-add-bank-save-and-restore-flows.patch @@ -0,0 +1,419 @@ +From fe33dc7d602fc3dc85aa55640b4cc75c5860fc51 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 9 Apr 2024 21:03:53 +0800 +Subject: [PATCH 05/28] crypto: qat - add bank save and restore flows + +derived from bbfdde7d195ffc9c10598055c449b24c50a0cd25 + +Signed-off-by: Zelin Deng +--- + .../crypto/qat/qat_common/adf_accel_devices.h | 37 +++ + .../crypto/qat/qat_common/adf_gen4_hw_data.c | 289 ++++++++++++++++++ + .../crypto/qat/qat_common/adf_gen4_hw_data.h | 26 ++ + 3 files changed, 352 insertions(+) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +index b75255f..be3ec24 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +@@ -319,6 +319,39 @@ struct admin_info { + u32 mailbox_offset; + } __packed; + ++struct ring_config { ++ u64 base; ++ u32 config; ++ u32 head; ++ u32 tail; ++ u32 reserved0; ++}; ++ ++struct bank_state { ++ u32 ringstat0; ++ u32 ringstat1; ++ u32 ringuostat; ++ u32 ringestat; ++ u32 ringnestat; ++ u32 ringnfstat; ++ u32 ringfstat; ++ u32 ringcstat0; ++ u32 ringcstat1; ++ u32 ringcstat2; ++ u32 ringcstat3; ++ u32 iaintflagen; ++ u32 iaintflagreg; ++ u32 iaintflagsrcsel0; ++ u32 iaintflagsrcsel1; ++ u32 iaintcolen; ++ u32 iaintcolctl; ++ u32 iaintflagandcolen; ++ u32 ringexpstat; ++ u32 ringexpintenable; ++ u32 ringsrvarben; ++ u32 reserved0; ++ struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK]; ++}; + + struct adf_cfg_device_data; + struct adf_accel_dev; +@@ -585,6 +618,10 @@ struct adf_hw_device_data { + u32 bank_number); + int (*ring_pair_drain)(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_ms); ++ int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number, ++ struct bank_state *state); ++ int (*bank_state_restore)(struct adf_accel_dev *accel_dev, ++ u32 bank_number, struct bank_state *state); + void (*config_ring_irq)(struct adf_accel_dev *accel_dev, + u32 bank_number, u16 ring_mask); + int (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev, +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +index 1dac69a..fb1bd36 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +@@ -10,6 +10,8 @@ + #include "adf_gen4_hw_csr_data.h" + #include "adf_cfg.h" + #include "qat_crypto.h" ++#include ++#include + + #define ADF_CONST_TABLE_SIZE 1024 + +@@ -656,3 +658,290 @@ void adf_gen4_cfg_get_accel_algo_cap(struct adf_accel_dev *accel_dev) + #endif + } + EXPORT_SYMBOL_GPL(adf_gen4_cfg_get_accel_algo_cap); ++ ++/* ++ * adf_gen4_bank_quiesce_coal_timer() - quiesce bank coalesced interrupt timer ++ * @accel_dev: Pointer to the device structure ++ * @bank_idx: Offset to the bank within this device ++ * @timeout_ms: Timeout in milliseconds for the operation ++ * ++ * This function tries to quiesce the coalesced interrupt timer of a bank if ++ * it has been enabled and triggered. ++ * ++ * Returns 0 on success, error code otherwise ++ * ++ */ ++int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, ++ u32 bank_idx, int timeout_ms) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; ++ struct adf_bar *pmisc = ++ &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; ++ void __iomem *csr_misc = pmisc->virt_addr; ++ void __iomem *csr_etr = adf_get_etr_base(accel_dev); ++ u32 int_col_ctl, int_col_mask, int_col_en; ++ u32 e_stat, intsrc; ++ u64 wait_us; ++ int ret; ++ ++ if (timeout_ms < 0) ++ return -EINVAL; ++ ++ int_col_ctl = csr_ops->read_csr_int_col_ctl(csr_etr, bank_idx); ++ int_col_mask = csr_ops->get_int_col_ctl_enable_mask(); ++ if (!(int_col_ctl & int_col_mask)) ++ return 0; ++ ++ int_col_en = csr_ops->read_csr_int_col_en(csr_etr, bank_idx); ++ int_col_en &= BIT(ADF_WQM_CSR_RP_IDX_RX); ++ ++ e_stat = csr_ops->read_csr_e_stat(csr_etr, bank_idx); ++ if (!(~e_stat & int_col_en)) ++ return 0; ++ ++ wait_us = 2 * ((int_col_ctl & ~int_col_mask) << 8) * USEC_PER_SEC; ++ do_div(wait_us, hw_data->clock_frequency); ++ wait_us = min(wait_us, (u64)timeout_ms * USEC_PER_MSEC); ++ dev_dbg(&GET_DEV(accel_dev), ++ "wait for bank %d - coalesced timer expires in %llu us (max=%u ms estat=0x%x intcolen=0x%x)\n", ++ bank_idx, wait_us, timeout_ms, e_stat, int_col_en); ++ ++ ret = read_poll_timeout(ADF_CSR_RD, intsrc, intsrc, ++ ADF_COALESCED_POLL_DELAY_US, wait_us, true, ++ csr_misc, ADF_WQM_CSR_RPINTSTS(bank_idx)); ++ if (ret) ++ dev_warn(&GET_DEV(accel_dev), ++ "coalesced timer for bank %d expired (%llu us)\n", ++ bank_idx, wait_us); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(adf_gen4_bank_quiesce_coal_timer); ++ ++static int drain_bank(void __iomem *csr, u32 bank_number, int timeout_us) ++{ ++ u32 status; ++ ++ ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number), ++ ADF_WQM_CSR_RPRESETCTL_DRAIN_SHIFT); ++ ++ return read_poll_timeout(ADF_CSR_RD, status, ++ status & ADF_WQM_CSR_RPRESETSTS_STATUS, ++ ADF_RPRESET_POLL_DELAY_US, timeout_us, true, ++ csr, ADF_WQM_CSR_RPRESETSTS(bank_number)); ++} ++ ++void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, ++ u32 bank_number) ++{ ++ void __iomem *csr = adf_get_etr_base(accel_dev); ++ ++ ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), ++ ADF_WQM_CSR_RPRESETSTS_STATUS); ++} ++ ++int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, ++ u32 bank_number, int timeout_us) ++{ ++ void __iomem *csr = adf_get_etr_base(accel_dev); ++ int ret; ++ ++ dev_dbg(&GET_DEV(accel_dev), "Drain bank %d\n", bank_number); ++ ++ ret = drain_bank(csr, bank_number, timeout_us); ++ if (ret) ++ dev_err(&GET_DEV(accel_dev), "Bank drain failed (timeout)\n"); ++ else ++ dev_dbg(&GET_DEV(accel_dev), "Bank drain successful\n"); ++ ++ return ret; ++} ++ ++static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, ++ u32 bank, struct bank_state *state, u32 num_rings) ++{ ++ u32 i; ++ ++ state->ringstat0 = ops->read_csr_stat(base, bank); ++ state->ringuostat = ops->read_csr_uo_stat(base, bank); ++ state->ringestat = ops->read_csr_e_stat(base, bank); ++ state->ringnestat = ops->read_csr_ne_stat(base, bank); ++ state->ringnfstat = ops->read_csr_nf_stat(base, bank); ++ state->ringfstat = ops->read_csr_f_stat(base, bank); ++ state->ringcstat0 = ops->read_csr_c_stat(base, bank); ++ state->iaintflagen = ops->read_csr_int_en(base, bank); ++ state->iaintflagreg = ops->read_csr_int_flag(base, bank); ++ state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank, 0); ++ state->iaintcolen = ops->read_csr_int_col_en(base, bank); ++ state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); ++ state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); ++ state->ringexpstat = ops->read_csr_exp_stat(base, bank); ++ state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); ++ state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); ++ ++ for (i = 0; i < num_rings; i++) { ++ state->rings[i].head = ops->read_csr_ring_head(base, bank, i); ++ state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); ++ state->rings[i].config = ops->read_csr_ring_config(base, bank, i); ++ state->rings[i].base = ops->read_csr_ring_base(base, bank, i); ++ } ++} ++ ++#define CHECK_STAT(op, expect_val, name, args...) \ ++({ \ ++ u32 __expect_val = (expect_val); \ ++ u32 actual_val = op(args); \ ++ (__expect_val == actual_val) ? 0 : \ ++ (pr_err("QAT: Fail to restore %s register. Expected 0x%x, actual 0x%x\n", \ ++ name, __expect_val, actual_val), -EINVAL); \ ++}) ++ ++static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, ++ u32 bank, struct bank_state *state, u32 num_rings, ++ int tx_rx_gap) ++{ ++ u32 val, tmp_val, i; ++ int ret; ++ ++ for (i = 0; i < num_rings; i++) ++ ops->write_csr_ring_base(base, bank, i, state->rings[i].base); ++ ++ for (i = 0; i < num_rings; i++) ++ ops->write_csr_ring_config(base, bank, i, state->rings[i].config); ++ ++ for (i = 0; i < num_rings / 2; i++) { ++ int tx = i * (tx_rx_gap + 1); ++ int rx = tx + tx_rx_gap; ++ u32 tx_idx = tx / ADF_RINGS_PER_INT_SRCSEL_GEN4; ++ u32 rx_idx = rx / ADF_RINGS_PER_INT_SRCSEL_GEN4; ++ ++ ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); ++ ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); ++ ++ /* ++ * The TX ring head needs to be updated again to make sure that ++ * the HW will not consider the ring as full when it is empty ++ * and the correct state flags are set to match the recovered state. ++ */ ++ if (state->ringestat & BIT(tx)) { ++ val = ops->read_csr_int_srcsel(base, bank, 0); ++ val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; ++ ops->write_csr_int_srcsel(base, bank, tx_idx, val); ++ ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); ++ } ++ ++ ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); ++ ++ /* Set iaintflagsrcsel.src(2n+1) (src of RX ring) to F_RISE */ ++ val = ops->read_csr_int_srcsel(base, bank, rx_idx); ++ val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; ++ ops->write_csr_int_srcsel(base, bank, rx_idx, val); ++ ++ /* Restore RX ring head */ ++ ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); ++ ++ /* Set iaintflagsrcsel.src(2n+1) (src of RX ring) to F_FALL */ ++ val = ops->read_csr_int_srcsel(base, bank, rx_idx); ++ val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; ++ ops->write_csr_int_srcsel(base, bank, rx_idx, val); ++ ++ /* ++ * The RX ring tail needs to be updated again to make sure that ++ * the HW will not consider the ring as empty when it is full ++ * and the correct state flags are set to match the recovered state. ++ */ ++ if (state->ringfstat & BIT(rx)) ++ ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); ++ } ++ ++ ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); ++ ops->write_csr_int_en(base, bank, state->iaintflagen); ++ ops->write_csr_int_col_en(base, bank, state->iaintcolen); ++ ops->write_csr_int_srcsel(base, bank, 0, state->iaintflagsrcsel0); ++ ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); ++ ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); ++ ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); ++ ++ /* Check that all ring statuses match the saved state. */ ++ ret = CHECK_STAT(ops->read_csr_stat, state->ringstat0, "ringstat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ ret = CHECK_STAT(ops->read_csr_e_stat, state->ringestat, "ringestat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ ret = CHECK_STAT(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ ret = CHECK_STAT(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ ret = CHECK_STAT(ops->read_csr_f_stat, state->ringfstat, "ringfstat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ ret = CHECK_STAT(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", ++ base, bank); ++ if (ret) ++ return ret; ++ ++ tmp_val = ops->read_csr_exp_stat(base, bank); ++ val = state->ringexpstat; ++ if (tmp_val && !val) { ++ pr_err("QAT: Bank was restored with exception: 0x%x\n", val); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, ++ struct bank_state *state) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; ++ void __iomem *csr_base = adf_get_etr_base(accel_dev); ++ ++ if (bank_number >= hw_data->num_banks || !state) ++ return -EINVAL; ++ ++ dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); ++ ++ bank_state_save(csr_ops, csr_base, bank_number, state, ++ hw_data->num_rings_per_bank); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(adf_gen4_bank_state_save); ++ ++int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, ++ struct bank_state *state) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops; ++ void __iomem *csr_base = adf_get_etr_base(accel_dev); ++ int ret; ++ ++ if (bank_number >= hw_data->num_banks || !state) ++ return -EINVAL; ++ ++ dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); ++ ++ ret = bank_state_restore(csr_ops, csr_base, bank_number, state, ++ hw_data->num_rings_per_bank, hw_data->tx_rx_gap); ++ if (ret) ++ dev_err(&GET_DEV(accel_dev), ++ "Unable to restore state of bank %d\n", bank_number); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore); +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +index ff528b2..cabb13a 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +@@ -51,6 +51,22 @@ + /* UQ Base */ + #define ADF_GEN4_UQ_BASE 0x180000 + ++#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC) ++#define ADF_RPRESET_POLL_DELAY_US 20 ++#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0) ++#define ADF_WQM_CSR_RPRESETCTL_DRAIN BIT(2) ++#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3)) ++#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0) ++#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4) ++ ++/* Ring interrupt */ ++#define ADF_RP_INT_SRC_SEL_F_RISE_MASK BIT(2) ++#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) ++#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 ++#define ADF_COALESCED_POLL_DELAY_US 1000 ++#define ADF_WQM_CSR_RPINTSOU(bank) (0x200000 ((bank) << 12)) ++#define ADF_WQM_CSR_RP_IDX_RX 1 ++ + int adf_init_chaining(struct adf_accel_dev *accel_dev); + int adf_gen4_send_admin_init(struct adf_accel_dev *accel_dev); + int adf_gen4_get_uq_base_addr(struct adf_accel_dev *accel_dev, +@@ -72,5 +88,15 @@ enum adf_accel_unit_services + int adf_gen4_check_svc_to_hw_capabilities(struct adf_accel_dev *accel_dev, + u32 required_capability); + void adf_gen4_cfg_get_accel_algo_cap(struct adf_accel_dev *accel_dev); ++int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev, ++ u32 bank_idx, int timeout_ms); ++int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, ++ u32 bank_number, int timeout_us); ++void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, ++ u32 bank_number); ++int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, ++ struct bank_state *state); ++int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, ++ u32 bank_number, struct bank_state *state); + + #endif /* ADF_GEN4_HW_DATA_H_ */ +-- +2.39.3 + diff --git a/1006-crypto-qat-add-interface-for-live-migration.patch b/1006-crypto-qat-add-interface-for-live-migration.patch new file mode 100644 index 0000000000000000000000000000000000000000..51fd1d046138bdc1d91bab9b4d801e29f73ebb25 --- /dev/null +++ b/1006-crypto-qat-add-interface-for-live-migration.patch @@ -0,0 +1,273 @@ +From 6ba2a663a5571ec4adcef8fefdd0219b05b1bc0b Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 26 Mar 2024 16:34:13 +0800 +Subject: [PATCH 06/28] crypto: qat - add interface for live migration + +derived from 0fce55e5334d380d8a09f80ba9c9b68eeea6971d + +Signed-off-by: Zelin Deng +--- + .../drivers/crypto/qat/qat_common/Makefile | 2 +- + .../crypto/qat/qat_common/adf_accel_devices.h | 17 +++ + .../crypto/qat/qat_common/adf_gen4_vf_mig.h | 10 ++ + .../crypto/qat/qat_common/qat_mig_dev.c | 130 ++++++++++++++++++ + .../crypto/qat/qat_common/qat_mig_dev.h | 31 +++++ + 5 files changed, 189 insertions(+), 1 deletion(-) + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.h + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.h + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +index 46393ab..c53ab18 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +@@ -57,7 +57,7 @@ intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o + intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \ + adf_vf2pf_msg.o adf_vf_isr.o \ + adf_pf2vf_ring_to_svc_map.o \ +- adf_pf2vf_capabilities.o ++ adf_pf2vf_capabilities.o qat_mig_dev.o + + #Check kernel version and update sign if not defined yet + KERNELLESSTHAN4_13 := $(shell set -e; \ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +index be3ec24..db4f342 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +@@ -13,6 +13,7 @@ + #include + #endif + #include "adf_cfg_common.h" ++#include "qat_mig_dev.h" + #else + #include + #endif /* USER_SPACE */ +@@ -358,6 +359,20 @@ struct adf_accel_dev; + struct adf_etr_data; + struct adf_etr_ring_data; + ++struct qat_migdev_ops { ++ int (*init)(struct qat_mig_dev *mdev); ++ void (*cleanup)(struct qat_mig_dev *mdev); ++ void (*reset)(struct qat_mig_dev *mdev); ++ int (*open)(struct qat_mig_dev *mdev); ++ void (*close)(struct qat_mig_dev *mdev); ++ int (*suspend)(struct qat_mig_dev *mdev); ++ int (*resume)(struct qat_mig_dev *mdev); ++ int (*save_state)(struct qat_mig_dev *mdev); ++ int (*save_setup)(struct qat_mig_dev *mdev); ++ int (*load_state)(struct qat_mig_dev *mdev); ++ int (*load_setup)(struct qat_mig_dev *mdev, int size); ++}; ++ + struct adf_hw_csr_ops { + u32 (*build_ring_config)(u32 size); + u64 (*build_resp_ring_config)(u32 size, +@@ -655,6 +670,7 @@ struct adf_hw_device_data { + int (*get_uq_base_addr)(struct adf_accel_dev *accel_dev, + void **uq_base_addr, + u32 bank_number); ++ struct qat_migdev_ops vfmig_ops; + const char *fw_name; + const char *fw_mmp_name; + const char *fw_aux_obj; +@@ -758,6 +774,7 @@ enum operation { + #define accel_to_pci_dev(accel_ptr) ((accel_ptr)->accel_pci_dev.pci_dev) + #define GET_SRV_TYPE(ena_srv_mask, srv) \ + (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) ++#define GET_VFMIG_OPS(accel_dev) (&(accel_dev)->hw_device->vfmig_ops) + #define ADF_NUM_THREADS_PER_AE (8) + #define ADF_AE_ADMIN_THREAD (7) + #define ADF_NUM_PKE_STRAND (2) +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.h +new file mode 100644 +index 0000000..72216d0 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.h +@@ -0,0 +1,10 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++#ifndef ADF_GEN4_VF_MIG_H_ ++#define ADF_GEN4_VF_MIG_H_ ++ ++#include "adf_accel_devices.h" ++ ++void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); ++ ++#endif +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.c b/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.c +new file mode 100644 +index 0000000..b66e6de +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.c +@@ -0,0 +1,130 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++#include ++#include ++#include ++#include ++#include "qat_mig_dev.h" ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++ ++struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id) ++{ ++ struct adf_accel_dev *accel_dev; ++ struct qat_migdev_ops *ops; ++ struct qat_mig_dev *mdev; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(pdev); ++ if (!accel_dev) ++ return ERR_PTR(-ENODEV); ++ ++ ops = GET_VFMIG_OPS(accel_dev); ++ if (!ops || !ops->init || !ops->cleanup || !ops->reset || !ops->open || ++ !ops->close || !ops->suspend || !ops->resume || !ops->save_state || ++ !ops->load_state || !ops->save_setup || !ops->load_setup) ++ return ERR_PTR(-EINVAL); ++ ++ mdev = kmalloc(sizeof(*mdev), GFP_KERNEL); ++ if (!mdev) ++ return ERR_PTR(-ENOMEM); ++ ++ mdev->vf_id = vf_id; ++ mdev->parent_accel_dev = accel_dev; ++ ++ return mdev; ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_create); ++ ++int qat_vfmig_init(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->init(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_init); ++ ++void qat_vfmig_cleanup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->cleanup(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_cleanup); ++ ++void qat_vfmig_reset(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->reset(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_reset); ++ ++int qat_vfmig_open(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->open(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_open); ++ ++void qat_vfmig_close(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ GET_VFMIG_OPS(accel_dev)->close(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_close); ++ ++int qat_vfmig_suspend(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->suspend(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_suspend); ++ ++int qat_vfmig_resume(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->resume(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_resume); ++ ++int qat_vfmig_save_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->save_state(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_save_state); ++ ++int qat_vfmig_save_setup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->save_setup(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_save_setup); ++ ++int qat_vfmig_load_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->load_state(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_load_state); ++ ++int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->load_setup(mdev, size); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_load_setup); ++ ++void qat_vfmig_destroy(struct qat_mig_dev *mdev) ++{ ++ kfree(mdev); ++} ++EXPORT_SYMBOL_GPL(qat_vfmig_destroy); +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.h b/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.h +new file mode 100644 +index 0000000..dbbb6a0 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/qat_mig_dev.h +@@ -0,0 +1,31 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++#ifndef QAT_MIG_DEV_H_ ++#define QAT_MIG_DEV_H_ ++ ++struct pci_dev; ++ ++struct qat_mig_dev { ++ void *parent_accel_dev; ++ u8 *state; ++ u32 setup_size; ++ u32 remote_setup_size; ++ u32 state_size; ++ s32 vf_id; ++}; ++ ++struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id); ++int qat_vfmig_init(struct qat_mig_dev *mdev); ++void qat_vfmig_cleanup(struct qat_mig_dev *mdev); ++void qat_vfmig_reset(struct qat_mig_dev *mdev); ++int qat_vfmig_open(struct qat_mig_dev *mdev); ++void qat_vfmig_close(struct qat_mig_dev *mdev); ++int qat_vfmig_suspend(struct qat_mig_dev *mdev); ++int qat_vfmig_resume(struct qat_mig_dev *mdev); ++int qat_vfmig_save_state(struct qat_mig_dev *mdev); ++int qat_vfmig_save_setup(struct qat_mig_dev *mdev); ++int qat_vfmig_load_state(struct qat_mig_dev *mdev); ++int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size); ++void qat_vfmig_destroy(struct qat_mig_dev *mdev); ++ ++#endif /*QAT_MIG_DEV_H_*/ +-- +2.39.3 + diff --git a/1007-crypto-qat-implement-interface-for-live-migration.patch b/1007-crypto-qat-implement-interface-for-live-migration.patch new file mode 100644 index 0000000000000000000000000000000000000000..a0d25d2f6d0d212d4edac397348b7368909d89bd --- /dev/null +++ b/1007-crypto-qat-implement-interface-for-live-migration.patch @@ -0,0 +1,1616 @@ +From 627a2630226db264616e501b56e7982d5be38291 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Tue, 26 Mar 2024 17:05:26 +0800 +Subject: [PATCH 07/28] crypto: qat - implement interface for live migration + +derived from 0fce55e5334d380d8a09f80ba9c9b68eeea6971d + +NOTE: sla is not adapted correctly, need to be refactored later + +Signed-off-by: Zelin Deng +--- + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 2 + + .../drivers/crypto/qat/qat_common/Makefile | 2 + + .../crypto/qat/qat_common/adf_accel_devices.h | 7 + + .../qat/qat_common/adf_gen4_hw_csr_data.h | 11 +- + .../crypto/qat/qat_common/adf_gen4_rl.h | 1 + + .../crypto/qat/qat_common/adf_gen4_vf_mig.c | 1016 +++++++++++++++++ + .../crypto/qat/qat_common/adf_mstate_mgr.c | 318 ++++++ + .../crypto/qat/qat_common/adf_mstate_mgr.h | 89 ++ + .../drivers/crypto/qat/qat_common/adf_sriov.c | 4 + + .../adf_transport_access_macros_gen4.h | 2 + + 10 files changed, 1451 insertions(+), 1 deletion(-) + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.h + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +index c170d97..d6a1cac 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +@@ -23,6 +23,7 @@ + #include "adf_gen4_timer.h" + #include "adf_pasid.h" + #include "adf_gen4_rl.h" ++#include "adf_gen4_vf_mig.h" + + #define MAX_CLUSTER 4 + +@@ -1076,6 +1077,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id) + hw_data->get_uq_base_addr = adf_gen4_get_uq_base_addr; + gen4_init_hw_csr_info(&hw_data->csr_info); + gen4_init_adi_ops(&hw_data->adi_ops); ++ adf_gen4_init_vf_mig_ops(&hw_data->vfmig_ops); + /* KAT test enabled by default for qat_4xxx */ + hw_data->fw_integr_selftest = true; + hw_data->init_chaining = adf_init_chaining; +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +index c53ab18..4e137b3 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +@@ -27,6 +27,8 @@ intel_qat-objs := adf_cfg.o \ + adf_gen2_hw_csr_data.o \ + adf_gen4_hw_csr_data.o \ + adf_gen4vf_hw_csr_data.o \ ++ adf_gen4_vf_mig.o \ ++ adf_mstate_mgr.o \ + adf_gen4_ras.o \ + adf_vqat_hw_csr_data.o \ + adf_adi.o \ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +index db4f342..e1ad617 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +@@ -883,12 +883,19 @@ struct adf_fw_loader_data { + struct adf_accel_vf_info { + struct adf_accel_dev *accel_dev; + struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ ++ struct mutex pfvf_mig_lock; /* protects PFVF state for migration */ + struct ratelimit_state vf2pf_ratelimit; + u32 vf_nr; + bool init; + bool restarting; + u8 compat_ver; + struct pfvf_stats pfvf_counters; ++ ++ /* ++ * Private area used for device migration. ++ * Memory allocation and free is managed by migration driver. ++ */ ++ void *mig_priv; + }; + + struct adf_fw_versions { +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_csr_data.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_csr_data.h +index 1314ea9..4fc41c8 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_csr_data.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_csr_data.h +@@ -3,6 +3,7 @@ + #ifndef ADF_GEN4_HW_CSR_DATA_H_ + #define ADF_GEN4_HW_CSR_DATA_H_ + #include ++#include "adf_transport_access_macros_gen4.h" + + #define ADF_GEN4_PM_INTERRUPT (0x50A028) + +@@ -30,7 +31,15 @@ + + #define ADF_GEN4_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20)) + #define ADF_GEN4_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20)) +-#define ADF_GEN4_VINTMSK_OFFSET(i) (0x40B004 + ((i) * 0x20)) ++#define ADF_GEN4_VINTMSKPF2VM_OFFSET(i) (0x40B00C + (i) * 0x20) ++#define ADF_GEN4_VINTSOUPF2VM_OFFSET(i) (0x40B008 + (i) * 0x20) ++#define ADF_GEN4_VINTMSK_OFFSET(i) (0x40B004 + (i) * 0x20) ++#define ADF_GEN4_VINTSOU_OFFSET(i) (0x40B000 + (i) * 0x20) ++ ++struct adf_gen4_vfmig { ++ struct adf_mstate_mgr *mstate_mgr; ++ bool bank_stopped[ADF_GEN4_NUM_BANKS_PER_VF]; ++}; + + struct adf_hw_csr_info; + void gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info); +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h +index 65e48da..b83e017 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h +@@ -17,6 +17,7 @@ + #define RL_PCI_DEV_OFFSET 0x3 + #define RL_ROUND_MULTIPLE_MB 100 + #define RL_ROUND_MULTIPLE_KO 1000 ++#define RL_RP_CNT_PER_LEAF_MAX 4U + + #define RL_TOKEN_PCIE_SIZE 64 + #define RL_ASYM_TOKEN_SIZE 1024 +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +new file mode 100644 +index 0000000..a510274 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +@@ -0,0 +1,1016 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++#include "adf_gen4_hw_data.h" ++#include "adf_mstate_mgr.h" ++#include "adf_gen4_vf_mig.h" ++#include "adf_pf2vf_msg.h" ++#include "adf_gen4_hw_csr_data.h" ++#include "adf_gen4_rl.h" ++ ++#define ADF_GEN4_VF_MSTATE_SIZE 4096 ++#define ADF_GEN4_PFVF_RSP_TIMEOUT_US 5000 ++ ++static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev); ++static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len); ++ ++static int adf_gen4_vfmig_init_device(struct qat_mig_dev *mdev) ++{ ++ u8 *state; ++ ++ state = kmalloc(ADF_GEN4_VF_MSTATE_SIZE, GFP_KERNEL); ++ if (!state) ++ return -ENOMEM; ++ ++ mdev->state = state; ++ mdev->state_size = ADF_GEN4_VF_MSTATE_SIZE; ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++ ++ return 0; ++} ++ ++static void adf_gen4_vfmig_cleanup_device(struct qat_mig_dev *mdev) ++{ ++ kfree(mdev->state); ++ mdev->state = NULL; ++} ++ ++static void adf_gen4_vfmig_reset_device(struct qat_mig_dev *mdev) ++{ ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++} ++ ++static int adf_gen4_vfmig_open_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ ++ vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; ++ ++ vfmig = kzalloc(sizeof(*vfmig), GFP_KERNEL); ++ if (!vfmig) ++ return -ENOMEM; ++ ++ vfmig->mstate_mgr = adf_mstate_mgr_new(mdev->state, mdev->state_size); ++ if (!vfmig->mstate_mgr) { ++ kfree(vfmig); ++ return -ENOMEM; ++ } ++ vf_info->mig_priv = vfmig; ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++ ++ return 0; ++} ++ ++static void adf_gen4_vfmig_close_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ ++ vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; ++ if (vf_info->mig_priv) { ++ vfmig = vf_info->mig_priv; ++ adf_mstate_mgr_destroy(vfmig->mstate_mgr); ++ kfree(vfmig); ++ vf_info->mig_priv = NULL; ++ } ++} ++ ++static int adf_gen4_vfmig_suspend_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vf_mig; ++ u32 vf_nr = mdev->vf_id; ++ int ret, i; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vf_mig = vf_info->mig_priv; ++ ++ /* Stop all inflight jobs */ ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; ++ ++ ret = adf_gen4_bank_drain_start(accel_dev, pf_bank_nr, ++ ADF_RPRESET_POLL_TIMEOUT_US); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to drain bank %d for vf_nr %d\n", i, ++ vf_nr); ++ return ret; ++ } ++ vf_mig->bank_stopped[i] = true; ++ ++ adf_gen4_bank_quiesce_coal_timer(accel_dev, pf_bank_nr, ++ ADF_COALESCED_POLL_TIMEOUT_US); ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_resume_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vf_mig; ++ u32 vf_nr = mdev->vf_id; ++ int i; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vf_mig = vf_info->mig_priv; ++ ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; ++ ++ if (vf_mig->bank_stopped[i]) { ++ adf_gen4_bank_drain_finish(accel_dev, pf_bank_nr); ++ vf_mig->bank_stopped[i] = false; ++ } ++ } ++ ++ return 0; ++} ++ ++struct adf_vf_bank_info { ++ struct adf_accel_dev *accel_dev; ++ u32 vf_nr; ++ u32 bank_nr; ++}; ++ ++struct mig_user_sla { ++ //enum adf_base_services srv; ++ enum adf_svc_type srv; ++ u64 rp_mask; ++ u32 cir; ++ u32 pir; ++}; ++ ++static int adf_mstate_sla_check(struct adf_mstate_mgr *sub_mgr, u8 *src_buf, ++ u32 src_size, void *opaque) ++{ ++ struct adf_mstate_vreginfo _sinfo = { src_buf, src_size }; ++ struct adf_mstate_vreginfo *sinfo = &_sinfo, *dinfo = opaque; ++ u32 src_sla_cnt = sinfo->size / sizeof(struct mig_user_sla); ++ u32 dst_sla_cnt = dinfo->size / sizeof(struct mig_user_sla); ++ struct mig_user_sla *src_slas = sinfo->addr; ++ struct mig_user_sla *dst_slas = dinfo->addr; ++ int i, j; ++ ++ for (i = 0; i < src_sla_cnt; i++) { ++ for (j = 0; j < dst_sla_cnt; j++) { ++ if (src_slas[i].srv != dst_slas[j].srv || ++ src_slas[i].rp_mask != dst_slas[j].rp_mask) ++ continue; ++ ++ if (src_slas[i].cir > dst_slas[j].cir || ++ src_slas[i].pir > dst_slas[j].pir) { ++ pr_err("QAT: DST VF rate limiting mismatch.\n"); ++ return -EINVAL; ++ } ++ break; ++ } ++ ++ if (j == dst_sla_cnt) { ++ pr_err("QAT: SRC VF rate limiting mismatch - SRC srv %d and rp_mask 0x%llx.\n", ++ src_slas[i].srv, src_slas[i].rp_mask); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static inline int adf_mstate_check_cap_size(u32 src_sz, u32 dst_sz, u32 max_sz) ++{ ++ if (src_sz > max_sz || dst_sz > max_sz) ++ return -EINVAL; ++ else ++ return 0; ++} ++ ++static int adf_mstate_compatver_check(struct adf_mstate_mgr *sub_mgr, ++ u8 *src_buf, u32 src_sz, void *opaque) ++{ ++ struct adf_mstate_vreginfo *info = opaque; ++ u8 compat = 0; ++ u8 *pcompat; ++ ++ if (src_sz != info->size) { ++ pr_debug("QAT: State mismatch (compat version size), current %u, expected %u\n", ++ src_sz, info->size); ++ return -EINVAL; ++ } ++ ++ memcpy(info->addr, src_buf, info->size); ++ pcompat = info->addr; ++ if (*pcompat == 0) { ++ pr_warn("QAT: Unable to determine the version of VF\n"); ++ return 0; ++ } ++ ++ compat = adf_vf_compat_checker(*pcompat); ++ if (compat == ADF_PF2VF_VF_INCOMPATIBLE) { ++ pr_debug("QAT: SRC VF driver (ver=%u) is incompatible with DST PF driver (ver=%u)\n", ++ *pcompat, ADF_PFVF_COMPATIBILITY_VERSION); ++ return -EINVAL; ++ } ++ ++ if (compat == ADF_PF2VF_VF_COMPAT_UNKNOWN) ++ pr_debug("QAT: SRC VF driver (ver=%u) is newer than DST PF driver (ver=%u)\n", ++ *pcompat, ADF_PFVF_COMPATIBILITY_VERSION); ++ ++ return 0; ++} ++ ++/* ++ * adf_mstate_capmask_compare() - compare QAT device capability mask ++ * @sinfo: Pointer to source capability info ++ * @dinfo: Pointer to target capability info ++ * ++ * This function compares the capability mask between source VF and target VF ++ * ++ * Returns: 0 if target capability mask is identical to source capability mask, ++ * 1 if target mask can represent all the capabilities represented by source mask, ++ * -1 if target mask can't represent all the capabilities represented by source ++ * mask. ++ */ ++static int adf_mstate_capmask_compare(struct adf_mstate_vreginfo *sinfo, ++ struct adf_mstate_vreginfo *dinfo) ++{ ++ u64 src = 0, dst = 0; ++ ++ if (adf_mstate_check_cap_size(sinfo->size, dinfo->size, sizeof(u64))) { ++ pr_debug("QAT: Unexpected capability size %u %u %zu\n", ++ sinfo->size, dinfo->size, sizeof(u64)); ++ return -1; ++ } ++ ++ memcpy(&src, sinfo->addr, sinfo->size); ++ memcpy(&dst, dinfo->addr, dinfo->size); ++ ++ pr_debug("QAT: Check cap compatibility of cap %llu %llu\n", src, dst); ++ ++ if (src == dst) ++ return 0; ++ ++ if ((src | dst) == dst) ++ return 1; ++ ++ return -1; ++} ++ ++static int adf_mstate_capmask_superset(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo sinfo = { buf, size }; ++ ++ if (adf_mstate_capmask_compare(&sinfo, opa) >= 0) ++ return 0; ++ ++ return -EINVAL; ++} ++ ++static int adf_mstate_capmask_equal(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo sinfo = { buf, size }; ++ ++ if (adf_mstate_capmask_compare(&sinfo, opa) == 0) ++ return 0; ++ ++ return -EINVAL; ++} ++ ++static int adf_mstate_set_vreg(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo *info = opa; ++ ++ if (size != info->size) { ++ pr_debug("QAT: Unexpected cap size %u %u\n", size, info->size); ++ return -EINVAL; ++ } ++ memcpy(info->addr, buf, info->size); ++ ++ return 0; ++} ++ ++static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, ++ struct mig_user_sla *pmig_slas) ++{ ++//TODO ++#if 0 ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla **sla_type_arr = NULL; ++ u64 rp_mask, rp_index; ++ u32 max_num_sla; ++ u32 sla_cnt = 0; ++ int i, j; ++ ++ if (!accel_dev->rate_limiting) ++ return 0; ++ ++ rp_index = vf_nr * hw_data->num_banks_per_vf; ++ max_num_sla = adf_rl_get_sla_arr_of_type(rl_data, RL_LEAF, &sla_type_arr); ++ ++ for (i = 0; i < max_num_sla; i++) { ++ if (!sla_type_arr[i]) ++ continue; ++ ++ rp_mask = 0; ++ for (j = 0; j < sla_type_arr[i]->ring_pairs_cnt; j++) ++ rp_mask |= BIT(sla_type_arr[i]->ring_pairs_ids[j]); ++ ++ if (rp_mask & GENMASK_ULL(rp_index + 3, rp_index)) { ++ pmig_slas->rp_mask = rp_mask; ++ pmig_slas->cir = sla_type_arr[i]->cir; ++ pmig_slas->pir = sla_type_arr[i]->pir; ++ pmig_slas->srv = sla_type_arr[i]->srv; ++ pmig_slas++; ++ sla_cnt++; ++ } ++ } ++ ++ return sla_cnt; ++#endif ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, ++ u8 *state, u32 size, void *opa) ++{ ++ struct adf_vf_bank_info *vf_bank_info = opa; ++ struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ u32 pf_bank_nr; ++ int ret; ++ ++ pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ++ ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr, ++ (struct bank_state *)state); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load regs for vf%d bank%d\n", ++ vf_bank_info->vf_nr, vf_bank_info->bank_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_etr_bank(struct adf_accel_dev *accel_dev, ++ u32 vf_nr, u32 bank_nr, ++ struct adf_mstate_mgr *mstate_mgr) ++{ ++ struct adf_vf_bank_info vf_bank_info = {accel_dev, vf_nr, bank_nr}; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ char bank_ids[ADF_MSTATE_ID_LEN]; ++ ++ snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); ++ subsec = adf_mstate_sect_lookup(mstate_mgr, bank_ids, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to lookup sec %s for vf%d bank%d\n", ++ ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, ++ adf_gen4_vfmig_load_etr_regs, ++ &vf_bank_info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec; ++ int ret, i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, ++ NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_ETRB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ ret = adf_gen4_vfmig_load_etr_bank(accel_dev, vf_nr, i, ++ &sub_sects_mgr); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ void __iomem *csr = adf_get_pmisc_base(accel_dev); ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ u64 ofs; ++ } misc_states[] = { ++ {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, ++ NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_MISCB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(misc_states); i++) { ++ struct adf_mstate_vreginfo info; ++ u32 regv; ++ ++ info.addr = ®v; ++ info.size = sizeof(regv); ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ++ misc_states[i].id, ++ adf_mstate_set_vreg, ++ &info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load sec %s\n", misc_states[i].id); ++ return -EINVAL; ++ } ++ ADF_CSR_WR(csr, misc_states[i].ofs, regv); ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct mig_user_sla dst_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ u32 dst_sla_cnt; ++ struct { ++ char *id; ++ int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); ++ struct adf_mstate_vreginfo info; ++ } gen_states[] = { ++ {ADF_MSTATE_IOV_INIT_IDS, adf_mstate_set_vreg, ++ {&vf_info->init, sizeof(vf_info->init)}}, ++ {ADF_MSTATE_COMPAT_VER_IDS, adf_mstate_compatver_check, ++ {&vf_info->compat_ver, sizeof(vf_info->compat_ver)}}, ++ {ADF_MSTATE_SLA_IDS, adf_mstate_sla_check, {dst_slas, 0}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_GEN_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(gen_states); i++) { ++ if (gen_states[i].info.addr == dst_slas) { ++ dst_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, dst_slas); ++ gen_states[i].info.size = dst_sla_cnt * sizeof(struct mig_user_sla); ++ } ++ ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ++ gen_states[i].id, ++ gen_states[i].action, ++ &gen_states[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ gen_states[i].id); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_config(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); ++ struct adf_mstate_vreginfo info; ++ } setups[] = { ++ {ADF_MSTATE_GEN_CAP_IDS, adf_mstate_capmask_superset, ++ {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, ++ {ADF_MSTATE_GEN_SVCMAP_IDS, adf_mstate_capmask_equal, ++ {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, ++ {ADF_MSTATE_GEN_EXTDC_IDS, adf_mstate_capmask_superset, ++ {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_CONFIG_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(setups); i++) { ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, setups[i].id, ++ setups[i].action, &setups[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ setups[i].id); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, ++ u32 size, void *opa) ++{ ++ struct adf_vf_bank_info *vf_bank_info = opa; ++ struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ u32 pf_bank_nr; ++ int ret; ++ ++ pf_bank_nr = vf_bank_info->bank_nr; ++ pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ++ ++ ret = hw_data->bank_state_save(accel_dev, pf_bank_nr, ++ (struct bank_state *)state); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save regs for vf%d bank%d\n", ++ vf_bank_info->vf_nr, vf_bank_info->bank_nr); ++ return ret; ++ } ++ ++ return sizeof(struct bank_state); ++} ++ ++static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev, ++ u32 vf_nr, u32 bank_nr, ++ struct adf_mstate_mgr *mstate_mgr) ++{ ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_vf_bank_info vf_bank_info; ++ struct adf_mstate_mgr sub_sects_mgr; ++ char bank_ids[ADF_MSTATE_ID_LEN]; ++ ++ snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, bank_ids, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ vf_bank_info.accel_dev = accel_dev; ++ vf_bank_info.vf_nr = vf_nr; ++ vf_bank_info.bank_nr = bank_nr; ++ l2_subsec = adf_mstate_sect_add(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, ++ adf_gen4_vfmig_save_etr_regs, ++ &vf_bank_info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec; ++ int ret, i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_ETRB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ ret = adf_gen4_vfmig_save_etr_bank(accel_dev, vf_nr, i, ++ &sub_sects_mgr); ++ if (ret) ++ return ret; ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ void __iomem *csr = adf_get_pmisc_base(accel_dev); ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ u64 offset; ++ } misc_states[] = { ++ {ADF_MSTATE_VINTSRC_IDS, ADF_GEN4_VINTSOU_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTSRC_PF2VM_IDS, ADF_GEN4_VINTSOUPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, ++ }; ++ ktime_t time_exp; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_MISCB_IDS); ++ return -EINVAL; ++ } ++ ++ time_exp = ktime_add_us(ktime_get(), ADF_GEN4_PFVF_RSP_TIMEOUT_US); ++ while (!mutex_trylock(&vf_info->pfvf_mig_lock)) { ++ if (ktime_after(ktime_get(), time_exp)) { ++ dev_err(&GET_DEV(accel_dev), "Failed to get pfvf mig lock\n"); ++ return -ETIMEDOUT; ++ } ++ usleep_range(500, 1000); ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(misc_states); i++) { ++ struct adf_mstate_vreginfo info; ++ u32 regv; ++ ++ info.addr = ®v; ++ info.size = sizeof(regv); ++ regv = ADF_CSR_RD(csr, misc_states[i].offset); ++ ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, ++ misc_states[i].id, ++ &info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ misc_states[i].id); ++ mutex_unlock(&vf_info->pfvf_mig_lock); ++ return -EINVAL; ++ } ++ } ++ ++ mutex_unlock(&vf_info->pfvf_mig_lock); ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct mig_user_sla src_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; ++ u32 src_sla_cnt; ++ struct { ++ char *id; ++ struct adf_mstate_vreginfo info; ++ } gen_states[] = { ++ {ADF_MSTATE_IOV_INIT_IDS, ++ {&vf_info->init, sizeof(vf_info->init)}}, ++ {ADF_MSTATE_COMPAT_VER_IDS, ++ {&vf_info->compat_ver, sizeof(vf_info->compat_ver)}}, ++ {ADF_MSTATE_SLA_IDS, {src_slas, 0}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_GEN_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(gen_states); i++) { ++ if (gen_states[i].info.addr == src_slas) { ++ src_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, src_slas); ++ gen_states[i].info.size = src_sla_cnt * sizeof(struct mig_user_sla); ++ } ++ ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, ++ gen_states[i].id, ++ &gen_states[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ gen_states[i].id); ++ return -EINVAL; ++ } ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_config(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct { ++ char *id; ++ struct adf_mstate_vreginfo info; ++ } setups[] = { ++ {ADF_MSTATE_GEN_CAP_IDS, ++ {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, ++ {ADF_MSTATE_GEN_SVCMAP_IDS, ++ {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, ++ {ADF_MSTATE_GEN_EXTDC_IDS, ++ {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_CONFIG_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(setups); i++) { ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, setups[i].id, ++ &setups[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ setups[i].id); ++ return -EINVAL; ++ } ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ ret = adf_gen4_vfmig_save_setup(mdev); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save setup for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state + mdev->setup_size, ++ mdev->state_size - mdev->setup_size); ++ if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) ++ return -EINVAL; ++ ++ ret = adf_gen4_vfmig_save_generic(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save generic state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_save_misc(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save misc bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_save_etr(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save etr bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ adf_mstate_preamble_update(vfmig->mstate_mgr); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ ret = adf_gen4_vfmig_load_setup(mdev, mdev->state_size); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load setup for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, ++ mdev->state + mdev->remote_setup_size, ++ mdev->state_size - mdev->remote_setup_size, ++ NULL, NULL); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Invalid state for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_generic(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load general state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_misc(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load misc bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_etr(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load etr bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ if (mdev->setup_size) ++ return 0; ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); ++ if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) ++ return -EINVAL; ++ ++ ret = adf_gen4_vfmig_save_config(accel_dev, mdev->vf_id); ++ if (ret) ++ return ret; ++ ++ adf_mstate_preamble_update(vfmig->mstate_mgr); ++ mdev->setup_size = adf_mstate_state_size(vfmig->mstate_mgr); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ u32 setup_size; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ if (mdev->remote_setup_size) ++ return 0; ++ ++ if (len < sizeof(struct adf_mstate_preh)) ++ return -EAGAIN; ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); ++ setup_size = adf_mstate_state_size_from_remote(vfmig->mstate_mgr); ++ if (setup_size > mdev->state_size) ++ return -EINVAL; ++ ++ if (len < setup_size) ++ return -EAGAIN; ++ ++ ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, mdev->state, ++ setup_size, NULL, NULL); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Invalide setup for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ mdev->remote_setup_size = setup_size; ++ ++ ret = adf_gen4_vfmig_load_config(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load config for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) ++{ ++ vfmig_ops->init = adf_gen4_vfmig_init_device; ++ vfmig_ops->cleanup = adf_gen4_vfmig_cleanup_device; ++ vfmig_ops->reset = adf_gen4_vfmig_reset_device; ++ vfmig_ops->open = adf_gen4_vfmig_open_device; ++ vfmig_ops->close = adf_gen4_vfmig_close_device; ++ vfmig_ops->suspend = adf_gen4_vfmig_suspend_device; ++ vfmig_ops->resume = adf_gen4_vfmig_resume_device; ++ vfmig_ops->save_state = adf_gen4_vfmig_save_state; ++ vfmig_ops->load_state = adf_gen4_vfmig_load_state; ++ vfmig_ops->load_setup = adf_gen4_vfmig_load_setup; ++ vfmig_ops->save_setup = adf_gen4_vfmig_save_setup; ++} ++EXPORT_SYMBOL_GPL(adf_gen4_init_vf_mig_ops); +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.c +new file mode 100644 +index 0000000..41cc763 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.c +@@ -0,0 +1,318 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++ ++#include ++#include ++#include "adf_mstate_mgr.h" ++ ++#define ADF_MSTATE_MAGIC 0xADF5CAEA ++#define ADF_MSTATE_VERSION 0x1 ++ ++struct adf_mstate_sect_h { ++ u8 id[ADF_MSTATE_ID_LEN]; ++ u32 size; ++ u32 sub_sects; ++ u8 state[]; ++}; ++ ++u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr) ++{ ++ return mgr->state - mgr->buf; ++} ++ ++static inline u32 adf_mstate_avail_room(struct adf_mstate_mgr *mgr) ++{ ++ return mgr->buf + mgr->size - mgr->state; ++} ++ ++void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size) ++{ ++ mgr->buf = buf; ++ mgr->state = buf; ++ mgr->size = size; ++ mgr->n_sects = 0; ++}; ++ ++struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size) ++{ ++ struct adf_mstate_mgr *mgr; ++ ++ mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); ++ if (!mgr) ++ return NULL; ++ ++ adf_mstate_mgr_init(mgr, buf, size); ++ ++ return mgr; ++} ++ ++void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr) ++{ ++ kfree(mgr); ++} ++ ++void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_mgr *p_mgr) ++{ ++ adf_mstate_mgr_init(mgr, p_mgr->state, ++ p_mgr->size - adf_mstate_state_size(p_mgr)); ++} ++ ++void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *p_sect) ++{ ++ adf_mstate_mgr_init(mgr, p_sect->state, p_sect->size); ++ mgr->n_sects = p_sect->sub_sects; ++} ++ ++static void adf_mstate_preamble_init(struct adf_mstate_preh *preamble) ++{ ++ preamble->magic = ADF_MSTATE_MAGIC; ++ preamble->version = ADF_MSTATE_VERSION; ++ preamble->preh_len = sizeof(*preamble); ++ preamble->size = 0; ++ preamble->n_sects = 0; ++} ++ ++/* default preambles checker */ ++static int adf_mstate_preamble_def_checker(struct adf_mstate_preh *preamble, ++ void *opaque) ++{ ++ struct adf_mstate_mgr *mgr = opaque; ++ ++ if (preamble->magic != ADF_MSTATE_MAGIC || ++ preamble->version > ADF_MSTATE_VERSION || ++ preamble->preh_len > mgr->size) { ++ pr_debug("QAT: LM - Invalid state (magic=%#x, version=%#x, hlen=%u), state_size=%u\n", ++ preamble->magic, preamble->version, preamble->preh_len, ++ mgr->size); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *pre = (struct adf_mstate_preh *)mgr->buf; ++ ++ if (adf_mstate_avail_room(mgr) < sizeof(*pre)) { ++ pr_err("QAT: LM - Not enough space for preamble\n"); ++ return NULL; ++ } ++ ++ adf_mstate_preamble_init(pre); ++ mgr->state += pre->preh_len; ++ ++ return pre; ++} ++ ++int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *preamble = (struct adf_mstate_preh *)mgr->buf; ++ ++ preamble->size = adf_mstate_state_size(mgr) - preamble->preh_len; ++ preamble->n_sects = mgr->n_sects; ++ ++ return 0; ++} ++ ++static void adf_mstate_dump_sect(struct adf_mstate_sect_h *sect, ++ const char *prefix) ++{ ++ pr_debug("QAT: LM - %s QAT state section %s\n", prefix, sect->id); ++ print_hex_dump_debug("h-", DUMP_PREFIX_OFFSET, 16, 2, sect, ++ sizeof(*sect), true); ++ print_hex_dump_debug("s-", DUMP_PREFIX_OFFSET, 16, 2, sect->state, ++ sect->size, true); ++} ++ ++static inline void __adf_mstate_sect_update(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *sect, ++ u32 size, ++ u32 n_subsects) ++{ ++ sect->size += size; ++ sect->sub_sects += n_subsects; ++ mgr->n_sects++; ++ mgr->state += sect->size; ++ ++ adf_mstate_dump_sect(sect, "Add"); ++} ++ ++void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, ++ struct adf_mstate_mgr *curr_mgr, ++ struct adf_mstate_sect_h *sect) ++{ ++ __adf_mstate_sect_update(p_mgr, sect, adf_mstate_state_size(curr_mgr), ++ curr_mgr->n_sects); ++} ++ ++static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mgr *mgr, ++ const char *id) ++{ ++ struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)(mgr->state); ++ ++ if (adf_mstate_avail_room(mgr) < sizeof(*sect)) { ++ pr_debug("QAT: LM - Not enough space for header of QAT state sect %s\n", id); ++ return NULL; ++ } ++ ++ strscpy(sect->id, id, sizeof(sect->id)); ++ sect->size = 0; ++ sect->sub_sects = 0; ++ mgr->state += sizeof(*sect); ++ ++ return sect; ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, ++ const char *id, ++ struct adf_mstate_vreginfo *info) ++{ ++ struct adf_mstate_sect_h *sect; ++ ++ sect = adf_mstate_sect_add_header(mgr, id); ++ if (!sect) ++ return NULL; ++ ++ if (adf_mstate_avail_room(mgr) < info->size) { ++ pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", ++ id, info->size); ++ return NULL; ++ } ++ ++ memcpy(sect->state, info->addr, info->size); ++ __adf_mstate_sect_update(mgr, sect, info->size, 0); ++ ++ return sect; ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_populate populate, ++ void *opaque) ++{ ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *sect; ++ int avail_room, size; ++ ++ sect = adf_mstate_sect_add_header(mgr, id); ++ if (!sect) ++ return NULL; ++ ++ if (!populate) ++ return sect; ++ ++ avail_room = adf_mstate_avail_room(mgr); ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mgr); ++ ++ size = (*populate)(&sub_sects_mgr, sect->state, avail_room, opaque); ++ if (size < 0) ++ return NULL; ++ ++ size += adf_mstate_state_size(&sub_sects_mgr); ++ if (avail_room < size) { ++ pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", ++ id, size); ++ return NULL; ++ } ++ __adf_mstate_sect_update(mgr, sect, size, sub_sects_mgr.n_sects); ++ ++ return sect; ++} ++ ++static int adf_mstate_sect_validate(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_sect_h *start = (struct adf_mstate_sect_h *)mgr->state; ++ struct adf_mstate_sect_h *sect = start; ++ u64 end; ++ int i; ++ ++ end = (uintptr_t)mgr->buf + mgr->size; ++ for (i = 0; i < mgr->n_sects; i++) { ++ uintptr_t s_start = (uintptr_t)sect->state; ++ uintptr_t s_end = s_start + sect->size; ++ ++ if (s_end < s_start || s_end > end) { ++ pr_debug("QAT: LM - Corrupted state section (index=%u, size=%u) in state_mgr (size=%u, secs=%u)\n", ++ i, sect->size, mgr->size, mgr->n_sects); ++ return -EINVAL; ++ } ++ sect = (struct adf_mstate_sect_h *)s_end; ++ } ++ ++ pr_debug("QAT: LM - Scanned section (last child=%s, size=%lu) in state_mgr (size=%u, secs=%u)\n", ++ start->id, sizeof(struct adf_mstate_sect_h) * (ulong)(sect - start), ++ mgr->size, mgr->n_sects); ++ ++ return 0; ++} ++ ++u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *preh = (struct adf_mstate_preh *)mgr->buf; ++ ++ return preh->preh_len + preh->size; ++} ++ ++int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, u8 *buf, u32 size, ++ adf_mstate_preamble_checker pre_checker, ++ void *opaque) ++{ ++ struct adf_mstate_preh *pre; ++ int ret; ++ ++ adf_mstate_mgr_init(mgr, buf, size); ++ pre = (struct adf_mstate_preh *)(mgr->buf); ++ ++ pr_debug("QAT: LM - Dump state preambles\n"); ++ print_hex_dump_debug("", DUMP_PREFIX_OFFSET, 16, 2, pre, pre->preh_len, 0); ++ ++ if (pre_checker) ++ ret = (*pre_checker)(pre, opaque); ++ else ++ ret = adf_mstate_preamble_def_checker(pre, mgr); ++ if (ret) ++ return ret; ++ ++ mgr->state = mgr->buf + pre->preh_len; ++ mgr->n_sects = pre->n_sects; ++ ++ return adf_mstate_sect_validate(mgr); ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_action action, ++ void *opaque) ++{ ++ struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)mgr->state; ++ struct adf_mstate_mgr sub_sects_mgr; ++ int i, ret; ++ ++ for (i = 0; i < mgr->n_sects; i++) { ++ if (!strncmp(sect->id, id, sizeof(sect->id))) ++ goto found; ++ ++ sect = (struct adf_mstate_sect_h *)(sect->state + sect->size); ++ } ++ ++ return NULL; ++ ++found: ++ adf_mstate_dump_sect(sect, "Found"); ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, sect); ++ if (sect->sub_sects && adf_mstate_sect_validate(&sub_sects_mgr)) ++ return NULL; ++ ++ if (!action) ++ return sect; ++ ++ ret = (*action)(&sub_sects_mgr, sect->state, sect->size, opaque); ++ if (ret) ++ return NULL; ++ ++ return sect; ++} +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.h +new file mode 100644 +index 0000000..81d263a +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_mstate_mgr.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++ ++#ifndef ADF_MSTATE_MGR_H ++#define ADF_MSTATE_MGR_H ++ ++#define ADF_MSTATE_ID_LEN 8 ++ ++#define ADF_MSTATE_ETRB_IDS "ETRBAR" ++#define ADF_MSTATE_MISCB_IDS "MISCBAR" ++#define ADF_MSTATE_EXTB_IDS "EXTBAR" ++#define ADF_MSTATE_GEN_IDS "GENER" ++#define ADF_MSTATE_CONFIG_IDS "CONFIG" ++#define ADF_MSTATE_SECTION_NUM 5 ++ ++#define ADF_MSTATE_BANK_IDX_IDS "bnk" ++ ++#define ADF_MSTATE_ETR_REGS_IDS "mregs" ++#define ADF_MSTATE_VINTSRC_IDS "visrc" ++#define ADF_MSTATE_VINTMSK_IDS "vimsk" ++#define ADF_MSTATE_SLA_IDS "sla" ++#define ADF_MSTATE_IOV_INIT_IDS "iovinit" ++#define ADF_MSTATE_COMPAT_VER_IDS "compver" ++#define ADF_MSTATE_GEN_CAP_IDS "gencap" ++#define ADF_MSTATE_GEN_SVCMAP_IDS "svcmap" ++#define ADF_MSTATE_GEN_EXTDC_IDS "extdc" ++#define ADF_MSTATE_VINTSRC_PF2VM_IDS "vispv" ++#define ADF_MSTATE_VINTMSK_PF2VM_IDS "vimpv" ++#define ADF_MSTATE_VM2PF_IDS "vm2pf" ++#define ADF_MSTATE_PF2VM_IDS "pf2vm" ++ ++struct adf_mstate_mgr { ++ u8 *buf; ++ u8 *state; ++ u32 size; ++ u32 n_sects; ++}; ++ ++struct adf_mstate_preh { ++ u32 magic; ++ u32 version; ++ u16 preh_len; ++ u16 n_sects; ++ u32 size; ++}; ++ ++struct adf_mstate_vreginfo { ++ void *addr; ++ u32 size; ++}; ++ ++struct adf_mstate_sect_h; ++ ++typedef int (*adf_mstate_preamble_checker)(struct adf_mstate_preh *preamble, void *opa); ++typedef int (*adf_mstate_populate)(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa); ++typedef int (*adf_mstate_action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, ++ void *opa); ++ ++struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size); ++void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr); ++void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size); ++void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_mgr *p_mgr); ++void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *p_sect); ++int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, ++ u8 *buf, u32 size, ++ adf_mstate_preamble_checker checker, ++ void *opaque); ++struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr); ++int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr); ++u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr); ++u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr); ++void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, ++ struct adf_mstate_mgr *curr_mgr, ++ struct adf_mstate_sect_h *sect); ++struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, ++ const char *id, ++ struct adf_mstate_vreginfo *info); ++struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_populate populate, ++ void *opaque); ++struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_action action, ++ void *opaque); ++#endif +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c +index cda3f11..e91d008 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c +@@ -90,7 +90,9 @@ static void adf_iov_send_resp(struct work_struct *work) + struct adf_pf2vf_resp *pf2vf_resp = + container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); + ++ mutex_lock(&pf2vf_resp->vf_info->pfvf_mig_lock); + adf_vf2pf_req_hndl(pf2vf_resp->vf_info); ++ mutex_lock(&pf2vf_resp->vf_info->pfvf_mig_lock); + kfree(pf2vf_resp); + } + +@@ -132,6 +134,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev, const int numvfs) + vf_info->compat_ver = 0; + + mutex_init(&vf_info->pf2vf_lock); ++ mutex_init(&vf_info->pfvf_mig_lock); + ratelimit_state_init(&vf_info->vf2pf_ratelimit, + ADF_IOV_RATELIMIT_INTERVAL, + ADF_IOV_RATELIMIT_BURST); +@@ -198,6 +201,7 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) + + for (i = 0, vf = accel_dev->pf.vf_info; i < numvfs; i++, vf++) { + mutex_destroy(&vf->pf2vf_lock); ++ mutex_destroy(&vf->pfvf_mig_lock); + } + + /* destroy workqueue in PF */ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport_access_macros_gen4.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport_access_macros_gen4.h +index 084efad..1353e82 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport_access_macros_gen4.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_transport_access_macros_gen4.h +@@ -18,6 +18,7 @@ + #define ADF_RING_CSR_EXP_INT_EN_GEN4 0x18C + #define ADF_UQ_WINDOW_SIZE_GEN4 0x2000 + #define ADF_UQ_OFFSET_UNPRIV_GEN4 0 ++#define ADF_GEN4_NUM_BANKS_PER_VF 4 + + #define BUILD_RING_BASE_ADDR_GEN4(addr, size) \ + ((((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) << 6) +@@ -252,4 +253,5 @@ static inline u64 read_base_gen4(void __iomem *csr_base_addr, + #define ADF_WQM_CSR_RINGMODECTL(bank) (0x9000 + ((bank) << 2)) + #define ADF_RINGMODECTL_ENABLE_UQ BIT(0) + ++#define ADF_COALESCED_POLL_TIMEOUT_US (1 * USEC_PER_SEC) + #endif +-- +2.39.3 + diff --git a/1008-vfio-qat-Add-vfio_pci-driver-for-Intel-QAT-VF-device.patch b/1008-vfio-qat-Add-vfio_pci-driver-for-Intel-QAT-VF-device.patch new file mode 100644 index 0000000000000000000000000000000000000000..806bb0194814d204bff5f55078f8dfdc751334ee --- /dev/null +++ b/1008-vfio-qat-Add-vfio_pci-driver-for-Intel-QAT-VF-device.patch @@ -0,0 +1,717 @@ +From 0cda2e30594c3e843180fdbf316cd5602a61f5b8 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Wed, 27 Mar 2024 14:51:22 +0800 +Subject: [PATCH 08/28] vfio/qat: Add vfio_pci driver for Intel QAT VF devices + +derived from f0bbfc391aa7eaa796f09ee40dd1cd78c6c81960 + +NOTE: driver_managed_dma field of pci_driver needs to be evaluated if it +has to be backported + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/vfio/pci/qat/Kconfig | 12 + + quickassist/qat/drivers/vfio/pci/qat/Makefile | 3 + + quickassist/qat/drivers/vfio/pci/qat/main.c | 661 ++++++++++++++++++ + 3 files changed, 676 insertions(+) + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/Kconfig + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/Makefile + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/main.c + +diff --git a/quickassist/qat/drivers/vfio/pci/qat/Kconfig b/quickassist/qat/drivers/vfio/pci/qat/Kconfig +new file mode 100644 +index 0000000..bf52cfa +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/Kconfig +@@ -0,0 +1,12 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++config QAT_VFIO_PCI ++ tristate "VFIO support for QAT VF PCI devices" ++ select VFIO_PCI_CORE ++ depends on CRYPTO_DEV_QAT_4XXX ++ help ++ This provides migration support for Intel(R) QAT Virtual Function ++ using the VFIO framework. ++ ++ To compile this as a module, choose M here: the module ++ will be called qat_vfio_pci. If you don't know what to do here, ++ say N. +diff --git a/quickassist/qat/drivers/vfio/pci/qat/Makefile b/quickassist/qat/drivers/vfio/pci/qat/Makefile +new file mode 100644 +index 0000000..873308b +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/Makefile +@@ -0,0 +1,3 @@ ++# SPDX-License-Identifier: GPL-2.0-only ++obj-$(CONFIG_QAT_VFIO_PCI) += qat_vfio_pci.o ++qat_vfio_pci-y := main.o dummy.o +diff --git a/quickassist/qat/drivers/vfio/pci/qat/main.c b/quickassist/qat/drivers/vfio/pci/qat/main.c +new file mode 100644 +index 0000000..b5ad197 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/main.c +@@ -0,0 +1,661 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "vfio_pci_core.h" ++#include "qat_mig_dev.h" ++ ++struct qat_vf_migration_file { ++ struct file *filp; ++ /* protects migration region context */ ++ struct mutex lock; ++ bool disabled; ++ struct qat_vf_core_device *qat_vdev; ++ ssize_t filled_size; ++}; ++ ++struct qat_vf_core_device { ++ struct vfio_pci_core_device core_device; ++ struct qat_mig_dev *mdev; ++ /* protects migration state */ ++ struct mutex state_mutex; ++ enum vfio_device_mig_state mig_state; ++ struct qat_vf_migration_file *resuming_migf; ++ struct qat_vf_migration_file *saving_migf; ++}; ++ ++static int qat_vf_pci_open_device(struct vfio_device *core_vdev) ++{ ++ struct qat_vf_core_device *qat_vdev = ++ container_of(core_vdev, struct qat_vf_core_device, ++ core_device.vdev); ++ struct vfio_pci_core_device *vdev = &qat_vdev->core_device; ++ int ret; ++ ++ ret = vfio_pci_core_enable(vdev); ++ if (ret) ++ return ret; ++ ++ ret = qat_vfmig_open(qat_vdev->mdev); ++ if (ret) { ++ vfio_pci_core_disable(vdev); ++ return ret; ++ } ++ qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; ++ ++ vfio_pci_core_finish_enable(vdev); ++ ++ return 0; ++} ++ ++static void qat_vf_disable_fd(struct qat_vf_migration_file *migf) ++{ ++ mutex_lock(&migf->lock); ++ migf->disabled = true; ++ migf->filp->f_pos = 0; ++ migf->filled_size = 0; ++ mutex_unlock(&migf->lock); ++} ++ ++static void qat_vf_disable_fds(struct qat_vf_core_device *qat_vdev) ++{ ++ if (qat_vdev->resuming_migf) { ++ qat_vf_disable_fd(qat_vdev->resuming_migf); ++ fput(qat_vdev->resuming_migf->filp); ++ qat_vdev->resuming_migf = NULL; ++ } ++ ++ if (qat_vdev->saving_migf) { ++ qat_vf_disable_fd(qat_vdev->saving_migf); ++ fput(qat_vdev->saving_migf->filp); ++ qat_vdev->saving_migf = NULL; ++ } ++} ++ ++static void qat_vf_pci_close_device(struct vfio_device *core_vdev) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(core_vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ ++ qat_vfmig_close(qat_vdev->mdev); ++ qat_vf_disable_fds(qat_vdev); ++ vfio_pci_core_close_device(core_vdev); ++} ++ ++static long qat_vf_precopy_ioctl(struct file *filp, unsigned int cmd, ++ unsigned long arg) ++{ ++ struct qat_vf_migration_file *migf = filp->private_data; ++ struct qat_vf_core_device *qat_vdev = migf->qat_vdev; ++ struct qat_mig_dev *mig_dev = qat_vdev->mdev; ++ struct vfio_precopy_info info; ++ loff_t *pos = &filp->f_pos; ++ unsigned long minsz; ++ int ret = 0; ++ ++ if (cmd != VFIO_MIG_GET_PRECOPY_INFO) ++ return -ENOTTY; ++ ++ minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); ++ ++ if (copy_from_user(&info, (void __user *)arg, minsz)) ++ return -EFAULT; ++ if (info.argsz < minsz) ++ return -EINVAL; ++ ++ mutex_lock(&qat_vdev->state_mutex); ++ if (qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY && ++ qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY_P2P) { ++ mutex_unlock(&qat_vdev->state_mutex); ++ return -EINVAL; ++ } ++ ++ mutex_lock(&migf->lock); ++ if (migf->disabled) { ++ ret = -ENODEV; ++ goto out; ++ } ++ ++ if (*pos > mig_dev->setup_size) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ info.dirty_bytes = 0; ++ info.initial_bytes = mig_dev->setup_size - *pos; ++ ++out: ++ mutex_unlock(&migf->lock); ++ mutex_unlock(&qat_vdev->state_mutex); ++ if (ret) ++ return ret; ++ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; ++} ++ ++static ssize_t qat_vf_save_read(struct file *filp, char __user *buf, ++ size_t len, loff_t *pos) ++{ ++ struct qat_vf_migration_file *migf = filp->private_data; ++ struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev; ++ ssize_t done = 0; ++ loff_t *offs; ++ int ret; ++ ++ if (pos) ++ return -ESPIPE; ++ offs = &filp->f_pos; ++ ++ mutex_lock(&migf->lock); ++ if (*offs > migf->filled_size || *offs < 0) { ++ done = -EINVAL; ++ goto out_unlock; ++ } ++ ++ if (migf->disabled) { ++ done = -ENODEV; ++ goto out_unlock; ++ } ++ ++ len = min_t(size_t, migf->filled_size - *offs, len); ++ if (len) { ++ ret = copy_to_user(buf, mig_dev->state + *offs, len); ++ if (ret) { ++ done = -EFAULT; ++ goto out_unlock; ++ } ++ *offs += len; ++ done = len; ++ } ++ ++out_unlock: ++ mutex_unlock(&migf->lock); ++ return done; ++} ++ ++static int qat_vf_release_file(struct inode *inode, struct file *filp) ++{ ++ struct qat_vf_migration_file *migf = filp->private_data; ++ ++ qat_vf_disable_fd(migf); ++ mutex_destroy(&migf->lock); ++ kfree(migf); ++ ++ return 0; ++} ++ ++static const struct file_operations qat_vf_save_fops = { ++ .owner = THIS_MODULE, ++ .read = qat_vf_save_read, ++ .unlocked_ioctl = qat_vf_precopy_ioctl, ++ .compat_ioctl = compat_ptr_ioctl, ++ .release = qat_vf_release_file, ++ .llseek = no_llseek, ++}; ++ ++static int qat_vf_save_state(struct qat_vf_core_device *qat_vdev, ++ struct qat_vf_migration_file *migf) ++{ ++ int ret; ++ ++ ret = qat_vfmig_save_state(qat_vdev->mdev); ++ if (ret) ++ return ret; ++ migf->filled_size = qat_vdev->mdev->state_size; ++ ++ return 0; ++} ++ ++static int qat_vf_save_setup(struct qat_vf_core_device *qat_vdev, ++ struct qat_vf_migration_file *migf) ++{ ++ int ret; ++ ++ ret = qat_vfmig_save_setup(qat_vdev->mdev); ++ if (ret) ++ return ret; ++ migf->filled_size = qat_vdev->mdev->setup_size; ++ ++ return 0; ++} ++ ++static struct qat_vf_migration_file * ++qat_vf_save_device_data(struct qat_vf_core_device *qat_vdev, bool pre_copy) ++{ ++ struct qat_vf_migration_file *migf; ++ int ret; ++ ++ migf = kzalloc(sizeof(*migf), GFP_KERNEL); ++ if (!migf) ++ return ERR_PTR(-ENOMEM); ++ ++ migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_save_fops, ++ migf, O_RDONLY); ++ ret = PTR_ERR_OR_ZERO(migf->filp); ++ if (ret) { ++ kfree(migf); ++ return ERR_PTR(ret); ++ } ++ ++ stream_open(migf->filp->f_inode, migf->filp); ++ mutex_init(&migf->lock); ++ ++ if (pre_copy) ++ ret = qat_vf_save_setup(qat_vdev, migf); ++ else ++ ret = qat_vf_save_state(qat_vdev, migf); ++ if (ret) { ++ fput(migf->filp); ++ return ERR_PTR(ret); ++ } ++ ++ migf->qat_vdev = qat_vdev; ++ ++ return migf; ++} ++ ++static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf, ++ size_t len, loff_t *pos) ++{ ++ struct qat_vf_migration_file *migf = filp->private_data; ++ struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev; ++ loff_t end, *offs; ++ ssize_t done = 0; ++ int ret; ++ ++ if (pos) ++ return -ESPIPE; ++ offs = &filp->f_pos; ++ ++ if (*offs < 0 || ++ check_add_overflow((loff_t)len, *offs, &end)) ++ return -EOVERFLOW; ++ ++ if (end > mig_dev->state_size) ++ return -ENOMEM; ++ ++ mutex_lock(&migf->lock); ++ if (migf->disabled) { ++ done = -ENODEV; ++ goto out_unlock; ++ } ++ ++ ret = copy_from_user(mig_dev->state + *offs, buf, len); ++ if (ret) { ++ done = -EFAULT; ++ goto out_unlock; ++ } ++ *offs += len; ++ migf->filled_size += len; ++ ++ ret = qat_vfmig_load_setup(mig_dev, migf->filled_size); ++ if (ret && ret != -EAGAIN) { ++ done = ret; ++ goto out_unlock; ++ } ++ done = len; ++ ++out_unlock: ++ mutex_unlock(&migf->lock); ++ return done; ++} ++ ++static const struct file_operations qat_vf_resume_fops = { ++ .owner = THIS_MODULE, ++ .write = qat_vf_resume_write, ++ .release = qat_vf_release_file, ++ .llseek = no_llseek, ++}; ++ ++static struct qat_vf_migration_file * ++qat_vf_resume_device_data(struct qat_vf_core_device *qat_vdev) ++{ ++ struct qat_vf_migration_file *migf; ++ int ret; ++ ++ migf = kzalloc(sizeof(*migf), GFP_KERNEL); ++ if (!migf) ++ return ERR_PTR(-ENOMEM); ++ ++ migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_resume_fops, migf, O_WRONLY); ++ ret = PTR_ERR_OR_ZERO(migf->filp); ++ if (ret) { ++ kfree(migf); ++ return ERR_PTR(ret); ++ } ++ ++ migf->qat_vdev = qat_vdev; ++ migf->filled_size = 0; ++ stream_open(migf->filp->f_inode, migf->filp); ++ mutex_init(&migf->lock); ++ ++ return migf; ++} ++ ++static int qat_vf_load_device_data(struct qat_vf_core_device *qat_vdev) ++{ ++ return qat_vfmig_load_state(qat_vdev->mdev); ++} ++ ++static struct file *qat_vf_pci_step_device_state(struct qat_vf_core_device *qat_vdev, u32 new) ++{ ++ u32 cur = qat_vdev->mig_state; ++ int ret; ++ ++ if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) || ++ (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) { ++ ret = qat_vfmig_suspend(qat_vdev->mdev); ++ if (ret) ++ return ERR_PTR(ret); ++ return NULL; ++ } ++ ++ if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) || ++ (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_PRE_COPY)) { ++ qat_vfmig_resume(qat_vdev->mdev); ++ return NULL; ++ } ++ ++ if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) || ++ (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P)) ++ return NULL; ++ ++ if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { ++ struct qat_vf_migration_file *migf; ++ ++ migf = qat_vf_save_device_data(qat_vdev, false); ++ if (IS_ERR(migf)) ++ return ERR_CAST(migf); ++ get_file(migf->filp); ++ qat_vdev->saving_migf = migf; ++ return migf->filp; ++ } ++ ++ if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { ++ struct qat_vf_migration_file *migf; ++ ++ migf = qat_vf_resume_device_data(qat_vdev); ++ if (IS_ERR(migf)) ++ return ERR_CAST(migf); ++ get_file(migf->filp); ++ qat_vdev->resuming_migf = migf; ++ return migf->filp; ++ } ++ ++ if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) || ++ (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) || ++ (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_RUNNING_P2P)) { ++ qat_vf_disable_fds(qat_vdev); ++ return NULL; ++ } ++ ++ if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) || ++ (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) { ++ struct qat_vf_migration_file *migf; ++ ++ migf = qat_vf_save_device_data(qat_vdev, true); ++ if (IS_ERR(migf)) ++ return ERR_CAST(migf); ++ get_file(migf->filp); ++ qat_vdev->saving_migf = migf; ++ return migf->filp; ++ } ++ ++ if (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_STOP_COPY) { ++ struct qat_vf_migration_file *migf = qat_vdev->saving_migf; ++ ++ if (!migf) ++ return ERR_PTR(-EINVAL); ++ ret = qat_vf_save_state(qat_vdev, migf); ++ if (ret) ++ return ERR_PTR(ret); ++ return NULL; ++ } ++ ++ if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { ++ ret = qat_vf_load_device_data(qat_vdev); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ qat_vf_disable_fds(qat_vdev); ++ return NULL; ++ } ++ ++ /* vfio_mig_get_next_state() does not use arcs other than the above */ ++ WARN_ON(true); ++ return ERR_PTR(-EINVAL); ++} ++ ++static void qat_vf_reset_done(struct qat_vf_core_device *qat_vdev) ++{ ++ qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; ++ qat_vfmig_reset(qat_vdev->mdev); ++ qat_vf_disable_fds(qat_vdev); ++} ++ ++static struct file *qat_vf_pci_set_device_state(struct vfio_device *vdev, ++ enum vfio_device_mig_state new_state) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ enum vfio_device_mig_state next_state; ++ struct file *res = NULL; ++ int ret; ++ ++ mutex_lock(&qat_vdev->state_mutex); ++ while (new_state != qat_vdev->mig_state) { ++ ret = vfio_mig_get_next_state(vdev, qat_vdev->mig_state, ++ new_state, &next_state); ++ if (ret) { ++ res = ERR_PTR(ret); ++ break; ++ } ++ res = qat_vf_pci_step_device_state(qat_vdev, next_state); ++ if (IS_ERR(res)) ++ break; ++ qat_vdev->mig_state = next_state; ++ if (WARN_ON(res && new_state != qat_vdev->mig_state)) { ++ fput(res); ++ res = ERR_PTR(-EINVAL); ++ break; ++ } ++ } ++ mutex_unlock(&qat_vdev->state_mutex); ++ ++ return res; ++} ++ ++static int qat_vf_pci_get_device_state(struct vfio_device *vdev, ++ enum vfio_device_mig_state *curr_state) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ ++ mutex_lock(&qat_vdev->state_mutex); ++ *curr_state = qat_vdev->mig_state; ++ mutex_unlock(&qat_vdev->state_mutex); ++ ++ return 0; ++} ++ ++static int qat_vf_pci_get_data_size(struct vfio_device *vdev, ++ unsigned long *stop_copy_length) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ ++ mutex_lock(&qat_vdev->state_mutex); ++ *stop_copy_length = qat_vdev->mdev->state_size; ++ mutex_unlock(&qat_vdev->state_mutex); ++ ++ return 0; ++} ++ ++static const struct vfio_migration_ops qat_vf_pci_mig_ops = { ++ .migration_set_state = qat_vf_pci_set_device_state, ++ .migration_get_state = qat_vf_pci_get_device_state, ++ .migration_get_data_size = qat_vf_pci_get_data_size, ++}; ++ ++static void qat_vf_pci_release_dev(struct vfio_device *core_vdev) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(core_vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ ++ qat_vfmig_cleanup(qat_vdev->mdev); ++ qat_vfmig_destroy(qat_vdev->mdev); ++ mutex_destroy(&qat_vdev->state_mutex); ++ vfio_pci_core_release_dev(core_vdev); ++} ++ ++static int qat_vf_pci_init_dev(struct vfio_device *core_vdev) ++{ ++ struct qat_vf_core_device *qat_vdev = container_of(core_vdev, ++ struct qat_vf_core_device, core_device.vdev); ++ struct qat_mig_dev *mdev; ++ struct pci_dev *parent; ++ int ret, vf_id; ++ ++ core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P | ++ VFIO_MIGRATION_PRE_COPY; ++ core_vdev->mig_ops = &qat_vf_pci_mig_ops; ++ ++ ret = vfio_pci_core_init_dev(core_vdev); ++ if (ret) ++ return ret; ++ ++ mutex_init(&qat_vdev->state_mutex); ++ ++ parent = pci_physfn(qat_vdev->core_device.pdev); ++ vf_id = pci_iov_vf_id(qat_vdev->core_device.pdev); ++ if (vf_id < 0) { ++ ret = -ENODEV; ++ goto err_rel; ++ } ++ ++ mdev = qat_vfmig_create(parent, vf_id); ++ if (IS_ERR(mdev)) { ++ ret = PTR_ERR(mdev); ++ goto err_rel; ++ } ++ ++ ret = qat_vfmig_init(mdev); ++ if (ret) ++ goto err_destroy; ++ ++ qat_vdev->mdev = mdev; ++ ++ return 0; ++ ++err_destroy: ++ qat_vfmig_destroy(mdev); ++err_rel: ++ vfio_pci_core_release_dev(core_vdev); ++ return ret; ++} ++ ++static const struct vfio_device_ops qat_vf_pci_ops = { ++ .name = "qat-vf-vfio-pci", ++ .init = qat_vf_pci_init_dev, ++ .release = qat_vf_pci_release_dev, ++ .open_device = qat_vf_pci_open_device, ++ .close_device = qat_vf_pci_close_device, ++ .ioctl = vfio_pci_core_ioctl, ++ .read = vfio_pci_core_read, ++ .write = vfio_pci_core_write, ++ .mmap = vfio_pci_core_mmap, ++ .request = vfio_pci_core_request, ++ .match = vfio_pci_core_match, ++ .bind_iommufd = vfio_iommufd_physical_bind, ++ .unbind_iommufd = vfio_iommufd_physical_unbind, ++ .attach_ioas = vfio_iommufd_physical_attach_ioas, ++ .detach_ioas = vfio_iommufd_physical_detach_ioas, ++}; ++ ++static struct qat_vf_core_device *qat_vf_drvdata(struct pci_dev *pdev) ++{ ++ struct vfio_pci_core_device *core_device = pci_get_drvdata(pdev); ++ ++ return container_of(core_device, struct qat_vf_core_device, core_device); ++} ++ ++static void qat_vf_pci_aer_reset_done(struct pci_dev *pdev) ++{ ++ struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev); ++ ++ if (!qat_vdev->mdev) ++ return; ++ ++ mutex_lock(&qat_vdev->state_mutex); ++ qat_vf_reset_done(qat_vdev); ++ mutex_unlock(&qat_vdev->state_mutex); ++} ++ ++static int ++qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) ++{ ++ struct device *dev = &pdev->dev; ++ struct qat_vf_core_device *qat_vdev; ++ int ret; ++ ++ qat_vdev = vfio_alloc_device(qat_vf_core_device, core_device.vdev, dev, &qat_vf_pci_ops); ++ if (IS_ERR(qat_vdev)) ++ return PTR_ERR(qat_vdev); ++ ++ pci_set_drvdata(pdev, &qat_vdev->core_device); ++ ret = vfio_pci_core_register_device(&qat_vdev->core_device); ++ if (ret) ++ goto out_put_device; ++ ++ return 0; ++ ++out_put_device: ++ vfio_put_device(&qat_vdev->core_device.vdev); ++ return ret; ++} ++ ++static void qat_vf_vfio_pci_remove(struct pci_dev *pdev) ++{ ++ struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev); ++ ++ vfio_pci_core_unregister_device(&qat_vdev->core_device); ++ vfio_put_device(&qat_vdev->core_device.vdev); ++} ++ ++static const struct pci_device_id qat_vf_vfio_pci_table[] = { ++ /* Intel QAT GEN4 4xxx VF device */ ++ { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4941) }, ++ { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4943) }, ++ { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4945) }, ++ {} ++}; ++MODULE_DEVICE_TABLE(pci, qat_vf_vfio_pci_table); ++ ++static const struct pci_error_handlers qat_vf_err_handlers = { ++ .reset_done = qat_vf_pci_aer_reset_done, ++ .error_detected = vfio_pci_core_aer_err_detected, ++}; ++ ++static struct pci_driver qat_vf_vfio_pci_driver = { ++ .name = "qat_vfio_pci", ++ .id_table = qat_vf_vfio_pci_table, ++ .probe = qat_vf_vfio_pci_probe, ++ .remove = qat_vf_vfio_pci_remove, ++ .err_handler = &qat_vf_err_handlers, ++ //.driver_managed_dma = true, ++}; ++module_pci_driver(qat_vf_vfio_pci_driver); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Xin Zeng "); ++MODULE_DESCRIPTION("QAT VFIO PCI - VFIO PCI driver with live migration support for Intel(R) QAT GEN4 device family"); ++MODULE_IMPORT_NS(CRYPTO_QAT); +-- +2.39.3 + diff --git a/1009-add-dummy-helpers-and-c-files-to-make-build-successf.patch b/1009-add-dummy-helpers-and-c-files-to-make-build-successf.patch new file mode 100644 index 0000000000000000000000000000000000000000..e8ecde1726343a6a7660aac7f58c0090da08a2f0 --- /dev/null +++ b/1009-add-dummy-helpers-and-c-files-to-make-build-successf.patch @@ -0,0 +1,2509 @@ +From cf52b97797cae4429148c0b70eaef3dc7375ad1d Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Wed, 27 Mar 2024 14:52:51 +0800 +Subject: [PATCH 09/28] add dummy helpers and c files to make build + successfully + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/vfio/pci/qat/dummy.c | 73 + + .../qat/drivers/vfio/pci/qat/uapi/vfio.h | 1826 +++++++++++++++++ + quickassist/qat/drivers/vfio/pci/qat/vfio.h | 370 ++++ + .../qat/drivers/vfio/pci/qat/vfio_pci_core.h | 195 ++ + 4 files changed, 2464 insertions(+) + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/dummy.c + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/uapi/vfio.h + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/vfio.h + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h + +diff --git a/quickassist/qat/drivers/vfio/pci/qat/dummy.c b/quickassist/qat/drivers/vfio/pci/qat/dummy.c +new file mode 100644 +index 0000000..b2461ac +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/dummy.c +@@ -0,0 +1,73 @@ ++#include "vfio.h" ++#include "vfio_pci_core.h" ++ ++int pci_iov_vf_id(struct pci_dev *dev) ++{ ++ struct pci_dev *pf; ++ ++ if (!dev->is_virtfn) ++ return -EINVAL; ++ ++ pf = pci_physfn(dev); ++ return (pci_dev_id(dev) - (pci_dev_id(pf) + pf->sriov->offset)) / ++ pf->sriov->stride; ++} ++ ++int vfio_mig_get_next_state(struct vfio_device *device, ++ enum vfio_device_mig_state cur_fsm, ++ enum vfio_device_mig_state new_fsm, ++ enum vfio_device_mig_state *next_fsm) ++{ ++ return 0; ++} ++ ++struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, ++ const struct vfio_device_ops *ops) ++{ ++ return NULL; ++} ++ ++/* Will be exported for vfio pci drivers usage */ ++int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, ++ unsigned int type, unsigned int subtype, ++ const struct vfio_pci_regops *ops, ++ size_t size, u32 flags, void *data) ++{ ++ return 0;; ++} ++void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, ++ bool is_disable_idle_d3) {} ++void vfio_pci_core_close_device(struct vfio_device *core_vdev) {} ++int vfio_pci_core_init_dev(struct vfio_device *core_vdev) { return 0; } ++void vfio_pci_core_release_dev(struct vfio_device *core_vdev) {} ++int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) { return 0; } ++void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev) {} ++//extern const struct pci_error_handlers vfio_pci_core_err_handlers; ++int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, ++ int nr_virtfn) { return 0; } ++long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, ++ unsigned long arg) { return 0; } ++int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, ++ void __user *arg, size_t argsz) { return 0; } ++ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, ++ size_t count, loff_t *ppos) { return 0; } ++ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, ++ size_t count, loff_t *ppos) { return 0; } ++int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) { return 0; } ++void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count) {} ++int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf) { return 0; } ++int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) { return 0; } ++void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) { } ++void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev) {} ++int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar) { return 0; } ++pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, ++ pci_channel_state_t state) { return 0; } ; ++ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, ++ void __iomem *io, char __user *buf, ++ loff_t off, size_t count, size_t x_start, ++ size_t x_end, bool iswrite) { return 0; } ++bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, ++ loff_t reg_start, size_t reg_cnt, ++ loff_t *buf_offset, ++ size_t *intersect_count, ++ size_t *register_offset) { return true; } +diff --git a/quickassist/qat/drivers/vfio/pci/qat/uapi/vfio.h b/quickassist/qat/drivers/vfio/pci/qat/uapi/vfio.h +new file mode 100644 +index 0000000..2b68e6c +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/uapi/vfio.h +@@ -0,0 +1,1826 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* ++ * VFIO API definition ++ * ++ * Copyright (C) 2012 Red Hat, Inc. All rights reserved. ++ * Author: Alex Williamson ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License version 2 as ++ * published by the Free Software Foundation. ++ */ ++#ifndef _UAPIVFIO_H ++#define _UAPIVFIO_H ++ ++#include ++#include ++ ++#define VFIO_API_VERSION 0 ++ ++ ++/* Kernel & User level defines for VFIO IOCTLs. */ ++ ++/* Extensions */ ++ ++#define VFIO_TYPE1_IOMMU 1 ++#define VFIO_SPAPR_TCE_IOMMU 2 ++#define VFIO_TYPE1v2_IOMMU 3 ++/* ++ * IOMMU enforces DMA cache coherence (ex. PCIe NoSnoop stripping). This ++ * capability is subject to change as groups are added or removed. ++ */ ++#define VFIO_DMA_CC_IOMMU 4 ++ ++/* Check if EEH is supported */ ++#define VFIO_EEH 5 ++ ++/* Two-stage IOMMU */ ++#define VFIO_TYPE1_NESTING_IOMMU 6 /* Implies v2 */ ++ ++#define VFIO_SPAPR_TCE_v2_IOMMU 7 ++ ++/* ++ * The No-IOMMU IOMMU offers no translation or isolation for devices and ++ * supports no ioctls outside of VFIO_CHECK_EXTENSION. Use of VFIO's No-IOMMU ++ * code will taint the host kernel and should be used with extreme caution. ++ */ ++#define VFIO_NOIOMMU_IOMMU 8 ++ ++/* Supports VFIO_DMA_UNMAP_FLAG_ALL */ ++#define VFIO_UNMAP_ALL 9 ++ ++/* ++ * Supports the vaddr flag for DMA map and unmap. Not supported for mediated ++ * devices, so this capability is subject to change as groups are added or ++ * removed. ++ */ ++#define VFIO_UPDATE_VADDR 10 ++ ++/* ++ * The IOCTL interface is designed for extensibility by embedding the ++ * structure length (argsz) and flags into structures passed between ++ * kernel and userspace. We therefore use the _IO() macro for these ++ * defines to avoid implicitly embedding a size into the ioctl request. ++ * As structure fields are added, argsz will increase to match and flag ++ * bits will be defined to indicate additional fields with valid data. ++ * It's *always* the caller's responsibility to indicate the size of ++ * the structure passed by setting argsz appropriately. ++ */ ++ ++#define VFIO_TYPE (';') ++#define VFIO_BASE 100 ++ ++/* ++ * For extension of INFO ioctls, VFIO makes use of a capability chain ++ * designed after PCI/e capabilities. A flag bit indicates whether ++ * this capability chain is supported and a field defined in the fixed ++ * structure defines the offset of the first capability in the chain. ++ * This field is only valid when the corresponding bit in the flags ++ * bitmap is set. This offset field is relative to the start of the ++ * INFO buffer, as is the next field within each capability header. ++ * The id within the header is a shared address space per INFO ioctl, ++ * while the version field is specific to the capability id. The ++ * contents following the header are specific to the capability id. ++ */ ++struct vfio_info_cap_header { ++ __u16 id; /* Identifies capability */ ++ __u16 version; /* Version specific to the capability ID */ ++ __u32 next; /* Offset of next capability */ ++}; ++ ++/* ++ * Callers of INFO ioctls passing insufficiently sized buffers will see ++ * the capability chain flag bit set, a zero value for the first capability ++ * offset (if available within the provided argsz), and argsz will be ++ * updated to report the necessary buffer size. For compatibility, the ++ * INFO ioctl will not report error in this case, but the capability chain ++ * will not be available. ++ */ ++ ++/* -------- IOCTLs for VFIO file descriptor (/dev/vfio/vfio) -------- */ ++ ++/** ++ * VFIO_GET_API_VERSION - _IO(VFIO_TYPE, VFIO_BASE + 0) ++ * ++ * Report the version of the VFIO API. This allows us to bump the entire ++ * API version should we later need to add or change features in incompatible ++ * ways. ++ * Return: VFIO_API_VERSION ++ * Availability: Always ++ */ ++#define VFIO_GET_API_VERSION _IO(VFIO_TYPE, VFIO_BASE + 0) ++ ++/** ++ * VFIO_CHECK_EXTENSION - _IOW(VFIO_TYPE, VFIO_BASE + 1, __u32) ++ * ++ * Check whether an extension is supported. ++ * Return: 0 if not supported, 1 (or some other positive integer) if supported. ++ * Availability: Always ++ */ ++#define VFIO_CHECK_EXTENSION _IO(VFIO_TYPE, VFIO_BASE + 1) ++ ++/** ++ * VFIO_SET_IOMMU - _IOW(VFIO_TYPE, VFIO_BASE + 2, __s32) ++ * ++ * Set the iommu to the given type. The type must be supported by an ++ * iommu driver as verified by calling CHECK_EXTENSION using the same ++ * type. A group must be set to this file descriptor before this ++ * ioctl is available. The IOMMU interfaces enabled by this call are ++ * specific to the value set. ++ * Return: 0 on success, -errno on failure ++ * Availability: When VFIO group attached ++ */ ++#define VFIO_SET_IOMMU _IO(VFIO_TYPE, VFIO_BASE + 2) ++ ++/* -------- IOCTLs for GROUP file descriptors (/dev/vfio/$GROUP) -------- */ ++ ++/** ++ * VFIO_GROUP_GET_STATUS - _IOR(VFIO_TYPE, VFIO_BASE + 3, ++ * struct vfio_group_status) ++ * ++ * Retrieve information about the group. Fills in provided ++ * struct vfio_group_info. Caller sets argsz. ++ * Return: 0 on succes, -errno on failure. ++ * Availability: Always ++ */ ++struct vfio_group_status { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_GROUP_FLAGS_VIABLE (1 << 0) ++#define VFIO_GROUP_FLAGS_CONTAINER_SET (1 << 1) ++}; ++#define VFIO_GROUP_GET_STATUS _IO(VFIO_TYPE, VFIO_BASE + 3) ++ ++/** ++ * VFIO_GROUP_SET_CONTAINER - _IOW(VFIO_TYPE, VFIO_BASE + 4, __s32) ++ * ++ * Set the container for the VFIO group to the open VFIO file ++ * descriptor provided. Groups may only belong to a single ++ * container. Containers may, at their discretion, support multiple ++ * groups. Only when a container is set are all of the interfaces ++ * of the VFIO file descriptor and the VFIO group file descriptor ++ * available to the user. ++ * Return: 0 on success, -errno on failure. ++ * Availability: Always ++ */ ++#define VFIO_GROUP_SET_CONTAINER _IO(VFIO_TYPE, VFIO_BASE + 4) ++ ++/** ++ * VFIO_GROUP_UNSET_CONTAINER - _IO(VFIO_TYPE, VFIO_BASE + 5) ++ * ++ * Remove the group from the attached container. This is the ++ * opposite of the SET_CONTAINER call and returns the group to ++ * an initial state. All device file descriptors must be released ++ * prior to calling this interface. When removing the last group ++ * from a container, the IOMMU will be disabled and all state lost, ++ * effectively also returning the VFIO file descriptor to an initial ++ * state. ++ * Return: 0 on success, -errno on failure. ++ * Availability: When attached to container ++ */ ++#define VFIO_GROUP_UNSET_CONTAINER _IO(VFIO_TYPE, VFIO_BASE + 5) ++ ++/** ++ * VFIO_GROUP_GET_DEVICE_FD - _IOW(VFIO_TYPE, VFIO_BASE + 6, char) ++ * ++ * Return a new file descriptor for the device object described by ++ * the provided string. The string should match a device listed in ++ * the devices subdirectory of the IOMMU group sysfs entry. The ++ * group containing the device must already be added to this context. ++ * Return: new file descriptor on success, -errno on failure. ++ * Availability: When attached to container ++ */ ++#define VFIO_GROUP_GET_DEVICE_FD _IO(VFIO_TYPE, VFIO_BASE + 6) ++ ++/* --------------- IOCTLs for DEVICE file descriptors --------------- */ ++ ++/** ++ * VFIO_DEVICE_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 7, ++ * struct vfio_device_info) ++ * ++ * Retrieve information about the device. Fills in provided ++ * struct vfio_device_info. Caller sets argsz. ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_device_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_DEVICE_FLAGS_RESET (1 << 0) /* Device supports reset */ ++#define VFIO_DEVICE_FLAGS_PCI (1 << 1) /* vfio-pci device */ ++#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2) /* vfio-platform device */ ++#define VFIO_DEVICE_FLAGS_AMBA (1 << 3) /* vfio-amba device */ ++#define VFIO_DEVICE_FLAGS_CCW (1 << 4) /* vfio-ccw device */ ++#define VFIO_DEVICE_FLAGS_AP (1 << 5) /* vfio-ap device */ ++#define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6) /* vfio-fsl-mc device */ ++#define VFIO_DEVICE_FLAGS_CAPS (1 << 7) /* Info supports caps */ ++#define VFIO_DEVICE_FLAGS_CDX (1 << 8) /* vfio-cdx device */ ++ __u32 num_regions; /* Max region index + 1 */ ++ __u32 num_irqs; /* Max IRQ index + 1 */ ++ __u32 cap_offset; /* Offset within info struct of first cap */ ++ __u32 pad; ++}; ++#define VFIO_DEVICE_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 7) ++ ++/* ++ * Vendor driver using Mediated device framework should provide device_api ++ * attribute in supported type attribute groups. Device API string should be one ++ * of the following corresponding to device flags in vfio_device_info structure. ++ */ ++ ++#define VFIO_DEVICE_API_PCI_STRING "vfio-pci" ++#define VFIO_DEVICE_API_PLATFORM_STRING "vfio-platform" ++#define VFIO_DEVICE_API_AMBA_STRING "vfio-amba" ++#define VFIO_DEVICE_API_CCW_STRING "vfio-ccw" ++#define VFIO_DEVICE_API_AP_STRING "vfio-ap" ++ ++/* ++ * The following capabilities are unique to s390 zPCI devices. Their contents ++ * are further-defined in vfio_zdev.h ++ */ ++#define VFIO_DEVICE_INFO_CAP_ZPCI_BASE 1 ++#define VFIO_DEVICE_INFO_CAP_ZPCI_GROUP 2 ++#define VFIO_DEVICE_INFO_CAP_ZPCI_UTIL 3 ++#define VFIO_DEVICE_INFO_CAP_ZPCI_PFIP 4 ++ ++/* ++ * The following VFIO_DEVICE_INFO capability reports support for PCIe AtomicOp ++ * completion to the root bus with supported widths provided via flags. ++ */ ++#define VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP 5 ++struct vfio_device_info_cap_pci_atomic_comp { ++ struct vfio_info_cap_header header; ++ __u32 flags; ++#define VFIO_PCI_ATOMIC_COMP32 (1 << 0) ++#define VFIO_PCI_ATOMIC_COMP64 (1 << 1) ++#define VFIO_PCI_ATOMIC_COMP128 (1 << 2) ++ __u32 reserved; ++}; ++ ++/** ++ * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8, ++ * struct vfio_region_info) ++ * ++ * Retrieve information about a device region. Caller provides ++ * struct vfio_region_info with index value set. Caller sets argsz. ++ * Implementation of region mapping is bus driver specific. This is ++ * intended to describe MMIO, I/O port, as well as bus specific ++ * regions (ex. PCI config space). Zero sized regions may be used ++ * to describe unimplemented regions (ex. unimplemented PCI BARs). ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_region_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_REGION_INFO_FLAG_READ (1 << 0) /* Region supports read */ ++#define VFIO_REGION_INFO_FLAG_WRITE (1 << 1) /* Region supports write */ ++#define VFIO_REGION_INFO_FLAG_MMAP (1 << 2) /* Region supports mmap */ ++#define VFIO_REGION_INFO_FLAG_CAPS (1 << 3) /* Info supports caps */ ++ __u32 index; /* Region index */ ++ __u32 cap_offset; /* Offset within info struct of first cap */ ++ __aligned_u64 size; /* Region size (bytes) */ ++ __aligned_u64 offset; /* Region offset from start of device fd */ ++}; ++#define VFIO_DEVICE_GET_REGION_INFO _IO(VFIO_TYPE, VFIO_BASE + 8) ++ ++/* ++ * The sparse mmap capability allows finer granularity of specifying areas ++ * within a region with mmap support. When specified, the user should only ++ * mmap the offset ranges specified by the areas array. mmaps outside of the ++ * areas specified may fail (such as the range covering a PCI MSI-X table) or ++ * may result in improper device behavior. ++ * ++ * The structures below define version 1 of this capability. ++ */ ++#define VFIO_REGION_INFO_CAP_SPARSE_MMAP 1 ++ ++struct vfio_region_sparse_mmap_area { ++ __aligned_u64 offset; /* Offset of mmap'able area within region */ ++ __aligned_u64 size; /* Size of mmap'able area */ ++}; ++ ++struct vfio_region_info_cap_sparse_mmap { ++ struct vfio_info_cap_header header; ++ __u32 nr_areas; ++ __u32 reserved; ++ struct vfio_region_sparse_mmap_area areas[]; ++}; ++ ++/* ++ * The device specific type capability allows regions unique to a specific ++ * device or class of devices to be exposed. This helps solve the problem for ++ * vfio bus drivers of defining which region indexes correspond to which region ++ * on the device, without needing to resort to static indexes, as done by ++ * vfio-pci. For instance, if we were to go back in time, we might remove ++ * VFIO_PCI_VGA_REGION_INDEX and let vfio-pci simply define that all indexes ++ * greater than or equal to VFIO_PCI_NUM_REGIONS are device specific and we'd ++ * make a "VGA" device specific type to describe the VGA access space. This ++ * means that non-VGA devices wouldn't need to waste this index, and thus the ++ * address space associated with it due to implementation of device file ++ * descriptor offsets in vfio-pci. ++ * ++ * The current implementation is now part of the user ABI, so we can't use this ++ * for VGA, but there are other upcoming use cases, such as opregions for Intel ++ * IGD devices and framebuffers for vGPU devices. We missed VGA, but we'll ++ * use this for future additions. ++ * ++ * The structure below defines version 1 of this capability. ++ */ ++#define VFIO_REGION_INFO_CAP_TYPE 2 ++ ++struct vfio_region_info_cap_type { ++ struct vfio_info_cap_header header; ++ __u32 type; /* global per bus driver */ ++ __u32 subtype; /* type specific */ ++}; ++ ++/* ++ * List of region types, global per bus driver. ++ * If you introduce a new type, please add it here. ++ */ ++ ++/* PCI region type containing a PCI vendor part */ ++#define VFIO_REGION_TYPE_PCI_VENDOR_TYPE (1 << 31) ++#define VFIO_REGION_TYPE_PCI_VENDOR_MASK (0xffff) ++#define VFIO_REGION_TYPE_GFX (1) ++#define VFIO_REGION_TYPE_CCW (2) ++#define VFIO_REGION_TYPE_MIGRATION_DEPRECATED (3) ++ ++/* sub-types for VFIO_REGION_TYPE_PCI_* */ ++ ++/* 8086 vendor PCI sub-types */ ++#define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION (1) ++#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG (2) ++#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG (3) ++ ++/* 10de vendor PCI sub-types */ ++/* ++ * NVIDIA GPU NVlink2 RAM is coherent RAM mapped onto the host address space. ++ * ++ * Deprecated, region no longer provided ++ */ ++#define VFIO_REGION_SUBTYPE_NVIDIA_NVLINK2_RAM (1) ++ ++/* 1014 vendor PCI sub-types */ ++/* ++ * IBM NPU NVlink2 ATSD (Address Translation Shootdown) register of NPU ++ * to do TLB invalidation on a GPU. ++ * ++ * Deprecated, region no longer provided ++ */ ++#define VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD (1) ++ ++/* sub-types for VFIO_REGION_TYPE_GFX */ ++#define VFIO_REGION_SUBTYPE_GFX_EDID (1) ++ ++/** ++ * struct vfio_region_gfx_edid - EDID region layout. ++ * ++ * Set display link state and EDID blob. ++ * ++ * The EDID blob has monitor information such as brand, name, serial ++ * number, physical size, supported video modes and more. ++ * ++ * This special region allows userspace (typically qemu) set a virtual ++ * EDID for the virtual monitor, which allows a flexible display ++ * configuration. ++ * ++ * For the edid blob spec look here: ++ * https://en.wikipedia.org/wiki/Extended_Display_Identification_Data ++ * ++ * On linux systems you can find the EDID blob in sysfs: ++ * /sys/class/drm/${card}/${connector}/edid ++ * ++ * You can use the edid-decode ulility (comes with xorg-x11-utils) to ++ * decode the EDID blob. ++ * ++ * @edid_offset: location of the edid blob, relative to the ++ * start of the region (readonly). ++ * @edid_max_size: max size of the edid blob (readonly). ++ * @edid_size: actual edid size (read/write). ++ * @link_state: display link state (read/write). ++ * VFIO_DEVICE_GFX_LINK_STATE_UP: Monitor is turned on. ++ * VFIO_DEVICE_GFX_LINK_STATE_DOWN: Monitor is turned off. ++ * @max_xres: max display width (0 == no limitation, readonly). ++ * @max_yres: max display height (0 == no limitation, readonly). ++ * ++ * EDID update protocol: ++ * (1) set link-state to down. ++ * (2) update edid blob and size. ++ * (3) set link-state to up. ++ */ ++struct vfio_region_gfx_edid { ++ __u32 edid_offset; ++ __u32 edid_max_size; ++ __u32 edid_size; ++ __u32 max_xres; ++ __u32 max_yres; ++ __u32 link_state; ++#define VFIO_DEVICE_GFX_LINK_STATE_UP 1 ++#define VFIO_DEVICE_GFX_LINK_STATE_DOWN 2 ++}; ++ ++/* sub-types for VFIO_REGION_TYPE_CCW */ ++#define VFIO_REGION_SUBTYPE_CCW_ASYNC_CMD (1) ++#define VFIO_REGION_SUBTYPE_CCW_SCHIB (2) ++#define VFIO_REGION_SUBTYPE_CCW_CRW (3) ++ ++/* sub-types for VFIO_REGION_TYPE_MIGRATION */ ++#define VFIO_REGION_SUBTYPE_MIGRATION_DEPRECATED (1) ++ ++struct vfio_device_migration_info { ++ __u32 device_state; /* VFIO device state */ ++#define VFIO_DEVICE_STATE_V1_STOP (0) ++#define VFIO_DEVICE_STATE_V1_RUNNING (1 << 0) ++#define VFIO_DEVICE_STATE_V1_SAVING (1 << 1) ++#define VFIO_DEVICE_STATE_V1_RESUMING (1 << 2) ++#define VFIO_DEVICE_STATE_MASK (VFIO_DEVICE_STATE_V1_RUNNING | \ ++ VFIO_DEVICE_STATE_V1_SAVING | \ ++ VFIO_DEVICE_STATE_V1_RESUMING) ++ ++#define VFIO_DEVICE_STATE_VALID(state) \ ++ (state & VFIO_DEVICE_STATE_V1_RESUMING ? \ ++ (state & VFIO_DEVICE_STATE_MASK) == VFIO_DEVICE_STATE_V1_RESUMING : 1) ++ ++#define VFIO_DEVICE_STATE_IS_ERROR(state) \ ++ ((state & VFIO_DEVICE_STATE_MASK) == (VFIO_DEVICE_STATE_V1_SAVING | \ ++ VFIO_DEVICE_STATE_V1_RESUMING)) ++ ++#define VFIO_DEVICE_STATE_SET_ERROR(state) \ ++ ((state & ~VFIO_DEVICE_STATE_MASK) | VFIO_DEVICE_STATE_V1_SAVING | \ ++ VFIO_DEVICE_STATE_V1_RESUMING) ++ ++ __u32 reserved; ++ __aligned_u64 pending_bytes; ++ __aligned_u64 data_offset; ++ __aligned_u64 data_size; ++}; ++ ++/* ++ * The MSIX mappable capability informs that MSIX data of a BAR can be mmapped ++ * which allows direct access to non-MSIX registers which happened to be within ++ * the same system page. ++ * ++ * Even though the userspace gets direct access to the MSIX data, the existing ++ * VFIO_DEVICE_SET_IRQS interface must still be used for MSIX configuration. ++ */ ++#define VFIO_REGION_INFO_CAP_MSIX_MAPPABLE 3 ++ ++/* ++ * Capability with compressed real address (aka SSA - small system address) ++ * where GPU RAM is mapped on a system bus. Used by a GPU for DMA routing ++ * and by the userspace to associate a NVLink bridge with a GPU. ++ * ++ * Deprecated, capability no longer provided ++ */ ++#define VFIO_REGION_INFO_CAP_NVLINK2_SSATGT 4 ++ ++struct vfio_region_info_cap_nvlink2_ssatgt { ++ struct vfio_info_cap_header header; ++ __aligned_u64 tgt; ++}; ++ ++/* ++ * Capability with an NVLink link speed. The value is read by ++ * the NVlink2 bridge driver from the bridge's "ibm,nvlink-speed" ++ * property in the device tree. The value is fixed in the hardware ++ * and failing to provide the correct value results in the link ++ * not working with no indication from the driver why. ++ * ++ * Deprecated, capability no longer provided ++ */ ++#define VFIO_REGION_INFO_CAP_NVLINK2_LNKSPD 5 ++ ++struct vfio_region_info_cap_nvlink2_lnkspd { ++ struct vfio_info_cap_header header; ++ __u32 link_speed; ++ __u32 __pad; ++}; ++ ++/** ++ * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9, ++ * struct vfio_irq_info) ++ * ++ * Retrieve information about a device IRQ. Caller provides ++ * struct vfio_irq_info with index value set. Caller sets argsz. ++ * Implementation of IRQ mapping is bus driver specific. Indexes ++ * using multiple IRQs are primarily intended to support MSI-like ++ * interrupt blocks. Zero count irq blocks may be used to describe ++ * unimplemented interrupt types. ++ * ++ * The EVENTFD flag indicates the interrupt index supports eventfd based ++ * signaling. ++ * ++ * The MASKABLE flags indicates the index supports MASK and UNMASK ++ * actions described below. ++ * ++ * AUTOMASKED indicates that after signaling, the interrupt line is ++ * automatically masked by VFIO and the user needs to unmask the line ++ * to receive new interrupts. This is primarily intended to distinguish ++ * level triggered interrupts. ++ * ++ * The NORESIZE flag indicates that the interrupt lines within the index ++ * are setup as a set and new subindexes cannot be enabled without first ++ * disabling the entire index. This is used for interrupts like PCI MSI ++ * and MSI-X where the driver may only use a subset of the available ++ * indexes, but VFIO needs to enable a specific number of vectors ++ * upfront. In the case of MSI-X, where the user can enable MSI-X and ++ * then add and unmask vectors, it's up to userspace to make the decision ++ * whether to allocate the maximum supported number of vectors or tear ++ * down setup and incrementally increase the vectors as each is enabled. ++ * Absence of the NORESIZE flag indicates that vectors can be enabled ++ * and disabled dynamically without impacting other vectors within the ++ * index. ++ */ ++struct vfio_irq_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_IRQ_INFO_EVENTFD (1 << 0) ++#define VFIO_IRQ_INFO_MASKABLE (1 << 1) ++#define VFIO_IRQ_INFO_AUTOMASKED (1 << 2) ++#define VFIO_IRQ_INFO_NORESIZE (1 << 3) ++ __u32 index; /* IRQ index */ ++ __u32 count; /* Number of IRQs within this index */ ++}; ++#define VFIO_DEVICE_GET_IRQ_INFO _IO(VFIO_TYPE, VFIO_BASE + 9) ++ ++/** ++ * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set) ++ * ++ * Set signaling, masking, and unmasking of interrupts. Caller provides ++ * struct vfio_irq_set with all fields set. 'start' and 'count' indicate ++ * the range of subindexes being specified. ++ * ++ * The DATA flags specify the type of data provided. If DATA_NONE, the ++ * operation performs the specified action immediately on the specified ++ * interrupt(s). For example, to unmask AUTOMASKED interrupt [0,0]: ++ * flags = (DATA_NONE|ACTION_UNMASK), index = 0, start = 0, count = 1. ++ * ++ * DATA_BOOL allows sparse support for the same on arrays of interrupts. ++ * For example, to mask interrupts [0,1] and [0,3] (but not [0,2]): ++ * flags = (DATA_BOOL|ACTION_MASK), index = 0, start = 1, count = 3, ++ * data = {1,0,1} ++ * ++ * DATA_EVENTFD binds the specified ACTION to the provided __s32 eventfd. ++ * A value of -1 can be used to either de-assign interrupts if already ++ * assigned or skip un-assigned interrupts. For example, to set an eventfd ++ * to be trigger for interrupts [0,0] and [0,2]: ++ * flags = (DATA_EVENTFD|ACTION_TRIGGER), index = 0, start = 0, count = 3, ++ * data = {fd1, -1, fd2} ++ * If index [0,1] is previously set, two count = 1 ioctls calls would be ++ * required to set [0,0] and [0,2] without changing [0,1]. ++ * ++ * Once a signaling mechanism is set, DATA_BOOL or DATA_NONE can be used ++ * with ACTION_TRIGGER to perform kernel level interrupt loopback testing ++ * from userspace (ie. simulate hardware triggering). ++ * ++ * Setting of an event triggering mechanism to userspace for ACTION_TRIGGER ++ * enables the interrupt index for the device. Individual subindex interrupts ++ * can be disabled using the -1 value for DATA_EVENTFD or the index can be ++ * disabled as a whole with: flags = (DATA_NONE|ACTION_TRIGGER), count = 0. ++ * ++ * Note that ACTION_[UN]MASK specify user->kernel signaling (irqfds) while ++ * ACTION_TRIGGER specifies kernel->user signaling. ++ */ ++struct vfio_irq_set { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_IRQ_SET_DATA_NONE (1 << 0) /* Data not present */ ++#define VFIO_IRQ_SET_DATA_BOOL (1 << 1) /* Data is bool (u8) */ ++#define VFIO_IRQ_SET_DATA_EVENTFD (1 << 2) /* Data is eventfd (s32) */ ++#define VFIO_IRQ_SET_ACTION_MASK (1 << 3) /* Mask interrupt */ ++#define VFIO_IRQ_SET_ACTION_UNMASK (1 << 4) /* Unmask interrupt */ ++#define VFIO_IRQ_SET_ACTION_TRIGGER (1 << 5) /* Trigger interrupt */ ++ __u32 index; ++ __u32 start; ++ __u32 count; ++ __u8 data[]; ++}; ++#define VFIO_DEVICE_SET_IRQS _IO(VFIO_TYPE, VFIO_BASE + 10) ++ ++#define VFIO_IRQ_SET_DATA_TYPE_MASK (VFIO_IRQ_SET_DATA_NONE | \ ++ VFIO_IRQ_SET_DATA_BOOL | \ ++ VFIO_IRQ_SET_DATA_EVENTFD) ++#define VFIO_IRQ_SET_ACTION_TYPE_MASK (VFIO_IRQ_SET_ACTION_MASK | \ ++ VFIO_IRQ_SET_ACTION_UNMASK | \ ++ VFIO_IRQ_SET_ACTION_TRIGGER) ++/** ++ * VFIO_DEVICE_RESET - _IO(VFIO_TYPE, VFIO_BASE + 11) ++ * ++ * Reset a device. ++ */ ++#define VFIO_DEVICE_RESET _IO(VFIO_TYPE, VFIO_BASE + 11) ++ ++/* ++ * The VFIO-PCI bus driver makes use of the following fixed region and ++ * IRQ index mapping. Unimplemented regions return a size of zero. ++ * Unimplemented IRQ types return a count of zero. ++ */ ++ ++enum { ++ VFIO_PCI_BAR0_REGION_INDEX, ++ VFIO_PCI_BAR1_REGION_INDEX, ++ VFIO_PCI_BAR2_REGION_INDEX, ++ VFIO_PCI_BAR3_REGION_INDEX, ++ VFIO_PCI_BAR4_REGION_INDEX, ++ VFIO_PCI_BAR5_REGION_INDEX, ++ VFIO_PCI_ROM_REGION_INDEX, ++ VFIO_PCI_CONFIG_REGION_INDEX, ++ /* ++ * Expose VGA regions defined for PCI base class 03, subclass 00. ++ * This includes I/O port ranges 0x3b0 to 0x3bb and 0x3c0 to 0x3df ++ * as well as the MMIO range 0xa0000 to 0xbffff. Each implemented ++ * range is found at it's identity mapped offset from the region ++ * offset, for example 0x3b0 is region_info.offset + 0x3b0. Areas ++ * between described ranges are unimplemented. ++ */ ++ VFIO_PCI_VGA_REGION_INDEX, ++ VFIO_PCI_NUM_REGIONS = 9 /* Fixed user ABI, region indexes >=9 use */ ++ /* device specific cap to define content. */ ++}; ++ ++enum { ++ VFIO_PCI_INTX_IRQ_INDEX, ++ VFIO_PCI_MSI_IRQ_INDEX, ++ VFIO_PCI_MSIX_IRQ_INDEX, ++ VFIO_PCI_ERR_IRQ_INDEX, ++ VFIO_PCI_REQ_IRQ_INDEX, ++ VFIO_PCI_NUM_IRQS ++}; ++ ++/* ++ * The vfio-ccw bus driver makes use of the following fixed region and ++ * IRQ index mapping. Unimplemented regions return a size of zero. ++ * Unimplemented IRQ types return a count of zero. ++ */ ++ ++enum { ++ VFIO_CCW_CONFIG_REGION_INDEX, ++ VFIO_CCW_NUM_REGIONS ++}; ++ ++enum { ++ VFIO_CCW_IO_IRQ_INDEX, ++ VFIO_CCW_CRW_IRQ_INDEX, ++ VFIO_CCW_REQ_IRQ_INDEX, ++ VFIO_CCW_NUM_IRQS ++}; ++ ++/* ++ * The vfio-ap bus driver makes use of the following IRQ index mapping. ++ * Unimplemented IRQ types return a count of zero. ++ */ ++enum { ++ VFIO_AP_REQ_IRQ_INDEX, ++ VFIO_AP_NUM_IRQS ++}; ++ ++/** ++ * VFIO_DEVICE_GET_PCI_HOT_RESET_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 12, ++ * struct vfio_pci_hot_reset_info) ++ * ++ * This command is used to query the affected devices in the hot reset for ++ * a given device. ++ * ++ * This command always reports the segment, bus, and devfn information for ++ * each affected device, and selectively reports the group_id or devid per ++ * the way how the calling device is opened. ++ * ++ * - If the calling device is opened via the traditional group/container ++ * API, group_id is reported. User should check if it has owned all ++ * the affected devices and provides a set of group fds to prove the ++ * ownership in VFIO_DEVICE_PCI_HOT_RESET ioctl. ++ * ++ * - If the calling device is opened as a cdev, devid is reported. ++ * Flag VFIO_PCI_HOT_RESET_FLAG_DEV_ID is set to indicate this ++ * data type. All the affected devices should be represented in ++ * the dev_set, ex. bound to a vfio driver, and also be owned by ++ * this interface which is determined by the following conditions: ++ * 1) Has a valid devid within the iommufd_ctx of the calling device. ++ * Ownership cannot be determined across separate iommufd_ctx and ++ * the cdev calling conventions do not support a proof-of-ownership ++ * model as provided in the legacy group interface. In this case ++ * valid devid with value greater than zero is provided in the return ++ * structure. ++ * 2) Does not have a valid devid within the iommufd_ctx of the calling ++ * device, but belongs to the same IOMMU group as the calling device ++ * or another opened device that has a valid devid within the ++ * iommufd_ctx of the calling device. This provides implicit ownership ++ * for devices within the same DMA isolation context. In this case ++ * the devid value of VFIO_PCI_DEVID_OWNED is provided in the return ++ * structure. ++ * ++ * A devid value of VFIO_PCI_DEVID_NOT_OWNED is provided in the return ++ * structure for affected devices where device is NOT represented in the ++ * dev_set or ownership is not available. Such devices prevent the use ++ * of VFIO_DEVICE_PCI_HOT_RESET ioctl outside of the proof-of-ownership ++ * calling conventions (ie. via legacy group accessed devices). Flag ++ * VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED would be set when all the ++ * affected devices are represented in the dev_set and also owned by ++ * the user. This flag is available only when ++ * flag VFIO_PCI_HOT_RESET_FLAG_DEV_ID is set, otherwise reserved. ++ * When set, user could invoke VFIO_DEVICE_PCI_HOT_RESET with a zero ++ * length fd array on the calling device as the ownership is validated ++ * by iommufd_ctx. ++ * ++ * Return: 0 on success, -errno on failure: ++ * -enospc = insufficient buffer, -enodev = unsupported for device. ++ */ ++struct vfio_pci_dependent_device { ++ union { ++ __u32 group_id; ++ __u32 devid; ++#define VFIO_PCI_DEVID_OWNED 0 ++#define VFIO_PCI_DEVID_NOT_OWNED -1 ++ }; ++ __u16 segment; ++ __u8 bus; ++ __u8 devfn; /* Use PCI_SLOT/PCI_FUNC */ ++}; ++ ++struct vfio_pci_hot_reset_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_PCI_HOT_RESET_FLAG_DEV_ID (1 << 0) ++#define VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED (1 << 1) ++ __u32 count; ++ struct vfio_pci_dependent_device devices[]; ++}; ++ ++#define VFIO_DEVICE_GET_PCI_HOT_RESET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12) ++ ++/** ++ * VFIO_DEVICE_PCI_HOT_RESET - _IOW(VFIO_TYPE, VFIO_BASE + 13, ++ * struct vfio_pci_hot_reset) ++ * ++ * A PCI hot reset results in either a bus or slot reset which may affect ++ * other devices sharing the bus/slot. The calling user must have ++ * ownership of the full set of affected devices as determined by the ++ * VFIO_DEVICE_GET_PCI_HOT_RESET_INFO ioctl. ++ * ++ * When called on a device file descriptor acquired through the vfio ++ * group interface, the user is required to provide proof of ownership ++ * of those affected devices via the group_fds array in struct ++ * vfio_pci_hot_reset. ++ * ++ * When called on a direct cdev opened vfio device, the flags field of ++ * struct vfio_pci_hot_reset_info reports the ownership status of the ++ * affected devices and this ioctl must be called with an empty group_fds ++ * array. See above INFO ioctl definition for ownership requirements. ++ * ++ * Mixed usage of legacy groups and cdevs across the set of affected ++ * devices is not supported. ++ * ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_pci_hot_reset { ++ __u32 argsz; ++ __u32 flags; ++ __u32 count; ++ __s32 group_fds[]; ++}; ++ ++#define VFIO_DEVICE_PCI_HOT_RESET _IO(VFIO_TYPE, VFIO_BASE + 13) ++ ++/** ++ * VFIO_DEVICE_QUERY_GFX_PLANE - _IOW(VFIO_TYPE, VFIO_BASE + 14, ++ * struct vfio_device_query_gfx_plane) ++ * ++ * Set the drm_plane_type and flags, then retrieve the gfx plane info. ++ * ++ * flags supported: ++ * - VFIO_GFX_PLANE_TYPE_PROBE and VFIO_GFX_PLANE_TYPE_DMABUF are set ++ * to ask if the mdev supports dma-buf. 0 on support, -EINVAL on no ++ * support for dma-buf. ++ * - VFIO_GFX_PLANE_TYPE_PROBE and VFIO_GFX_PLANE_TYPE_REGION are set ++ * to ask if the mdev supports region. 0 on support, -EINVAL on no ++ * support for region. ++ * - VFIO_GFX_PLANE_TYPE_DMABUF or VFIO_GFX_PLANE_TYPE_REGION is set ++ * with each call to query the plane info. ++ * - Others are invalid and return -EINVAL. ++ * ++ * Note: ++ * 1. Plane could be disabled by guest. In that case, success will be ++ * returned with zero-initialized drm_format, size, width and height ++ * fields. ++ * 2. x_hot/y_hot is set to 0xFFFFFFFF if no hotspot information available ++ * ++ * Return: 0 on success, -errno on other failure. ++ */ ++struct vfio_device_gfx_plane_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_GFX_PLANE_TYPE_PROBE (1 << 0) ++#define VFIO_GFX_PLANE_TYPE_DMABUF (1 << 1) ++#define VFIO_GFX_PLANE_TYPE_REGION (1 << 2) ++ /* in */ ++ __u32 drm_plane_type; /* type of plane: DRM_PLANE_TYPE_* */ ++ /* out */ ++ __u32 drm_format; /* drm format of plane */ ++ __aligned_u64 drm_format_mod; /* tiled mode */ ++ __u32 width; /* width of plane */ ++ __u32 height; /* height of plane */ ++ __u32 stride; /* stride of plane */ ++ __u32 size; /* size of plane in bytes, align on page*/ ++ __u32 x_pos; /* horizontal position of cursor plane */ ++ __u32 y_pos; /* vertical position of cursor plane*/ ++ __u32 x_hot; /* horizontal position of cursor hotspot */ ++ __u32 y_hot; /* vertical position of cursor hotspot */ ++ union { ++ __u32 region_index; /* region index */ ++ __u32 dmabuf_id; /* dma-buf id */ ++ }; ++ __u32 reserved; ++}; ++ ++#define VFIO_DEVICE_QUERY_GFX_PLANE _IO(VFIO_TYPE, VFIO_BASE + 14) ++ ++/** ++ * VFIO_DEVICE_GET_GFX_DMABUF - _IOW(VFIO_TYPE, VFIO_BASE + 15, __u32) ++ * ++ * Return a new dma-buf file descriptor for an exposed guest framebuffer ++ * described by the provided dmabuf_id. The dmabuf_id is returned from VFIO_ ++ * DEVICE_QUERY_GFX_PLANE as a token of the exposed guest framebuffer. ++ */ ++ ++#define VFIO_DEVICE_GET_GFX_DMABUF _IO(VFIO_TYPE, VFIO_BASE + 15) ++ ++/** ++ * VFIO_DEVICE_IOEVENTFD - _IOW(VFIO_TYPE, VFIO_BASE + 16, ++ * struct vfio_device_ioeventfd) ++ * ++ * Perform a write to the device at the specified device fd offset, with ++ * the specified data and width when the provided eventfd is triggered. ++ * vfio bus drivers may not support this for all regions, for all widths, ++ * or at all. vfio-pci currently only enables support for BAR regions, ++ * excluding the MSI-X vector table. ++ * ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_device_ioeventfd { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_DEVICE_IOEVENTFD_8 (1 << 0) /* 1-byte write */ ++#define VFIO_DEVICE_IOEVENTFD_16 (1 << 1) /* 2-byte write */ ++#define VFIO_DEVICE_IOEVENTFD_32 (1 << 2) /* 4-byte write */ ++#define VFIO_DEVICE_IOEVENTFD_64 (1 << 3) /* 8-byte write */ ++#define VFIO_DEVICE_IOEVENTFD_SIZE_MASK (0xf) ++ __aligned_u64 offset; /* device fd offset of write */ ++ __aligned_u64 data; /* data to be written */ ++ __s32 fd; /* -1 for de-assignment */ ++ __u32 reserved; ++}; ++ ++#define VFIO_DEVICE_IOEVENTFD _IO(VFIO_TYPE, VFIO_BASE + 16) ++ ++/** ++ * VFIO_DEVICE_FEATURE - _IOWR(VFIO_TYPE, VFIO_BASE + 17, ++ * struct vfio_device_feature) ++ * ++ * Get, set, or probe feature data of the device. The feature is selected ++ * using the FEATURE_MASK portion of the flags field. Support for a feature ++ * can be probed by setting both the FEATURE_MASK and PROBE bits. A probe ++ * may optionally include the GET and/or SET bits to determine read vs write ++ * access of the feature respectively. Probing a feature will return success ++ * if the feature is supported and all of the optionally indicated GET/SET ++ * methods are supported. The format of the data portion of the structure is ++ * specific to the given feature. The data portion is not required for ++ * probing. GET and SET are mutually exclusive, except for use with PROBE. ++ * ++ * Return 0 on success, -errno on failure. ++ */ ++struct vfio_device_feature { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_DEVICE_FEATURE_MASK (0xffff) /* 16-bit feature index */ ++#define VFIO_DEVICE_FEATURE_GET (1 << 16) /* Get feature into data[] */ ++#define VFIO_DEVICE_FEATURE_SET (1 << 17) /* Set feature from data[] */ ++#define VFIO_DEVICE_FEATURE_PROBE (1 << 18) /* Probe feature support */ ++ __u8 data[]; ++}; ++ ++#define VFIO_DEVICE_FEATURE _IO(VFIO_TYPE, VFIO_BASE + 17) ++ ++/* ++ * VFIO_DEVICE_BIND_IOMMUFD - _IOR(VFIO_TYPE, VFIO_BASE + 18, ++ * struct vfio_device_bind_iommufd) ++ * @argsz: User filled size of this data. ++ * @flags: Must be 0. ++ * @iommufd: iommufd to bind. ++ * @out_devid: The device id generated by this bind. devid is a handle for ++ * this device/iommufd bond and can be used in IOMMUFD commands. ++ * ++ * Bind a vfio_device to the specified iommufd. ++ * ++ * User is restricted from accessing the device before the binding operation ++ * is completed. Only allowed on cdev fds. ++ * ++ * Unbind is automatically conducted when device fd is closed. ++ * ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_device_bind_iommufd { ++ __u32 argsz; ++ __u32 flags; ++ __s32 iommufd; ++ __u32 out_devid; ++}; ++ ++#define VFIO_DEVICE_BIND_IOMMUFD _IO(VFIO_TYPE, VFIO_BASE + 18) ++ ++/* ++ * VFIO_DEVICE_ATTACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 19, ++ * struct vfio_device_attach_iommufd_pt) ++ * @argsz: User filled size of this data. ++ * @flags: Must be 0. ++ * @pt_id: Input the target id which can represent an ioas or a hwpt ++ * allocated via iommufd subsystem. ++ * Output the input ioas id or the attached hwpt id which could ++ * be the specified hwpt itself or a hwpt automatically created ++ * for the specified ioas by kernel during the attachment. ++ * ++ * Associate the device with an address space within the bound iommufd. ++ * Undo by VFIO_DEVICE_DETACH_IOMMUFD_PT or device fd close. This is only ++ * allowed on cdev fds. ++ * ++ * If a vfio device is currently attached to a valid hw_pagetable, without doing ++ * a VFIO_DEVICE_DETACH_IOMMUFD_PT, a second VFIO_DEVICE_ATTACH_IOMMUFD_PT ioctl ++ * passing in another hw_pagetable (hwpt) id is allowed. This action, also known ++ * as a hw_pagetable replacement, will replace the device's currently attached ++ * hw_pagetable with a new hw_pagetable corresponding to the given pt_id. ++ * ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_device_attach_iommufd_pt { ++ __u32 argsz; ++ __u32 flags; ++ __u32 pt_id; ++}; ++ ++#define VFIO_DEVICE_ATTACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 19) ++ ++/* ++ * VFIO_DEVICE_DETACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 20, ++ * struct vfio_device_detach_iommufd_pt) ++ * @argsz: User filled size of this data. ++ * @flags: Must be 0. ++ * ++ * Remove the association of the device and its current associated address ++ * space. After it, the device should be in a blocking DMA state. This is only ++ * allowed on cdev fds. ++ * ++ * Return: 0 on success, -errno on failure. ++ */ ++struct vfio_device_detach_iommufd_pt { ++ __u32 argsz; ++ __u32 flags; ++}; ++ ++#define VFIO_DEVICE_DETACH_IOMMUFD_PT _IO(VFIO_TYPE, VFIO_BASE + 20) ++ ++/* ++ * Provide support for setting a PCI VF Token, which is used as a shared ++ * secret between PF and VF drivers. This feature may only be set on a ++ * PCI SR-IOV PF when SR-IOV is enabled on the PF and there are no existing ++ * open VFs. Data provided when setting this feature is a 16-byte array ++ * (__u8 b[16]), representing a UUID. ++ */ ++#define VFIO_DEVICE_FEATURE_PCI_VF_TOKEN (0) ++ ++/* ++ * Indicates the device can support the migration API through ++ * VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE. If this GET succeeds, the RUNNING and ++ * ERROR states are always supported. Support for additional states is ++ * indicated via the flags field; at least VFIO_MIGRATION_STOP_COPY must be ++ * set. ++ * ++ * VFIO_MIGRATION_STOP_COPY means that STOP, STOP_COPY and ++ * RESUMING are supported. ++ * ++ * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P means that RUNNING_P2P ++ * is supported in addition to the STOP_COPY states. ++ * ++ * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY means that ++ * PRE_COPY is supported in addition to the STOP_COPY states. ++ * ++ * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P | VFIO_MIGRATION_PRE_COPY ++ * means that RUNNING_P2P, PRE_COPY and PRE_COPY_P2P are supported ++ * in addition to the STOP_COPY states. ++ * ++ * Other combinations of flags have behavior to be defined in the future. ++ */ ++struct vfio_device_feature_migration { ++ __aligned_u64 flags; ++#define VFIO_MIGRATION_STOP_COPY (1 << 0) ++#define VFIO_MIGRATION_P2P (1 << 1) ++#define VFIO_MIGRATION_PRE_COPY (1 << 2) ++}; ++#define VFIO_DEVICE_FEATURE_MIGRATION 1 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_SET, execute a migration state change on the VFIO ++ * device. The new state is supplied in device_state, see enum ++ * vfio_device_mig_state for details ++ * ++ * The kernel migration driver must fully transition the device to the new state ++ * value before the operation returns to the user. ++ * ++ * The kernel migration driver must not generate asynchronous device state ++ * transitions outside of manipulation by the user or the VFIO_DEVICE_RESET ++ * ioctl as described above. ++ * ++ * If this function fails then current device_state may be the original ++ * operating state or some other state along the combination transition path. ++ * The user can then decide if it should execute a VFIO_DEVICE_RESET, attempt ++ * to return to the original state, or attempt to return to some other state ++ * such as RUNNING or STOP. ++ * ++ * If the new_state starts a new data transfer session then the FD associated ++ * with that session is returned in data_fd. The user is responsible to close ++ * this FD when it is finished. The user must consider the migration data stream ++ * carried over the FD to be opaque and must preserve the byte order of the ++ * stream. The user is not required to preserve buffer segmentation when writing ++ * the data stream during the RESUMING operation. ++ * ++ * Upon VFIO_DEVICE_FEATURE_GET, get the current migration state of the VFIO ++ * device, data_fd will be -1. ++ */ ++struct vfio_device_feature_mig_state { ++ __u32 device_state; /* From enum vfio_device_mig_state */ ++ __s32 data_fd; ++}; ++#define VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE 2 ++ ++/* ++ * The device migration Finite State Machine is described by the enum ++ * vfio_device_mig_state. Some of the FSM arcs will create a migration data ++ * transfer session by returning a FD, in this case the migration data will ++ * flow over the FD using read() and write() as discussed below. ++ * ++ * There are 5 states to support VFIO_MIGRATION_STOP_COPY: ++ * RUNNING - The device is running normally ++ * STOP - The device does not change the internal or external state ++ * STOP_COPY - The device internal state can be read out ++ * RESUMING - The device is stopped and is loading a new internal state ++ * ERROR - The device has failed and must be reset ++ * ++ * And optional states to support VFIO_MIGRATION_P2P: ++ * RUNNING_P2P - RUNNING, except the device cannot do peer to peer DMA ++ * And VFIO_MIGRATION_PRE_COPY: ++ * PRE_COPY - The device is running normally but tracking internal state ++ * changes ++ * And VFIO_MIGRATION_P2P | VFIO_MIGRATION_PRE_COPY: ++ * PRE_COPY_P2P - PRE_COPY, except the device cannot do peer to peer DMA ++ * ++ * The FSM takes actions on the arcs between FSM states. The driver implements ++ * the following behavior for the FSM arcs: ++ * ++ * RUNNING_P2P -> STOP ++ * STOP_COPY -> STOP ++ * While in STOP the device must stop the operation of the device. The device ++ * must not generate interrupts, DMA, or any other change to external state. ++ * It must not change its internal state. When stopped the device and kernel ++ * migration driver must accept and respond to interaction to support external ++ * subsystems in the STOP state, for example PCI MSI-X and PCI config space. ++ * Failure by the user to restrict device access while in STOP must not result ++ * in error conditions outside the user context (ex. host system faults). ++ * ++ * The STOP_COPY arc will terminate a data transfer session. ++ * ++ * RESUMING -> STOP ++ * Leaving RESUMING terminates a data transfer session and indicates the ++ * device should complete processing of the data delivered by write(). The ++ * kernel migration driver should complete the incorporation of data written ++ * to the data transfer FD into the device internal state and perform ++ * final validity and consistency checking of the new device state. If the ++ * user provided data is found to be incomplete, inconsistent, or otherwise ++ * invalid, the migration driver must fail the SET_STATE ioctl and ++ * optionally go to the ERROR state as described below. ++ * ++ * While in STOP the device has the same behavior as other STOP states ++ * described above. ++ * ++ * To abort a RESUMING session the device must be reset. ++ * ++ * PRE_COPY -> RUNNING ++ * RUNNING_P2P -> RUNNING ++ * While in RUNNING the device is fully operational, the device may generate ++ * interrupts, DMA, respond to MMIO, all vfio device regions are functional, ++ * and the device may advance its internal state. ++ * ++ * The PRE_COPY arc will terminate a data transfer session. ++ * ++ * PRE_COPY_P2P -> RUNNING_P2P ++ * RUNNING -> RUNNING_P2P ++ * STOP -> RUNNING_P2P ++ * While in RUNNING_P2P the device is partially running in the P2P quiescent ++ * state defined below. ++ * ++ * The PRE_COPY_P2P arc will terminate a data transfer session. ++ * ++ * RUNNING -> PRE_COPY ++ * RUNNING_P2P -> PRE_COPY_P2P ++ * STOP -> STOP_COPY ++ * PRE_COPY, PRE_COPY_P2P and STOP_COPY form the "saving group" of states ++ * which share a data transfer session. Moving between these states alters ++ * what is streamed in session, but does not terminate or otherwise affect ++ * the associated fd. ++ * ++ * These arcs begin the process of saving the device state and will return a ++ * new data_fd. The migration driver may perform actions such as enabling ++ * dirty logging of device state when entering PRE_COPY or PER_COPY_P2P. ++ * ++ * Each arc does not change the device operation, the device remains ++ * RUNNING, P2P quiesced or in STOP. The STOP_COPY state is described below ++ * in PRE_COPY_P2P -> STOP_COPY. ++ * ++ * PRE_COPY -> PRE_COPY_P2P ++ * Entering PRE_COPY_P2P continues all the behaviors of PRE_COPY above. ++ * However, while in the PRE_COPY_P2P state, the device is partially running ++ * in the P2P quiescent state defined below, like RUNNING_P2P. ++ * ++ * PRE_COPY_P2P -> PRE_COPY ++ * This arc allows returning the device to a full RUNNING behavior while ++ * continuing all the behaviors of PRE_COPY. ++ * ++ * PRE_COPY_P2P -> STOP_COPY ++ * While in the STOP_COPY state the device has the same behavior as STOP ++ * with the addition that the data transfers session continues to stream the ++ * migration state. End of stream on the FD indicates the entire device ++ * state has been transferred. ++ * ++ * The user should take steps to restrict access to vfio device regions while ++ * the device is in STOP_COPY or risk corruption of the device migration data ++ * stream. ++ * ++ * STOP -> RESUMING ++ * Entering the RESUMING state starts a process of restoring the device state ++ * and will return a new data_fd. The data stream fed into the data_fd should ++ * be taken from the data transfer output of a single FD during saving from ++ * a compatible device. The migration driver may alter/reset the internal ++ * device state for this arc if required to prepare the device to receive the ++ * migration data. ++ * ++ * STOP_COPY -> PRE_COPY ++ * STOP_COPY -> PRE_COPY_P2P ++ * These arcs are not permitted and return error if requested. Future ++ * revisions of this API may define behaviors for these arcs, in this case ++ * support will be discoverable by a new flag in ++ * VFIO_DEVICE_FEATURE_MIGRATION. ++ * ++ * any -> ERROR ++ * ERROR cannot be specified as a device state, however any transition request ++ * can be failed with an errno return and may then move the device_state into ++ * ERROR. In this case the device was unable to execute the requested arc and ++ * was also unable to restore the device to any valid device_state. ++ * To recover from ERROR VFIO_DEVICE_RESET must be used to return the ++ * device_state back to RUNNING. ++ * ++ * The optional peer to peer (P2P) quiescent state is intended to be a quiescent ++ * state for the device for the purposes of managing multiple devices within a ++ * user context where peer-to-peer DMA between devices may be active. The ++ * RUNNING_P2P and PRE_COPY_P2P states must prevent the device from initiating ++ * any new P2P DMA transactions. If the device can identify P2P transactions ++ * then it can stop only P2P DMA, otherwise it must stop all DMA. The migration ++ * driver must complete any such outstanding operations prior to completing the ++ * FSM arc into a P2P state. For the purpose of specification the states ++ * behave as though the device was fully running if not supported. Like while in ++ * STOP or STOP_COPY the user must not touch the device, otherwise the state ++ * can be exited. ++ * ++ * The remaining possible transitions are interpreted as combinations of the ++ * above FSM arcs. As there are multiple paths through the FSM arcs the path ++ * should be selected based on the following rules: ++ * - Select the shortest path. ++ * - The path cannot have saving group states as interior arcs, only ++ * starting/end states. ++ * Refer to vfio_mig_get_next_state() for the result of the algorithm. ++ * ++ * The automatic transit through the FSM arcs that make up the combination ++ * transition is invisible to the user. When working with combination arcs the ++ * user may see any step along the path in the device_state if SET_STATE ++ * fails. When handling these types of errors users should anticipate future ++ * revisions of this protocol using new states and those states becoming ++ * visible in this case. ++ * ++ * The optional states cannot be used with SET_STATE if the device does not ++ * support them. The user can discover if these states are supported by using ++ * VFIO_DEVICE_FEATURE_MIGRATION. By using combination transitions the user can ++ * avoid knowing about these optional states if the kernel driver supports them. ++ * ++ * Arcs touching PRE_COPY and PRE_COPY_P2P are removed if support for PRE_COPY ++ * is not present. ++ */ ++enum vfio_device_mig_state { ++ VFIO_DEVICE_STATE_ERROR = 0, ++ VFIO_DEVICE_STATE_STOP = 1, ++ VFIO_DEVICE_STATE_RUNNING = 2, ++ VFIO_DEVICE_STATE_STOP_COPY = 3, ++ VFIO_DEVICE_STATE_RESUMING = 4, ++ VFIO_DEVICE_STATE_RUNNING_P2P = 5, ++ VFIO_DEVICE_STATE_PRE_COPY = 6, ++ VFIO_DEVICE_STATE_PRE_COPY_P2P = 7, ++ VFIO_DEVICE_STATE_NR, ++}; ++ ++/** ++ * VFIO_MIG_GET_PRECOPY_INFO - _IO(VFIO_TYPE, VFIO_BASE + 21) ++ * ++ * This ioctl is used on the migration data FD in the precopy phase of the ++ * migration data transfer. It returns an estimate of the current data sizes ++ * remaining to be transferred. It allows the user to judge when it is ++ * appropriate to leave PRE_COPY for STOP_COPY. ++ * ++ * This ioctl is valid only in PRE_COPY states and kernel driver should ++ * return -EINVAL from any other migration state. ++ * ++ * The vfio_precopy_info data structure returned by this ioctl provides ++ * estimates of data available from the device during the PRE_COPY states. ++ * This estimate is split into two categories, initial_bytes and ++ * dirty_bytes. ++ * ++ * The initial_bytes field indicates the amount of initial precopy ++ * data available from the device. This field should have a non-zero initial ++ * value and decrease as migration data is read from the device. ++ * It is recommended to leave PRE_COPY for STOP_COPY only after this field ++ * reaches zero. Leaving PRE_COPY earlier might make things slower. ++ * ++ * The dirty_bytes field tracks device state changes relative to data ++ * previously retrieved. This field starts at zero and may increase as ++ * the internal device state is modified or decrease as that modified ++ * state is read from the device. ++ * ++ * Userspace may use the combination of these fields to estimate the ++ * potential data size available during the PRE_COPY phases, as well as ++ * trends relative to the rate the device is dirtying its internal ++ * state, but these fields are not required to have any bearing relative ++ * to the data size available during the STOP_COPY phase. ++ * ++ * Drivers have a lot of flexibility in when and what they transfer during the ++ * PRE_COPY phase, and how they report this from VFIO_MIG_GET_PRECOPY_INFO. ++ * ++ * During pre-copy the migration data FD has a temporary "end of stream" that is ++ * reached when both initial_bytes and dirty_byte are zero. For instance, this ++ * may indicate that the device is idle and not currently dirtying any internal ++ * state. When read() is done on this temporary end of stream the kernel driver ++ * should return ENOMSG from read(). Userspace can wait for more data (which may ++ * never come) by using poll. ++ * ++ * Once in STOP_COPY the migration data FD has a permanent end of stream ++ * signaled in the usual way by read() always returning 0 and poll always ++ * returning readable. ENOMSG may not be returned in STOP_COPY. ++ * Support for this ioctl is mandatory if a driver claims to support ++ * VFIO_MIGRATION_PRE_COPY. ++ * ++ * Return: 0 on success, -1 and errno set on failure. ++ */ ++struct vfio_precopy_info { ++ __u32 argsz; ++ __u32 flags; ++ __aligned_u64 initial_bytes; ++ __aligned_u64 dirty_bytes; ++}; ++ ++#define VFIO_MIG_GET_PRECOPY_INFO _IO(VFIO_TYPE, VFIO_BASE + 21) ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_SET, allow the device to be moved into a low power ++ * state with the platform-based power management. Device use of lower power ++ * states depends on factors managed by the runtime power management core, ++ * including system level support and coordinating support among dependent ++ * devices. Enabling device low power entry does not guarantee lower power ++ * usage by the device, nor is a mechanism provided through this feature to ++ * know the current power state of the device. If any device access happens ++ * (either from the host or through the vfio uAPI) when the device is in the ++ * low power state, then the host will move the device out of the low power ++ * state as necessary prior to the access. Once the access is completed, the ++ * device may re-enter the low power state. For single shot low power support ++ * with wake-up notification, see ++ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP below. Access to mmap'd ++ * device regions is disabled on LOW_POWER_ENTRY and may only be resumed after ++ * calling LOW_POWER_EXIT. ++ */ ++#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY 3 ++ ++/* ++ * This device feature has the same behavior as ++ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY with the exception that the user ++ * provides an eventfd for wake-up notification. When the device moves out of ++ * the low power state for the wake-up, the host will not allow the device to ++ * re-enter a low power state without a subsequent user call to one of the low ++ * power entry device feature IOCTLs. Access to mmap'd device regions is ++ * disabled on LOW_POWER_ENTRY_WITH_WAKEUP and may only be resumed after the ++ * low power exit. The low power exit can happen either through LOW_POWER_EXIT ++ * or through any other access (where the wake-up notification has been ++ * generated). The access to mmap'd device regions will not trigger low power ++ * exit. ++ * ++ * The notification through the provided eventfd will be generated only when ++ * the device has entered and is resumed from a low power state after ++ * calling this device feature IOCTL. A device that has not entered low power ++ * state, as managed through the runtime power management core, will not ++ * generate a notification through the provided eventfd on access. Calling the ++ * LOW_POWER_EXIT feature is optional in the case where notification has been ++ * signaled on the provided eventfd that a resume from low power has occurred. ++ */ ++struct vfio_device_low_power_entry_with_wakeup { ++ __s32 wakeup_eventfd; ++ __u32 reserved; ++}; ++ ++#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP 4 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_SET, disallow use of device low power states as ++ * previously enabled via VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY or ++ * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP device features. ++ * This device feature IOCTL may itself generate a wakeup eventfd notification ++ * in the latter case if the device had previously entered a low power state. ++ */ ++#define VFIO_DEVICE_FEATURE_LOW_POWER_EXIT 5 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_SET start/stop device DMA logging. ++ * VFIO_DEVICE_FEATURE_PROBE can be used to detect if the device supports ++ * DMA logging. ++ * ++ * DMA logging allows a device to internally record what DMAs the device is ++ * initiating and report them back to userspace. It is part of the VFIO ++ * migration infrastructure that allows implementing dirty page tracking ++ * during the pre copy phase of live migration. Only DMA WRITEs are logged, ++ * and this API is not connected to VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE. ++ * ++ * When DMA logging is started a range of IOVAs to monitor is provided and the ++ * device can optimize its logging to cover only the IOVA range given. Each ++ * DMA that the device initiates inside the range will be logged by the device ++ * for later retrieval. ++ * ++ * page_size is an input that hints what tracking granularity the device ++ * should try to achieve. If the device cannot do the hinted page size then ++ * it's the driver choice which page size to pick based on its support. ++ * On output the device will return the page size it selected. ++ * ++ * ranges is a pointer to an array of ++ * struct vfio_device_feature_dma_logging_range. ++ * ++ * The core kernel code guarantees to support by minimum num_ranges that fit ++ * into a single kernel page. User space can try higher values but should give ++ * up if the above can't be achieved as of some driver limitations. ++ * ++ * A single call to start device DMA logging can be issued and a matching stop ++ * should follow at the end. Another start is not allowed in the meantime. ++ */ ++struct vfio_device_feature_dma_logging_control { ++ __aligned_u64 page_size; ++ __u32 num_ranges; ++ __u32 __reserved; ++ __aligned_u64 ranges; ++}; ++ ++struct vfio_device_feature_dma_logging_range { ++ __aligned_u64 iova; ++ __aligned_u64 length; ++}; ++ ++#define VFIO_DEVICE_FEATURE_DMA_LOGGING_START 6 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_SET stop device DMA logging that was started ++ * by VFIO_DEVICE_FEATURE_DMA_LOGGING_START ++ */ ++#define VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP 7 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_GET read back and clear the device DMA log ++ * ++ * Query the device's DMA log for written pages within the given IOVA range. ++ * During querying the log is cleared for the IOVA range. ++ * ++ * bitmap is a pointer to an array of u64s that will hold the output bitmap ++ * with 1 bit reporting a page_size unit of IOVA. The mapping of IOVA to bits ++ * is given by: ++ * bitmap[(addr - iova)/page_size] & (1ULL << (addr % 64)) ++ * ++ * The input page_size can be any power of two value and does not have to ++ * match the value given to VFIO_DEVICE_FEATURE_DMA_LOGGING_START. The driver ++ * will format its internal logging to match the reporting page size, possibly ++ * by replicating bits if the internal page size is lower than requested. ++ * ++ * The LOGGING_REPORT will only set bits in the bitmap and never clear or ++ * perform any initialization of the user provided bitmap. ++ * ++ * If any error is returned userspace should assume that the dirty log is ++ * corrupted. Error recovery is to consider all memory dirty and try to ++ * restart the dirty tracking, or to abort/restart the whole migration. ++ * ++ * If DMA logging is not enabled, an error will be returned. ++ * ++ */ ++struct vfio_device_feature_dma_logging_report { ++ __aligned_u64 iova; ++ __aligned_u64 length; ++ __aligned_u64 page_size; ++ __aligned_u64 bitmap; ++}; ++ ++#define VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT 8 ++ ++/* ++ * Upon VFIO_DEVICE_FEATURE_GET read back the estimated data length that will ++ * be required to complete stop copy. ++ * ++ * Note: Can be called on each device state. ++ */ ++ ++struct vfio_device_feature_mig_data_size { ++ __aligned_u64 stop_copy_length; ++}; ++ ++#define VFIO_DEVICE_FEATURE_MIG_DATA_SIZE 9 ++ ++/** ++ * Upon VFIO_DEVICE_FEATURE_SET, set or clear the BUS mastering for the device ++ * based on the operation specified in op flag. ++ * ++ * The functionality is incorporated for devices that needs bus master control, ++ * but the in-band device interface lacks the support. Consequently, it is not ++ * applicable to PCI devices, as bus master control for PCI devices is managed ++ * in-band through the configuration space. At present, this feature is supported ++ * only for CDX devices. ++ * When the device's BUS MASTER setting is configured as CLEAR, it will result in ++ * blocking all incoming DMA requests from the device. On the other hand, configuring ++ * the device's BUS MASTER setting as SET (enable) will grant the device the ++ * capability to perform DMA to the host memory. ++ */ ++struct vfio_device_feature_bus_master { ++ __u32 op; ++#define VFIO_DEVICE_FEATURE_CLEAR_MASTER 0 /* Clear Bus Master */ ++#define VFIO_DEVICE_FEATURE_SET_MASTER 1 /* Set Bus Master */ ++}; ++#define VFIO_DEVICE_FEATURE_BUS_MASTER 10 ++ ++/* -------- API for Type1 VFIO IOMMU -------- */ ++ ++/** ++ * VFIO_IOMMU_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 12, struct vfio_iommu_info) ++ * ++ * Retrieve information about the IOMMU object. Fills in provided ++ * struct vfio_iommu_info. Caller sets argsz. ++ * ++ * XXX Should we do these by CHECK_EXTENSION too? ++ */ ++struct vfio_iommu_type1_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_IOMMU_INFO_PGSIZES (1 << 0) /* supported page sizes info */ ++#define VFIO_IOMMU_INFO_CAPS (1 << 1) /* Info supports caps */ ++ __aligned_u64 iova_pgsizes; /* Bitmap of supported page sizes */ ++ __u32 cap_offset; /* Offset within info struct of first cap */ ++ __u32 pad; ++}; ++ ++/* ++ * The IOVA capability allows to report the valid IOVA range(s) ++ * excluding any non-relaxable reserved regions exposed by ++ * devices attached to the container. Any DMA map attempt ++ * outside the valid iova range will return error. ++ * ++ * The structures below define version 1 of this capability. ++ */ ++#define VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE 1 ++ ++struct vfio_iova_range { ++ __u64 start; ++ __u64 end; ++}; ++ ++struct vfio_iommu_type1_info_cap_iova_range { ++ struct vfio_info_cap_header header; ++ __u32 nr_iovas; ++ __u32 reserved; ++ struct vfio_iova_range iova_ranges[]; ++}; ++ ++/* ++ * The migration capability allows to report supported features for migration. ++ * ++ * The structures below define version 1 of this capability. ++ * ++ * The existence of this capability indicates that IOMMU kernel driver supports ++ * dirty page logging. ++ * ++ * pgsize_bitmap: Kernel driver returns bitmap of supported page sizes for dirty ++ * page logging. ++ * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty bitmap ++ * size in bytes that can be used by user applications when getting the dirty ++ * bitmap. ++ */ ++#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION 2 ++ ++struct vfio_iommu_type1_info_cap_migration { ++ struct vfio_info_cap_header header; ++ __u32 flags; ++ __u64 pgsize_bitmap; ++ __u64 max_dirty_bitmap_size; /* in bytes */ ++}; ++ ++/* ++ * The DMA available capability allows to report the current number of ++ * simultaneously outstanding DMA mappings that are allowed. ++ * ++ * The structure below defines version 1 of this capability. ++ * ++ * avail: specifies the current number of outstanding DMA mappings allowed. ++ */ ++#define VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL 3 ++ ++struct vfio_iommu_type1_info_dma_avail { ++ struct vfio_info_cap_header header; ++ __u32 avail; ++}; ++ ++#define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12) ++ ++/** ++ * VFIO_IOMMU_MAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 13, struct vfio_dma_map) ++ * ++ * Map process virtual addresses to IO virtual addresses using the ++ * provided struct vfio_dma_map. Caller sets argsz. READ &/ WRITE required. ++ * ++ * If flags & VFIO_DMA_MAP_FLAG_VADDR, update the base vaddr for iova. The vaddr ++ * must have previously been invalidated with VFIO_DMA_UNMAP_FLAG_VADDR. To ++ * maintain memory consistency within the user application, the updated vaddr ++ * must address the same memory object as originally mapped. Failure to do so ++ * will result in user memory corruption and/or device misbehavior. iova and ++ * size must match those in the original MAP_DMA call. Protection is not ++ * changed, and the READ & WRITE flags must be 0. ++ */ ++struct vfio_iommu_type1_dma_map { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_DMA_MAP_FLAG_READ (1 << 0) /* readable from device */ ++#define VFIO_DMA_MAP_FLAG_WRITE (1 << 1) /* writable from device */ ++#define VFIO_DMA_MAP_FLAG_VADDR (1 << 2) ++ __u64 vaddr; /* Process virtual address */ ++ __u64 iova; /* IO virtual address */ ++ __u64 size; /* Size of mapping (bytes) */ ++}; ++ ++#define VFIO_IOMMU_MAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 13) ++ ++struct vfio_bitmap { ++ __u64 pgsize; /* page size for bitmap in bytes */ ++ __u64 size; /* in bytes */ ++ __u64 __user *data; /* one bit per page */ ++}; ++ ++/** ++ * VFIO_IOMMU_UNMAP_DMA - _IOWR(VFIO_TYPE, VFIO_BASE + 14, ++ * struct vfio_dma_unmap) ++ * ++ * Unmap IO virtual addresses using the provided struct vfio_dma_unmap. ++ * Caller sets argsz. The actual unmapped size is returned in the size ++ * field. No guarantee is made to the user that arbitrary unmaps of iova ++ * or size different from those used in the original mapping call will ++ * succeed. ++ * ++ * VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP should be set to get the dirty bitmap ++ * before unmapping IO virtual addresses. When this flag is set, the user must ++ * provide a struct vfio_bitmap in data[]. User must provide zero-allocated ++ * memory via vfio_bitmap.data and its size in the vfio_bitmap.size field. ++ * A bit in the bitmap represents one page, of user provided page size in ++ * vfio_bitmap.pgsize field, consecutively starting from iova offset. Bit set ++ * indicates that the page at that offset from iova is dirty. A Bitmap of the ++ * pages in the range of unmapped size is returned in the user-provided ++ * vfio_bitmap.data. ++ * ++ * If flags & VFIO_DMA_UNMAP_FLAG_ALL, unmap all addresses. iova and size ++ * must be 0. This cannot be combined with the get-dirty-bitmap flag. ++ * ++ * If flags & VFIO_DMA_UNMAP_FLAG_VADDR, do not unmap, but invalidate host ++ * virtual addresses in the iova range. DMA to already-mapped pages continues. ++ * Groups may not be added to the container while any addresses are invalid. ++ * This cannot be combined with the get-dirty-bitmap flag. ++ */ ++struct vfio_iommu_type1_dma_unmap { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP (1 << 0) ++#define VFIO_DMA_UNMAP_FLAG_ALL (1 << 1) ++#define VFIO_DMA_UNMAP_FLAG_VADDR (1 << 2) ++ __u64 iova; /* IO virtual address */ ++ __u64 size; /* Size of mapping (bytes) */ ++ __u8 data[]; ++}; ++ ++#define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14) ++ ++/* ++ * IOCTLs to enable/disable IOMMU container usage. ++ * No parameters are supported. ++ */ ++#define VFIO_IOMMU_ENABLE _IO(VFIO_TYPE, VFIO_BASE + 15) ++#define VFIO_IOMMU_DISABLE _IO(VFIO_TYPE, VFIO_BASE + 16) ++ ++/** ++ * VFIO_IOMMU_DIRTY_PAGES - _IOWR(VFIO_TYPE, VFIO_BASE + 17, ++ * struct vfio_iommu_type1_dirty_bitmap) ++ * IOCTL is used for dirty pages logging. ++ * Caller should set flag depending on which operation to perform, details as ++ * below: ++ * ++ * Calling the IOCTL with VFIO_IOMMU_DIRTY_PAGES_FLAG_START flag set, instructs ++ * the IOMMU driver to log pages that are dirtied or potentially dirtied by ++ * the device; designed to be used when a migration is in progress. Dirty pages ++ * are logged until logging is disabled by user application by calling the IOCTL ++ * with VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP flag. ++ * ++ * Calling the IOCTL with VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP flag set, instructs ++ * the IOMMU driver to stop logging dirtied pages. ++ * ++ * Calling the IOCTL with VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP flag set ++ * returns the dirty pages bitmap for IOMMU container for a given IOVA range. ++ * The user must specify the IOVA range and the pgsize through the structure ++ * vfio_iommu_type1_dirty_bitmap_get in the data[] portion. This interface ++ * supports getting a bitmap of the smallest supported pgsize only and can be ++ * modified in future to get a bitmap of any specified supported pgsize. The ++ * user must provide a zeroed memory area for the bitmap memory and specify its ++ * size in bitmap.size. One bit is used to represent one page consecutively ++ * starting from iova offset. The user should provide page size in bitmap.pgsize ++ * field. A bit set in the bitmap indicates that the page at that offset from ++ * iova is dirty. The caller must set argsz to a value including the size of ++ * structure vfio_iommu_type1_dirty_bitmap_get, but excluding the size of the ++ * actual bitmap. If dirty pages logging is not enabled, an error will be ++ * returned. ++ * ++ * Only one of the flags _START, _STOP and _GET may be specified at a time. ++ * ++ */ ++struct vfio_iommu_type1_dirty_bitmap { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_IOMMU_DIRTY_PAGES_FLAG_START (1 << 0) ++#define VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP (1 << 1) ++#define VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP (1 << 2) ++ __u8 data[]; ++}; ++ ++struct vfio_iommu_type1_dirty_bitmap_get { ++ __u64 iova; /* IO virtual address */ ++ __u64 size; /* Size of iova range */ ++ struct vfio_bitmap bitmap; ++}; ++ ++#define VFIO_IOMMU_DIRTY_PAGES _IO(VFIO_TYPE, VFIO_BASE + 17) ++ ++/* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */ ++ ++/* ++ * The SPAPR TCE DDW info struct provides the information about ++ * the details of Dynamic DMA window capability. ++ * ++ * @pgsizes contains a page size bitmask, 4K/64K/16M are supported. ++ * @max_dynamic_windows_supported tells the maximum number of windows ++ * which the platform can create. ++ * @levels tells the maximum number of levels in multi-level IOMMU tables; ++ * this allows splitting a table into smaller chunks which reduces ++ * the amount of physically contiguous memory required for the table. ++ */ ++struct vfio_iommu_spapr_tce_ddw_info { ++ __u64 pgsizes; /* Bitmap of supported page sizes */ ++ __u32 max_dynamic_windows_supported; ++ __u32 levels; ++}; ++ ++/* ++ * The SPAPR TCE info struct provides the information about the PCI bus ++ * address ranges available for DMA, these values are programmed into ++ * the hardware so the guest has to know that information. ++ * ++ * The DMA 32 bit window start is an absolute PCI bus address. ++ * The IOVA address passed via map/unmap ioctls are absolute PCI bus ++ * addresses too so the window works as a filter rather than an offset ++ * for IOVA addresses. ++ * ++ * Flags supported: ++ * - VFIO_IOMMU_SPAPR_INFO_DDW: informs the userspace that dynamic DMA windows ++ * (DDW) support is present. @ddw is only supported when DDW is present. ++ */ ++struct vfio_iommu_spapr_tce_info { ++ __u32 argsz; ++ __u32 flags; ++#define VFIO_IOMMU_SPAPR_INFO_DDW (1 << 0) /* DDW supported */ ++ __u32 dma32_window_start; /* 32 bit window start (bytes) */ ++ __u32 dma32_window_size; /* 32 bit window size (bytes) */ ++ struct vfio_iommu_spapr_tce_ddw_info ddw; ++}; ++ ++#define VFIO_IOMMU_SPAPR_TCE_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12) ++ ++/* ++ * EEH PE operation struct provides ways to: ++ * - enable/disable EEH functionality; ++ * - unfreeze IO/DMA for frozen PE; ++ * - read PE state; ++ * - reset PE; ++ * - configure PE; ++ * - inject EEH error. ++ */ ++struct vfio_eeh_pe_err { ++ __u32 type; ++ __u32 func; ++ __u64 addr; ++ __u64 mask; ++}; ++ ++struct vfio_eeh_pe_op { ++ __u32 argsz; ++ __u32 flags; ++ __u32 op; ++ union { ++ struct vfio_eeh_pe_err err; ++ }; ++}; ++ ++#define VFIO_EEH_PE_DISABLE 0 /* Disable EEH functionality */ ++#define VFIO_EEH_PE_ENABLE 1 /* Enable EEH functionality */ ++#define VFIO_EEH_PE_UNFREEZE_IO 2 /* Enable IO for frozen PE */ ++#define VFIO_EEH_PE_UNFREEZE_DMA 3 /* Enable DMA for frozen PE */ ++#define VFIO_EEH_PE_GET_STATE 4 /* PE state retrieval */ ++#define VFIO_EEH_PE_STATE_NORMAL 0 /* PE in functional state */ ++#define VFIO_EEH_PE_STATE_RESET 1 /* PE reset in progress */ ++#define VFIO_EEH_PE_STATE_STOPPED 2 /* Stopped DMA and IO */ ++#define VFIO_EEH_PE_STATE_STOPPED_DMA 4 /* Stopped DMA only */ ++#define VFIO_EEH_PE_STATE_UNAVAIL 5 /* State unavailable */ ++#define VFIO_EEH_PE_RESET_DEACTIVATE 5 /* Deassert PE reset */ ++#define VFIO_EEH_PE_RESET_HOT 6 /* Assert hot reset */ ++#define VFIO_EEH_PE_RESET_FUNDAMENTAL 7 /* Assert fundamental reset */ ++#define VFIO_EEH_PE_CONFIGURE 8 /* PE configuration */ ++#define VFIO_EEH_PE_INJECT_ERR 9 /* Inject EEH error */ ++ ++#define VFIO_EEH_PE_OP _IO(VFIO_TYPE, VFIO_BASE + 21) ++ ++/** ++ * VFIO_IOMMU_SPAPR_REGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 17, struct vfio_iommu_spapr_register_memory) ++ * ++ * Registers user space memory where DMA is allowed. It pins ++ * user pages and does the locked memory accounting so ++ * subsequent VFIO_IOMMU_MAP_DMA/VFIO_IOMMU_UNMAP_DMA calls ++ * get faster. ++ */ ++struct vfio_iommu_spapr_register_memory { ++ __u32 argsz; ++ __u32 flags; ++ __u64 vaddr; /* Process virtual address */ ++ __u64 size; /* Size of mapping (bytes) */ ++}; ++#define VFIO_IOMMU_SPAPR_REGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 17) ++ ++/** ++ * VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 18, struct vfio_iommu_spapr_register_memory) ++ * ++ * Unregisters user space memory registered with ++ * VFIO_IOMMU_SPAPR_REGISTER_MEMORY. ++ * Uses vfio_iommu_spapr_register_memory for parameters. ++ */ ++#define VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 18) ++ ++/** ++ * VFIO_IOMMU_SPAPR_TCE_CREATE - _IOWR(VFIO_TYPE, VFIO_BASE + 19, struct vfio_iommu_spapr_tce_create) ++ * ++ * Creates an additional TCE table and programs it (sets a new DMA window) ++ * to every IOMMU group in the container. It receives page shift, window ++ * size and number of levels in the TCE table being created. ++ * ++ * It allocates and returns an offset on a PCI bus of the new DMA window. ++ */ ++struct vfio_iommu_spapr_tce_create { ++ __u32 argsz; ++ __u32 flags; ++ /* in */ ++ __u32 page_shift; ++ __u32 __resv1; ++ __u64 window_size; ++ __u32 levels; ++ __u32 __resv2; ++ /* out */ ++ __u64 start_addr; ++}; ++#define VFIO_IOMMU_SPAPR_TCE_CREATE _IO(VFIO_TYPE, VFIO_BASE + 19) ++ ++/** ++ * VFIO_IOMMU_SPAPR_TCE_REMOVE - _IOW(VFIO_TYPE, VFIO_BASE + 20, struct vfio_iommu_spapr_tce_remove) ++ * ++ * Unprograms a TCE table from all groups in the container and destroys it. ++ * It receives a PCI bus offset as a window id. ++ */ ++struct vfio_iommu_spapr_tce_remove { ++ __u32 argsz; ++ __u32 flags; ++ /* in */ ++ __u64 start_addr; ++}; ++#define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20) ++ ++/* ***************************************************************** */ ++ ++#endif /* _UAPIVFIO_H */ +diff --git a/quickassist/qat/drivers/vfio/pci/qat/vfio.h b/quickassist/qat/drivers/vfio/pci/qat/vfio.h +new file mode 100644 +index 0000000..603ddc2 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/vfio.h +@@ -0,0 +1,370 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * VFIO API definition ++ * ++ * Copyright (C) 2012 Red Hat, Inc. All rights reserved. ++ * Author: Alex Williamson ++ */ ++#ifndef VFIO_H ++#define VFIO_H ++ ++ ++#include ++#include ++#include ++#include ++#include ++#include "uapi/vfio.h" ++//#include ++ ++struct kvm; ++struct iommufd_ctx; ++struct iommufd_device; ++struct iommufd_access; ++ ++/* ++ * VFIO devices can be placed in a set, this allows all devices to share this ++ * structure and the VFIO core will provide a lock that is held around ++ * open_device()/close_device() for all devices in the set. ++ */ ++struct vfio_device_set { ++ void *set_id; ++ struct mutex lock; ++ struct list_head device_list; ++ unsigned int device_count; ++}; ++ ++struct vfio_device { ++ struct device *dev; ++ const struct vfio_device_ops *ops; ++ /* ++ * mig_ops/log_ops is a static property of the vfio_device which must ++ * be set prior to registering the vfio_device. ++ */ ++ const struct vfio_migration_ops *mig_ops; ++ const struct vfio_log_ops *log_ops; ++#if IS_ENABLED(CONFIG_VFIO_GROUP) ++ struct vfio_group *group; ++ struct list_head group_next; ++ struct list_head iommu_entry; ++#endif ++ struct vfio_device_set *dev_set; ++ struct list_head dev_set_list; ++ unsigned int migration_flags; ++ struct kvm *kvm; ++ ++ /* Members below here are private, not for driver use */ ++ unsigned int index; ++ struct device device; /* device.kref covers object life circle */ ++#if IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) ++ struct cdev cdev; ++#endif ++ refcount_t refcount; /* user count on registered device*/ ++ unsigned int open_count; ++ struct completion comp; ++ struct iommufd_access *iommufd_access; ++ void (*put_kvm)(struct kvm *kvm); ++#if IS_ENABLED(CONFIG_IOMMUFD) ++ struct iommufd_device *iommufd_device; ++ u8 iommufd_attached:1; ++#endif ++ u8 cdev_opened:1; ++#ifdef CONFIG_DEBUG_FS ++ /* ++ * debug_root is a static property of the vfio_device ++ * which must be set prior to registering the vfio_device. ++ */ ++ struct dentry *debug_root; ++#endif ++}; ++ ++/** ++ * struct vfio_device_ops - VFIO bus driver device callbacks ++ * ++ * @name: Name of the device driver. ++ * @init: initialize private fields in device structure ++ * @release: Reclaim private fields in device structure ++ * @bind_iommufd: Called when binding the device to an iommufd ++ * @unbind_iommufd: Opposite of bind_iommufd ++ * @attach_ioas: Called when attaching device to an IOAS/HWPT managed by the ++ * bound iommufd. Undo in unbind_iommufd if @detach_ioas is not ++ * called. ++ * @detach_ioas: Opposite of attach_ioas ++ * @open_device: Called when the first file descriptor is opened for this device ++ * @close_device: Opposite of open_device ++ * @read: Perform read(2) on device file descriptor ++ * @write: Perform write(2) on device file descriptor ++ * @ioctl: Perform ioctl(2) on device file descriptor, supporting VFIO_DEVICE_* ++ * operations documented below ++ * @mmap: Perform mmap(2) on a region of the device file descriptor ++ * @request: Request for the bus driver to release the device ++ * @match: Optional device name match callback (return: 0 for no-match, >0 for ++ * match, -errno for abort (ex. match with insufficient or incorrect ++ * additional args) ++ * @dma_unmap: Called when userspace unmaps IOVA from the container ++ * this device is attached to. ++ * @device_feature: Optional, fill in the VFIO_DEVICE_FEATURE ioctl ++ */ ++struct vfio_device_ops { ++ char *name; ++ int (*init)(struct vfio_device *vdev); ++ void (*release)(struct vfio_device *vdev); ++ int (*bind_iommufd)(struct vfio_device *vdev, ++ struct iommufd_ctx *ictx, u32 *out_device_id); ++ void (*unbind_iommufd)(struct vfio_device *vdev); ++ int (*attach_ioas)(struct vfio_device *vdev, u32 *pt_id); ++ void (*detach_ioas)(struct vfio_device *vdev); ++ int (*open_device)(struct vfio_device *vdev); ++ void (*close_device)(struct vfio_device *vdev); ++ ssize_t (*read)(struct vfio_device *vdev, char __user *buf, ++ size_t count, loff_t *ppos); ++ ssize_t (*write)(struct vfio_device *vdev, const char __user *buf, ++ size_t count, loff_t *size); ++ long (*ioctl)(struct vfio_device *vdev, unsigned int cmd, ++ unsigned long arg); ++ int (*mmap)(struct vfio_device *vdev, struct vm_area_struct *vma); ++ void (*request)(struct vfio_device *vdev, unsigned int count); ++ int (*match)(struct vfio_device *vdev, char *buf); ++ void (*dma_unmap)(struct vfio_device *vdev, u64 iova, u64 length); ++ int (*device_feature)(struct vfio_device *device, u32 flags, ++ void __user *arg, size_t argsz); ++}; ++ ++#if IS_ENABLED(CONFIG_IOMMUFD) ++struct iommufd_ctx *vfio_iommufd_device_ictx(struct vfio_device *vdev); ++int vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx); ++int vfio_iommufd_physical_bind(struct vfio_device *vdev, ++ struct iommufd_ctx *ictx, u32 *out_device_id); ++void vfio_iommufd_physical_unbind(struct vfio_device *vdev); ++int vfio_iommufd_physical_attach_ioas(struct vfio_device *vdev, u32 *pt_id); ++void vfio_iommufd_physical_detach_ioas(struct vfio_device *vdev); ++int vfio_iommufd_emulated_bind(struct vfio_device *vdev, ++ struct iommufd_ctx *ictx, u32 *out_device_id); ++void vfio_iommufd_emulated_unbind(struct vfio_device *vdev); ++int vfio_iommufd_emulated_attach_ioas(struct vfio_device *vdev, u32 *pt_id); ++void vfio_iommufd_emulated_detach_ioas(struct vfio_device *vdev); ++#else ++static inline struct iommufd_ctx * ++vfio_iommufd_device_ictx(struct vfio_device *vdev) ++{ ++ return NULL; ++} ++#define VFIO_PCI_DEVID_NOT_OWNED -1 ++static inline int ++vfio_iommufd_get_dev_id(struct vfio_device *vdev, struct iommufd_ctx *ictx) ++{ ++ return VFIO_PCI_DEVID_NOT_OWNED; ++} ++ ++#define vfio_iommufd_physical_bind \ ++ ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \ ++ u32 *out_device_id)) NULL) ++#define vfio_iommufd_physical_unbind \ ++ ((void (*)(struct vfio_device *vdev)) NULL) ++#define vfio_iommufd_physical_attach_ioas \ ++ ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL) ++#define vfio_iommufd_physical_detach_ioas \ ++ ((void (*)(struct vfio_device *vdev)) NULL) ++#define vfio_iommufd_emulated_bind \ ++ ((int (*)(struct vfio_device *vdev, struct iommufd_ctx *ictx, \ ++ u32 *out_device_id)) NULL) ++#define vfio_iommufd_emulated_unbind \ ++ ((void (*)(struct vfio_device *vdev)) NULL) ++#define vfio_iommufd_emulated_attach_ioas \ ++ ((int (*)(struct vfio_device *vdev, u32 *pt_id)) NULL) ++#define vfio_iommufd_emulated_detach_ioas \ ++ ((void (*)(struct vfio_device *vdev)) NULL) ++#endif ++ ++static inline bool vfio_device_cdev_opened(struct vfio_device *device) ++{ ++ return device->cdev_opened; ++} ++ ++/** ++ * struct vfio_migration_ops - VFIO bus device driver migration callbacks ++ * ++ * @migration_set_state: Optional callback to change the migration state for ++ * devices that support migration. It's mandatory for ++ * VFIO_DEVICE_FEATURE_MIGRATION migration support. ++ * The returned FD is used for data transfer according to the FSM ++ * definition. The driver is responsible to ensure that FD reaches end ++ * of stream or error whenever the migration FSM leaves a data transfer ++ * state or before close_device() returns. ++ * @migration_get_state: Optional callback to get the migration state for ++ * devices that support migration. It's mandatory for ++ * VFIO_DEVICE_FEATURE_MIGRATION migration support. ++ * @migration_get_data_size: Optional callback to get the estimated data ++ * length that will be required to complete stop copy. It's mandatory for ++ * VFIO_DEVICE_FEATURE_MIGRATION migration support. ++ */ ++struct vfio_migration_ops { ++ struct file *(*migration_set_state)( ++ struct vfio_device *device, ++ enum vfio_device_mig_state new_state); ++ int (*migration_get_state)(struct vfio_device *device, ++ enum vfio_device_mig_state *curr_state); ++ int (*migration_get_data_size)(struct vfio_device *device, ++ unsigned long *stop_copy_length); ++}; ++ ++/** ++ * struct vfio_log_ops - VFIO bus device driver logging callbacks ++ * ++ * @log_start: Optional callback to ask the device start DMA logging. ++ * @log_stop: Optional callback to ask the device stop DMA logging. ++ * @log_read_and_clear: Optional callback to ask the device read ++ * and clear the dirty DMAs in some given range. ++ * ++ * The vfio core implementation of the DEVICE_FEATURE_DMA_LOGGING_ set ++ * of features does not track logging state relative to the device, ++ * therefore the device implementation of vfio_log_ops must handle ++ * arbitrary user requests. This includes rejecting subsequent calls ++ * to log_start without an intervening log_stop, as well as graceful ++ * handling of log_stop and log_read_and_clear from invalid states. ++ */ ++#if 0 ++struct vfio_log_ops { ++ int (*log_start)(struct vfio_device *device, ++ struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); ++ int (*log_stop)(struct vfio_device *device); ++ int (*log_read_and_clear)(struct vfio_device *device, ++ unsigned long iova, unsigned long length, ++ struct iova_bitmap *dirty); ++}; ++#endif ++/** ++ * vfio_check_feature - Validate user input for the VFIO_DEVICE_FEATURE ioctl ++ * @flags: Arg from the device_feature op ++ * @argsz: Arg from the device_feature op ++ * @supported_ops: Combination of VFIO_DEVICE_FEATURE_GET and SET the driver ++ * supports ++ * @minsz: Minimum data size the driver accepts ++ * ++ * For use in a driver's device_feature op. Checks that the inputs to the ++ * VFIO_DEVICE_FEATURE ioctl are correct for the driver's feature. Returns 1 if ++ * the driver should execute the get or set, otherwise the relevant ++ * value should be returned. ++ */ ++static inline int vfio_check_feature(u32 flags, size_t argsz, u32 supported_ops, ++ size_t minsz) ++{ ++ if ((flags & (VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_SET)) & ++ ~supported_ops) ++ return -EINVAL; ++ if (flags & VFIO_DEVICE_FEATURE_PROBE) ++ return 0; ++ /* Without PROBE one of GET or SET must be requested */ ++ if (!(flags & (VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_SET))) ++ return -EINVAL; ++ if (argsz < minsz) ++ return -EINVAL; ++ return 1; ++} ++ ++struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, ++ const struct vfio_device_ops *ops); ++#define vfio_alloc_device(dev_struct, member, dev, ops) \ ++ container_of(_vfio_alloc_device(sizeof(struct dev_struct) + \ ++ BUILD_BUG_ON_ZERO(offsetof( \ ++ struct dev_struct, member)), \ ++ dev, ops), \ ++ struct dev_struct, member) ++ ++static inline void vfio_put_device(struct vfio_device *device) ++{ ++ put_device(&device->device); ++} ++ ++int vfio_register_group_dev(struct vfio_device *device); ++int vfio_register_emulated_iommu_dev(struct vfio_device *device); ++void vfio_unregister_group_dev(struct vfio_device *device); ++ ++int vfio_assign_device_set(struct vfio_device *device, void *set_id); ++unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set); ++struct vfio_device * ++vfio_find_device_in_devset(struct vfio_device_set *dev_set, ++ struct device *dev); ++ ++int vfio_mig_get_next_state(struct vfio_device *device, ++ enum vfio_device_mig_state cur_fsm, ++ enum vfio_device_mig_state new_fsm, ++ enum vfio_device_mig_state *next_fsm); ++ ++void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, ++ u32 req_nodes); ++ ++/* ++ * External user API ++ */ ++struct iommu_group *vfio_file_iommu_group(struct file *file); ++ ++#if IS_ENABLED(CONFIG_VFIO_GROUP) ++bool vfio_file_is_group(struct file *file); ++bool vfio_file_has_dev(struct file *file, struct vfio_device *device); ++#else ++static inline bool vfio_file_is_group(struct file *file) ++{ ++ return false; ++} ++ ++static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *device) ++{ ++ return false; ++} ++#endif ++bool vfio_file_is_valid(struct file *file); ++bool vfio_file_enforced_coherent(struct file *file); ++void vfio_file_set_kvm(struct file *file, struct kvm *kvm); ++ ++#define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) ++ ++int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, ++ int npage, int prot, struct page **pages); ++void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage); ++int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, ++ void *data, size_t len, bool write); ++ ++/* ++ * Sub-module helpers ++ */ ++struct vfio_info_cap { ++ struct vfio_info_cap_header *buf; ++ size_t size; ++}; ++struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps, ++ size_t size, u16 id, ++ u16 version); ++void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset); ++ ++int vfio_info_add_capability(struct vfio_info_cap *caps, ++ struct vfio_info_cap_header *cap, size_t size); ++ ++int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, ++ int num_irqs, int max_irq_type, ++ size_t *data_size); ++ ++/* ++ * IRQfd - generic ++ */ ++struct virqfd { ++ void *opaque; ++ struct eventfd_ctx *eventfd; ++ int (*handler)(void *, void *); ++ void (*thread)(void *, void *); ++ void *data; ++ struct work_struct inject; ++ wait_queue_entry_t wait; ++ poll_table pt; ++ struct work_struct shutdown; ++ struct work_struct flush_inject; ++ struct virqfd **pvirqfd; ++}; ++ ++int vfio_virqfd_enable(void *opaque, int (*handler)(void *, void *), ++ void (*thread)(void *, void *), void *data, ++ struct virqfd **pvirqfd, int fd); ++void vfio_virqfd_disable(struct virqfd **pvirqfd); ++void vfio_virqfd_flush_thread(struct virqfd **pvirqfd); ++ ++#endif /* VFIO_H */ +diff --git a/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h b/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h +new file mode 100644 +index 0000000..c77f85b +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h +@@ -0,0 +1,195 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Copyright (C) 2012 Red Hat, Inc. All rights reserved. ++ * Author: Alex Williamson ++ * ++ * Derived from original vfio: ++ * Copyright 2010 Cisco Systems, Inc. All rights reserved. ++ * Author: Tom Lyon, pugs@cisco.com ++ */ ++ ++#include ++#include ++#include "vfio.h" ++#include ++#include ++#include ++#include ++ ++#ifndef VFIO_PCI_CORE_H ++#define VFIO_PCI_CORE_H ++ ++#define VFIO_PCI_OFFSET_SHIFT 40 ++#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) ++#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) ++#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) ++ ++struct vfio_pci_core_device; ++struct vfio_pci_region; ++ ++#define PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \ ++ .vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \ ++ .subdevice = PCI_ANY_ID, ++#define PCI_DRIVER_OVERRIDE_DEVICE_VFIO(vend, dev) \ ++ PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, PCI_ID_F_VFIO_DRIVER_OVERRIDE) ++ ++/* Single Root I/O Virtualization */ ++struct pci_sriov { ++ int pos; /* Capability position */ ++ int nres; /* Number of resources */ ++ u32 cap; /* SR-IOV Capabilities */ ++ u16 ctrl; /* SR-IOV Control */ ++ u16 total_VFs; /* Total VFs associated with the PF */ ++ u16 initial_VFs; /* Initial VFs associated with the PF */ ++ u16 num_VFs; /* Number of VFs available */ ++ u16 offset; /* First VF Routing ID offset */ ++ u16 stride; /* Following VF stride */ ++ u16 vf_device; /* VF device ID */ ++ u32 pgsz; /* Page size for BAR alignment */ ++ u8 link; /* Function Dependency Link */ ++ u8 max_VF_buses; /* Max buses consumed by VFs */ ++ u16 driver_max_VFs; /* Max num VFs driver supports */ ++ struct pci_dev *dev; /* Lowest numbered PF */ ++ struct pci_dev *self; /* This PF */ ++ u32 class; /* VF device */ ++ u8 hdr_type; /* VF header type */ ++ u16 subsystem_vendor; /* VF subsystem vendor */ ++ u16 subsystem_device; /* VF subsystem device */ ++ resource_size_t barsz[PCI_SRIOV_NUM_BARS]; /* VF BAR size */ ++ bool drivers_autoprobe; /* Auto probing of VFs by driver */ ++}; ++ ++int pci_iov_vf_id(struct pci_dev *dev); ++ ++struct vfio_pci_regops { ++ ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf, ++ size_t count, loff_t *ppos, bool iswrite); ++ void (*release)(struct vfio_pci_core_device *vdev, ++ struct vfio_pci_region *region); ++ int (*mmap)(struct vfio_pci_core_device *vdev, ++ struct vfio_pci_region *region, ++ struct vm_area_struct *vma); ++ int (*add_capability)(struct vfio_pci_core_device *vdev, ++ struct vfio_pci_region *region, ++ struct vfio_info_cap *caps); ++}; ++ ++struct vfio_pci_region { ++ u32 type; ++ u32 subtype; ++ const struct vfio_pci_regops *ops; ++ void *data; ++ size_t size; ++ u32 flags; ++}; ++ ++struct vfio_pci_core_device { ++ struct vfio_device vdev; ++ struct pci_dev *pdev; ++ void __iomem *barmap[PCI_STD_NUM_BARS]; ++ bool bar_mmap_supported[PCI_STD_NUM_BARS]; ++ u8 *pci_config_map; ++ u8 *vconfig; ++ struct perm_bits *msi_perm; ++ spinlock_t irqlock; ++ struct mutex igate; ++ struct xarray ctx; ++ int irq_type; ++ int num_regions; ++ struct vfio_pci_region *region; ++ u8 msi_qmax; ++ u8 msix_bar; ++ u16 msix_size; ++ u32 msix_offset; ++ u32 rbar[7]; ++ bool has_dyn_msix:1; ++ bool pci_2_3:1; ++ bool virq_disabled:1; ++ bool reset_works:1; ++ bool extended_caps:1; ++ bool bardirty:1; ++ bool has_vga:1; ++ bool needs_reset:1; ++ bool nointx:1; ++ bool needs_pm_restore:1; ++ bool pm_intx_masked:1; ++ bool pm_runtime_engaged:1; ++ struct pci_saved_state *pci_saved_state; ++ struct pci_saved_state *pm_save; ++ int ioeventfds_nr; ++ struct eventfd_ctx *err_trigger; ++ struct eventfd_ctx *req_trigger; ++ struct eventfd_ctx *pm_wake_eventfd_ctx; ++ struct list_head dummy_resources_list; ++ struct mutex ioeventfds_lock; ++ struct list_head ioeventfds_list; ++ struct vfio_pci_vf_token *vf_token; ++ struct list_head sriov_pfs_item; ++ struct vfio_pci_core_device *sriov_pf_core_dev; ++ struct notifier_block nb; ++ struct mutex vma_lock; ++ struct list_head vma_list; ++ struct rw_semaphore memory_lock; ++}; ++ ++/* Will be exported for vfio pci drivers usage */ ++int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, ++ unsigned int type, unsigned int subtype, ++ const struct vfio_pci_regops *ops, ++ size_t size, u32 flags, void *data); ++void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, ++ bool is_disable_idle_d3); ++void vfio_pci_core_close_device(struct vfio_device *core_vdev); ++int vfio_pci_core_init_dev(struct vfio_device *core_vdev); ++void vfio_pci_core_release_dev(struct vfio_device *core_vdev); ++int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); ++void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); ++extern const struct pci_error_handlers vfio_pci_core_err_handlers; ++int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, ++ int nr_virtfn); ++long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, ++ unsigned long arg); ++int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, ++ void __user *arg, size_t argsz); ++ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, ++ size_t count, loff_t *ppos); ++ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, ++ size_t count, loff_t *ppos); ++int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma); ++void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count); ++int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf); ++int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); ++void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); ++void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); ++int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar); ++pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, ++ pci_channel_state_t state); ++ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, ++ void __iomem *io, char __user *buf, ++ loff_t off, size_t count, size_t x_start, ++ size_t x_end, bool iswrite); ++bool vfio_pci_core_range_intersect_range(loff_t buf_start, size_t buf_cnt, ++ loff_t reg_start, size_t reg_cnt, ++ loff_t *buf_offset, ++ size_t *intersect_count, ++ size_t *register_offset); ++#define VFIO_IOWRITE_DECLATION(size) \ ++int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ ++ bool test_mem, u##size val, void __iomem *io); ++ ++VFIO_IOWRITE_DECLATION(8) ++VFIO_IOWRITE_DECLATION(16) ++VFIO_IOWRITE_DECLATION(32) ++#ifdef iowrite64 ++VFIO_IOWRITE_DECLATION(64) ++#endif ++ ++#define VFIO_IOREAD_DECLATION(size) \ ++int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \ ++ bool test_mem, u##size *val, void __iomem *io); ++ ++VFIO_IOREAD_DECLATION(8) ++VFIO_IOREAD_DECLATION(16) ++VFIO_IOREAD_DECLATION(32) ++ ++#endif /* VFIO_PCI_CORE_H */ +-- +2.39.3 + diff --git a/1010-make-qat-vfio-pci-drvier-to-be-built.patch b/1010-make-qat-vfio-pci-drvier-to-be-built.patch new file mode 100644 index 0000000000000000000000000000000000000000..dccbb55ff25620054508127e3c11d1eabb864f5f --- /dev/null +++ b/1010-make-qat-vfio-pci-drvier-to-be-built.patch @@ -0,0 +1,33 @@ +From c3d2dd8014bf2244b60f2e01e9b9640f8d3f33dd Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Wed, 27 Mar 2024 14:52:25 +0800 +Subject: [PATCH 10/28] make qat vfio pci drvier to be built + +Signed-off-by: Zelin Deng +--- + quickassist/qat/Makefile | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/quickassist/qat/Makefile b/quickassist/qat/Makefile +index cd0dc01..9f59a84 100644 +--- a/quickassist/qat/Makefile ++++ b/quickassist/qat/Makefile +@@ -16,6 +16,7 @@ export CONFIG_CRYPTO_DEV_QAT_D15XXVF=m + export CONFIG_CRYPTO_DEV_QAT_4XXX=m + export CONFIG_CRYPTO_DEV_QAT_4XXXVF=m + export CONFIG_CRYPTO_DEV_QAT_VQAT=m ++export CONFIG_QAT_VFIO_PCI=m + export QAT_UIO?=y + export ICP_HB_FAIL_SIM?=n + export QAT_LEGACY_ALGORITHMS?=n +@@ -102,5 +103,7 @@ help: + else + subdir-ccflags-y += -include $(src)/compat/qat_compat.h + subdir-ccflags-y += -include $(src)/compat/qat_compat_aux.h ++subdir-ccflags-y += -I $(src)/drivers/crypto/qat/qat_common/ + obj-m := drivers/crypto/qat/ ++obj-m += drivers/vfio/pci/qat/ + endif +-- +2.39.3 + diff --git a/1011-Use-self-defined-vfio-core-device-init-release-funct.patch b/1011-Use-self-defined-vfio-core-device-init-release-funct.patch new file mode 100644 index 0000000000000000000000000000000000000000..d3c08521c82308086f789e154e4ed05910abcd89 --- /dev/null +++ b/1011-Use-self-defined-vfio-core-device-init-release-funct.patch @@ -0,0 +1,61 @@ +From 5aface96b7bc6a714c3f96e316915e0076b5e8b2 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 12 Apr 2024 10:10:31 +0800 +Subject: [PATCH 11/28] Use self-defined vfio core device {init,release} + function + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/vfio/pci/qat/main.c | 36 +++++++++++++++++++++ + 1 file changed, 36 insertions(+) + +diff --git a/quickassist/qat/drivers/vfio/pci/qat/main.c b/quickassist/qat/drivers/vfio/pci/qat/main.c +index b5ad197..99e28aa 100644 +--- a/quickassist/qat/drivers/vfio/pci/qat/main.c ++++ b/quickassist/qat/drivers/vfio/pci/qat/main.c +@@ -15,6 +15,42 @@ + #include "vfio_pci_core.h" + #include "qat_mig_dev.h" + ++#ifndef vfio_pci_core_init_dev ++static int vfio_pci_core_init_dev(struct vfio_device *core_vdev) ++{ ++ struct vfio_pci_core_device *vdev = ++ container_of(core_vdev, struct vfio_pci_core_device, vdev); ++ ++ vdev->pdev = to_pci_dev(core_vdev->dev); ++ vdev->irq_type = VFIO_PCI_NUM_IRQS; ++ mutex_init(&vdev->igate); ++ spin_lock_init(&vdev->irqlock); ++ mutex_init(&vdev->ioeventfds_lock); ++ INIT_LIST_HEAD(&vdev->dummy_resources_list); ++ INIT_LIST_HEAD(&vdev->ioeventfds_list); ++ mutex_init(&vdev->vma_lock); ++ INIT_LIST_HEAD(&vdev->vma_list); ++ INIT_LIST_HEAD(&vdev->sriov_pfs_item); ++ init_rwsem(&vdev->memory_lock); ++ ++ return 0; ++} ++#endif ++ ++#ifndef vfio_pci_core_release_dev ++static void vfio_pci_core_release_dev(struct vfio_device *core_vdev) ++{ ++ struct vfio_pci_core_device *vdev = ++ container_of(core_vdev, struct vfio_pci_core_device, vdev); ++ ++ mutex_destroy(&vdev->igate); ++ mutex_destroy(&vdev->ioeventfds_lock); ++ mutex_destroy(&vdev->vma_lock); ++ kfree(vdev->region); ++ kfree(vdev->pm_save); ++} ++#endif ++ + struct qat_vf_migration_file { + struct file *filp; + /* protects migration region context */ +-- +2.39.3 + diff --git a/1012-Remove-vfio-core-pci-init-release-definition-in-dumm.patch b/1012-Remove-vfio-core-pci-init-release-definition-in-dumm.patch new file mode 100644 index 0000000000000000000000000000000000000000..f367d96bceb6991d3d10923d27412187c71a0041 --- /dev/null +++ b/1012-Remove-vfio-core-pci-init-release-definition-in-dumm.patch @@ -0,0 +1,29 @@ +From ec5512dd413417e651c8fa602f6fe7155d762a2b Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 12 Apr 2024 10:12:00 +0800 +Subject: [PATCH 12/28] Remove vfio core pci {init,release} definition in dummy + *.h + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h b/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h +index c77f85b..08e8a81 100644 +--- a/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h ++++ b/quickassist/qat/drivers/vfio/pci/qat/vfio_pci_core.h +@@ -140,8 +140,8 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, + void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, + bool is_disable_idle_d3); + void vfio_pci_core_close_device(struct vfio_device *core_vdev); +-int vfio_pci_core_init_dev(struct vfio_device *core_vdev); +-void vfio_pci_core_release_dev(struct vfio_device *core_vdev); ++//int vfio_pci_core_init_dev(struct vfio_device *core_vdev); ++//void vfio_pci_core_release_dev(struct vfio_device *core_vdev); + int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); + void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); + extern const struct pci_error_handlers vfio_pci_core_err_handlers; +-- +2.39.3 + diff --git a/1013-Add-new-rl-full-support.patch b/1013-Add-new-rl-full-support.patch new file mode 100644 index 0000000000000000000000000000000000000000..eac52b92ffb013acd120324f576b3ea5b948ec87 --- /dev/null +++ b/1013-Add-new-rl-full-support.patch @@ -0,0 +1,2428 @@ +From ddc32e0bb8f44e08b999d2ce5b6ee55b1d586a5b Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 12 Apr 2024 12:38:07 +0800 +Subject: [PATCH 13/28] Add new rl full support + +Signed-off-by: Zelin Deng +--- + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 26 +- + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 9 + + .../qat/drivers/crypto/qat/qat_common/Kbuild | 5 +- + .../drivers/crypto/qat/qat_common/Makefile | 8 +- + .../crypto/qat/qat_common/adf_accel_devices.h | 3 + + .../drivers/crypto/qat/qat_common/adf_admin.c | 47 + + .../crypto/qat/qat_common/adf_cfg_device.c | 3 +- + .../crypto/qat/qat_common/adf_common_drv.h | 7 + + .../crypto/qat/qat_common/adf_ctl_drv.c | 2 + + .../crypto/qat/qat_common/adf_gen4_hw_data.h | 7 + + .../crypto/qat/qat_common/adf_gen4_rl.h | 2 +- + .../crypto/qat/qat_common/adf_gen4_vf_mig.c | 1 - + .../drivers/crypto/qat/qat_common/adf_init.c | 14 +- + .../drivers/crypto/qat/qat_common/adf_rl.c | 1197 +++++++++++++++++ + .../drivers/crypto/qat/qat_common/adf_rl.h | 179 +++ + .../crypto/qat/qat_common/adf_rl_admin.c | 98 ++ + .../crypto/qat/qat_common/adf_rl_admin.h | 18 + + .../crypto/qat/qat_common/adf_sysfs_rl.c | 461 +++++++ + .../crypto/qat/qat_common/adf_sysfs_rl.h | 11 + + .../qat/qat_common/icp_qat_fw_init_admin.h | 13 + + 20 files changed, 2096 insertions(+), 15 deletions(-) + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.h + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.h + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.c + create mode 100644 quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.h + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +index d6a1cac..0a6efe9 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +@@ -22,7 +22,8 @@ + #include "adf_gen4_adi_hal.h" + #include "adf_gen4_timer.h" + #include "adf_pasid.h" +-#include "adf_gen4_rl.h" ++//#include "adf_gen4_rl.h" ++#include "adf_rl.h" + #include "adf_gen4_vf_mig.h" + + #define MAX_CLUSTER 4 +@@ -551,6 +552,24 @@ static u32 get_ae_clock(struct adf_hw_device_data *self) + return self->clock_frequency / 16; + } + ++static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) ++{ ++ rl_data->pciout_tb_offset = ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; ++ rl_data->pciin_tb_offset = ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET; ++ rl_data->r2l_offset = ADF_GEN4_RL_R2L_OFFSET; ++ rl_data->l2c_offset = ADF_GEN4_RL_L2C_OFFSET; ++ rl_data->c2s_offset = ADF_GEN4_RL_C2S_OFFSET; ++ ++ rl_data->pcie_scale_div = ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV; ++ rl_data->pcie_scale_mul = ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL; ++ rl_data->dcpr_correction = ADF_4XXX_RL_DCPR_CORRECTION; ++ rl_data->max_tp[ADF_SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM; ++ rl_data->max_tp[ADF_SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM; ++ rl_data->max_tp[ADF_SVC_DC] = ADF_4XXX_RL_MAX_TP_DC; ++ rl_data->scan_interval = ADF_4XXX_RL_SCANS_PER_SEC; ++ rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF; ++} ++ + static int adf_4xxx_configure_accel_units(struct adf_accel_dev *accel_dev) + { + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = {0}; +@@ -1049,8 +1068,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id) + #ifdef NON_GPL_COMMON + hw_data->get_accel_algo_cap = adf_gen4_cfg_get_accel_algo_cap; + #endif +- hw_data->init_rl_v2 = adf_rl_v2_init; +- hw_data->exit_rl_v2 = adf_rl_v2_exit; ++ //hw_data->init_rl_v2 = adf_rl_v2_init; ++ //hw_data->exit_rl_v2 = adf_rl_v2_exit; + hw_data->config_bank_pasid = adf_pasid_config_bank; + hw_data->telemetry_init = adf_4xxx_init_tl; + hw_data->telemetry_exit = adf_4xxx_exit_tl; +@@ -1086,6 +1105,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id) + hw_data->coalescing_min_time = ADF_4XXX_COALESCING_MIN_TIME; + hw_data->coalescing_max_time = ADF_4XXX_COALESCING_MAX_TIME; + hw_data->coalescing_def_time = ADF_4XXX_COALESCING_DEF_TIME; ++ adf_init_rl_data(&hw_data->rl_data); + } + + void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +diff --git a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +index e282b7f..1e592f4 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +@@ -1581,6 +1581,15 @@ void adf_4xxx_handle_slice_hang_error(struct adf_accel_dev *accel_dev, + u32 accel_num, + void __iomem *csr); + ++/* RL constants */ ++#define ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV 100 ++#define ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL 102 ++#define ADF_4XXX_RL_DCPR_CORRECTION 1 ++#define ADF_4XXX_RL_SCANS_PER_SEC 954 ++#define ADF_4XXX_RL_MAX_TP_ASYM 173750UL ++#define ADF_4XXX_RL_MAX_TP_SYM 95000UL ++#define ADF_4XXX_RL_MAX_TP_DC 45000UL ++#define ADF_4XXX_RL_SLICE_REF 1000UL + + #define ADF_4XXX_AE_FREQ (1000 * 1000000) + #define ADF_4XXX_KPT_COUNTER_FREQ (100 * 1000000) +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/Kbuild b/quickassist/qat/drivers/crypto/qat/qat_common/Kbuild +index a26f8ee..a3cfbc4 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/Kbuild ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/Kbuild +@@ -25,8 +25,9 @@ intel_qat-$(QAT_UIO) += \ + adf_cfg_bundle.o \ + adf_cfg_instance.o \ + adf_cfg_section.o \ +- adf_fw_counters.o \ +- adf_ctl_rl.o ++ adf_fw_counters.o ++ ++#intel_qat-$(QAT_UIO) += adf_ctl_rl.o + + ccflags-$(QAT_UIO) += -DQAT_RSA_SUPPORT=y -DQAT_AEAD_OLD_SUPPORT=y -DQAT_FW_AUTH_CONFIG=y + ccflags-$(ICP_HB_FAIL_SIM) += -DQAT_HB_FAIL_SIM=y +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +index 4e137b3..0714173 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/Makefile ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/Makefile +@@ -37,11 +37,11 @@ intel_qat-objs := adf_cfg.o \ + adf_vdcm_iov_msg.o \ + adf_vqat_isr.o \ + adf_svm.o \ +- adf_pasid.o \ +- adf_sla.o ++ adf_pasid.o + +-intel_qat-objs += adf_ctl_rl.o +-intel_qat-objs += adf_gen4_rl.o ++#intel_qat-objs += adf_gen4_rl.o ++#intel_qat-objs += adf_sla.o ++intel_qat-objs += adf_rl.o adf_rl_admin.o adf_sysfs_rl.o + intel_qat-objs += adf_gen4_hw_data.o + intel_qat-objs += adf_gen4_timer.o + intel_qat-objs += adf_uq.o +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +index e1ad617..b43cbe0 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_accel_devices.h +@@ -14,6 +14,7 @@ + #endif + #include "adf_cfg_common.h" + #include "qat_mig_dev.h" ++#include "adf_rl.h" + #else + #include + #endif /* USER_SPACE */ +@@ -734,6 +735,7 @@ struct adf_hw_device_data { + struct adf_adi_ops *adi_ops; + int (*get_capabilities_ex)(struct adf_accel_dev *accel_dev); + void *priv_data; ++ struct adf_rl_hw_data rl_data; + struct adf_aux_ops *aux_ops; + u32 rl_max_tp[ADF_SVC_NONE + 1]; + u32 rl_slice_ref; +@@ -965,6 +967,7 @@ struct adf_accel_dev { + struct adf_int_timer *int_timer; + unsigned int autoreset_on_error; + struct adf_rl_v2 *rl_v2; ++ struct adf_rl *rate_limiting; + #ifdef QAT_UIO + struct adf_fw_counters_data *fw_counters_data; + struct dentry *debugfs_inline_dir; +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_admin.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_admin.c +index c874a22..72926f3 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_admin.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_admin.c +@@ -514,3 +514,50 @@ void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) + accel_dev->admin = NULL; + } + EXPORT_SYMBOL_GPL(adf_exit_admin_comms); ++ ++int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, ++ struct icp_qat_fw_init_admin_slice_cnt *slices) ++{ ++ u32 ae_mask = accel_dev->hw_device->admin_ae_mask; ++ struct icp_qat_fw_init_admin_resp resp = { }; ++ struct icp_qat_fw_init_admin_req req = { }; ++ int ret; ++ ++ req.cmd_id = ICP_QAT_FW_RL_INIT; ++ ++ ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); ++ if (ret) ++ return ret; ++ ++ memcpy(slices, &resp.slices, sizeof(*slices)); ++ ++ return 0; ++} ++ ++int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, ++ struct icp_qat_fw_init_admin_req *req) ++{ ++ u32 ae_mask = accel_dev->hw_device->admin_ae_mask; ++ struct icp_qat_fw_init_admin_resp resp = { }; ++ ++ /* ++ * req struct filled in rl implementation. Used commands ++ * ICP_QAT_FW_RL_ADD for a new SLA ++ * ICP_QAT_FW_RL_UPDATE for update SLA ++ */ ++ return adf_send_admin(accel_dev, req, &resp, ae_mask); ++} ++ ++int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, ++ u8 node_type) ++{ ++ u32 ae_mask = accel_dev->hw_device->admin_ae_mask; ++ struct icp_qat_fw_init_admin_resp resp = { }; ++ struct icp_qat_fw_init_admin_req req = { }; ++ ++ req.cmd_id = ICP_QAT_FW_RL_REMOVE; ++ req.node_id = node_id; ++ req.node_type = node_type; ++ ++ return adf_send_admin(accel_dev, &req, &resp, ae_mask); ++} +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_device.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_device.c +index 7856ae4..6a15c25 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_device.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_device.c +@@ -5,6 +5,7 @@ + #include "adf_cfg_device.h" + #include "adf_cfg_section.h" + ++#if 0 + enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), +@@ -39,7 +40,7 @@ enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT = BIT(30), + ICP_ACCEL_CAPABILITIES_AUX = BIT(31) + }; +- ++#endif + + int adf_cfg_get_ring_pairs(struct adf_cfg_device *device, + struct adf_cfg_instance *inst, +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h +index e0ab348..a38265f 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_common_drv.h +@@ -11,6 +11,7 @@ + #include "adf_accel_devices.h" + #include "icp_qat_fw_loader_handle.h" + #include "icp_qat_hal.h" ++#include "icp_qat_fw_init_admin.h" + #endif + #define SET_BIT(byte, bit) ((byte) |= ((uint32_t)1 << (bit))) + #define CLEAR_BIT(byte, bit) ((byte) &= ~((uint32_t)1 << (bit))) +@@ -206,6 +207,12 @@ int adf_send_admin(struct adf_accel_dev *accel_dev, + struct icp_qat_fw_init_admin_resp *resp, + u32 ae_mask); + int adf_send_admin_init(struct adf_accel_dev *accel_dev); ++int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, ++ struct icp_qat_fw_init_admin_slice_cnt *slices); ++int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, ++ struct icp_qat_fw_init_admin_req *req); ++int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, ++ u8 node_type); + int adf_get_fw_pke_stats(struct adf_accel_dev *accel_dev, + u64 *suc_counter, + u64 *unsuc_counter); +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_ctl_drv.c +index df435ad..dfd51ec 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_ctl_drv.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_ctl_drv.c +@@ -841,6 +841,7 @@ static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) + case IOCTL_DISABLE_RING: + ret = adf_ctl_ioctl_disable_ring(arg); + break; ++#if 0 + case IOCTL_SLA_GET_CAPS: + ret = adf_ctl_ioctl_sla_get_caps(arg); + break; +@@ -862,6 +863,7 @@ static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) + case IOCTL_SLA_GET_LIST: + ret = adf_ctl_ioctl_sla_get_list(arg); + break; ++#endif + case IOCTL_GET_DEV_REAL_ID: + ret = adf_ctl_ioctl_get_real_id(arg); + break; +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +index cabb13a..341e2e9 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +@@ -42,6 +42,13 @@ + /* Number of heartbeat counter pairs */ + #define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE) + ++/* Rate Limiting */ ++#define ADF_GEN4_RL_R2L_OFFSET 0x508000 ++#define ADF_GEN4_RL_L2C_OFFSET 0x509000 ++#define ADF_GEN4_RL_C2S_OFFSET 0x508818 ++#define ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800 ++#define ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804 ++ + /* Clock Gating Control IOSF Primary Register */ + #define ADF_GEN4_PFCGC_IOSF_PRIR (0x2C0) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h +index b83e017..251252c 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_rl.h +@@ -97,7 +97,7 @@ struct rl_node_count { + }; + + /* Structure for slice numbering - generic for all 2.x products */ +-struct rl_slice_cnt { ++struct rl_slice_cnt_gen4 { + u8 rl_slice_cnt; + u8 rl_dcpr_slice_cnt; + u8 rl_pke_slice_cnt; +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +index a510274..be7c78f 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +@@ -15,7 +15,6 @@ + #include "adf_gen4_vf_mig.h" + #include "adf_pf2vf_msg.h" + #include "adf_gen4_hw_csr_data.h" +-#include "adf_gen4_rl.h" + + #define ADF_GEN4_VF_MSTATE_SIZE 4096 + #define ADF_GEN4_PFVF_RSP_TIMEOUT_US 5000 +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_init.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_init.c +index 747c854..4e53434 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_init.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_init.c +@@ -487,6 +487,10 @@ static int adf_dev_init_locked(struct adf_accel_dev *accel_dev) + if (hw_data->add_misc_error && hw_data->add_misc_error(accel_dev)) + return -EFAULT; + ++ ret = adf_rl_init(accel_dev); ++ if (ret && ret != -EOPNOTSUPP) ++ return ret; ++ + /* + * Subservice initialisation is divided into two stages: init and start. + * This is to facilitate any ordering dependencies between services +@@ -533,6 +537,7 @@ static int adf_dev_start_locked(struct adf_accel_dev *accel_dev) + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service = NULL; + struct list_head *list_itr = NULL; ++ int ret; + + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + +@@ -598,8 +603,9 @@ static int adf_dev_start_locked(struct adf_accel_dev *accel_dev) + } + } + +- if (adf_rate_limiting_init(accel_dev)) { +- dev_err(&GET_DEV(accel_dev), "Failed to init RL\n"); ++ ret = adf_rl_start(accel_dev); ++ if (ret && ret != -EOPNOTSUPP) { ++ dev_err(&GET_DEV(accel_dev), "Failed to start RL\n"); + return -EFAULT; + } + +@@ -701,7 +707,7 @@ static void adf_dev_stop_locked(struct adf_accel_dev *accel_dev) + clear_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + +- adf_rate_limiting_exit(accel_dev); ++ adf_rl_stop(accel_dev); + + if (hw_data->int_timer_exit) + hw_data->int_timer_exit(accel_dev); +@@ -855,6 +861,8 @@ static void adf_dev_shutdown_locked(struct adf_accel_dev *accel_dev) + #endif + } + ++ adf_rl_exit(accel_dev); ++ + if (hw_data->remove_pke_stats) + hw_data->remove_pke_stats(accel_dev); + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c +new file mode 100644 +index 0000000..a56ae16 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c +@@ -0,0 +1,1197 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2023 Intel Corporation */ ++ ++#ifndef dev_fmt ++#define dev_fmt(fmt) "RateLimiting: " fmt ++#endif ++ ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++#include "adf_rl_admin.h" ++#include "adf_rl.h" ++#include "adf_sysfs_rl.h" ++#include "icp_qat_hw.h" ++ ++#define RL_TOKEN_GRANULARITY_PCIEIN_BUCKET 0U ++#define RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET 0U ++#define RL_TOKEN_PCIE_SIZE 64 ++#define RL_TOKEN_ASYM_SIZE 1024 ++#define RL_CSR_SIZE 4U ++#define RL_CAPABILITY_MASK GENMASK(6, 4) ++#define RL_CAPABILITY_VALUE 0x70 ++#define RL_VALIDATE_NON_ZERO(input) ((input) == 0) ++#define ROOT_MASK GENMASK(1, 0) ++#define CLUSTER_MASK GENMASK(3, 0) ++#define LEAF_MASK GENMASK(5, 0) ++ ++#ifndef BITS_PER_BYTE ++#define BITS_PER_BYTE 8 ++#endif ++ ++#ifndef BYTES_PER_MBIT ++#define BYTES_PER_MBIT 125000 ++#endif ++ ++static int validate_user_input(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in, ++ bool is_update) ++{ ++ const unsigned long rp_mask = sla_in->rp_mask; ++ size_t rp_mask_size; ++ int i, cnt; ++ ++ if (sla_in->pir < sla_in->cir) { ++ dev_notice(&GET_DEV(accel_dev), ++ "PIR must be >= CIR, setting PIR to CIR\n"); ++ sla_in->pir = sla_in->cir; ++ } ++ ++ if (!is_update) { ++ cnt = 0; ++ rp_mask_size = sizeof(sla_in->rp_mask) * BITS_PER_BYTE; ++ for_each_set_bit(i, &rp_mask, rp_mask_size) { ++ if (++cnt > RL_RP_CNT_PER_LEAF_MAX) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Too many ring pairs selected for this SLA\n"); ++ return -EINVAL; ++ } ++ } ++ ++ if (sla_in->srv >= ADF_SVC_NONE) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Wrong service type\n"); ++ return -EINVAL; ++ } ++ ++ if (sla_in->type > RL_LEAF) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Wrong node type\n"); ++ return -EINVAL; ++ } ++ ++ if (sla_in->parent_id < RL_PARENT_DEFAULT_ID || ++ sla_in->parent_id >= RL_NODES_CNT_MAX) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Wrong parent ID\n"); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static int validate_sla_id(struct adf_accel_dev *accel_dev, int sla_id) ++{ ++ struct rl_sla *sla; ++ ++ if (sla_id <= RL_SLA_EMPTY_ID || sla_id >= RL_NODES_CNT_MAX) { ++ dev_notice(&GET_DEV(accel_dev), "Provided ID is out of bounds\n"); ++ return -EINVAL; ++ } ++ ++ sla = accel_dev->rate_limiting->sla[sla_id]; ++ ++ if (!sla) { ++ dev_notice(&GET_DEV(accel_dev), "SLA with provided ID does not exist\n"); ++ return -EINVAL; ++ } ++ ++ if (sla->type != RL_LEAF) { ++ dev_notice(&GET_DEV(accel_dev), "This ID is reserved for internal use\n"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/** ++ * find_parent() - Find the parent for a new SLA ++ * @rl_data: pointer to ratelimiting data ++ * @sla_in: pointer to user input data for a new SLA ++ * ++ * Function returns a pointer to the parent SLA. If the parent ID is provided ++ * as input in the user data, then such ID is validated and the parent SLA ++ * is returned. ++ * Otherwise, it returns the default parent SLA (root or cluster) for ++ * the new object. ++ * ++ * Return: ++ * * Pointer to the parent SLA object ++ * * NULL - when parent cannot be found ++ */ ++static struct rl_sla *find_parent(struct adf_rl *rl_data, ++ struct adf_rl_sla_input_data *sla_in) ++{ ++ int input_parent_id = sla_in->parent_id; ++ struct rl_sla *root = NULL; ++ struct rl_sla *parent_sla; ++ int i; ++ ++ if (sla_in->type == RL_ROOT) ++ return NULL; ++ ++ if (input_parent_id > RL_PARENT_DEFAULT_ID) { ++ parent_sla = rl_data->sla[input_parent_id]; ++ /* ++ * SLA can be a parent if it has the same service as the child ++ * and its type is higher in the hierarchy, ++ * for example the parent type of a LEAF must be a CLUSTER. ++ */ ++ if (parent_sla && parent_sla->srv == sla_in->srv && ++ parent_sla->type == sla_in->type - 1) ++ return parent_sla; ++ ++ return NULL; ++ } ++ ++ /* If input_parent_id is not valid, get root for this service type. */ ++ for (i = 0; i < RL_ROOT_MAX; i++) { ++ if (rl_data->root[i] && rl_data->root[i]->srv == sla_in->srv) { ++ root = rl_data->root[i]; ++ break; ++ } ++ } ++ ++ if (!root) ++ return NULL; ++ ++ /* ++ * If the type of this SLA is cluster, then return the root. ++ * Otherwise, find the default (i.e. first) cluster for this service. ++ */ ++ if (sla_in->type == RL_CLUSTER) ++ return root; ++ ++ for (i = 0; i < RL_CLUSTER_MAX; i++) { ++ if (rl_data->cluster[i] && rl_data->cluster[i]->parent == root) ++ return rl_data->cluster[i]; ++ } ++ ++ return NULL; ++} ++ ++static enum adf_cfg_service_type srv_to_cfg_svc_type(enum adf_svc_type rl_srv) ++{ ++ switch (rl_srv) { ++ case ADF_SVC_ASYM: ++ return ASYM; ++ case ADF_SVC_SYM: ++ return SYM; ++ case ADF_SVC_DC: ++ return COMP; ++ default: ++ return NA; ++ } ++} ++ ++/** ++ * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array ++ * @rl_data: pointer to ratelimiting data ++ * @type: SLA type ++ * @sla_arr: pointer to variable where requested pointer will be stored ++ * ++ * Return: Max number of elements allowed for the returned array ++ */ ++u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, ++ struct rl_sla ***sla_arr) ++{ ++ switch (type) { ++ case RL_LEAF: ++ *sla_arr = rl_data->leaf; ++ return RL_LEAF_MAX; ++ case RL_CLUSTER: ++ *sla_arr = rl_data->cluster; ++ return RL_CLUSTER_MAX; ++ case RL_ROOT: ++ *sla_arr = rl_data->root; ++ return RL_ROOT_MAX; ++ default: ++ *sla_arr = NULL; ++ return 0; ++ } ++} ++ ++static bool is_service_enabled(struct adf_accel_dev *accel_dev, ++ enum adf_svc_type rl_srv) ++{ ++ enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(rl_srv); ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ u8 rps_per_bundle = hw_data->num_banks_per_vf; ++ int i; ++ ++ for (i = 0; i < rps_per_bundle; i++) { ++ if (GET_SRV_TYPE(accel_dev->hw_device->ring_to_svc_map, i) == arb_srv) ++ return true; ++ } ++ ++ return false; ++} ++ ++/** ++ * prepare_rp_ids() - Creates an array of ring pair IDs from bitmask ++ * @accel_dev: pointer to acceleration device structure ++ * @sla: SLA object data where result will be written ++ * @rp_mask: bitmask of ring pair IDs ++ * ++ * Function tries to convert provided bitmap to an array of IDs. It checks if ++ * RPs aren't in use, are assigned to SLA service or if a number of provided ++ * IDs is not too big. If successful, writes the result into the field ++ * sla->ring_pairs_cnt. ++ * ++ * Return: ++ * * 0 - ok ++ * * -EINVAL - ring pairs array cannot be created from provided mask ++ */ ++static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla, ++ const unsigned long rp_mask) ++{ ++ enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(sla->srv); ++ u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf; ++ bool *rp_in_use = accel_dev->rate_limiting->rp_in_use; ++ size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids); ++ u16 rp_id_max = GET_HW_DATA(accel_dev)->num_banks; ++ u16 cnt = 0; ++ u16 rp_id; ++ ++ for_each_set_bit(rp_id, &rp_mask, rp_id_max) { ++ if (cnt >= rp_cnt_max) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Assigned more ring pairs than supported"); ++ return -EINVAL; ++ } ++ ++ if (rp_in_use[rp_id]) { ++ dev_notice(&GET_DEV(accel_dev), ++ "RP %u already assigned to other SLA", rp_id); ++ return -EINVAL; ++ } ++ ++ if (GET_SRV_TYPE(accel_dev->hw_device->ring_to_svc_map, rp_id % rps_per_bundle) != arb_srv) { ++ dev_notice(&GET_DEV(accel_dev), ++ "RP %u does not support SLA service", rp_id); ++ return -EINVAL; ++ } ++ ++ sla->ring_pairs_ids[cnt++] = rp_id; ++ } ++ ++ sla->ring_pairs_cnt = cnt; ++ ++ return 0; ++} ++ ++static void mark_rps_usage(struct rl_sla *sla, bool *rp_in_use, bool used) ++{ ++ u16 rp_id; ++ int i; ++ ++ for (i = 0; i < sla->ring_pairs_cnt; i++) { ++ rp_id = sla->ring_pairs_ids[i]; ++ rp_in_use[rp_id] = used; ++ } ++} ++ ++static void assign_rps_to_leaf(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool clear) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); ++ u32 base_offset = hw_data->rl_data.r2l_offset; ++ u32 node_id = clear ? 0U : (sla->node_id & LEAF_MASK); ++ u32 offset; ++ int i; ++ ++ for (i = 0; i < sla->ring_pairs_cnt; i++) { ++ offset = base_offset + (RL_CSR_SIZE * sla->ring_pairs_ids[i]); ++ ADF_CSR_WR(pmisc_addr, offset, node_id); ++ } ++} ++ ++static void assign_leaf_to_cluster(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool clear) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); ++ u32 base_offset = hw_data->rl_data.l2c_offset; ++ u32 node_id = sla->node_id & LEAF_MASK; ++ u32 parent_id = clear ? 0U : (sla->parent->node_id & CLUSTER_MASK); ++ u32 offset; ++ ++ offset = base_offset + (RL_CSR_SIZE * node_id); ++ ADF_CSR_WR(pmisc_addr, offset, parent_id); ++} ++ ++static void assign_cluster_to_root(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool clear) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); ++ u32 base_offset = hw_data->rl_data.c2s_offset; ++ u32 node_id = sla->node_id & CLUSTER_MASK; ++ u32 parent_id = clear ? 0U : (sla->parent->node_id & ROOT_MASK); ++ u32 offset; ++ ++ offset = base_offset + (RL_CSR_SIZE * node_id); ++ ADF_CSR_WR(pmisc_addr, offset, parent_id); ++} ++ ++static void assign_node_to_parent(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool clear_assignment) ++{ ++ switch (sla->type) { ++ case RL_LEAF: ++ assign_rps_to_leaf(accel_dev, sla, clear_assignment); ++ assign_leaf_to_cluster(accel_dev, sla, clear_assignment); ++ break; ++ case RL_CLUSTER: ++ assign_cluster_to_root(accel_dev, sla, clear_assignment); ++ break; ++ default: ++ break; ++ } ++} ++ ++/** ++ * can_parent_afford_sla() - Verifies if parent allows to create an SLA ++ * @sla_in: pointer to user input data for a new SLA ++ * @sla_parent: pointer to parent SLA object ++ * @sla_cir: current child CIR value (only for update) ++ * @is_update: request is a update ++ * ++ * Algorithm verifies if parent has enough remaining budget to take assignment ++ * of a child with provided parameters. In update case current CIR value must be ++ * returned to budget first. ++ * PIR value cannot exceed the PIR assigned to parent. ++ * ++ * Return: ++ * * true - SLA can be created ++ * * false - SLA cannot be created ++ */ ++static bool can_parent_afford_sla(struct adf_rl_sla_input_data *sla_in, ++ struct rl_sla *sla_parent, u32 sla_cir, ++ bool is_update) ++{ ++ u32 rem_cir = sla_parent->rem_cir; ++ ++ if (is_update) ++ rem_cir += sla_cir; ++ ++ if (sla_in->cir > rem_cir || sla_in->pir > sla_parent->pir) ++ return false; ++ ++ return true; ++} ++ ++/** ++ * can_node_afford_update() - Verifies if SLA can be updated with input data ++ * @sla_in: pointer to user input data for a new SLA ++ * @sla: pointer to SLA object selected for update ++ * ++ * Algorithm verifies if a new CIR value is big enough to satisfy currently ++ * assigned child SLAs and if PIR can be updated ++ * ++ * Return: ++ * * true - SLA can be updated ++ * * false - SLA cannot be updated ++ */ ++static bool can_node_afford_update(struct adf_rl_sla_input_data *sla_in, ++ struct rl_sla *sla) ++{ ++ u32 cir_in_use = sla->cir - sla->rem_cir; ++ ++ /* new CIR cannot be smaller then currently consumed value */ ++ if (cir_in_use > sla_in->cir) ++ return false; ++ ++ /* PIR of root/cluster cannot be reduced in node with assigned children */ ++ if (sla_in->pir < sla->pir && sla->type != RL_LEAF && cir_in_use > 0) ++ return false; ++ ++ return true; ++} ++ ++static bool is_enough_budget(struct adf_rl *rl_data, struct rl_sla *sla, ++ struct adf_rl_sla_input_data *sla_in, ++ bool is_update) ++{ ++ u32 max_val = rl_data->device_data->scale_ref; ++ struct rl_sla *parent = sla->parent; ++ bool ret = true; ++ ++ if (sla_in->cir > max_val || sla_in->pir > max_val) ++ ret = false; ++ ++ switch (sla->type) { ++ case RL_LEAF: ++ ret &= can_parent_afford_sla(sla_in, parent, sla->cir, ++ is_update); ++ break; ++ case RL_CLUSTER: ++ ret &= can_parent_afford_sla(sla_in, parent, sla->cir, ++ is_update); ++ ++ if (is_update) ++ ret &= can_node_afford_update(sla_in, sla); ++ ++ break; ++ case RL_ROOT: ++ if (is_update) ++ ret &= can_node_afford_update(sla_in, sla); ++ ++ break; ++ default: ++ ret = false; ++ break; ++ } ++ ++ return ret; ++} ++ ++static void update_budget(struct rl_sla *sla, u32 old_cir, bool is_update) ++{ ++ switch (sla->type) { ++ case RL_LEAF: ++ if (is_update) ++ sla->parent->rem_cir += old_cir; ++ ++ sla->parent->rem_cir -= sla->cir; ++ sla->rem_cir = 0; ++ break; ++ case RL_CLUSTER: ++ if (is_update) { ++ sla->parent->rem_cir += old_cir; ++ sla->rem_cir = sla->cir - (old_cir - sla->rem_cir); ++ } else { ++ sla->rem_cir = sla->cir; ++ } ++ ++ sla->parent->rem_cir -= sla->cir; ++ break; ++ case RL_ROOT: ++ if (is_update) ++ sla->rem_cir = sla->cir - (old_cir - sla->rem_cir); ++ else ++ sla->rem_cir = sla->cir; ++ break; ++ default: ++ break; ++ } ++} ++ ++/** ++ * get_next_free_sla_id() - finds next free ID in the SLA array ++ * @rl_data: Pointer to ratelimiting data structure ++ * ++ * Return: ++ * * 0 : RL_NODES_CNT_MAX - correct ID ++ * * -ENOSPC - all SLA slots are in use ++ */ ++static int get_next_free_sla_id(struct adf_rl *rl_data) ++{ ++ int i = 0; ++ ++ while (i < RL_NODES_CNT_MAX && rl_data->sla[i++]) ++ ; ++ ++ if (i == RL_NODES_CNT_MAX) ++ return -ENOSPC; ++ ++ return i - 1; ++} ++ ++/** ++ * get_next_free_node_id() - finds next free ID in the array of that node type ++ * @rl_data: Pointer to ratelimiting data structure ++ * @sla: Pointer to SLA object for which the ID is searched ++ * ++ * Return: ++ * * 0 : RL_[NODE_TYPE]_MAX - correct ID ++ * * -ENOSPC - all slots of that type are in use ++ */ ++static int get_next_free_node_id(struct adf_rl *rl_data, struct rl_sla *sla) ++{ ++ struct adf_hw_device_data *hw_device = GET_HW_DATA(rl_data->accel_dev); ++ int max_id, i, step, rp_per_leaf; ++ struct rl_sla **sla_list; ++ ++ rp_per_leaf = hw_device->num_banks / hw_device->num_banks_per_vf; ++ ++ /* ++ * Static nodes mapping: ++ * root0 - cluster[0,4,8,12] - leaf[0-15] ++ * root1 - cluster[1,5,9,13] - leaf[16-31] ++ * root2 - cluster[2,6,10,14] - leaf[32-47] ++ */ ++ switch (sla->type) { ++ case RL_LEAF: ++ i = sla->srv * rp_per_leaf; ++ step = 1; ++ max_id = i + rp_per_leaf; ++ sla_list = rl_data->leaf; ++ break; ++ case RL_CLUSTER: ++ i = sla->srv; ++ step = 4; ++ max_id = RL_CLUSTER_MAX; ++ sla_list = rl_data->cluster; ++ break; ++ case RL_ROOT: ++ return sla->srv; ++ default: ++ return -EINVAL; ++ } ++ ++ while (i < max_id && sla_list[i]) ++ i += step; ++ ++ if (i >= max_id) ++ return -ENOSPC; ++ ++ return i; ++} ++ ++u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type) ++{ ++ struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ u64 avail_slice_cycles, allocated_tokens; ++ ++ if (!sla_val) ++ return 0; ++ ++ avail_slice_cycles = hw_data->clock_frequency; ++ ++ switch (svc_type) { ++ case ADF_SVC_ASYM: ++ avail_slice_cycles *= device_data->slices.pke_cnt; ++ break; ++ case ADF_SVC_SYM: ++ avail_slice_cycles *= device_data->slices.cph_cnt; ++ break; ++ case ADF_SVC_DC: ++ avail_slice_cycles *= device_data->slices.dcpr_cnt; ++ break; ++ default: ++ break; ++ } ++ ++ do_div(avail_slice_cycles, device_data->scan_interval); ++ allocated_tokens = avail_slice_cycles * sla_val; ++ do_div(allocated_tokens, device_data->scale_ref); ++ ++ return allocated_tokens; ++} ++ ++u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type) ++{ ++ struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ u64 allocated_ae_cycles, avail_ae_cycles; ++ ++ if (!sla_val) ++ return 0; ++ ++ avail_ae_cycles = hw_data->clock_frequency; ++ avail_ae_cycles *= hw_data->get_num_aes(hw_data) - 1; ++ do_div(avail_ae_cycles, device_data->scan_interval); ++ ++ sla_val *= device_data->max_tp[svc_type]; ++ sla_val /= device_data->scale_ref; ++ ++ allocated_ae_cycles = (sla_val * avail_ae_cycles); ++ do_div(allocated_ae_cycles, device_data->max_tp[svc_type]); ++ ++ return allocated_ae_cycles; ++} ++ ++u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type, bool is_bw_out) ++{ ++ struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data; ++ u64 sla_to_bytes, allocated_bw, sla_scaled; ++ ++ if (!sla_val) ++ return 0; ++ ++ sla_to_bytes = sla_val; ++ sla_to_bytes *= device_data->max_tp[svc_type]; ++ do_div(sla_to_bytes, device_data->scale_ref); ++ ++ sla_to_bytes *= (svc_type == ADF_SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : ++ BYTES_PER_MBIT; ++ if (svc_type == ADF_SVC_DC && is_bw_out) ++ sla_to_bytes *= device_data->slices.dcpr_cnt - ++ device_data->dcpr_correction; ++ ++ sla_scaled = sla_to_bytes * device_data->pcie_scale_mul; ++ do_div(sla_scaled, device_data->pcie_scale_div); ++ allocated_bw = sla_scaled; ++ do_div(allocated_bw, RL_TOKEN_PCIE_SIZE); ++ do_div(allocated_bw, device_data->scan_interval); ++ ++ return allocated_bw; ++} ++ ++/** ++ * add_new_sla_entry() - creates a new SLA object and fills it with user data ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_in: pointer to user input data for a new SLA ++ * @sla_out: Pointer to variable that will contain the address of a new ++ * SLA object if the operation succeeds ++ * ++ * Return: ++ * * 0 - ok ++ * * -ENOMEM - memory allocation failed ++ * * -EINVAL - invalid user input ++ * * -ENOSPC - all available SLAs are in use ++ */ ++static int add_new_sla_entry(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in, ++ struct rl_sla **sla_out) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla *sla; ++ int ret = 0; ++ ++ sla = kzalloc(sizeof(*sla), GFP_KERNEL); ++ if (!sla) { ++ ret = -ENOMEM; ++ goto ret_err; ++ } ++ *sla_out = sla; ++ ++ if (!is_service_enabled(accel_dev, sla_in->srv)) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Provided service is not enabled\n"); ++ ret = -EINVAL; ++ goto ret_err; ++ } ++ ++ sla->srv = sla_in->srv; ++ sla->type = sla_in->type; ++ ret = get_next_free_node_id(rl_data, sla); ++ if (ret < 0) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Exceeded number of available nodes for that service\n"); ++ goto ret_err; ++ } ++ sla->node_id = ret; ++ ++ ret = get_next_free_sla_id(rl_data); ++ if (ret < 0) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Allocated maximum SLAs number\n"); ++ goto ret_err; ++ } ++ sla->sla_id = ret; ++ ++ sla->parent = find_parent(rl_data, sla_in); ++ if (!sla->parent && sla->type != RL_ROOT) { ++ if (sla_in->parent_id != RL_PARENT_DEFAULT_ID) ++ dev_notice(&GET_DEV(accel_dev), ++ "Provided parent ID does not exist or cannot be parent for this SLA."); ++ else ++ dev_notice(&GET_DEV(accel_dev), ++ "Unable to find parent node for this service. Is service enabled?"); ++ ret = -EINVAL; ++ goto ret_err; ++ } ++ ++ if (sla->type == RL_LEAF) { ++ ret = prepare_rp_ids(accel_dev, sla, sla_in->rp_mask); ++ if (!sla->ring_pairs_cnt || ret) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Unable to find ring pairs to assign to the leaf"); ++ if (!ret) ++ ret = -EINVAL; ++ ++ goto ret_err; ++ } ++ } ++ ++ return 0; ++ ++ret_err: ++ kfree(sla); ++ *sla_out = NULL; ++ ++ return ret; ++} ++ ++static int initialize_default_nodes(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct adf_rl_hw_data *device_data = rl_data->device_data; ++ struct adf_rl_sla_input_data sla_in = { }; ++ int ret = 0; ++ int i; ++ ++ /* Init root for each enabled service */ ++ sla_in.type = RL_ROOT; ++ sla_in.parent_id = RL_PARENT_DEFAULT_ID; ++ ++ for (i = 0; i < ADF_SVC_NONE; i++) { ++ if (!is_service_enabled(accel_dev, i)) ++ continue; ++ ++ sla_in.cir = device_data->scale_ref; ++ sla_in.pir = sla_in.cir; ++ sla_in.srv = i; ++ ++ ret = adf_rl_add_sla(accel_dev, &sla_in); ++ if (ret) ++ return ret; ++ } ++ ++ /* Init default cluster for each root */ ++ sla_in.type = RL_CLUSTER; ++ for (i = 0; i < ADF_SVC_NONE; i++) { ++ if (!rl_data->root[i]) ++ continue; ++ ++ sla_in.cir = rl_data->root[i]->cir; ++ sla_in.pir = sla_in.cir; ++ sla_in.srv = rl_data->root[i]->srv; ++ ++ ret = adf_rl_add_sla(accel_dev, &sla_in); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void clear_sla(struct adf_rl *rl_data, struct rl_sla *sla) ++{ ++ bool *rp_in_use = rl_data->rp_in_use; ++ struct rl_sla **sla_type_arr = NULL; ++ int i, sla_id, node_id; ++ u32 old_cir; ++ ++ sla_id = sla->sla_id; ++ node_id = sla->node_id; ++ old_cir = sla->cir; ++ sla->cir = 0; ++ sla->pir = 0; ++ ++ for (i = 0; i < sla->ring_pairs_cnt; i++) ++ rp_in_use[sla->ring_pairs_ids[i]] = false; ++ ++ update_budget(sla, old_cir, true); ++ adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); ++ assign_node_to_parent(rl_data->accel_dev, sla, true); ++ adf_rl_send_admin_delete_msg(rl_data->accel_dev, node_id, sla->type); ++ mark_rps_usage(sla, rl_data->rp_in_use, false); ++ ++ kfree(sla); ++ rl_data->sla[sla_id] = NULL; ++ sla_type_arr[node_id] = NULL; ++} ++ ++static void free_all_sla(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ int sla_id; ++ ++ mutex_lock(&rl_data->rl_lock); ++ ++ for (sla_id = 0; sla_id < RL_NODES_CNT_MAX; sla_id++) { ++ if (!rl_data->sla[sla_id]) ++ continue; ++ ++ kfree(rl_data->sla[sla_id]); ++ rl_data->sla[sla_id] = NULL; ++ } ++ ++ mutex_unlock(&rl_data->rl_lock); ++} ++ ++/** ++ * add_update_sla() - handles the creation and the update of an SLA ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_in: pointer to user input data for a new/updated SLA ++ * @is_update: flag to indicate if this is an update or an add operation ++ * ++ * Return: ++ * * 0 - ok ++ * * -ENOMEM - memory allocation failed ++ * * -EINVAL - user input data cannot be used to create SLA ++ * * -ENOSPC - all available SLAs are in use ++ */ ++static int add_update_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in, bool is_update) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla **sla_type_arr = NULL; ++ struct rl_sla *sla = NULL; ++ u32 old_cir = 0; ++ int ret; ++ ++ if (!sla_in) { ++ dev_warn(&GET_DEV(accel_dev), ++ "SLA input data pointer is missing\n"); ++ return -EFAULT; ++ } ++ ++ mutex_lock(&rl_data->rl_lock); ++ ++ /* Input validation */ ++ ret = validate_user_input(accel_dev, sla_in, is_update); ++ if (ret) ++ goto ret_err; ++ ++ if (is_update) { ++ ret = validate_sla_id(accel_dev, sla_in->sla_id); ++ if (ret) ++ goto ret_err; ++ ++ sla = rl_data->sla[sla_in->sla_id]; ++ old_cir = sla->cir; ++ } else { ++ ret = add_new_sla_entry(accel_dev, sla_in, &sla); ++ if (ret) ++ goto ret_err; ++ } ++ ++ if (!is_enough_budget(rl_data, sla, sla_in, is_update)) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Input value exceeds the remaining budget%s\n", ++ is_update ? " or more budget is already in use" : ""); ++ ret = -EINVAL; ++ goto ret_err; ++ } ++ sla->cir = sla_in->cir; ++ sla->pir = sla_in->pir; ++ ++ /* Apply SLA */ ++ assign_node_to_parent(accel_dev, sla, false); ++ ret = adf_rl_send_admin_add_update_msg(accel_dev, sla, is_update); ++ if (ret) { ++ dev_notice(&GET_DEV(accel_dev), ++ "Failed to apply an SLA\n"); ++ goto ret_err; ++ } ++ update_budget(sla, old_cir, is_update); ++ ++ if (!is_update) { ++ mark_rps_usage(sla, rl_data->rp_in_use, true); ++ adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr); ++ sla_type_arr[sla->node_id] = sla; ++ rl_data->sla[sla->sla_id] = sla; ++ } ++ ++ sla_in->sla_id = sla->sla_id; ++ goto ret_ok; ++ ++ret_err: ++ if (!is_update) { ++ sla_in->sla_id = -1; ++ kfree(sla); ++ } ++ret_ok: ++ mutex_unlock(&rl_data->rl_lock); ++ return ret; ++} ++ ++/** ++ * adf_rl_add_sla() - handles the creation of an SLA ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_in: pointer to user input data required to add an SLA ++ * ++ * Return: ++ * * 0 - ok ++ * * -ENOMEM - memory allocation failed ++ * * -EINVAL - invalid user input ++ * * -ENOSPC - all available SLAs are in use ++ */ ++int adf_rl_add_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in) ++{ ++ return add_update_sla(accel_dev, sla_in, false); ++} ++ ++/** ++ * adf_rl_update_sla() - handles the update of an SLA ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_in: pointer to user input data required to update an SLA ++ * ++ * Return: ++ * * 0 - ok ++ * * -EINVAL - user input data cannot be used to update SLA ++ */ ++int adf_rl_update_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in) ++{ ++ return add_update_sla(accel_dev, sla_in, true); ++} ++ ++/** ++ * adf_rl_get_sla() - returns an existing SLA data ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_in: pointer to user data where SLA info will be stored ++ * ++ * The sla_id for which data are requested should be set in sla_id structure ++ * ++ * Return: ++ * * 0 - ok ++ * * -EINVAL - provided sla_id does not exist ++ */ ++int adf_rl_get_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in) ++{ ++ struct rl_sla *sla; ++ int ret, i; ++ ++ ret = validate_sla_id(accel_dev, sla_in->sla_id); ++ if (ret) ++ return ret; ++ ++ sla = accel_dev->rate_limiting->sla[sla_in->sla_id]; ++ sla_in->type = sla->type; ++ sla_in->srv = sla->srv; ++ sla_in->cir = sla->cir; ++ sla_in->pir = sla->pir; ++ sla_in->rp_mask = 0U; ++ if (sla->parent) ++ sla_in->parent_id = sla->parent->sla_id; ++ else ++ sla_in->parent_id = RL_PARENT_DEFAULT_ID; ++ ++ for (i = 0; i < sla->ring_pairs_cnt; i++) ++ sla_in->rp_mask |= BIT(sla->ring_pairs_ids[i]); ++ ++ return 0; ++} ++ ++/** ++ * adf_rl_get_capability_remaining() - returns the remaining SLA value (CIR) for ++ * selected service or provided sla_id ++ * @accel_dev: pointer to acceleration device structure ++ * @srv: service ID for which capability is requested ++ * @sla_id: ID of the cluster or root to which we want assign a new SLA ++ * ++ * Check if the provided SLA id is valid. If it is and the service matches ++ * the requested service and the type is cluster or root, return the remaining ++ * capability. ++ * If the provided ID does not match the service or type, return the remaining ++ * capacity of the default cluster for that service. ++ * ++ * Return: ++ * * Positive value - correct remaining value ++ * * -EINVAL - algorithm cannot find a remaining value for provided data ++ */ ++int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev, ++ enum adf_svc_type srv, int sla_id) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla *sla = NULL; ++ int i; ++ ++ if (srv >= ADF_SVC_NONE) ++ return -EINVAL; ++ ++ if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) { ++ sla = rl_data->sla[sla_id]; ++ ++ if (sla->srv == srv && sla->type <= RL_CLUSTER) ++ goto ret_ok; ++ } ++ ++ for (i = 0; i < RL_CLUSTER_MAX; i++) { ++ if (!rl_data->cluster[i]) ++ continue; ++ ++ if (rl_data->cluster[i]->srv == srv) { ++ sla = rl_data->cluster[i]; ++ goto ret_ok; ++ } ++ } ++ ++ return -EINVAL; ++ret_ok: ++ return sla->rem_cir; ++} ++ ++/** ++ * adf_rl_remove_sla() - removes provided sla_id ++ * @accel_dev: pointer to acceleration device structure ++ * @sla_id: ID of the cluster or root to which we want assign an new SLA ++ * ++ * Return: ++ * * 0 - ok ++ * * -EINVAL - wrong sla_id or it still have assigned children ++ */ ++int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla *sla; ++ int ret = 0; ++ ++ mutex_lock(&rl_data->rl_lock); ++ ret = validate_sla_id(accel_dev, sla_id); ++ if (ret) ++ goto err_ret; ++ ++ sla = rl_data->sla[sla_id]; ++ ++ if (sla->type < RL_LEAF && sla->rem_cir != sla->cir) { ++ dev_notice(&GET_DEV(accel_dev), ++ "To remove parent SLA all its children must be removed first"); ++ ret = -EINVAL; ++ goto err_ret; ++ } ++ ++ clear_sla(rl_data, sla); ++ ++err_ret: ++ mutex_unlock(&rl_data->rl_lock); ++ return ret; ++} ++ ++/** ++ * adf_rl_remove_sla_all() - removes all SLAs from device ++ * @accel_dev: pointer to acceleration device structure ++ * @incl_default: set to true if default SLAs also should be removed ++ */ ++void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default) ++{ ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ int end_type = incl_default ? RL_ROOT : RL_LEAF; ++ struct rl_sla **sla_type_arr = NULL; ++ u32 max_id; ++ int i, j; ++ ++ mutex_lock(&rl_data->rl_lock); ++ ++ /* Unregister and remove all SLAs */ ++ for (j = RL_LEAF; j >= end_type; j--) { ++ max_id = adf_rl_get_sla_arr_of_type(rl_data, j, &sla_type_arr); ++ ++ for (i = 0; i < max_id; i++) { ++ if (!sla_type_arr[i]) ++ continue; ++ ++ clear_sla(rl_data, sla_type_arr[i]); ++ } ++ } ++ ++ mutex_unlock(&rl_data->rl_lock); ++} ++ ++int adf_rl_init(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); ++ struct adf_rl_hw_data *rl_hw_data = &hw_data->rl_data; ++ struct adf_rl *rl; ++ int ret = 0; ++ ++ /* Validate device parameters */ ++ if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_ASYM]) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_SYM]) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_DC]) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) || ++ RL_VALIDATE_NON_ZERO(rl_hw_data->scale_ref)) { ++ ret = -EOPNOTSUPP; ++ goto err_ret; ++ } ++ ++ rl = kzalloc(sizeof(*rl), GFP_KERNEL); ++ if (!rl) { ++ ret = -ENOMEM; ++ goto err_ret; ++ } ++ ++ mutex_init(&rl->rl_lock); ++ rl->device_data = &accel_dev->hw_device->rl_data; ++ rl->accel_dev = accel_dev; ++ accel_dev->rate_limiting = rl; ++ ++err_ret: ++ return ret; ++} ++ ++int adf_rl_start(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_rl_hw_data *rl_hw_data = &GET_HW_DATA(accel_dev)->rl_data; ++ void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); ++ u16 fw_caps = GET_HW_DATA(accel_dev)->accel_capabilities_mask; ++ int ret; ++ ++ if (!accel_dev->rate_limiting) { ++ ret = -EOPNOTSUPP; ++ goto ret_err; ++ } ++ ++ if (!(fw_caps & ICP_ACCEL_CAPABILITIES_RL)) { ++ dev_info(&GET_DEV(accel_dev), "not supported\n"); ++ ret = -EOPNOTSUPP; ++ goto ret_free; ++ } ++ ++ ADF_CSR_WR(pmisc_addr, rl_hw_data->pciin_tb_offset, ++ RL_TOKEN_GRANULARITY_PCIEIN_BUCKET); ++ ADF_CSR_WR(pmisc_addr, rl_hw_data->pciout_tb_offset, ++ RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET); ++ ++ ret = adf_rl_send_admin_init_msg(accel_dev, &rl_hw_data->slices); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "initialization failed\n"); ++ goto ret_free; ++ } ++ ++ ret = initialize_default_nodes(accel_dev); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "failed to initialize default SLAs\n"); ++ goto ret_sla_rm; ++ } ++ ++ ret = adf_sysfs_rl_add(accel_dev); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "failed to add sysfs interface\n"); ++ goto ret_sysfs_rm; ++ } ++ ++ return 0; ++ ++ret_sysfs_rm: ++ adf_sysfs_rl_rm(accel_dev); ++ret_sla_rm: ++ adf_rl_remove_sla_all(accel_dev, true); ++ret_free: ++ kfree(accel_dev->rate_limiting); ++ accel_dev->rate_limiting = NULL; ++ret_err: ++ return ret; ++} ++ ++void adf_rl_stop(struct adf_accel_dev *accel_dev) ++{ ++ if (!accel_dev->rate_limiting) ++ return; ++ ++ adf_sysfs_rl_rm(accel_dev); ++ free_all_sla(accel_dev); ++} ++ ++void adf_rl_exit(struct adf_accel_dev *accel_dev) ++{ ++ if (!accel_dev->rate_limiting) ++ return; ++ ++ kfree(accel_dev->rate_limiting); ++ accel_dev->rate_limiting = NULL; ++} +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.h +new file mode 100644 +index 0000000..ec9579f +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.h +@@ -0,0 +1,179 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2023 Intel Corporation */ ++ ++#ifndef ADF_RL_H_ ++#define ADF_RL_H_ ++ ++#include ++#include ++ ++struct adf_accel_dev; ++ ++#define RL_ROOT_MAX 4 ++#define RL_CLUSTER_MAX 16 ++#define RL_LEAF_MAX 64 ++#define RL_NODES_CNT_MAX (RL_ROOT_MAX + RL_CLUSTER_MAX + RL_LEAF_MAX) ++#define RL_RP_CNT_PER_LEAF_MAX 4U ++#define RL_RP_CNT_MAX 64 ++#define RL_SLA_EMPTY_ID -1 ++#define RL_PARENT_DEFAULT_ID -1 ++ ++enum rl_node_type { ++ RL_ROOT, ++ RL_CLUSTER, ++ RL_LEAF, ++}; ++#if 0 ++enum adf_svc_type { ++ ADF_SVC_ASYM = 0, ++ ADF_SVC_SYM, ++ ADF_SVC_DC, ++ ADF_SVC_NONE, ++}; ++#endif ++/** ++ * struct adf_rl_sla_input_data - ratelimiting user input data structure ++ * @rp_mask: 64 bit bitmask of ring pair IDs which will be assigned to SLA. ++ * Eg. 0x5 -> RP0 and RP2 assigned; 0xA005 -> RP0,2,13,15 assigned. ++ * @sla_id: ID of current SLA for operations update, rm, get. For the add ++ * operation, this field will be updated with the ID of the newly ++ * added SLA ++ * @parent_id: ID of the SLA to which the current one should be assigned. ++ * Set to -1 to refer to the default parent. ++ * @cir: Committed information rate. Rate guaranteed to be achieved. Input value ++ * is expressed in permille scale, i.e. 1000 refers to the maximum ++ * device throughput for a selected service. ++ * @pir: Peak information rate. Maximum rate available that the SLA can achieve. ++ * Input value is expressed in permille scale, i.e. 1000 refers to ++ * the maximum device throughput for a selected service. ++ * @type: SLA type: root, cluster, node ++ * @srv: Service associated to the SLA: asym, sym dc. ++ * ++ * This structure is used to perform operations on an SLA. ++ * Depending on the operation, some of the parameters are ignored. ++ * The following list reports which parameters should be set for each operation. ++ * - add: all except sla_id ++ * - update: cir, pir, sla_id ++ * - rm: sla_id ++ * - rm_all: - ++ * - get: sla_id ++ * - get_capability_rem: srv, sla_id ++ */ ++struct adf_rl_sla_input_data { ++ u64 rp_mask; ++ int sla_id; ++ int parent_id; ++ unsigned int cir; ++ unsigned int pir; ++ enum rl_node_type type; ++ enum adf_svc_type srv; ++}; ++ ++struct rl_slice_cnt { ++ u8 dcpr_cnt; ++ u8 pke_cnt; ++ u8 cph_cnt; ++}; ++ ++struct adf_rl_interface_data { ++ struct adf_rl_sla_input_data input; ++ enum adf_svc_type cap_rem_srv; ++ struct rw_semaphore lock; ++ bool sysfs_added; ++}; ++ ++struct adf_rl_hw_data { ++ u32 scale_ref; ++ u32 scan_interval; ++ u32 r2l_offset; ++ u32 l2c_offset; ++ u32 c2s_offset; ++ u32 pciin_tb_offset; ++ u32 pciout_tb_offset; ++ u32 pcie_scale_mul; ++ u32 pcie_scale_div; ++ u32 dcpr_correction; ++ u32 max_tp[RL_ROOT_MAX]; ++ struct rl_slice_cnt slices; ++}; ++ ++/** ++ * struct adf_rl - ratelimiting data structure ++ * @accel_dev: pointer to acceleration device data ++ * @device_data: pointer to rate limiting data specific to a device type (or revision) ++ * @sla: array of pointers to SLA objects ++ * @root: array of pointers to root type SLAs, element number reflects node_id ++ * @cluster: array of pointers to cluster type SLAs, element number reflects node_id ++ * @leaf: array of pointers to leaf type SLAs, element number reflects node_id ++ * @rp_in_use: array of ring pair IDs already used in one of SLAs ++ * @rl_lock: mutex object which is protecting data in this structure ++ * @input: structure which is used for holding the data received from user ++ */ ++struct adf_rl { ++ struct adf_accel_dev *accel_dev; ++ struct adf_rl_hw_data *device_data; ++ /* mapping sla_id to SLA objects */ ++ struct rl_sla *sla[RL_NODES_CNT_MAX]; ++ struct rl_sla *root[RL_ROOT_MAX]; ++ struct rl_sla *cluster[RL_CLUSTER_MAX]; ++ struct rl_sla *leaf[RL_LEAF_MAX]; ++ bool rp_in_use[RL_RP_CNT_MAX]; ++ /* Mutex protecting writing to SLAs lists */ ++ struct mutex rl_lock; ++ struct adf_rl_interface_data user_input; ++}; ++ ++/** ++ * struct rl_sla - SLA object data structure ++ * @parent: pointer to the parent SLA (root/cluster) ++ * @type: SLA type ++ * @srv: service associated with this SLA ++ * @sla_id: ID of the SLA, used as element number in SLA array and as identifier ++ * shared with the user ++ * @node_id: ID of node, each of SLA type have a separate ID list ++ * @cir: committed information rate ++ * @pir: peak information rate (PIR >= CIR) ++ * @rem_cir: if this SLA is a parent then this field represents a remaining ++ * value to be used by child SLAs. ++ * @ring_pairs_ids: array with numeric ring pairs IDs assigned to this SLA ++ * @ring_pairs_cnt: number of assigned ring pairs listed in the array above ++ */ ++struct rl_sla { ++ struct rl_sla *parent; ++ enum rl_node_type type; ++ enum adf_svc_type srv; ++ u32 sla_id; ++ u32 node_id; ++ u32 cir; ++ u32 pir; ++ u32 rem_cir; ++ u16 ring_pairs_ids[RL_RP_CNT_PER_LEAF_MAX]; ++ u16 ring_pairs_cnt; ++}; ++ ++u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, ++ struct rl_sla ***sla_arr); ++int adf_rl_add_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in); ++int adf_rl_update_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in); ++int adf_rl_get_sla(struct adf_accel_dev *accel_dev, ++ struct adf_rl_sla_input_data *sla_in); ++int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev, ++ enum adf_svc_type srv, int sla_id); ++int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id); ++void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default); ++ ++int adf_rl_init(struct adf_accel_dev *accel_dev); ++int adf_rl_start(struct adf_accel_dev *accel_dev); ++void adf_rl_stop(struct adf_accel_dev *accel_dev); ++void adf_rl_exit(struct adf_accel_dev *accel_dev); ++ ++u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type, bool is_bw_out); ++u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type); ++u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val, ++ enum adf_svc_type svc_type); ++ ++#endif /* ADF_RL_H_ */ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c +new file mode 100644 +index 0000000..a47c399 +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c +@@ -0,0 +1,98 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2023 Intel Corporation */ ++ ++#include ++#include ++ ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++#include "icp_qat_fw_init_admin.h" ++#include "adf_rl_admin.h" ++ ++static void ++prep_admin_req_msg(struct rl_sla *sla, dma_addr_t dma_addr, ++ struct icp_qat_fw_init_admin_sla_config_params *fw_params, ++ struct icp_qat_fw_init_admin_req *req, bool is_update) ++{ ++ req->cmd_id = is_update ? ICP_QAT_FW_RL_UPDATE : ICP_QAT_FW_RL_ADD; ++ req->init_cfg_ptr = dma_addr; ++ req->init_cfg_sz = sizeof(*fw_params); ++ req->node_id = sla->node_id; ++ req->node_type = sla->type; ++ req->rp_count = sla->ring_pairs_cnt; ++ req->svc_type = sla->srv; ++} ++ ++static void ++prep_admin_req_params(struct adf_accel_dev *accel_dev, struct rl_sla *sla, ++ struct icp_qat_fw_init_admin_sla_config_params *fw_params) ++{ ++ fw_params->pcie_in_cir = ++ adf_rl_calculate_pci_bw(accel_dev, sla->cir, sla->srv, false); ++ fw_params->pcie_in_pir = ++ adf_rl_calculate_pci_bw(accel_dev, sla->pir, sla->srv, false); ++ fw_params->pcie_out_cir = ++ adf_rl_calculate_pci_bw(accel_dev, sla->cir, sla->srv, true); ++ fw_params->pcie_out_pir = ++ adf_rl_calculate_pci_bw(accel_dev, sla->pir, sla->srv, true); ++ ++ fw_params->slice_util_cir = ++ adf_rl_calculate_slice_tokens(accel_dev, sla->cir, sla->srv); ++ fw_params->slice_util_pir = ++ adf_rl_calculate_slice_tokens(accel_dev, sla->pir, sla->srv); ++ ++ fw_params->ae_util_cir = ++ adf_rl_calculate_ae_cycles(accel_dev, sla->cir, sla->srv); ++ fw_params->ae_util_pir = ++ adf_rl_calculate_ae_cycles(accel_dev, sla->pir, sla->srv); ++ ++ memcpy(fw_params->rp_ids, sla->ring_pairs_ids, ++ sizeof(sla->ring_pairs_ids)); ++} ++ ++int adf_rl_send_admin_init_msg(struct adf_accel_dev *accel_dev, ++ struct rl_slice_cnt *slices_int) ++{ ++ struct icp_qat_fw_init_admin_slice_cnt slices_resp = { }; ++ int ret; ++ ++ ret = adf_send_admin_rl_init(accel_dev, &slices_resp); ++ if (ret) ++ return ret; ++ ++ slices_int->dcpr_cnt = slices_resp.dcpr_cnt; ++ slices_int->pke_cnt = slices_resp.pke_cnt; ++ /* For symmetric crypto, slice tokens are relative to the UCS slice */ ++ slices_int->cph_cnt = slices_resp.ucs_cnt; ++ ++ return 0; ++} ++ ++int adf_rl_send_admin_add_update_msg(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool is_update) ++{ ++ struct icp_qat_fw_init_admin_sla_config_params *fw_params; ++ struct icp_qat_fw_init_admin_req req = { }; ++ dma_addr_t dma_addr; ++ int ret; ++ ++ fw_params = dma_alloc_coherent(&GET_DEV(accel_dev), sizeof(*fw_params), ++ &dma_addr, GFP_KERNEL); ++ if (!fw_params) ++ return -ENOMEM; ++ ++ prep_admin_req_params(accel_dev, sla, fw_params); ++ prep_admin_req_msg(sla, dma_addr, fw_params, &req, is_update); ++ ret = adf_send_admin_rl_add_update(accel_dev, &req); ++ ++ dma_free_coherent(&GET_DEV(accel_dev), sizeof(*fw_params), fw_params, ++ dma_addr); ++ ++ return ret; ++} ++ ++int adf_rl_send_admin_delete_msg(struct adf_accel_dev *accel_dev, u16 node_id, ++ u8 node_type) ++{ ++ return adf_send_admin_rl_delete(accel_dev, node_id, node_type); ++} +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.h +new file mode 100644 +index 0000000..dd5419b +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.h +@@ -0,0 +1,18 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2023 Intel Corporation */ ++ ++#ifndef ADF_RL_ADMIN_H_ ++#define ADF_RL_ADMIN_H_ ++ ++#include ++ ++#include "adf_rl.h" ++ ++int adf_rl_send_admin_init_msg(struct adf_accel_dev *accel_dev, ++ struct rl_slice_cnt *slices_int); ++int adf_rl_send_admin_add_update_msg(struct adf_accel_dev *accel_dev, ++ struct rl_sla *sla, bool is_update); ++int adf_rl_send_admin_delete_msg(struct adf_accel_dev *accel_dev, u16 node_id, ++ u8 node_type); ++ ++#endif /* ADF_RL_ADMIN_H_ */ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.c +new file mode 100644 +index 0000000..5109d4f +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.c +@@ -0,0 +1,461 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2023 Intel Corporation */ ++ ++#ifndef dev_fmt ++#define dev_fmt(fmt) "RateLimiting: " fmt ++#endif ++ ++#include ++#include ++#include ++#include ++ ++#include "adf_common_drv.h" ++#include "adf_rl.h" ++#include "adf_sysfs_rl.h" ++ ++#define GET_RL_STRUCT(accel_dev) ((accel_dev)->rate_limiting->user_input) ++ ++enum rl_ops { ++ ADD, ++ UPDATE, ++ RM, ++ RM_ALL, ++ GET, ++}; ++ ++enum rl_params { ++ RP_MASK, ++ ID, ++ CIR, ++ PIR, ++ SRV, ++ CAP_REM_SRV, ++}; ++ ++static const char *const rl_services[] = { ++ [ADF_SVC_ASYM] = "asym", ++ [ADF_SVC_SYM] = "sym", ++ [ADF_SVC_DC] = "dc", ++}; ++ ++static const char *const rl_operations[] = { ++ [ADD] = "add", ++ [UPDATE] = "update", ++ [RM] = "rm", ++ [RM_ALL] = "rm_all", ++ [GET] = "get", ++}; ++ ++static int set_param_u(struct device *dev, enum rl_params param, u64 set) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ int ret = 0; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ down_write(&data->lock); ++ switch (param) { ++ case RP_MASK: ++ data->input.rp_mask = set; ++ break; ++ case CIR: ++ data->input.cir = set; ++ break; ++ case PIR: ++ data->input.pir = set; ++ break; ++ case SRV: ++ data->input.srv = set; ++ break; ++ case CAP_REM_SRV: ++ data->cap_rem_srv = set; ++ break; ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ up_write(&data->lock); ++ ++ return ret; ++} ++ ++static int set_param_s(struct device *dev, enum rl_params param, int set) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev || param != ID) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ down_write(&data->lock); ++ data->input.sla_id = set; ++ up_write(&data->lock); ++ ++ return 0; ++} ++ ++static int get_param_u(struct device *dev, enum rl_params param, u64 *get) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ int ret = 0; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ down_read(&data->lock); ++ switch (param) { ++ case RP_MASK: ++ *get = data->input.rp_mask; ++ break; ++ case CIR: ++ *get = data->input.cir; ++ break; ++ case PIR: ++ *get = data->input.pir; ++ break; ++ case SRV: ++ *get = data->input.srv; ++ break; ++ default: ++ ret = -EINVAL; ++ } ++ up_read(&data->lock); ++ ++ return ret; ++} ++ ++static int get_param_s(struct device *dev, enum rl_params param) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ int ret = 0; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ down_read(&data->lock); ++ if (param == ID) ++ ret = data->input.sla_id; ++ up_read(&data->lock); ++ ++ return ret; ++} ++ ++static ssize_t rp_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ int ret; ++ u64 get; ++ ++ ret = get_param_u(dev, RP_MASK, &get); ++ if (ret) ++ return ret; ++ ++ return sysfs_emit(buf, "%#llx\n", get); ++} ++ ++static ssize_t rp_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ int err; ++ u64 val; ++ ++ err = kstrtou64(buf, 16, &val); ++ if (err) ++ return err; ++ ++ err = set_param_u(dev, RP_MASK, val); ++ if (err) ++ return err; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(rp); ++ ++static ssize_t id_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ return sysfs_emit(buf, "%d\n", get_param_s(dev, ID)); ++} ++ ++static ssize_t id_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ int err; ++ int val; ++ ++ err = kstrtoint(buf, 10, &val); ++ if (err) ++ return err; ++ ++ err = set_param_s(dev, ID, val); ++ if (err) ++ return err; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(id); ++ ++static ssize_t cir_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ int ret; ++ u64 get; ++ ++ ret = get_param_u(dev, CIR, &get); ++ if (ret) ++ return ret; ++ ++ return sysfs_emit(buf, "%llu\n", get); ++} ++ ++static ssize_t cir_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int val; ++ int err; ++ ++ err = kstrtouint(buf, 10, &val); ++ if (err) ++ return err; ++ ++ err = set_param_u(dev, CIR, val); ++ if (err) ++ return err; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(cir); ++ ++static ssize_t pir_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ int ret; ++ u64 get; ++ ++ ret = get_param_u(dev, PIR, &get); ++ if (ret) ++ return ret; ++ ++ return sysfs_emit(buf, "%llu\n", get); ++} ++ ++static ssize_t pir_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int val; ++ int err; ++ ++ err = kstrtouint(buf, 10, &val); ++ if (err) ++ return err; ++ ++ err = set_param_u(dev, PIR, val); ++ if (err) ++ return err; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(pir); ++ ++static ssize_t srv_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ int ret; ++ u64 get; ++ ++ ret = get_param_u(dev, SRV, &get); ++ if (ret) ++ return ret; ++ ++ if (get == ADF_SVC_NONE) ++ return -EINVAL; ++ ++ return sysfs_emit(buf, "%s\n", rl_services[get]); ++} ++ ++static ssize_t srv_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int val; ++ int ret; ++ ++ ret = sysfs_match_string(rl_services, buf); ++ if (ret < 0) ++ return ret; ++ ++ val = ret; ++ ret = set_param_u(dev, SRV, val); ++ if (ret) ++ return ret; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(srv); ++ ++static ssize_t cap_rem_show(struct device *dev, struct device_attribute *attr, ++ char *buf) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ int ret, rem_cap; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ down_read(&data->lock); ++ rem_cap = adf_rl_get_capability_remaining(accel_dev, data->cap_rem_srv, ++ RL_SLA_EMPTY_ID); ++ up_read(&data->lock); ++ if (rem_cap < 0) ++ return rem_cap; ++ ++ ret = sysfs_emit(buf, "%u\n", rem_cap); ++ ++ return ret; ++} ++ ++static ssize_t cap_rem_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ unsigned int val; ++ int ret; ++ ++ ret = sysfs_match_string(rl_services, buf); ++ if (ret < 0) ++ return ret; ++ ++ val = ret; ++ ret = set_param_u(dev, CAP_REM_SRV, val); ++ if (ret) ++ return ret; ++ ++ return count; ++} ++static DEVICE_ATTR_RW(cap_rem); ++ ++static ssize_t sla_op_store(struct device *dev, struct device_attribute *attr, ++ const char *buf, size_t count) ++{ ++ struct adf_rl_interface_data *data; ++ struct adf_accel_dev *accel_dev; ++ int ret; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); ++ if (!accel_dev) ++ return -EINVAL; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ ret = sysfs_match_string(rl_operations, buf); ++ if (ret < 0) ++ return ret; ++ ++ down_write(&data->lock); ++ switch (ret) { ++ case ADD: ++ data->input.parent_id = RL_PARENT_DEFAULT_ID; ++ data->input.type = RL_LEAF; ++ data->input.sla_id = 0; ++ ret = adf_rl_add_sla(accel_dev, &data->input); ++ if (ret) ++ goto err_free_lock; ++ break; ++ case UPDATE: ++ ret = adf_rl_update_sla(accel_dev, &data->input); ++ if (ret) ++ goto err_free_lock; ++ break; ++ case RM: ++ ret = adf_rl_remove_sla(accel_dev, data->input.sla_id); ++ if (ret) ++ goto err_free_lock; ++ break; ++ case RM_ALL: ++ adf_rl_remove_sla_all(accel_dev, false); ++ break; ++ case GET: ++ ret = adf_rl_get_sla(accel_dev, &data->input); ++ if (ret) ++ goto err_free_lock; ++ break; ++ default: ++ ret = -EINVAL; ++ goto err_free_lock; ++ } ++ up_write(&data->lock); ++ ++ return count; ++ ++err_free_lock: ++ up_write(&data->lock); ++ ++ return ret; ++} ++static DEVICE_ATTR_WO(sla_op); ++ ++static struct attribute *qat_rl_attrs[] = { ++ &dev_attr_rp.attr, ++ &dev_attr_id.attr, ++ &dev_attr_cir.attr, ++ &dev_attr_pir.attr, ++ &dev_attr_srv.attr, ++ &dev_attr_cap_rem.attr, ++ &dev_attr_sla_op.attr, ++ NULL, ++}; ++ ++static struct attribute_group qat_rl_group = { ++ .attrs = qat_rl_attrs, ++ .name = "qat_rl", ++}; ++ ++int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_rl_interface_data *data; ++ int ret; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ ++ ret = device_add_group(&GET_DEV(accel_dev), &qat_rl_group); ++ if (ret) ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to create qat_rl attribute group\n"); ++ ++ data->cap_rem_srv = ADF_SVC_NONE; ++ data->input.srv = ADF_SVC_NONE; ++ data->sysfs_added = true; ++ ++ return ret; ++} ++ ++void adf_sysfs_rl_rm(struct adf_accel_dev *accel_dev) ++{ ++ struct adf_rl_interface_data *data; ++ ++ data = &GET_RL_STRUCT(accel_dev); ++ if (!data->sysfs_added) ++ return; ++ ++ device_remove_group(&GET_DEV(accel_dev), &qat_rl_group); ++ data->sysfs_added = false; ++} +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.h +new file mode 100644 +index 0000000..22d36aa +--- /dev/null ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sysfs_rl.h +@@ -0,0 +1,11 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2023 Intel Corporation */ ++#ifndef ADF_SYSFS_RL_H_ ++#define ADF_SYSFS_RL_H_ ++ ++struct adf_accel_dev; ++ ++int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev); ++void adf_sysfs_rl_rm(struct adf_accel_dev *accel_dev); ++ ++#endif /* ADF_SYSFS_RL_H_ */ +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/quickassist/qat/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +index 82ba4ef..95240fb 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +@@ -52,6 +52,18 @@ enum icp_qat_fw_init_admin_resp_status { + #endif + }; + ++struct icp_qat_fw_init_admin_slice_cnt { ++ __u8 cpr_cnt; ++ __u8 xlt_cnt; ++ __u8 dcpr_cnt; ++ __u8 pke_cnt; ++ __u8 wat_cnt; ++ __u8 wcp_cnt; ++ __u8 ucs_cnt; ++ __u8 cph_cnt; ++ __u8 ath_cnt; ++}; ++ + enum icp_qat_fw_cnv_error_type { + CNV_ERR_TYPE_NO_ERROR = 0, + CNV_ERR_TYPE_CHECKSUM_ERROR, +@@ -214,6 +226,7 @@ struct icp_qat_fw_init_admin_resp { + u16 reservd2; + u32 reservd3; + } slice_count; ++ struct icp_qat_fw_init_admin_slice_cnt slices; + }; + } __packed; + +-- +2.39.3 + diff --git a/1014-Uncomment-sla-in-vf-migs.patch b/1014-Uncomment-sla-in-vf-migs.patch new file mode 100644 index 0000000000000000000000000000000000000000..db05264bdabe7b577513f86358d4e5c36949df01 --- /dev/null +++ b/1014-Uncomment-sla-in-vf-migs.patch @@ -0,0 +1,35 @@ +From b9c3e2f46a1c4e0b236f6bab2bc4dda8c770b7d2 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 12 Apr 2024 12:44:42 +0800 +Subject: [PATCH 14/28] Uncomment sla in vf migs + +Signed-off-by: Zelin Deng +--- + .../qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c | 4 ---- + 1 file changed, 4 deletions(-) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +index be7c78f..00eb60b 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_vf_mig.c +@@ -313,8 +313,6 @@ static int adf_mstate_set_vreg(struct adf_mstate_mgr *sub_mgr, u8 *buf, + static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, + struct mig_user_sla *pmig_slas) + { +-//TODO +-#if 0 + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_rl *rl_data = accel_dev->rate_limiting; + struct rl_sla **sla_type_arr = NULL; +@@ -348,8 +346,6 @@ static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, + } + + return sla_cnt; +-#endif +- return 0; + } + + static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, +-- +2.39.3 + diff --git a/1015-OOT-fw-only-accept-sla-req-init_cfg_sz-64.patch b/1015-OOT-fw-only-accept-sla-req-init_cfg_sz-64.patch new file mode 100644 index 0000000000000000000000000000000000000000..18f79495488c0cf2dde37810bf02f5f08efe30b7 --- /dev/null +++ b/1015-OOT-fw-only-accept-sla-req-init_cfg_sz-64.patch @@ -0,0 +1,26 @@ +From 252eb83579c559b300081933fe997c6ea168a328 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 12 Apr 2024 16:50:23 +0800 +Subject: [PATCH 15/28] OOT fw only accept sla req 'init_cfg_sz == 64' + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c +index a47c399..a0e909c 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl_admin.c +@@ -16,7 +16,7 @@ prep_admin_req_msg(struct rl_sla *sla, dma_addr_t dma_addr, + { + req->cmd_id = is_update ? ICP_QAT_FW_RL_UPDATE : ICP_QAT_FW_RL_ADD; + req->init_cfg_ptr = dma_addr; +- req->init_cfg_sz = sizeof(*fw_params); ++ req->init_cfg_sz = 64; + req->node_id = sla->node_id; + req->node_type = sla->type; + req->rp_count = sla->ring_pairs_cnt; +-- +2.39.3 + diff --git a/1016-Fix-typo-when-merging-code.patch b/1016-Fix-typo-when-merging-code.patch new file mode 100644 index 0000000000000000000000000000000000000000..a728f71d3d68a81cacb44d0e3a5171cda07038cb --- /dev/null +++ b/1016-Fix-typo-when-merging-code.patch @@ -0,0 +1,26 @@ +From 9f972210bfc5114a7ee9009adc4f3950b1cc97f0 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Mon, 22 Apr 2024 14:08:44 +0800 +Subject: [PATCH 16/28] Fix typo when merging code + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c +index e91d008..48c4be5 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_sriov.c +@@ -92,7 +92,7 @@ static void adf_iov_send_resp(struct work_struct *work) + + mutex_lock(&pf2vf_resp->vf_info->pfvf_mig_lock); + adf_vf2pf_req_hndl(pf2vf_resp->vf_info); +- mutex_lock(&pf2vf_resp->vf_info->pfvf_mig_lock); ++ mutex_unlock(&pf2vf_resp->vf_info->pfvf_mig_lock); + kfree(pf2vf_resp); + } + +-- +2.39.3 + diff --git a/1017-Move-migration-relevant-logic-to-qat_vf_vfio-module.patch b/1017-Move-migration-relevant-logic-to-qat_vf_vfio-module.patch new file mode 100644 index 0000000000000000000000000000000000000000..e7accfb57fc340eb450d3a286ada5fea717ec968 --- /dev/null +++ b/1017-Move-migration-relevant-logic-to-qat_vf_vfio-module.patch @@ -0,0 +1,1724 @@ +From 8811a40db6db5a7baaeb32f1a8ca8e8c3446e17f Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Wed, 24 Apr 2024 14:39:32 +0800 +Subject: [PATCH 17/28] Move migration relevant logic to qat_vf_vfio module + +Also keep original logic in intel_qat.ko and qat_4xxx.ko as qat_vf_vfio +would override them. + +Signed-off-by: Zelin Deng +--- + .../crypto/qat/qat_common/adf_gen4_hw_data.c | 2 + + .../drivers/crypto/qat/qat_common/adf_rl.c | 1 + + quickassist/qat/drivers/vfio/pci/qat/Makefile | 2 +- + .../drivers/vfio/pci/qat/adf_gen4_vf_mig.c | 1011 +++++++++++++++++ + .../drivers/vfio/pci/qat/adf_gen4_vf_mig.h | 10 + + .../qat/drivers/vfio/pci/qat/adf_mstate_mgr.c | 318 ++++++ + .../qat/drivers/vfio/pci/qat/adf_mstate_mgr.h | 89 ++ + quickassist/qat/drivers/vfio/pci/qat/main.c | 7 + + .../qat/drivers/vfio/pci/qat/qat_mig_dev.c | 130 +++ + .../qat/drivers/vfio/pci/qat/qat_mig_dev.h | 31 + + 10 files changed, 1600 insertions(+), 1 deletion(-) + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.c + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.h + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.c + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.h + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.c + create mode 100644 quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.h + +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +index fb1bd36..a6fc618 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c +@@ -740,6 +740,7 @@ void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev, + ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number), + ADF_WQM_CSR_RPRESETSTS_STATUS); + } ++EXPORT_SYMBOL(adf_gen4_bank_drain_finish); + + int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + u32 bank_number, int timeout_us) +@@ -757,6 +758,7 @@ int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev, + + return ret; + } ++EXPORT_SYMBOL(adf_gen4_bank_drain_start); + + static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, + u32 bank, struct bank_state *state, u32 num_rings) +diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c +index a56ae16..2594d70 100644 +--- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c ++++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_rl.c +@@ -219,6 +219,7 @@ u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type, + return 0; + } + } ++EXPORT_SYMBOL(adf_rl_get_sla_arr_of_type); + + static bool is_service_enabled(struct adf_accel_dev *accel_dev, + enum adf_svc_type rl_srv) +diff --git a/quickassist/qat/drivers/vfio/pci/qat/Makefile b/quickassist/qat/drivers/vfio/pci/qat/Makefile +index 873308b..dde76ac 100644 +--- a/quickassist/qat/drivers/vfio/pci/qat/Makefile ++++ b/quickassist/qat/drivers/vfio/pci/qat/Makefile +@@ -1,3 +1,3 @@ + # SPDX-License-Identifier: GPL-2.0-only + obj-$(CONFIG_QAT_VFIO_PCI) += qat_vfio_pci.o +-qat_vfio_pci-y := main.o dummy.o ++qat_vfio_pci-y := main.o dummy.o adf_gen4_vf_mig.o adf_mstate_mgr.o qat_mig_dev.o +diff --git a/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.c b/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.c +new file mode 100644 +index 0000000..d79ab16 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.c +@@ -0,0 +1,1011 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++#include "adf_gen4_hw_data.h" ++#include "adf_mstate_mgr.h" ++#include "adf_gen4_vf_mig.h" ++#include "adf_pf2vf_msg.h" ++#include "adf_gen4_hw_csr_data.h" ++ ++#define ADF_GEN4_VF_MSTATE_SIZE 4096 ++#define ADF_GEN4_PFVF_RSP_TIMEOUT_US 5000 ++ ++static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev); ++static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len); ++ ++static int adf_gen4_vfmig_init_device(struct qat_mig_dev *mdev) ++{ ++ u8 *state; ++ ++ state = kmalloc(ADF_GEN4_VF_MSTATE_SIZE, GFP_KERNEL); ++ if (!state) ++ return -ENOMEM; ++ ++ mdev->state = state; ++ mdev->state_size = ADF_GEN4_VF_MSTATE_SIZE; ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++ ++ return 0; ++} ++ ++static void adf_gen4_vfmig_cleanup_device(struct qat_mig_dev *mdev) ++{ ++ kfree(mdev->state); ++ mdev->state = NULL; ++} ++ ++static void adf_gen4_vfmig_reset_device(struct qat_mig_dev *mdev) ++{ ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++} ++ ++static int adf_gen4_vfmig_open_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ ++ vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; ++ ++ vfmig = kzalloc(sizeof(*vfmig), GFP_KERNEL); ++ if (!vfmig) ++ return -ENOMEM; ++ ++ vfmig->mstate_mgr = adf_mstate_mgr_new(mdev->state, mdev->state_size); ++ if (!vfmig->mstate_mgr) { ++ kfree(vfmig); ++ return -ENOMEM; ++ } ++ vf_info->mig_priv = vfmig; ++ mdev->setup_size = 0; ++ mdev->remote_setup_size = 0; ++ ++ return 0; ++} ++ ++static void adf_gen4_vfmig_close_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ ++ vf_info = &accel_dev->pf.vf_info[mdev->vf_id]; ++ if (vf_info->mig_priv) { ++ vfmig = vf_info->mig_priv; ++ adf_mstate_mgr_destroy(vfmig->mstate_mgr); ++ kfree(vfmig); ++ vf_info->mig_priv = NULL; ++ } ++} ++ ++static int adf_gen4_vfmig_suspend_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vf_mig; ++ u32 vf_nr = mdev->vf_id; ++ int ret, i; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vf_mig = vf_info->mig_priv; ++ ++ /* Stop all inflight jobs */ ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; ++ ++ ret = adf_gen4_bank_drain_start(accel_dev, pf_bank_nr, ++ ADF_RPRESET_POLL_TIMEOUT_US); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to drain bank %d for vf_nr %d\n", i, ++ vf_nr); ++ return ret; ++ } ++ vf_mig->bank_stopped[i] = true; ++ ++ adf_gen4_bank_quiesce_coal_timer(accel_dev, pf_bank_nr, ++ ADF_COALESCED_POLL_TIMEOUT_US); ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_resume_device(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vf_mig; ++ u32 vf_nr = mdev->vf_id; ++ int i; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vf_mig = vf_info->mig_priv; ++ ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ u32 pf_bank_nr = i + vf_nr * hw_data->num_banks_per_vf; ++ ++ if (vf_mig->bank_stopped[i]) { ++ adf_gen4_bank_drain_finish(accel_dev, pf_bank_nr); ++ vf_mig->bank_stopped[i] = false; ++ } ++ } ++ ++ return 0; ++} ++ ++struct adf_vf_bank_info { ++ struct adf_accel_dev *accel_dev; ++ u32 vf_nr; ++ u32 bank_nr; ++}; ++ ++struct mig_user_sla { ++ //enum adf_base_services srv; ++ enum adf_svc_type srv; ++ u64 rp_mask; ++ u32 cir; ++ u32 pir; ++}; ++ ++static int adf_mstate_sla_check(struct adf_mstate_mgr *sub_mgr, u8 *src_buf, ++ u32 src_size, void *opaque) ++{ ++ struct adf_mstate_vreginfo _sinfo = { src_buf, src_size }; ++ struct adf_mstate_vreginfo *sinfo = &_sinfo, *dinfo = opaque; ++ u32 src_sla_cnt = sinfo->size / sizeof(struct mig_user_sla); ++ u32 dst_sla_cnt = dinfo->size / sizeof(struct mig_user_sla); ++ struct mig_user_sla *src_slas = sinfo->addr; ++ struct mig_user_sla *dst_slas = dinfo->addr; ++ int i, j; ++ ++ for (i = 0; i < src_sla_cnt; i++) { ++ for (j = 0; j < dst_sla_cnt; j++) { ++ if (src_slas[i].srv != dst_slas[j].srv || ++ src_slas[i].rp_mask != dst_slas[j].rp_mask) ++ continue; ++ ++ if (src_slas[i].cir > dst_slas[j].cir || ++ src_slas[i].pir > dst_slas[j].pir) { ++ pr_err("QAT: DST VF rate limiting mismatch.\n"); ++ return -EINVAL; ++ } ++ break; ++ } ++ ++ if (j == dst_sla_cnt) { ++ pr_err("QAT: SRC VF rate limiting mismatch - SRC srv %d and rp_mask 0x%llx.\n", ++ src_slas[i].srv, src_slas[i].rp_mask); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static inline int adf_mstate_check_cap_size(u32 src_sz, u32 dst_sz, u32 max_sz) ++{ ++ if (src_sz > max_sz || dst_sz > max_sz) ++ return -EINVAL; ++ else ++ return 0; ++} ++ ++static int adf_mstate_compatver_check(struct adf_mstate_mgr *sub_mgr, ++ u8 *src_buf, u32 src_sz, void *opaque) ++{ ++ struct adf_mstate_vreginfo *info = opaque; ++ u8 compat = 0; ++ u8 *pcompat; ++ ++ if (src_sz != info->size) { ++ pr_debug("QAT: State mismatch (compat version size), current %u, expected %u\n", ++ src_sz, info->size); ++ return -EINVAL; ++ } ++ ++ memcpy(info->addr, src_buf, info->size); ++ pcompat = info->addr; ++ if (*pcompat == 0) { ++ pr_warn("QAT: Unable to determine the version of VF\n"); ++ return 0; ++ } ++ ++ compat = adf_vf_compat_checker(*pcompat); ++ if (compat == ADF_PF2VF_VF_INCOMPATIBLE) { ++ pr_debug("QAT: SRC VF driver (ver=%u) is incompatible with DST PF driver (ver=%u)\n", ++ *pcompat, ADF_PFVF_COMPATIBILITY_VERSION); ++ return -EINVAL; ++ } ++ ++ if (compat == ADF_PF2VF_VF_COMPAT_UNKNOWN) ++ pr_debug("QAT: SRC VF driver (ver=%u) is newer than DST PF driver (ver=%u)\n", ++ *pcompat, ADF_PFVF_COMPATIBILITY_VERSION); ++ ++ return 0; ++} ++ ++/* ++ * adf_mstate_capmask_compare() - compare QAT device capability mask ++ * @sinfo: Pointer to source capability info ++ * @dinfo: Pointer to target capability info ++ * ++ * This function compares the capability mask between source VF and target VF ++ * ++ * Returns: 0 if target capability mask is identical to source capability mask, ++ * 1 if target mask can represent all the capabilities represented by source mask, ++ * -1 if target mask can't represent all the capabilities represented by source ++ * mask. ++ */ ++static int adf_mstate_capmask_compare(struct adf_mstate_vreginfo *sinfo, ++ struct adf_mstate_vreginfo *dinfo) ++{ ++ u64 src = 0, dst = 0; ++ ++ if (adf_mstate_check_cap_size(sinfo->size, dinfo->size, sizeof(u64))) { ++ pr_debug("QAT: Unexpected capability size %u %u %zu\n", ++ sinfo->size, dinfo->size, sizeof(u64)); ++ return -1; ++ } ++ ++ memcpy(&src, sinfo->addr, sinfo->size); ++ memcpy(&dst, dinfo->addr, dinfo->size); ++ ++ pr_debug("QAT: Check cap compatibility of cap %llu %llu\n", src, dst); ++ ++ if (src == dst) ++ return 0; ++ ++ if ((src | dst) == dst) ++ return 1; ++ ++ return -1; ++} ++ ++static int adf_mstate_capmask_superset(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo sinfo = { buf, size }; ++ ++ if (adf_mstate_capmask_compare(&sinfo, opa) >= 0) ++ return 0; ++ ++ return -EINVAL; ++} ++ ++static int adf_mstate_capmask_equal(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo sinfo = { buf, size }; ++ ++ if (adf_mstate_capmask_compare(&sinfo, opa) == 0) ++ return 0; ++ ++ return -EINVAL; ++} ++ ++static int adf_mstate_set_vreg(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa) ++{ ++ struct adf_mstate_vreginfo *info = opa; ++ ++ if (size != info->size) { ++ pr_debug("QAT: Unexpected cap size %u %u\n", size, info->size); ++ return -EINVAL; ++ } ++ memcpy(info->addr, buf, info->size); ++ ++ return 0; ++} ++ ++static u32 adf_gen4_vfmig_get_slas(struct adf_accel_dev *accel_dev, u32 vf_nr, ++ struct mig_user_sla *pmig_slas) ++{ ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_rl *rl_data = accel_dev->rate_limiting; ++ struct rl_sla **sla_type_arr = NULL; ++ u64 rp_mask, rp_index; ++ u32 max_num_sla; ++ u32 sla_cnt = 0; ++ int i, j; ++ ++ if (!accel_dev->rate_limiting) ++ return 0; ++ ++ rp_index = vf_nr * hw_data->num_banks_per_vf; ++ max_num_sla = adf_rl_get_sla_arr_of_type(rl_data, RL_LEAF, &sla_type_arr); ++ ++ for (i = 0; i < max_num_sla; i++) { ++ if (!sla_type_arr[i]) ++ continue; ++ ++ rp_mask = 0; ++ for (j = 0; j < sla_type_arr[i]->ring_pairs_cnt; j++) ++ rp_mask |= BIT(sla_type_arr[i]->ring_pairs_ids[j]); ++ ++ if (rp_mask & GENMASK_ULL(rp_index + 3, rp_index)) { ++ pmig_slas->rp_mask = rp_mask; ++ pmig_slas->cir = sla_type_arr[i]->cir; ++ pmig_slas->pir = sla_type_arr[i]->pir; ++ pmig_slas->srv = sla_type_arr[i]->srv; ++ pmig_slas++; ++ sla_cnt++; ++ } ++ } ++ ++ return sla_cnt; ++} ++ ++static int adf_gen4_vfmig_load_etr_regs(struct adf_mstate_mgr *sub_mgr, ++ u8 *state, u32 size, void *opa) ++{ ++ struct adf_vf_bank_info *vf_bank_info = opa; ++ struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ u32 pf_bank_nr; ++ int ret; ++ ++ pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ++ ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr, ++ (struct bank_state *)state); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load regs for vf%d bank%d\n", ++ vf_bank_info->vf_nr, vf_bank_info->bank_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_etr_bank(struct adf_accel_dev *accel_dev, ++ u32 vf_nr, u32 bank_nr, ++ struct adf_mstate_mgr *mstate_mgr) ++{ ++ struct adf_vf_bank_info vf_bank_info = {accel_dev, vf_nr, bank_nr}; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ char bank_ids[ADF_MSTATE_ID_LEN]; ++ ++ snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); ++ subsec = adf_mstate_sect_lookup(mstate_mgr, bank_ids, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to lookup sec %s for vf%d bank%d\n", ++ ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, ++ adf_gen4_vfmig_load_etr_regs, ++ &vf_bank_info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec; ++ int ret, i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, ++ NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_ETRB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ ret = adf_gen4_vfmig_load_etr_bank(accel_dev, vf_nr, i, ++ &sub_sects_mgr); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ void __iomem *csr = adf_get_pmisc_base(accel_dev); ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ u64 ofs; ++ } misc_states[] = { ++ {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, ++ NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_MISCB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(misc_states); i++) { ++ struct adf_mstate_vreginfo info; ++ u32 regv; ++ ++ info.addr = ®v; ++ info.size = sizeof(regv); ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ++ misc_states[i].id, ++ adf_mstate_set_vreg, ++ &info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load sec %s\n", misc_states[i].id); ++ return -EINVAL; ++ } ++ ADF_CSR_WR(csr, misc_states[i].ofs, regv); ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct mig_user_sla dst_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ u32 dst_sla_cnt; ++ struct { ++ char *id; ++ int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); ++ struct adf_mstate_vreginfo info; ++ } gen_states[] = { ++ {ADF_MSTATE_IOV_INIT_IDS, adf_mstate_set_vreg, ++ {&vf_info->init, sizeof(vf_info->init)}}, ++ {ADF_MSTATE_COMPAT_VER_IDS, adf_mstate_compatver_check, ++ {&vf_info->compat_ver, sizeof(vf_info->compat_ver)}}, ++ {ADF_MSTATE_SLA_IDS, adf_mstate_sla_check, {dst_slas, 0}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_GEN_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(gen_states); i++) { ++ if (gen_states[i].info.addr == dst_slas) { ++ dst_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, dst_slas); ++ gen_states[i].info.size = dst_sla_cnt * sizeof(struct mig_user_sla); ++ } ++ ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, ++ gen_states[i].id, ++ gen_states[i].action, ++ &gen_states[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ gen_states[i].id); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_config(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ int (*action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, void *opa); ++ struct adf_mstate_vreginfo info; ++ } setups[] = { ++ {ADF_MSTATE_GEN_CAP_IDS, adf_mstate_capmask_superset, ++ {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, ++ {ADF_MSTATE_GEN_SVCMAP_IDS, adf_mstate_capmask_equal, ++ {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, ++ {ADF_MSTATE_GEN_EXTDC_IDS, adf_mstate_capmask_superset, ++ {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_lookup(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ ADF_MSTATE_CONFIG_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, subsec); ++ for (i = 0; i < ARRAY_SIZE(setups); i++) { ++ l2_subsec = adf_mstate_sect_lookup(&sub_sects_mgr, setups[i].id, ++ setups[i].action, &setups[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load sec %s\n", ++ setups[i].id); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_etr_regs(struct adf_mstate_mgr *subs, u8 *state, ++ u32 size, void *opa) ++{ ++ struct adf_vf_bank_info *vf_bank_info = opa; ++ struct adf_accel_dev *accel_dev = vf_bank_info->accel_dev; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ u32 pf_bank_nr; ++ int ret; ++ ++ pf_bank_nr = vf_bank_info->bank_nr; ++ pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf; ++ ++ ret = hw_data->bank_state_save(accel_dev, pf_bank_nr, ++ (struct bank_state *)state); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save regs for vf%d bank%d\n", ++ vf_bank_info->vf_nr, vf_bank_info->bank_nr); ++ return ret; ++ } ++ ++ return sizeof(struct bank_state); ++} ++ ++static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev, ++ u32 vf_nr, u32 bank_nr, ++ struct adf_mstate_mgr *mstate_mgr) ++{ ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_vf_bank_info vf_bank_info; ++ struct adf_mstate_mgr sub_sects_mgr; ++ char bank_ids[ADF_MSTATE_ID_LEN]; ++ ++ snprintf(bank_ids, sizeof(bank_ids), ADF_MSTATE_BANK_IDX_IDS "%x", bank_nr); ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, bank_ids, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_BANK_IDX_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ vf_bank_info.accel_dev = accel_dev; ++ vf_bank_info.vf_nr = vf_nr; ++ vf_bank_info.bank_nr = bank_nr; ++ l2_subsec = adf_mstate_sect_add(&sub_sects_mgr, ADF_MSTATE_ETR_REGS_IDS, ++ adf_gen4_vfmig_save_etr_regs, ++ &vf_bank_info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to add sec %s for vf%d bank%d\n", ++ ADF_MSTATE_ETR_REGS_IDS, vf_nr, bank_nr); ++ return -EINVAL; ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_etr(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec; ++ int ret, i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_ETRB_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_ETRB_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < hw_data->num_banks_per_vf; i++) { ++ ret = adf_gen4_vfmig_save_etr_bank(accel_dev, vf_nr, i, ++ &sub_sects_mgr); ++ if (ret) ++ return ret; ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_misc(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ void __iomem *csr = adf_get_pmisc_base(accel_dev); ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct { ++ char *id; ++ u64 offset; ++ } misc_states[] = { ++ {ADF_MSTATE_VINTSRC_IDS, ADF_GEN4_VINTSOU_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_IDS, ADF_GEN4_VINTMSK_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTSRC_PF2VM_IDS, ADF_GEN4_VINTSOUPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VINTMSK_PF2VM_IDS, ADF_GEN4_VINTMSKPF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_PF2VM_IDS, ADF_GEN4_PF2VM_OFFSET(vf_nr)}, ++ {ADF_MSTATE_VM2PF_IDS, ADF_GEN4_VM2PF_OFFSET(vf_nr)}, ++ }; ++ ktime_t time_exp; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_MISCB_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_MISCB_IDS); ++ return -EINVAL; ++ } ++ ++ time_exp = ktime_add_us(ktime_get(), ADF_GEN4_PFVF_RSP_TIMEOUT_US); ++ while (!mutex_trylock(&vf_info->pfvf_mig_lock)) { ++ if (ktime_after(ktime_get(), time_exp)) { ++ dev_err(&GET_DEV(accel_dev), "Failed to get pfvf mig lock\n"); ++ return -ETIMEDOUT; ++ } ++ usleep_range(500, 1000); ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(misc_states); i++) { ++ struct adf_mstate_vreginfo info; ++ u32 regv; ++ ++ info.addr = ®v; ++ info.size = sizeof(regv); ++ regv = ADF_CSR_RD(csr, misc_states[i].offset); ++ ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, ++ misc_states[i].id, ++ &info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ misc_states[i].id); ++ mutex_unlock(&vf_info->pfvf_mig_lock); ++ return -EINVAL; ++ } ++ } ++ ++ mutex_unlock(&vf_info->pfvf_mig_lock); ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_generic(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct mig_user_sla src_slas[RL_RP_CNT_PER_LEAF_MAX] = { }; ++ u32 src_sla_cnt; ++ struct { ++ char *id; ++ struct adf_mstate_vreginfo info; ++ } gen_states[] = { ++ {ADF_MSTATE_IOV_INIT_IDS, ++ {&vf_info->init, sizeof(vf_info->init)}}, ++ {ADF_MSTATE_COMPAT_VER_IDS, ++ {&vf_info->compat_ver, sizeof(vf_info->compat_ver)}}, ++ {ADF_MSTATE_SLA_IDS, {src_slas, 0}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_GEN_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_GEN_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(gen_states); i++) { ++ if (gen_states[i].info.addr == src_slas) { ++ src_sla_cnt = adf_gen4_vfmig_get_slas(accel_dev, vf_nr, src_slas); ++ gen_states[i].info.size = src_sla_cnt * sizeof(struct mig_user_sla); ++ } ++ ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, ++ gen_states[i].id, ++ &gen_states[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ gen_states[i].id); ++ return -EINVAL; ++ } ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_config(struct adf_accel_dev *accel_dev, u32 vf_nr) ++{ ++ struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ struct adf_hw_device_data *hw_data = accel_dev->hw_device; ++ struct adf_gen4_vfmig *vfmig = vf_info->mig_priv; ++ struct adf_mstate_mgr *mstate_mgr = vfmig->mstate_mgr; ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *subsec, *l2_subsec; ++ struct { ++ char *id; ++ struct adf_mstate_vreginfo info; ++ } setups[] = { ++ {ADF_MSTATE_GEN_CAP_IDS, ++ {&hw_data->accel_capabilities_mask, sizeof(hw_data->accel_capabilities_mask)}}, ++ {ADF_MSTATE_GEN_SVCMAP_IDS, ++ {&hw_data->ring_to_svc_map, sizeof(hw_data->ring_to_svc_map)}}, ++ {ADF_MSTATE_GEN_EXTDC_IDS, ++ {&hw_data->extended_dc_capabilities, sizeof(hw_data->extended_dc_capabilities)}}, ++ }; ++ int i; ++ ++ subsec = adf_mstate_sect_add(mstate_mgr, ADF_MSTATE_CONFIG_IDS, NULL, NULL); ++ if (!subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ ADF_MSTATE_CONFIG_IDS); ++ return -EINVAL; ++ } ++ ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mstate_mgr); ++ for (i = 0; i < ARRAY_SIZE(setups); i++) { ++ l2_subsec = adf_mstate_sect_add_vreg(&sub_sects_mgr, setups[i].id, ++ &setups[i].info); ++ if (!l2_subsec) { ++ dev_err(&GET_DEV(accel_dev), "Failed to add sec %s\n", ++ setups[i].id); ++ return -EINVAL; ++ } ++ } ++ adf_mstate_sect_update(mstate_mgr, &sub_sects_mgr, subsec); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ ret = adf_gen4_vfmig_save_setup(mdev); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save setup for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state + mdev->setup_size, ++ mdev->state_size - mdev->setup_size); ++ if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) ++ return -EINVAL; ++ ++ ret = adf_gen4_vfmig_save_generic(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save generic state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_save_misc(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save misc bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_save_etr(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to save etr bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ adf_mstate_preamble_update(vfmig->mstate_mgr); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ ret = adf_gen4_vfmig_load_setup(mdev, mdev->state_size); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Failed to load setup for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, ++ mdev->state + mdev->remote_setup_size, ++ mdev->state_size - mdev->remote_setup_size, ++ NULL, NULL); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Invalid state for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_generic(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load general state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_misc(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load misc bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ ret = adf_gen4_vfmig_load_etr(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load etr bar state for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ if (mdev->setup_size) ++ return 0; ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); ++ if (!adf_mstate_preamble_add(vfmig->mstate_mgr)) ++ return -EINVAL; ++ ++ ret = adf_gen4_vfmig_save_config(accel_dev, mdev->vf_id); ++ if (ret) ++ return ret; ++ ++ adf_mstate_preamble_update(vfmig->mstate_mgr); ++ mdev->setup_size = adf_mstate_state_size(vfmig->mstate_mgr); ++ ++ return 0; ++} ++ ++static int adf_gen4_vfmig_load_setup(struct qat_mig_dev *mdev, int len) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ struct adf_accel_vf_info *vf_info; ++ struct adf_gen4_vfmig *vfmig; ++ u32 vf_nr = mdev->vf_id; ++ u32 setup_size; ++ int ret; ++ ++ vf_info = &accel_dev->pf.vf_info[vf_nr]; ++ vfmig = vf_info->mig_priv; ++ ++ if (mdev->remote_setup_size) ++ return 0; ++ ++ if (len < sizeof(struct adf_mstate_preh)) ++ return -EAGAIN; ++ ++ adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size); ++ setup_size = adf_mstate_state_size_from_remote(vfmig->mstate_mgr); ++ if (setup_size > mdev->state_size) ++ return -EINVAL; ++ ++ if (len < setup_size) ++ return -EAGAIN; ++ ++ ret = adf_mstate_mgr_init_from_remote(vfmig->mstate_mgr, mdev->state, ++ setup_size, NULL, NULL); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), "Invalide setup for vf_nr %d\n", ++ vf_nr); ++ return ret; ++ } ++ ++ mdev->remote_setup_size = setup_size; ++ ++ ret = adf_gen4_vfmig_load_config(accel_dev, vf_nr); ++ if (ret) { ++ dev_err(&GET_DEV(accel_dev), ++ "Failed to load config for vf_nr %d\n", vf_nr); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops) ++{ ++ vfmig_ops->init = adf_gen4_vfmig_init_device; ++ vfmig_ops->cleanup = adf_gen4_vfmig_cleanup_device; ++ vfmig_ops->reset = adf_gen4_vfmig_reset_device; ++ vfmig_ops->open = adf_gen4_vfmig_open_device; ++ vfmig_ops->close = adf_gen4_vfmig_close_device; ++ vfmig_ops->suspend = adf_gen4_vfmig_suspend_device; ++ vfmig_ops->resume = adf_gen4_vfmig_resume_device; ++ vfmig_ops->save_state = adf_gen4_vfmig_save_state; ++ vfmig_ops->load_state = adf_gen4_vfmig_load_state; ++ vfmig_ops->load_setup = adf_gen4_vfmig_load_setup; ++ vfmig_ops->save_setup = adf_gen4_vfmig_save_setup; ++} ++//EXPORT_SYMBOL_GPL(adf_gen4_init_vf_mig_ops); +diff --git a/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.h b/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.h +new file mode 100644 +index 0000000..f8ef10d +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/adf_gen4_vf_mig.h +@@ -0,0 +1,10 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++#ifndef ADF_GEN4_VF_MIG_H_ ++#define ADF_GEN4_VF_MIG_H_ ++ ++#include "adf_accel_devices.h" ++ ++extern void adf_gen4_init_vf_mig_ops(struct qat_migdev_ops *vfmig_ops); ++ ++#endif +diff --git a/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.c b/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.c +new file mode 100644 +index 0000000..41cc763 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.c +@@ -0,0 +1,318 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++ ++#include ++#include ++#include "adf_mstate_mgr.h" ++ ++#define ADF_MSTATE_MAGIC 0xADF5CAEA ++#define ADF_MSTATE_VERSION 0x1 ++ ++struct adf_mstate_sect_h { ++ u8 id[ADF_MSTATE_ID_LEN]; ++ u32 size; ++ u32 sub_sects; ++ u8 state[]; ++}; ++ ++u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr) ++{ ++ return mgr->state - mgr->buf; ++} ++ ++static inline u32 adf_mstate_avail_room(struct adf_mstate_mgr *mgr) ++{ ++ return mgr->buf + mgr->size - mgr->state; ++} ++ ++void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size) ++{ ++ mgr->buf = buf; ++ mgr->state = buf; ++ mgr->size = size; ++ mgr->n_sects = 0; ++}; ++ ++struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size) ++{ ++ struct adf_mstate_mgr *mgr; ++ ++ mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); ++ if (!mgr) ++ return NULL; ++ ++ adf_mstate_mgr_init(mgr, buf, size); ++ ++ return mgr; ++} ++ ++void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr) ++{ ++ kfree(mgr); ++} ++ ++void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_mgr *p_mgr) ++{ ++ adf_mstate_mgr_init(mgr, p_mgr->state, ++ p_mgr->size - adf_mstate_state_size(p_mgr)); ++} ++ ++void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *p_sect) ++{ ++ adf_mstate_mgr_init(mgr, p_sect->state, p_sect->size); ++ mgr->n_sects = p_sect->sub_sects; ++} ++ ++static void adf_mstate_preamble_init(struct adf_mstate_preh *preamble) ++{ ++ preamble->magic = ADF_MSTATE_MAGIC; ++ preamble->version = ADF_MSTATE_VERSION; ++ preamble->preh_len = sizeof(*preamble); ++ preamble->size = 0; ++ preamble->n_sects = 0; ++} ++ ++/* default preambles checker */ ++static int adf_mstate_preamble_def_checker(struct adf_mstate_preh *preamble, ++ void *opaque) ++{ ++ struct adf_mstate_mgr *mgr = opaque; ++ ++ if (preamble->magic != ADF_MSTATE_MAGIC || ++ preamble->version > ADF_MSTATE_VERSION || ++ preamble->preh_len > mgr->size) { ++ pr_debug("QAT: LM - Invalid state (magic=%#x, version=%#x, hlen=%u), state_size=%u\n", ++ preamble->magic, preamble->version, preamble->preh_len, ++ mgr->size); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *pre = (struct adf_mstate_preh *)mgr->buf; ++ ++ if (adf_mstate_avail_room(mgr) < sizeof(*pre)) { ++ pr_err("QAT: LM - Not enough space for preamble\n"); ++ return NULL; ++ } ++ ++ adf_mstate_preamble_init(pre); ++ mgr->state += pre->preh_len; ++ ++ return pre; ++} ++ ++int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *preamble = (struct adf_mstate_preh *)mgr->buf; ++ ++ preamble->size = adf_mstate_state_size(mgr) - preamble->preh_len; ++ preamble->n_sects = mgr->n_sects; ++ ++ return 0; ++} ++ ++static void adf_mstate_dump_sect(struct adf_mstate_sect_h *sect, ++ const char *prefix) ++{ ++ pr_debug("QAT: LM - %s QAT state section %s\n", prefix, sect->id); ++ print_hex_dump_debug("h-", DUMP_PREFIX_OFFSET, 16, 2, sect, ++ sizeof(*sect), true); ++ print_hex_dump_debug("s-", DUMP_PREFIX_OFFSET, 16, 2, sect->state, ++ sect->size, true); ++} ++ ++static inline void __adf_mstate_sect_update(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *sect, ++ u32 size, ++ u32 n_subsects) ++{ ++ sect->size += size; ++ sect->sub_sects += n_subsects; ++ mgr->n_sects++; ++ mgr->state += sect->size; ++ ++ adf_mstate_dump_sect(sect, "Add"); ++} ++ ++void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, ++ struct adf_mstate_mgr *curr_mgr, ++ struct adf_mstate_sect_h *sect) ++{ ++ __adf_mstate_sect_update(p_mgr, sect, adf_mstate_state_size(curr_mgr), ++ curr_mgr->n_sects); ++} ++ ++static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mgr *mgr, ++ const char *id) ++{ ++ struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)(mgr->state); ++ ++ if (adf_mstate_avail_room(mgr) < sizeof(*sect)) { ++ pr_debug("QAT: LM - Not enough space for header of QAT state sect %s\n", id); ++ return NULL; ++ } ++ ++ strscpy(sect->id, id, sizeof(sect->id)); ++ sect->size = 0; ++ sect->sub_sects = 0; ++ mgr->state += sizeof(*sect); ++ ++ return sect; ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, ++ const char *id, ++ struct adf_mstate_vreginfo *info) ++{ ++ struct adf_mstate_sect_h *sect; ++ ++ sect = adf_mstate_sect_add_header(mgr, id); ++ if (!sect) ++ return NULL; ++ ++ if (adf_mstate_avail_room(mgr) < info->size) { ++ pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", ++ id, info->size); ++ return NULL; ++ } ++ ++ memcpy(sect->state, info->addr, info->size); ++ __adf_mstate_sect_update(mgr, sect, info->size, 0); ++ ++ return sect; ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_populate populate, ++ void *opaque) ++{ ++ struct adf_mstate_mgr sub_sects_mgr; ++ struct adf_mstate_sect_h *sect; ++ int avail_room, size; ++ ++ sect = adf_mstate_sect_add_header(mgr, id); ++ if (!sect) ++ return NULL; ++ ++ if (!populate) ++ return sect; ++ ++ avail_room = adf_mstate_avail_room(mgr); ++ adf_mstate_mgr_init_from_parent(&sub_sects_mgr, mgr); ++ ++ size = (*populate)(&sub_sects_mgr, sect->state, avail_room, opaque); ++ if (size < 0) ++ return NULL; ++ ++ size += adf_mstate_state_size(&sub_sects_mgr); ++ if (avail_room < size) { ++ pr_debug("QAT: LM - Not enough space for QAT state sect %s, requires %u\n", ++ id, size); ++ return NULL; ++ } ++ __adf_mstate_sect_update(mgr, sect, size, sub_sects_mgr.n_sects); ++ ++ return sect; ++} ++ ++static int adf_mstate_sect_validate(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_sect_h *start = (struct adf_mstate_sect_h *)mgr->state; ++ struct adf_mstate_sect_h *sect = start; ++ u64 end; ++ int i; ++ ++ end = (uintptr_t)mgr->buf + mgr->size; ++ for (i = 0; i < mgr->n_sects; i++) { ++ uintptr_t s_start = (uintptr_t)sect->state; ++ uintptr_t s_end = s_start + sect->size; ++ ++ if (s_end < s_start || s_end > end) { ++ pr_debug("QAT: LM - Corrupted state section (index=%u, size=%u) in state_mgr (size=%u, secs=%u)\n", ++ i, sect->size, mgr->size, mgr->n_sects); ++ return -EINVAL; ++ } ++ sect = (struct adf_mstate_sect_h *)s_end; ++ } ++ ++ pr_debug("QAT: LM - Scanned section (last child=%s, size=%lu) in state_mgr (size=%u, secs=%u)\n", ++ start->id, sizeof(struct adf_mstate_sect_h) * (ulong)(sect - start), ++ mgr->size, mgr->n_sects); ++ ++ return 0; ++} ++ ++u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr) ++{ ++ struct adf_mstate_preh *preh = (struct adf_mstate_preh *)mgr->buf; ++ ++ return preh->preh_len + preh->size; ++} ++ ++int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, u8 *buf, u32 size, ++ adf_mstate_preamble_checker pre_checker, ++ void *opaque) ++{ ++ struct adf_mstate_preh *pre; ++ int ret; ++ ++ adf_mstate_mgr_init(mgr, buf, size); ++ pre = (struct adf_mstate_preh *)(mgr->buf); ++ ++ pr_debug("QAT: LM - Dump state preambles\n"); ++ print_hex_dump_debug("", DUMP_PREFIX_OFFSET, 16, 2, pre, pre->preh_len, 0); ++ ++ if (pre_checker) ++ ret = (*pre_checker)(pre, opaque); ++ else ++ ret = adf_mstate_preamble_def_checker(pre, mgr); ++ if (ret) ++ return ret; ++ ++ mgr->state = mgr->buf + pre->preh_len; ++ mgr->n_sects = pre->n_sects; ++ ++ return adf_mstate_sect_validate(mgr); ++} ++ ++struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_action action, ++ void *opaque) ++{ ++ struct adf_mstate_sect_h *sect = (struct adf_mstate_sect_h *)mgr->state; ++ struct adf_mstate_mgr sub_sects_mgr; ++ int i, ret; ++ ++ for (i = 0; i < mgr->n_sects; i++) { ++ if (!strncmp(sect->id, id, sizeof(sect->id))) ++ goto found; ++ ++ sect = (struct adf_mstate_sect_h *)(sect->state + sect->size); ++ } ++ ++ return NULL; ++ ++found: ++ adf_mstate_dump_sect(sect, "Found"); ++ ++ adf_mstate_mgr_init_from_psect(&sub_sects_mgr, sect); ++ if (sect->sub_sects && adf_mstate_sect_validate(&sub_sects_mgr)) ++ return NULL; ++ ++ if (!action) ++ return sect; ++ ++ ret = (*action)(&sub_sects_mgr, sect->state, sect->size, opaque); ++ if (ret) ++ return NULL; ++ ++ return sect; ++} +diff --git a/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.h b/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.h +new file mode 100644 +index 0000000..81d263a +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/adf_mstate_mgr.h +@@ -0,0 +1,89 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++ ++#ifndef ADF_MSTATE_MGR_H ++#define ADF_MSTATE_MGR_H ++ ++#define ADF_MSTATE_ID_LEN 8 ++ ++#define ADF_MSTATE_ETRB_IDS "ETRBAR" ++#define ADF_MSTATE_MISCB_IDS "MISCBAR" ++#define ADF_MSTATE_EXTB_IDS "EXTBAR" ++#define ADF_MSTATE_GEN_IDS "GENER" ++#define ADF_MSTATE_CONFIG_IDS "CONFIG" ++#define ADF_MSTATE_SECTION_NUM 5 ++ ++#define ADF_MSTATE_BANK_IDX_IDS "bnk" ++ ++#define ADF_MSTATE_ETR_REGS_IDS "mregs" ++#define ADF_MSTATE_VINTSRC_IDS "visrc" ++#define ADF_MSTATE_VINTMSK_IDS "vimsk" ++#define ADF_MSTATE_SLA_IDS "sla" ++#define ADF_MSTATE_IOV_INIT_IDS "iovinit" ++#define ADF_MSTATE_COMPAT_VER_IDS "compver" ++#define ADF_MSTATE_GEN_CAP_IDS "gencap" ++#define ADF_MSTATE_GEN_SVCMAP_IDS "svcmap" ++#define ADF_MSTATE_GEN_EXTDC_IDS "extdc" ++#define ADF_MSTATE_VINTSRC_PF2VM_IDS "vispv" ++#define ADF_MSTATE_VINTMSK_PF2VM_IDS "vimpv" ++#define ADF_MSTATE_VM2PF_IDS "vm2pf" ++#define ADF_MSTATE_PF2VM_IDS "pf2vm" ++ ++struct adf_mstate_mgr { ++ u8 *buf; ++ u8 *state; ++ u32 size; ++ u32 n_sects; ++}; ++ ++struct adf_mstate_preh { ++ u32 magic; ++ u32 version; ++ u16 preh_len; ++ u16 n_sects; ++ u32 size; ++}; ++ ++struct adf_mstate_vreginfo { ++ void *addr; ++ u32 size; ++}; ++ ++struct adf_mstate_sect_h; ++ ++typedef int (*adf_mstate_preamble_checker)(struct adf_mstate_preh *preamble, void *opa); ++typedef int (*adf_mstate_populate)(struct adf_mstate_mgr *sub_mgr, u8 *buf, ++ u32 size, void *opa); ++typedef int (*adf_mstate_action)(struct adf_mstate_mgr *sub_mgr, u8 *buf, u32 size, ++ void *opa); ++ ++struct adf_mstate_mgr *adf_mstate_mgr_new(u8 *buf, u32 size); ++void adf_mstate_mgr_destroy(struct adf_mstate_mgr *mgr); ++void adf_mstate_mgr_init(struct adf_mstate_mgr *mgr, u8 *buf, u32 size); ++void adf_mstate_mgr_init_from_parent(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_mgr *p_mgr); ++void adf_mstate_mgr_init_from_psect(struct adf_mstate_mgr *mgr, ++ struct adf_mstate_sect_h *p_sect); ++int adf_mstate_mgr_init_from_remote(struct adf_mstate_mgr *mgr, ++ u8 *buf, u32 size, ++ adf_mstate_preamble_checker checker, ++ void *opaque); ++struct adf_mstate_preh *adf_mstate_preamble_add(struct adf_mstate_mgr *mgr); ++int adf_mstate_preamble_update(struct adf_mstate_mgr *mgr); ++u32 adf_mstate_state_size(struct adf_mstate_mgr *mgr); ++u32 adf_mstate_state_size_from_remote(struct adf_mstate_mgr *mgr); ++void adf_mstate_sect_update(struct adf_mstate_mgr *p_mgr, ++ struct adf_mstate_mgr *curr_mgr, ++ struct adf_mstate_sect_h *sect); ++struct adf_mstate_sect_h *adf_mstate_sect_add_vreg(struct adf_mstate_mgr *mgr, ++ const char *id, ++ struct adf_mstate_vreginfo *info); ++struct adf_mstate_sect_h *adf_mstate_sect_add(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_populate populate, ++ void *opaque); ++struct adf_mstate_sect_h *adf_mstate_sect_lookup(struct adf_mstate_mgr *mgr, ++ const char *id, ++ adf_mstate_action action, ++ void *opaque); ++#endif +diff --git a/quickassist/qat/drivers/vfio/pci/qat/main.c b/quickassist/qat/drivers/vfio/pci/qat/main.c +index 99e28aa..449bc11 100644 +--- a/quickassist/qat/drivers/vfio/pci/qat/main.c ++++ b/quickassist/qat/drivers/vfio/pci/qat/main.c +@@ -12,8 +12,10 @@ + #include + #include + #include ++#include "adf_common_drv.h" + #include "vfio_pci_core.h" + #include "qat_mig_dev.h" ++#include "adf_gen4_vf_mig.h" + + #ifndef vfio_pci_core_init_dev + static int vfio_pci_core_init_dev(struct vfio_device *core_vdev) +@@ -642,6 +644,11 @@ qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) + struct device *dev = &pdev->dev; + struct qat_vf_core_device *qat_vdev; + int ret; ++ struct pci_dev *physfn = pdev->physfn; ++ struct adf_accel_dev *pf_accel_dev = adf_devmgr_pci_to_accel_dev(physfn); ++ struct adf_hw_device_data *pf_hw_data = GET_HW_DATA(pf_accel_dev); ++ ++ adf_gen4_init_vf_mig_ops(&pf_hw_data->vfmig_ops); + + qat_vdev = vfio_alloc_device(qat_vf_core_device, core_device.vdev, dev, &qat_vf_pci_ops); + if (IS_ERR(qat_vdev)) +diff --git a/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.c b/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.c +new file mode 100644 +index 0000000..1297a60 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.c +@@ -0,0 +1,130 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright(c) 2024 Intel Corporation */ ++#include ++#include ++#include ++#include ++#include "qat_mig_dev.h" ++#include "adf_accel_devices.h" ++#include "adf_common_drv.h" ++ ++struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id) ++{ ++ struct adf_accel_dev *accel_dev; ++ struct qat_migdev_ops *ops; ++ struct qat_mig_dev *mdev; ++ ++ accel_dev = adf_devmgr_pci_to_accel_dev(pdev); ++ if (!accel_dev) ++ return ERR_PTR(-ENODEV); ++ ++ ops = GET_VFMIG_OPS(accel_dev); ++ if (!ops || !ops->init || !ops->cleanup || !ops->reset || !ops->open || ++ !ops->close || !ops->suspend || !ops->resume || !ops->save_state || ++ !ops->load_state || !ops->save_setup || !ops->load_setup) ++ return ERR_PTR(-EINVAL); ++ ++ mdev = kmalloc(sizeof(*mdev), GFP_KERNEL); ++ if (!mdev) ++ return ERR_PTR(-ENOMEM); ++ ++ mdev->vf_id = vf_id; ++ mdev->parent_accel_dev = accel_dev; ++ ++ return mdev; ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_create); ++ ++int qat_vfmig_init(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->init(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_init); ++ ++void qat_vfmig_cleanup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->cleanup(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_cleanup); ++ ++void qat_vfmig_reset(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->reset(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_reset); ++ ++int qat_vfmig_open(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->open(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_open); ++ ++void qat_vfmig_close(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ GET_VFMIG_OPS(accel_dev)->close(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_close); ++ ++int qat_vfmig_suspend(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->suspend(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_suspend); ++ ++int qat_vfmig_resume(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->resume(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_resume); ++ ++int qat_vfmig_save_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->save_state(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_save_state); ++ ++int qat_vfmig_save_setup(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->save_setup(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_save_setup); ++ ++int qat_vfmig_load_state(struct qat_mig_dev *mdev) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->load_state(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_load_state); ++ ++int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size) ++{ ++ struct adf_accel_dev *accel_dev = mdev->parent_accel_dev; ++ ++ return GET_VFMIG_OPS(accel_dev)->load_setup(mdev, size); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_load_setup); ++ ++void qat_vfmig_destroy(struct qat_mig_dev *mdev) ++{ ++ kfree(mdev); ++} ++//EXPORT_SYMBOL_GPL(qat_vfmig_destroy); +diff --git a/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.h b/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.h +new file mode 100644 +index 0000000..dbbb6a0 +--- /dev/null ++++ b/quickassist/qat/drivers/vfio/pci/qat/qat_mig_dev.h +@@ -0,0 +1,31 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright(c) 2024 Intel Corporation */ ++#ifndef QAT_MIG_DEV_H_ ++#define QAT_MIG_DEV_H_ ++ ++struct pci_dev; ++ ++struct qat_mig_dev { ++ void *parent_accel_dev; ++ u8 *state; ++ u32 setup_size; ++ u32 remote_setup_size; ++ u32 state_size; ++ s32 vf_id; ++}; ++ ++struct qat_mig_dev *qat_vfmig_create(struct pci_dev *pdev, int vf_id); ++int qat_vfmig_init(struct qat_mig_dev *mdev); ++void qat_vfmig_cleanup(struct qat_mig_dev *mdev); ++void qat_vfmig_reset(struct qat_mig_dev *mdev); ++int qat_vfmig_open(struct qat_mig_dev *mdev); ++void qat_vfmig_close(struct qat_mig_dev *mdev); ++int qat_vfmig_suspend(struct qat_mig_dev *mdev); ++int qat_vfmig_resume(struct qat_mig_dev *mdev); ++int qat_vfmig_save_state(struct qat_mig_dev *mdev); ++int qat_vfmig_save_setup(struct qat_mig_dev *mdev); ++int qat_vfmig_load_state(struct qat_mig_dev *mdev); ++int qat_vfmig_load_setup(struct qat_mig_dev *mdev, int size); ++void qat_vfmig_destroy(struct qat_mig_dev *mdev); ++ ++#endif /*QAT_MIG_DEV_H_*/ +-- +2.39.3 + diff --git a/1018-Add-telemetry-python-script-to-monitor-ring-pairs.patch b/1018-Add-telemetry-python-script-to-monitor-ring-pairs.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca22df51407c29c434a2961da06321fb72360c0b --- /dev/null +++ b/1018-Add-telemetry-python-script-to-monitor-ring-pairs.patch @@ -0,0 +1,174 @@ +From 98233e676a9b8223cba3e6b96da880ec1125a50a Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Mon, 27 May 2024 14:25:54 +0800 +Subject: [PATCH 18/28] Add telemetry python script to monitor ring pairs + +Signed-off-by: Zelin Deng +--- + quickassist/utilities/telemetry/tl.py | 154 ++++++++++++++++++++++++++ + 1 file changed, 154 insertions(+) + create mode 100755 quickassist/utilities/telemetry/tl.py + +diff --git a/quickassist/utilities/telemetry/tl.py b/quickassist/utilities/telemetry/tl.py +new file mode 100755 +index 0000000..45276aa +--- /dev/null ++++ b/quickassist/utilities/telemetry/tl.py +@@ -0,0 +1,154 @@ ++#!/usr/bin/python3.6 ++import time ++import curses ++import subprocess ++import re ++devices=[] ++ ++def EnableTelemetry(): ++ devices.clear() ++ command = "adf_ctl status" ++ sp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) ++ # Store the return code in rc variable ++ rc=sp.wait() ++ # Separate the output and error. ++ # This is similar to Tuple where we store two values to two different variables ++ out,err=sp.communicate() ++ # Split string into list of strings ++ output = out.split() ++ i = 0 ++ state = "down" ++ name = None ++ bus = None ++ telemetry_supported = False ++ while i < len(output): ++ if "qat_dev" in output[i]: ++ name = output[i] ++ elif "type:" == output[i]: ++ if "4xxx," == output[i+1]: ++ telemetry_supported = True ++ elif "bsf:" == output[i]: ++ bus = output[i+1][5:7] ++ elif "state:" == output[i]: ++ if "up" == output[i+1]: ++ if telemetry_supported == True: ++ devices.append((name, bus)) ++ # Reset variables to ensure we only attempt to enable telemetery on devices that￿,support telemetry and are in up ++ state = "down" ++ name = None ++ bus = None ++ telemetry_supported = False ++ i += 1 ++ ++ for device in devices: ++ control_file_name="/sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/control" ++ command = "echo 1 > " + control_file_name ++ try: ++ str(subprocess.check_output(command, shell=True)) ++ except: ++ break ++ ++def pbar(window): ++ refresh_counter = 0 ++ while True: ++ try: ++ refresh_counter += 1 ++ window.addstr(0, 10, "Intel(R) QuickAssist Device Utilization") ++ window.addstr(2, 10, "Device\t%Comp\t%Decomp\t%PKE\t%Cipher\t%Auth\t%UCS\tLatency(ns)") ++ window.addstr(3, 10,"=========================================================================") ++ count = 0 ++ for device in devices: ++ command = "cat /sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/device_data" ++ sp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) ++ # Store the return code in rc variable ++ rc=sp.wait() ++ # Separate the output and error. ++ # This is similar to Tuple where we store two values to two different￿variables ++ out,err=sp.communicate() ++ # Split string into list of strings ++ output = out.split() ++ i = 0 ++ while i < len(output): ++ if "lat_acc_avg" == output[i]: ++ latency = output[i+1] ++ elif "util_cpr0" == output[i]: ++ compression = output[i+1] ++ elif "util_dcpr0" == output[i]: ++ decompression0 = output[i+1] ++ elif "util_dcpr1" == output[i]: ++ decompression1 = output[i+1] ++ elif "util_dcpr2" == output[i]: ++ decompression2 = output[i+1] ++ elif "util_pke0" == output[i]: ++ pke0 = output[i+1] ++ elif "util_pke1" == output[i]: ++ pke1 = output[i+1] ++ elif "util_pke2" == output[i]: ++ pke2 = output[i+1] ++ elif "util_pke3" == output[i]: ++ pke3 = output[i+1] ++ elif "util_pke4" == output[i]: ++ pke4 = output[i+1] ++ elif "util_pke5" == output[i]: ++ pke5 = output[i+1] ++ elif "util_cph0" == output[i]: ++ cph0 = output[i+1] ++ elif "util_cph1" == output[i]: ++ cph1 = output[i+1] ++ elif "util_cph2" == output[i]: ++ cph2 = output[i+1] ++ elif "util_cph3" == output[i]: ++ cph3 = output[i+1] ++ elif "util_ath0" == output[i]: ++ ath0 = output[i+1] ++ elif "util_ath1" == output[i]: ++ ath1 = output[i+1] ++ elif "util_ath2" == output[i]: ++ ath2 = output[i+1] ++ elif "util_ath3" == output[i]: ++ ath3 = output[i+1] ++ elif "util_ucs0" == output[i]: ++ ucs0 = output[i+1] ++ elif "util_ucs1" == output[i]: ++ ucs1 = output[i+1] ++ i += 1 ++ decompress_utilization = int(decompression0) + int(decompression1) + int(decompression2) ++ if decompress_utilization > 0: ++ decompress_utilization = decompress_utilization / 3 ++ decompress_utilization = round(decompress_utilization) ++ pke_utilization = int(pke0) + int(pke1) + int(pke2) + int(pke3) + int(pke4) + int(pke5) ++ if pke_utilization > 0: ++ pke_utilization = pke_utilization / 6 ++ pke_utilization = round(pke_utilization) ++ cph_utilization = int(cph0) + int(cph1) + int(cph2) + int(cph3) ++ if cph_utilization > 0: ++ cph_utilization = cph_utilization / 4 ++ cph_utilization = round(cph_utilization) ++ ath_utilization = int(ath0) + int(ath1) + int(ath2) + int(ath3) ++ if ath_utilization > 0: ++ ath_utilization = ath_utilization / 4 ++ ath_utilization = round(ath_utilization) ++ usc_utilization = int(ucs0) + int(ucs1) ++ if usc_utilization > 0: ++ usc_utilization = usc_utilization / 2 ++ usc_utilization = round(usc_utilization) ++ if int(latency) == 0: ++ window.addstr(4+count, 10, device[0] + '\t0\t0\t0\t0\t0\t00') ++ window.addstr(4+count, 10, device[0] + '\t' + compression + '\t' + str(decompress_utilization) + '\t' + str(pke_utilization) + '\t' + str(cph_utilization) + '\t' + str(ath_utilization) + '\t' + str(usc_utilization) + '\t'+ latency) ++ count += 1 ++ ++ window.addstr(4+count, 10,"=========================================================================") ++ window.refresh() ++ time.sleep(2) ++ if refresh_counter % 5 == 0: ++ window.clear() ++ EnableTelemetry() ++ ++ except KeyboardInterrupt: ++ break ++ except: ++ break ++ ++if __name__ == "__main__": ++ EnableTelemetry() ++ curses.wrapper(pbar) +-- +2.39.3 + diff --git a/1019-Add-per-rp-telemetry.patch b/1019-Add-per-rp-telemetry.patch new file mode 100644 index 0000000000000000000000000000000000000000..14d9ed32d4e81e60f8a5081efce80aee7003a4dc --- /dev/null +++ b/1019-Add-per-rp-telemetry.patch @@ -0,0 +1,258 @@ +From 7eec2c5a29acbfce6e7da278630c05ef29c322da Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Mon, 27 May 2024 16:32:49 +0800 +Subject: [PATCH 19/28] Add per rp telemetry + +Signed-off-by: Zelin Deng +--- + quickassist/utilities/telemetry/tl_per_rp.py | 238 +++++++++++++++++++ + 1 file changed, 238 insertions(+) + create mode 100755 quickassist/utilities/telemetry/tl_per_rp.py + +diff --git a/quickassist/utilities/telemetry/tl_per_rp.py b/quickassist/utilities/telemetry/tl_per_rp.py +new file mode 100755 +index 0000000..1c5fcf1 +--- /dev/null ++++ b/quickassist/utilities/telemetry/tl_per_rp.py +@@ -0,0 +1,238 @@ ++#!/usr/bin/python3.6 ++import time ++import curses ++import subprocess ++import re ++import sys ++ ++devices=[] ++args=sys.argv[1:] ++rps_data=["rp_A_data", "rp_B_data", "rp_C_data", "rp_D_data"] ++rps=[] ++ ++def ParseArgs(): ++ if (len(sys.argv) - 1) > 4: ++ print("args num more then 4, abort") ++ return -1 ++ for arg in args: ++ if int(arg) > 63: ++ print("arg larger then 64, abort") ++ return -1 ++ rps.append(arg) ++ return 0 ++ ++def EnableTelemetry(): ++ devices.clear() ++ command = "adf_ctl status" ++ sp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) ++ # Store the return code in rc variable ++ rc=sp.wait() ++ # Separate the output and error. ++ # This is similar to Tuple where we store two values to two different variables ++ out,err=sp.communicate() ++ # Split string into list of strings ++ output = out.split() ++ i = 0 ++ state = "down" ++ name = None ++ bus = None ++ telemetry_supported = False ++ while i < len(output): ++ if "qat_dev" in output[i]: ++ name = output[i] ++ elif "type:" == output[i]: ++ if "4xxx," == output[i+1]: ++ telemetry_supported = True ++ elif "bsf:" == output[i]: ++ bus = output[i+1][5:7] ++ elif "state:" == output[i]: ++ if "up" == output[i+1]: ++ if telemetry_supported == True: ++ devices.append((name, bus)) ++ # Reset variables to ensure we only attempt to enable telemetery on devices that￿,support telemetry and are in up ++ state = "down" ++ name = None ++ bus = None ++ telemetry_supported = False ++ i += 1 ++ ++ for device in devices: ++ control_file_name="/sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/control" ++ command = "echo 1 > " + control_file_name ++ try: ++ str(subprocess.check_output(command, shell=True)) ++ except: ++ break ++ i = 0 ++ for rp in rps: ++ control_file_name="/sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/" + rps_data[i] ++ command = "echo " + rp + " > " + control_file_name ++ print("command:", command) ++ #try: ++ # str(subprocess.check_output(command, shell=True)) ++ #except: ++ # continue ++ i += 1 ++ ++ ++def pbar(window): ++ refresh_counter = 0 ++ while True: ++ try: ++ refresh_counter += 1 ++ window.addstr(0, 10, "Intel(R) QuickAssist Device Utilization") ++ window.addstr(2, 10, "Device\t%Comp\t%Decomp\t%PKE\t%Cipher\t%Auth\t%UCS\tLatency(ns)") ++ window.addstr(3, 10,"=========================================================================") ++ count = 0 ++ for device in devices: ++ command = "cat /sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/device_data" ++ sp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) ++ # Store the return code in rc variable ++ rc=sp.wait() ++ # Separate the output and error. ++ # This is similar to Tuple where we store two values to two different￿variables ++ out,err=sp.communicate() ++ # Split string into list of strings ++ output = out.split() ++ i = 0 ++ while i < len(output): ++ if "lat_acc_avg" == output[i]: ++ latency = output[i+1] ++ elif "util_cpr0" == output[i]: ++ compression = output[i+1] ++ elif "util_dcpr0" == output[i]: ++ decompression0 = output[i+1] ++ elif "util_dcpr1" == output[i]: ++ decompression1 = output[i+1] ++ elif "util_dcpr2" == output[i]: ++ decompression2 = output[i+1] ++ elif "util_pke0" == output[i]: ++ pke0 = output[i+1] ++ elif "util_pke1" == output[i]: ++ pke1 = output[i+1] ++ elif "util_pke2" == output[i]: ++ pke2 = output[i+1] ++ elif "util_pke3" == output[i]: ++ pke3 = output[i+1] ++ elif "util_pke4" == output[i]: ++ pke4 = output[i+1] ++ elif "util_pke5" == output[i]: ++ pke5 = output[i+1] ++ elif "util_cph0" == output[i]: ++ cph0 = output[i+1] ++ elif "util_cph1" == output[i]: ++ cph1 = output[i+1] ++ elif "util_cph2" == output[i]: ++ cph2 = output[i+1] ++ elif "util_cph3" == output[i]: ++ cph3 = output[i+1] ++ elif "util_ath0" == output[i]: ++ ath0 = output[i+1] ++ elif "util_ath1" == output[i]: ++ ath1 = output[i+1] ++ elif "util_ath2" == output[i]: ++ ath2 = output[i+1] ++ elif "util_ath3" == output[i]: ++ ath3 = output[i+1] ++ elif "util_ucs0" == output[i]: ++ ucs0 = output[i+1] ++ elif "util_ucs1" == output[i]: ++ ucs1 = output[i+1] ++ i += 1 ++ decompress_utilization = int(decompression0) + int(decompression1) + int(decompression2) ++ if decompress_utilization > 0: ++ decompress_utilization = decompress_utilization / 3 ++ decompress_utilization = round(decompress_utilization) ++ pke_utilization = int(pke0) + int(pke1) + int(pke2) + int(pke3) + int(pke4) + int(pke5) ++ if pke_utilization > 0: ++ pke_utilization = pke_utilization / 6 ++ pke_utilization = round(pke_utilization) ++ cph_utilization = int(cph0) + int(cph1) + int(cph2) + int(cph3) ++ if cph_utilization > 0: ++ cph_utilization = cph_utilization / 4 ++ cph_utilization = round(cph_utilization) ++ ath_utilization = int(ath0) + int(ath1) + int(ath2) + int(ath3) ++ if ath_utilization > 0: ++ ath_utilization = ath_utilization / 4 ++ ath_utilization = round(ath_utilization) ++ usc_utilization = int(ucs0) + int(ucs1) ++ if usc_utilization > 0: ++ usc_utilization = usc_utilization / 2 ++ usc_utilization = round(usc_utilization) ++ if int(latency) == 0: ++ window.addstr(4+count, 10, device[0] + '\t0\t0\t0\t0\t0\t00') ++ window.addstr(4+count, 10, device[0] + '\t' + compression + '\t' + str(decompress_utilization) + '\t' + str(pke_utilization) + '\t' + str(cph_utilization) + '\t' + str(ath_utilization) + '\t' + str(usc_utilization) + '\t'+ latency) ++ count += 1 ++ ++ window.addstr(4+count, 10,"=========================================================================") ++ window.refresh() ++ time.sleep(2) ++ if refresh_counter % 5 == 0: ++ window.clear() ++ #EnableTelemetry() ++ ++ except KeyboardInterrupt: ++ break ++ except: ++ break ++ ++def per_rp(window): ++ refresh_counter = 0 ++ while True: ++ try: ++ refresh_counter += 1 ++ window.addstr(0, 10, "Intel(R) QuickAssist Device Utilization") ++ window.addstr(2, 10, "rp\tPTC\tLAT\tBWI\tBWO\tGTH\tGTM\tPTH\tPTM") ++ window.addstr(3, 10,"=========================================================================") ++ count = 0 ++ for device in devices: ++ for rpd in rps_data: ++ command = "cat /sys/devices/pci0000:" + device[1] + "/0000:" + device[1] + ":00.0/telemetry/" + rpd ++ sp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,universal_newlines=True) ++ rc=sp.wait() ++ out,err=sp.communicate() ++ output = out.split() ++ i = 0 ++ while i < len(output): ++ if "rp_num" == output[i]: ++ rp_num = output[i+1] ++ if "pci_trans_cnt" == output[i]: ++ pci_trans_cnt = output[i+1] ++ if "lat_acc_avg" == output[i]: ++ lat_acc_avg = output[i+1] ++ if "bw_in" == output[i]: ++ bw_in = output[i+1] ++ if "bw_out" == output[i]: ++ bw_out = output[i+1] ++ if "at_glob_devtlb_hit" == output[i]: ++ at_glob_devtlb_hit = output[i+1] ++ if "at_glob_devtlb_miss" == output[i]: ++ at_glob_devtlb_miss = output[i+1] ++ if "tl_at_payld_devtlb_hit" == output[i]: ++ tl_at_payld_devtlb_hit = output[i+1] ++ if "tl_at_payld_devtlb_miss" == output[i]: ++ tl_at_payld_devtlb_miss = output[i+1] ++ i += 1 ++ ++ window.addstr(4+count, 10, str(rp_num) + '\t' + str(pci_trans_cnt) + '\t' + str(lat_acc_avg) + '\t' + str(bw_in) + '\t' + str(bw_out) + '\t' + str(at_glob_devtlb_hit) + '\t' + str(at_glob_devtlb_miss) + '\t' + str(tl_at_payld_devtlb_hit) + '\t' + str(tl_at_payld_devtlb_miss)) ++ ++ count += 1 ++ ++ window.addstr(4+count, 10,"=========================================================================") ++ window.refresh() ++ time.sleep(2) ++ if refresh_counter % 5 == 0: ++ window.clear() ++ #EnableTelemetry() ++ ++ except KeyboardInterrupt: ++ break ++ except: ++ break ++ ++if __name__ == "__main__": ++ if ParseArgs() < 0: ++ sys.exit() ++ EnableTelemetry() ++ curses.wrapper(per_rp) ++ #curses.wrapper(pbar) +-- +2.39.3 + diff --git a/1000-bugfix-support-5.10.patch b/1020-Build-patch-on-anck-5.10.patch similarity index 67% rename from 1000-bugfix-support-5.10.patch rename to 1020-Build-patch-on-anck-5.10.patch index f10dec65b8cecd7cf995ee33e0a0c6f7977c7de1..76e7409de6d7f91a4c97c5876716a8abda4e0766 100644 --- a/1000-bugfix-support-5.10.patch +++ b/1020-Build-patch-on-anck-5.10.patch @@ -1,74 +1,74 @@ -commit 6d3281629dd2d2eb46a4e88964e79fd1fccfedc5 -Author: Jiayu Ni -Date: Mon Oct 10 09:14:12 2022 +0800 +From 10d853463fca5abd5b69d86d0c508060a684889d Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 15:15:01 +0800 +Subject: [PATCH 20/28] Build patch on anck 5.10 - Support 5.10 +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/crypto/qat/Makefile | 8 ++++++ + .../crypto/qat/qat_common/adf_vdcm_adi.c | 16 +++--------- + .../crypto/qat/qat_common/adf_vdcm_core.c | 26 +++++++++---------- + 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/quickassist/qat/drivers/crypto/qat/Makefile b/quickassist/qat/drivers/crypto/qat/Makefile -index e7280d8..5312e72 100644 +index d04daf6..268eb42 100644 --- a/quickassist/qat/drivers/crypto/qat/Makefile +++ b/quickassist/qat/drivers/crypto/qat/Makefile -@@ -11,11 +11,9 @@ else ifeq ($(shell test $(VERSION) -ge 4 -a $(PATCHLEVEL) -ge 10; echo $$?), 0) +@@ -21,6 +21,14 @@ else ifeq ($(shell test $(kernel) -ge $(call ver,4,10); echo $$?), 0) CONFIG_QAT_MDEV=1 endif endif --ifeq ($(CONFIG_QAT_MDEV), 1) --ifdef CONFIG_IRQ_BYPASS_MANAGER -- export CONFIG_CRYPTO_DEV_QAT_VDCM=m --endif --endif + -+export CONFIG_CRYPTO_DEV_QAT_VDCM=m ++# override CONFIG_QAT_MDEV on anck 5.10 kernel ++ifeq ($(shell test $(kernel) -eq $(call ver,5,10); echo $$?), 0) ++ ifdef CONFIG_VFIO_MDEV ++ CONFIG_QAT_MDEV=1 ++ endif ++endif + - - obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ - obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ + ifeq ($(CONFIG_QAT_MDEV), 1) + ifdef CONFIG_IRQ_BYPASS_MANAGER + export CONFIG_CRYPTO_DEV_QAT_VDCM=m diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_adi.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_adi.c -index 1315682..2173eb0 100644 +index 02706f6..241b0ed 100644 --- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_adi.c +++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_adi.c -@@ -15,7 +15,7 @@ - #include - #endif - #endif +@@ -10,7 +10,7 @@ + #include + #include + #include -#if (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE) +#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE) #include #endif #include "adf_common_drv.h" -@@ -409,7 +409,7 @@ void adf_vqat_release_irqs(struct adf_vdcm_vqat *vqat) +@@ -395,15 +395,7 @@ void adf_vqat_release_irqs(struct adf_vdcm_vqat *vqat) + } #endif - #ifdef INTEL_NEXT -#if (KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE) -+#if (KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE) - static int adf_vqat_get_pasid(struct adf_vdcm_vqat *vqat) - { - #if ((KERNEL_VERSION(5, 6, 0) == LINUX_VERSION_CODE) || \ -@@ -445,7 +445,7 @@ static int adf_vqat_get_pasid(struct adf_vdcm_vqat *vqat) - } - #endif - #endif --#if (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE) -+#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE) +-static int adf_vqat_get_pasid(struct adf_vdcm_vqat *vqat) +-{ +- dev_warn(&GET_DEV(vqat->parent), +- "%s : Please update your kernel to the one which supports sIOV\n", +- __func__); +- return -EINVAL; +-} +-#elif (KERNEL_VERSION(5, 16, 0) > LINUX_VERSION_CODE) ++#if (KERNEL_VERSION(5, 16, 0) > LINUX_VERSION_CODE) static int adf_vqat_get_pasid(struct adf_vdcm_vqat *vqat) { struct vfio_group *vfio_group; -@@ -536,12 +536,7 @@ static inline int adf_vqat_setup_iommu(struct adf_vdcm_vqat *vqat) +@@ -493,7 +485,7 @@ static inline int adf_vqat_setup_iommu(struct adf_vdcm_vqat *vqat) return -EINVAL; } /* Set mdev iommu device */ --#if KERNEL_VERSION(5, 16, 0) <= LINUX_VERSION_CODE -- dev_err(mdev_dev(vqat->mdev), -- "%s : SIOV is not supported with this kernel\n", __func__); -- return -EINVAL; -- --#elif KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE +-#if KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE +#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE mdev_set_iommu_device(vqat->mdev, &GET_DEV(vqat->parent)); return 0; -@@ -1000,7 +995,7 @@ static void adf_vdcm_adi_vqat_release(struct adf_vdcm_vqat *vqat) +@@ -943,7 +935,7 @@ static void adf_vdcm_adi_vqat_release(struct adf_vdcm_vqat *vqat) { dev_info(mdev_dev(vqat->mdev), "Release vqat %p\n", vqat); adf_vqat_release_irqs(vqat); @@ -78,10 +78,10 @@ index 1315682..2173eb0 100644 #endif adf_vdcm_adi_vqat_do_reset(vqat, false); diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_core.c b/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_core.c -index 616df63..5dc63ee 100644 +index 7fa258b..858cd29 100644 --- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_core.c +++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_vdcm_core.c -@@ -58,7 +58,7 @@ struct adf_vdcm_ctx_blk { +@@ -50,7 +50,7 @@ struct adf_vdcm_ctx_blk { static struct service_hndl adf_vdcm_srv_hndl; static struct adf_vdcm_vqat_type adf_vqat_types[QAT_VQAT_TYPES_MAX]; @@ -90,7 +90,7 @@ index 616df63..5dc63ee 100644 static struct adf_vdcm_vqat_type *adf_vdcm_lookup_vqat_type(struct device *dev, struct mdev_type *mtype); #else -@@ -428,7 +428,7 @@ void adf_vdcm_notify_vqat_iov(struct adf_vdcm_vqat *vqat, u32 queue) +@@ -232,7 +232,7 @@ void adf_vdcm_notify_vqat_iov(struct adf_vdcm_vqat *vqat, u32 queue) adf_vdcm_notify_vqat(vqat, ADF_VQAT_MISC_IRQ); } @@ -99,7 +99,7 @@ index 616df63..5dc63ee 100644 static int adf_vdcm_vqat_create(struct mdev_device *mdev) #else static int adf_vdcm_vqat_create(struct kobject *kobj, struct mdev_device *mdev) -@@ -441,7 +441,7 @@ static int adf_vdcm_vqat_create(struct kobject *kobj, struct mdev_device *mdev) +@@ -245,7 +245,7 @@ static int adf_vdcm_vqat_create(struct kobject *kobj, struct mdev_device *mdev) struct adf_accel_dev *parent; struct adf_vdcm_vqat_ops *ops; @@ -108,7 +108,7 @@ index 616df63..5dc63ee 100644 vqat_type = adf_vdcm_lookup_vqat_type(par_dev, mdev->type); if (!vqat_type || !vqat_type->ag) return -EOPNOTSUPP; -@@ -491,7 +491,7 @@ static int adf_vdcm_vqat_create(struct kobject *kobj, struct mdev_device *mdev) +@@ -295,7 +295,7 @@ static int adf_vdcm_vqat_create(struct kobject *kobj, struct mdev_device *mdev) return 0; } @@ -117,7 +117,7 @@ index 616df63..5dc63ee 100644 static void adf_vdcm_vqat_request(struct mdev_device *mdev, unsigned int count) { struct adf_vdcm_vqat *vqat = mdev_get_drvdata(mdev); -@@ -1342,7 +1342,7 @@ static long adf_vdcm_vqat_ioctl(struct mdev_device *mdev, unsigned int cmd, +@@ -1126,7 +1126,7 @@ static long adf_vdcm_vqat_ioctl(struct mdev_device *mdev, unsigned int cmd, return 0; } @@ -126,7 +126,7 @@ index 616df63..5dc63ee 100644 static ssize_t available_instances_show(struct mdev_type *mtype, struct mdev_type_attribute *attr, char *buf) -@@ -1351,14 +1351,14 @@ static ssize_t +@@ -1135,14 +1135,14 @@ static ssize_t available_instances_show(struct kobject *kobj, struct device *dev, char *buf) #endif { @@ -143,7 +143,7 @@ index 616df63..5dc63ee 100644 vqat_type = adf_vdcm_lookup_vqat_type(dev, mtype); #else vqat_type = adf_vdcm_lookup_vqat_type(dev, kobject_name(kobj)); -@@ -1392,7 +1392,7 @@ available_instances_show(struct kobject *kobj, struct device *dev, char *buf) +@@ -1176,7 +1176,7 @@ available_instances_show(struct kobject *kobj, struct device *dev, char *buf) MDEV_TYPE_ATTR_RO(available_instances); @@ -152,7 +152,7 @@ index 616df63..5dc63ee 100644 static ssize_t name_show(struct mdev_type *mtype, struct mdev_type_attribute *attr, char *buf) #else -@@ -1400,7 +1400,7 @@ static ssize_t +@@ -1184,7 +1184,7 @@ static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf) #endif { @@ -161,7 +161,7 @@ index 616df63..5dc63ee 100644 struct adf_vdcm_vqat_type *vqat_type; struct device *par_dev = mtype_get_parent_dev(mtype); -@@ -1417,7 +1417,7 @@ name_show(struct kobject *kobj, struct device *dev, char *buf) +@@ -1201,7 +1201,7 @@ name_show(struct kobject *kobj, struct device *dev, char *buf) MDEV_TYPE_ATTR_RO(name); @@ -170,7 +170,7 @@ index 616df63..5dc63ee 100644 static ssize_t device_api_show(struct mdev_type *mtype, struct mdev_type_attribute *attr, char *buf) #else -@@ -1450,7 +1450,7 @@ static struct mdev_parent_ops qat_vqat_vf_ops = { +@@ -1234,7 +1234,7 @@ static struct mdev_parent_ops qat_vqat_vf_ops = { .supported_type_groups = qat_vqat_vf_type_groups, .create = adf_vdcm_vqat_create, .remove = adf_vdcm_vqat_remove, @@ -179,7 +179,16 @@ index 616df63..5dc63ee 100644 .open_device = adf_vdcm_vqat_open, .close_device = adf_vdcm_vqat_release, #else -@@ -1527,7 +1527,7 @@ struct adf_vdcm_vqat_type *adf_vdcm_vqat_type_by_id(enum vqat_type type_id) +@@ -1273,7 +1273,7 @@ static struct mdev_parent_ops qat_vqat_adi_ops = { + .supported_type_groups = qat_vqat_adi_type_groups, + .create = adf_vdcm_vqat_create, + .remove = adf_vdcm_vqat_remove, +-#if (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE) ++#if (KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE) + .request = adf_vdcm_vqat_request, + #endif + #if KERNEL_VERSION(5, 15, 0) <= LINUX_VERSION_CODE +@@ -1311,7 +1311,7 @@ struct adf_vdcm_vqat_type *adf_vdcm_vqat_type_by_id(enum vqat_type type_id) return &adf_vqat_types[type_id]; } @@ -188,3 +197,6 @@ index 616df63..5dc63ee 100644 static struct adf_vdcm_vqat_type *adf_vdcm_lookup_vqat_type(struct device *dev, struct mdev_type *mtype) { +-- +2.39.3 + diff --git a/1001-bugfix-split-qat-vdcm-module.patch b/1021-qat_4xxx-split-qat_4xxx-module-with-qat_vdcm-module.patch similarity index 80% rename from 1001-bugfix-split-qat-vdcm-module.patch rename to 1021-qat_4xxx-split-qat_4xxx-module-with-qat_vdcm-module.patch index daa0233dee4ee7adca1bfca1f8a6ef14519af1eb..36089adbe9559bab6300b4921f4d843303c159ca 100644 --- a/1001-bugfix-split-qat-vdcm-module.patch +++ b/1021-qat_4xxx-split-qat_4xxx-module-with-qat_vdcm-module.patch @@ -1,23 +1,24 @@ -From 283fa83620686579aba4035b3c1b5df6bc21cf06 Mon Sep 17 00:00:00 2001 -From: Hao Xiang -Date: Mon, 10 Oct 2022 20:23:36 +0800 -Subject: [PATCH] qat_4xxx: split qat_4xxx module with qat_vdcm module +From edb4c2830c4af6627602f39a1bbb833900a6723a Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 16:19:28 +0800 +Subject: [PATCH 21/28] qat_4xxx: split qat_4xxx module with qat_vdcm module Remove qat_4xxx module dependency on qat_vdcm module by looking up adf_vdcm_register_vqat_parent/adf_vdcm_unregister_vqat_parent function symbols dynamically. Signed-off-by: Hao Xiang +Signed-off-by: Zelin Deng --- - quickassist/build_system/build_files/qat_service | 2 ++ - .../qat/drivers/crypto/qat/qat_4xxx/adf_drv.c | 32 +++++++++++++++++----- + .../build_system/build_files/qat_service | 2 ++ + .../qat/drivers/crypto/qat/qat_4xxx/adf_drv.c | 32 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/quickassist/build_system/build_files/qat_service b/quickassist/build_system/build_files/qat_service -index 3c00a24..2c2e95e 100755 +index f99415d..1f703a9 100755 --- a/quickassist/build_system/build_files/qat_service +++ b/quickassist/build_system/build_files/qat_service -@@ -241,6 +241,7 @@ case $1 in +@@ -251,6 +251,7 @@ case $1 in sleep 20 fi if [ $num4xxxDevicesPF != 0 ];then @@ -25,7 +26,7 @@ index 3c00a24..2c2e95e 100755 lsmod | grep qat_4xxx >/dev/null 2>&1 || modprobe qat_4xxx fi if [ `lsmod | grep "usdm_drv" | wc -l` == "0" ]; then -@@ -352,6 +353,7 @@ case $1 in +@@ -351,6 +352,7 @@ case $1 in modprobe -q -r qat_d15xx modprobe -q -r qat_c4xxx modprobe -q -r qat_4xxx @@ -34,7 +35,7 @@ index 3c00a24..2c2e95e 100755 modprobe -q -r usdm_drv modprobe -q -r intel_qat diff --git a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c -index 3d97831..309c8fe 100644 +index d011546..6e98b25 100644 --- a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -16,6 +16,7 @@ @@ -45,7 +46,7 @@ index 3d97831..309c8fe 100644 #ifdef CONFIG_CRYPTO_DEV_QAT_VDCM #include #endif -@@ -27,6 +28,12 @@ +@@ -29,6 +30,12 @@ #define ADF_SYSTEM_DEVICE(device_id) \ {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} @@ -58,7 +59,7 @@ index 3d97831..309c8fe 100644 static const struct pci_device_id adf_pci_tbl[] = { ADF_SYSTEM_DEVICE(ADF_4XXX_PCI_DEVICE_ID), ADF_SYSTEM_DEVICE(ADF_401XX_PCI_DEVICE_ID), -@@ -268,11 +275,16 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +@@ -263,11 +270,16 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_dev_stop; #ifdef CONFIG_CRYPTO_DEV_QAT_VDCM @@ -78,10 +79,10 @@ index 3d97831..309c8fe 100644 + pr_err("QAT: Function adf_vdcm_register_vqat_parent symbol not found\n"); + } #endif - return ret; -@@ -300,8 +312,14 @@ static void adf_remove(struct pci_dev *pdev) - } + /* Add accel device to accel table. +@@ -322,8 +334,14 @@ static void adf_remove(struct pci_dev *pdev) + adf_devmgr_rm_dev(accel_dev, NULL); #ifdef CONFIG_CRYPTO_DEV_QAT_VDCM if (accel_dev->vdcm) { - adf_vdcm_unregister_vqat_parent(accel_dev->vdcm, accel_dev); @@ -98,5 +99,5 @@ index 3d97831..309c8fe 100644 #endif adf_dev_stop(accel_dev); -- -1.8.3.1 +2.39.3 diff --git a/1002-bugfix-crash-of-rmmod-vdcm.patch b/1022-src-fix-crash-when-rmmod-qat_vdcm.patch similarity index 77% rename from 1002-bugfix-crash-of-rmmod-vdcm.patch rename to 1022-src-fix-crash-when-rmmod-qat_vdcm.patch index 6a92bc058f265aae4d445b3bc8c219f30908a3a4..55d3e22d89df3528636555a537ae4a8a1b767f9b 100644 --- a/1002-bugfix-crash-of-rmmod-vdcm.patch +++ b/1022-src-fix-crash-when-rmmod-qat_vdcm.patch @@ -1,18 +1,19 @@ -From a4d3b2b9d12e98ead5daafcbd476baf23c7f0eed Mon Sep 17 00:00:00 2001 -From: Xuchun Shang -Date: Tue, 25 Oct 2022 15:28:59 +0800 -Subject: [PATCH] src: fix crash when rmmod qat_vdcm +From 91a3c948abb5558809fce3391e64125a1fee31ff Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 16:20:26 +0800 +Subject: [PATCH 22/28] src: fix crash when rmmod qat_vdcm Signed-off-by: Xuchun Shang +Signed-off-by: Zelin Deng --- quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c -index 309c8fe..e683efb 100644 +index 6e98b25..4be9e89 100644 --- a/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/quickassist/qat/drivers/crypto/qat/qat_4xxx/adf_drv.c -@@ -32,6 +32,7 @@ +@@ -34,6 +34,7 @@ #include struct adf_vdcm_ctx_blk * (*adf_vdcm_register_vqat_parent_func)(struct adf_accel_dev *accel_dev,int total, enum vqat_type types[]); void (*adf_vdcm_unregister_vqat_parent_func)(struct adf_vdcm_ctx_blk *vdcm, struct adf_accel_dev *accel_dev); @@ -20,7 +21,7 @@ index 309c8fe..e683efb 100644 #endif static const struct pci_device_id adf_pci_tbl[] = { -@@ -275,6 +276,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +@@ -270,6 +271,11 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_dev_stop; #ifdef CONFIG_CRYPTO_DEV_QAT_VDCM @@ -32,7 +33,7 @@ index 309c8fe..e683efb 100644 adf_vdcm_register_vqat_parent_func = (void *)kallsyms_lookup_name("adf_vdcm_register_vqat_parent"); if (adf_vdcm_register_vqat_parent_func) { accel_dev->vdcm = adf_vdcm_register_vqat_parent_func(accel_dev, -@@ -316,6 +322,8 @@ static void adf_remove(struct pci_dev *pdev) +@@ -338,6 +344,8 @@ static void adf_remove(struct pci_dev *pdev) if (adf_vdcm_unregister_vqat_parent_func) { adf_vdcm_unregister_vqat_parent_func(accel_dev->vdcm, accel_dev); accel_dev->vdcm = NULL; @@ -42,5 +43,5 @@ index 309c8fe..e683efb 100644 pr_err("QAT: Function adf_vdcm_unregister_vqat_parent symbol not found\n"); return; -- -2.27.0 +2.39.3 diff --git a/1023-Fix-null-pointer-issue-if-qat_vf_vfio_pci-is-used-in.patch b/1023-Fix-null-pointer-issue-if-qat_vf_vfio_pci-is-used-in.patch new file mode 100644 index 0000000000000000000000000000000000000000..97b8d1b89c4e0da7534bbfe402916e4fda07f3f5 --- /dev/null +++ b/1023-Fix-null-pointer-issue-if-qat_vf_vfio_pci-is-used-in.patch @@ -0,0 +1,37 @@ +From 135fd37553f89bd676f20ad2a1b2f11ae2116a41 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 16:20:33 +0800 +Subject: [PATCH 23/28] Fix null pointer issue if qat_vf_vfio_pci is used in + guest + +Signed-off-by: Zelin Deng +--- + quickassist/qat/drivers/vfio/pci/qat/main.c | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/quickassist/qat/drivers/vfio/pci/qat/main.c b/quickassist/qat/drivers/vfio/pci/qat/main.c +index 449bc11..867e848 100644 +--- a/quickassist/qat/drivers/vfio/pci/qat/main.c ++++ b/quickassist/qat/drivers/vfio/pci/qat/main.c +@@ -645,8 +645,16 @@ qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) + struct qat_vf_core_device *qat_vdev; + int ret; + struct pci_dev *physfn = pdev->physfn; +- struct adf_accel_dev *pf_accel_dev = adf_devmgr_pci_to_accel_dev(physfn); +- struct adf_hw_device_data *pf_hw_data = GET_HW_DATA(pf_accel_dev); ++ struct adf_accel_dev *pf_accel_dev; ++ struct adf_hw_device_data *pf_hw_data; ++ ++ if (!physfn) { ++ dev_err(dev, "qat_vf_vfio_pci is not supported without PF\n"); ++ return -ENODEV; ++ } ++ ++ pf_accel_dev = adf_devmgr_pci_to_accel_dev(physfn); ++ pf_hw_data = GET_HW_DATA(pf_accel_dev); + + adf_gen4_init_vf_mig_ops(&pf_hw_data->vfmig_ops); + +-- +2.39.3 + diff --git a/1004-build-split-kernel-space-and-user-space-building.patch b/1024-build-split-kernel-space-and-user-space-building.patch similarity index 69% rename from 1004-build-split-kernel-space-and-user-space-building.patch rename to 1024-build-split-kernel-space-and-user-space-building.patch index 200bc45851a9ad86d45e253e7cc84e53fd01dce4..b61f1c3787676a280ceda31b58e586fc1483b3cf 100644 --- a/1004-build-split-kernel-space-and-user-space-building.patch +++ b/1024-build-split-kernel-space-and-user-space-building.patch @@ -1,30 +1,24 @@ -From 895209fd197f6fffc1d3b99b3a0bed55820bbfd4 Mon Sep 17 00:00:00 2001 -Message-Id: <895209fd197f6fffc1d3b99b3a0bed55820bbfd4.1692171093.git.llfl@linux.alibaba.com> -From: Artie Ding -Date: Mon, 17 Jul 2023 16:36:58 +0800 -Subject: [PATCH 1004/1004] build: split kernel space and user space building +From e9333bd396428945ed252049b01da8b85b125eeb Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 16:20:41 +0800 +Subject: [PATCH 24/28] build: split kernel space and user space building Signed-off-by: Artie Ding Signed-off-by: Kun(llfl) +Signed-off-by: Zelin Deng --- - Makefile.in | 82 ++++++++++++++++++++++++++++++++++++++------ - configure | 26 +++++++------- - quickassist/Makefile | 21 ++++++------ - 3 files changed, 96 insertions(+), 33 deletions(-) + Makefile.in | 72 +++++++++++++++++++++++++++++++++++++++++--- + configure | 26 ++++++++-------- + quickassist/Makefile | 17 ++++++----- + 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/Makefile.in b/Makefile.in -index ae48db8..90a0b0f 100644 +index f5a9052..0720803 100644 --- a/Makefile.in +++ b/Makefile.in -@@ -488,11 +488,17 @@ numVQATDevices := `lspci -vnd 8086: | egrep -c "$(INTEL_VENDORID):$(VQAT_DEVICE_ - isLegacyLoaded := `lsmod | egrep -c "icp_qa_al"` - - #Targets lists --ALL_TARGETS = qat-driver-all quickassist-all adf-ctl-all qat-service-all adf-rl-all -+ALL_TARGETS = quickassist-all adf-ctl-all qat-service-all adf-rl-all - INSTALL_TARGETS = qat-driver-install quickassist-install adf-ctl-install qat-service-install +@@ -405,6 +405,12 @@ INSTALL_TARGETS = qat-driver-install quickassist-install adf-ctl-install qat-ser UNINSTALL_TARGETS = qat-driver-uninstall quickassist-uninstall qat-service-uninstall adf-ctl-uninstall sample-uninstall - CLEAN_TARGETS = qat-driver-clean quickassist-clean adf-ctl-clean qat-service-clean sample-clean + CLEAN_TARGETS = qat-driver-clean quickassist-clean adf-ctl-clean qat-service-clean sample-clean adf-rl-clean +#User space targets lists +ALL_US_TARGETS = quickassist-us-all adf-ctl-all qat-service-us-all adf-rl-all @@ -35,7 +29,7 @@ index ae48db8..90a0b0f 100644 # Naming standards: # # Use XXX_DIR for directory paths for something -@@ -810,6 +816,9 @@ PRINT_ACC_DEVICE_INFO = $(PRINT_ACC_DEVICE_INFO_SH); PrintAccDeviceInfo +@@ -605,6 +611,9 @@ PRINT_ACC_DEVICE_INFO = $(PRINT_ACC_DEVICE_INFO_SH); PrintAccDeviceInfo all: config.h $(MAKE) $(AM_MAKEFLAGS) all-am @@ -45,7 +39,7 @@ index ae48db8..90a0b0f 100644 .SUFFIXES: am--refresh: Makefile @: -@@ -1088,15 +1097,21 @@ distcleancheck: distclean +@@ -882,15 +891,21 @@ distcleancheck: distclean check-am: all-am check: check-am all-am: Makefile config.h all-local @@ -67,7 +61,7 @@ index ae48db8..90a0b0f 100644 installcheck: installcheck-am install-strip: if test -z '$(STRIP)'; then \ -@@ -1123,6 +1138,8 @@ clean: clean-am +@@ -917,6 +932,8 @@ clean: clean-am clean-am: clean-generic clean-local mostlyclean-am @@ -76,7 +70,7 @@ index ae48db8..90a0b0f 100644 distclean: distclean-am -rm -f $(am__CONFIG_DISTCLEAN_FILES) -rm -f Makefile -@@ -1148,6 +1165,8 @@ install-dvi-am: +@@ -942,6 +959,8 @@ install-dvi-am: install-exec-am: install-exec-local @@ -85,7 +79,7 @@ index ae48db8..90a0b0f 100644 install-html: install-html-am install-html-am: -@@ -1188,23 +1207,25 @@ ps-am: +@@ -982,23 +1001,25 @@ ps-am: uninstall-am: uninstall-local @@ -115,16 +109,7 @@ index ae48db8..90a0b0f 100644 .PRECIOUS: Makefile -@@ -1212,7 +1233,7 @@ uninstall-am: uninstall-local - # QUICKASSIST Makefile # - ######################## - --quickassist-all: qat-driver-all -+quickassist-all: - @$(MAKE) -C $(QUICKASSIST_DIR) - - quickassist-install: -@@ -1222,6 +1243,20 @@ quickassist-uninstall: +@@ -1016,6 +1037,20 @@ quickassist-uninstall: quickassist-clean: @$(MAKE) clean -C $(QUICKASSIST_DIR) @@ -145,16 +130,7 @@ index ae48db8..90a0b0f 100644 ################### # QAT Sample Code # ################### -@@ -1266,7 +1301,7 @@ sample-clean: - # QAT Service # - ############### - --qat-service-all: qat-driver-all quickassist-all adf-ctl-all adf-rl-all -+qat-service-all: quickassist-all adf-ctl-all adf-rl-all - @$(INSTALL) -D -m 750 $(QUICKASSIST_DIR)/build_system/build_files/qat_service $(ICP_BUILD_OUTPUT)/qat_service - @$(INSTALL) -D -m 750 $(QUICKASSIST_DIR)/build_system/build_files/vqat_ctl $(ICP_BUILD_OUTPUT)/vqat_ctl - @$(SED) -i "s|/usr/sbin/adf_ctl|$(prefix)/bin/adf_ctl|g" $(ICP_BUILD_OUTPUT)/qat_service -@@ -1771,11 +1806,21 @@ qat-service-uninstall: +@@ -1341,6 +1376,16 @@ qat-service-uninstall: qat-service-clean: @$(ECHO) qat-service-clean @@ -171,22 +147,7 @@ index ae48db8..90a0b0f 100644 ########################### # RL related targets # ########################### - --adf-rl-all: qat-driver-all quickassist-all -+adf-rl-all: quickassist-all - @$(MAKE) -C $(ADF_RL_DIR) - @$(INSTALL) -D -m 750 $(ADF_RL_DIR)/sla_mgr/build/linux_2.6/user_space/sla_mgr $(ICP_BUILD_OUTPUT)/$(ADF_RL_SLA_MGR_BIN) - -@@ -1783,7 +1828,7 @@ adf-rl-all: qat-driver-all quickassist-all - # ADF_CTL related targets # - ########################### - --adf-ctl-all: qat-driver-all quickassist-all adf-rl-all -+adf-ctl-all: quickassist-all adf-rl-all - @$(MAKE) -C $(ADF_CTL_DIR) - @$(INSTALL) -D -m 750 $(QUICKASSIST_DIR)/utilities/adf_ctl/adf_ctl $(ICP_BUILD_OUTPUT)/adf_ctl - @$(foreach conf, $(CONFIG_LIST), $(INSTALL) -D -m 750 $(QUICKASSIST_DIR)/utilities/adf_ctl/conf_files/$(conf) $(ICP_BUILD_OUTPUT)/$(conf);) -@@ -1856,6 +1901,14 @@ qat-driver-uninstall: +@@ -1411,6 +1456,14 @@ qat-driver-uninstall: qat-driver-clean: @$(MAKE) KDIR=$(KERNEL_SOURCE_ROOT) -C $(QAT_DRIVER_DIR) clean @@ -201,7 +162,7 @@ index ae48db8..90a0b0f 100644 ########### # Targets # ########### -@@ -1869,6 +1922,15 @@ install-exec-local: $(INSTALL_TARGETS) +@@ -1424,6 +1477,15 @@ install-exec-local: $(INSTALL_TARGETS) uninstall-local: $(UNINSTALL_TARGETS) @@ -217,7 +178,7 @@ index ae48db8..90a0b0f 100644 default-local: all samples: sample-all -@@ -1881,7 +1943,7 @@ device-info: +@@ -1436,7 +1498,7 @@ device-info: .PHONY: device-info samples samples-install @@ -227,10 +188,10 @@ index ae48db8..90a0b0f 100644 export diff --git a/configure b/configure -index 42ee44d..6c36c19 100755 +index b9be804..530aa69 100755 --- a/configure +++ b/configure -@@ -9110,19 +9110,19 @@ fi +@@ -9111,19 +9111,19 @@ fi # Check for kernel sources @@ -264,10 +225,10 @@ index 42ee44d..6c36c19 100755 # ICP_DEBUG diff --git a/quickassist/Makefile b/quickassist/Makefile -index 808f67f..f1c2c43 100755 +index b5ce69b..bdcd856 100755 --- a/quickassist/Makefile +++ b/quickassist/Makefile -@@ -67,6 +67,7 @@ LAC_LIB_DIR=$(LAC_PATH)/build/libs +@@ -68,6 +68,7 @@ LAC_LIB_DIR=$(LAC_PATH)/build/libs #Release Package build steps ALL_TARGETS = clean lac_lib_dir qat_direct libosal_user lac_user @@ -275,7 +236,7 @@ index 808f67f..f1c2c43 100755 ifdef ICP_SRIOV -@@ -76,6 +77,8 @@ endif +@@ -77,6 +78,8 @@ endif all: $(ALL_TARGETS) @echo 'Build Done'; @@ -284,19 +245,7 @@ index 808f67f..f1c2c43 100755 user: lac_lib_dir libosal_user lac_user -@@ -89,10 +92,8 @@ install_scripts: - # - #userspace common memory library - cmn_user: clean output_dir lac_lib_dir -- @echo ; echo 'Building common mem driver for user space'; -- @cd $(CMN_MEM_PATH) && $(MAKE) ARCH=$(ICP_ARCH_USER) ICP_ENV_DIR=$(ICP_TOP_ENV) OS=linux ICP_OS?=linux_2.6 ICP_OS_LEVEL=user_space CPM_UPSTREAM=1 cm_user;\ - echo ; echo 'Copying Common mem library'; -- cp $(CMN_MEM_PATH)/libusdm_drv_s.so $(CMN_MEM_PATH)/libusdm_drv.a $(ICP_BUILD_OUTPUT)/; -+ cp /lib64/libusdm_drv_s.so /lib64/libusdm_drv.a $(ICP_BUILD_OUTPUT)/; - - #common mem driver ko - cmn_ko: clean output_dir -@@ -117,13 +118,19 @@ qat_direct: clean output_dir lac_lib_dir libosal_user cmn_user +@@ -118,13 +121,19 @@ qat_direct: clean output_dir lac_lib_dir libosal_user cmn_user cp $(QAT_DIRECT_ROOT)/src/build/linux_2.6/user_space/libadf.a $(ICP_BUILD_OUTPUT)/; @@ -317,7 +266,7 @@ index 808f67f..f1c2c43 100755 clean: ifeq ($(ICP_NO_CLEAN),) -@@ -159,12 +166,6 @@ lac_depend: clean +@@ -160,12 +169,6 @@ lac_depend: clean @sort ${ICP_BUILD_OUTPUT}/temp_list.txt | uniq > ${ICP_BUILD_OUTPUT}/kernel_header_file_list.txt;rm ${ICP_BUILD_OUTPUT}/temp_list.txt; depend_linux: clean output_dir osal_depend cmn_depend lac_depend @@ -331,5 +280,5 @@ index 808f67f..f1c2c43 100755 ifeq ("$(wildcard /lib64/libgcc_s.so.1 /lib/libgcc_s.so.1 /usr/libgcc_s.so.1 /lib64/*/libgcc_s.so.1 /lib/*/libgcc_s.so.1 /usr/*/libgcc_s.so.1)","") -- -2.39.0 +2.39.3 diff --git a/1005-optimise-qat_service-scripts.patch b/1025-optimise-qat_service-scripts.patch similarity index 84% rename from 1005-optimise-qat_service-scripts.patch rename to 1025-optimise-qat_service-scripts.patch index 993a3a8a07261f821723ec1be2d7e54c98c0c561..baac5379e9fa8d83e3e612da09602becf2d28b54 100644 --- a/1005-optimise-qat_service-scripts.patch +++ b/1025-optimise-qat_service-scripts.patch @@ -1,19 +1,19 @@ -From 96f682aabb551b482b505e587cc3b726cfac14dd Mon Sep 17 00:00:00 2001 -Message-Id: <96f682aabb551b482b505e587cc3b726cfac14dd.1692173679.git.llfl@linux.alibaba.com> -From: "Kun(llfl)" -Date: Wed, 16 Aug 2023 16:14:31 +0800 -Subject: [PATCH 1005/1005] optimise qat_service scripts +From bffe3c4ecf01635f263b0e6f0791b0e71841b359 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 16:32:02 +0800 +Subject: [PATCH 25/28] optimise qat_service scripts Signed-off-by: Kun(llfl) +Signed-off-by: Zelin Deng --- .../build_system/build_files/qat_service | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/quickassist/build_system/build_files/qat_service b/quickassist/build_system/build_files/qat_service -index 2c2e95e..0fc392e 100755 +index 1f703a9..b18623f 100755 --- a/quickassist/build_system/build_files/qat_service +++ b/quickassist/build_system/build_files/qat_service -@@ -59,7 +59,7 @@ LEGACY_LOADED=${LEGACY_LOADED-0} +@@ -58,7 +58,7 @@ LEGACY_LOADED=${LEGACY_LOADED-0} DO_ENABLE_SRIOV=${DO_ENABLE_SRIOV-0} # to protect parallel qat-service run instances @@ -22,8 +22,8 @@ index 2c2e95e..0fc392e 100755 if [ $pid != $$ ]; then echo "[$(date)] : qat_service : Process is already running with PID $pid" exit 1 -@@ -115,19 +115,19 @@ exit 1 - ADF_CTL=/usr/sbin/adf_ctl +@@ -117,19 +117,19 @@ ADF_CTL=/usr/sbin/adf_ctl + OOT_ONLY=/dev/qat_dev_processes # store the total number of each type of device -numDh895xDevicesPF=$(lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_PCI_ID") @@ -36,8 +36,8 @@ index 2c2e95e..0fc392e 100755 -numD15xxDevicesVF=$(lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_PCI_ID_VM") -numC4xxDevicesPF=$(lspci -n | egrep -c "$INTEL_VENDORID:$C4XX_DEVICE_PCI_ID") -numC4xxDevicesVF=$(lspci -n | egrep -c "$INTEL_VENDORID:$C4XX_DEVICE_PCI_ID_VM") --num4xxxDevicesPF=$(lspci -n | egrep -c "$INTEL_VENDORID:($QAT_4XXX_DEVICE_PCI_ID|$QAT_401XX_DEVICE_PCI_ID)") --num4xxxDevicesVF=$(lspci -n | egrep -c "$INTEL_VENDORID:($QAT_4XXX_DEVICE_PCI_ID_VM|$QAT_401XX_DEVICE_PCI_ID_VM)") +-num4xxxDevicesPF=$(lspci -n | egrep -c "$INTEL_VENDORID:($QAT_4XXX_DEVICE_PCI_ID|$QAT_401XX_DEVICE_PCI_ID|$QAT_402XX_DEVICE_PCI_ID)") +-num4xxxDevicesVF=$(lspci -n | egrep -c "$INTEL_VENDORID:($QAT_4XXX_DEVICE_PCI_ID_VM|$QAT_401XX_DEVICE_PCI_ID_VM|$QAT_402XX_DEVICE_PCI_ID_VM)") -numVQATDevices=$(lspci -n | egrep -c "$INTEL_VENDORID:$VQAT_DEVICE_PCI_ID") +numDh895xDevicesPF=$(lspci -n | grep -E -c "$INTEL_VENDORID:$DH895_DEVICE_PCI_ID") +numDh895xDevicesVF=$(lspci -n | grep -E -c "$INTEL_VENDORID:$DH895_DEVICE_PCI_ID_VM") @@ -56,5 +56,5 @@ index 2c2e95e..0fc392e 100755 disable_sriov() { PF_LIST=`$ADF_CTL $1 status | grep -e "^ *qat_dev" | grep -v "vf," | grep -v "vqat-adi," | awk '{print $1}'` -- -2.39.0 +2.39.3 diff --git a/1006-replace-usdm-with-udma.patch b/1026-replace-usdm-with-udma.patch similarity index 62% rename from 1006-replace-usdm-with-udma.patch rename to 1026-replace-usdm-with-udma.patch index 264723636bf4390f919422d520971dbcf6bcc12f..1ae47c13bd82c42f9186cb7b2675f7ab0f8d0fe9 100644 --- a/1006-replace-usdm-with-udma.patch +++ b/1026-replace-usdm-with-udma.patch @@ -1,49 +1,43 @@ -From f08c73e40e6b7a923ee28fe3e5d8050f205d2244 Mon Sep 17 00:00:00 2001 -From: Guanjun -Date: Wed, 13 Sep 2023 15:17:59 +0800 -Subject: [PATCH 1/1] replace usdm with udma +From ea65da8e6a99d5a2653e1b64195c41267a9b4933 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Fri, 14 Jun 2024 17:15:59 +0800 +Subject: [PATCH 26/28] replace usdm with udma Signed-off-by: Guanjun +Signed-off-by: Zelin Deng --- Makefile.in | 1 - - quickassist/Makefile | 3 +-- + quickassist/Makefile | 1 + quickassist/lookaside/access_layer/src/Makefile | 2 +- quickassist/utilities/apps/Makefile | 1 - - 4 files changed, 2 insertions(+), 5 deletions(-) + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile.in b/Makefile.in -index 90a0b0f..10d5644 100644 +index 0720803..597b996 100644 --- a/Makefile.in +++ b/Makefile.in -@@ -1822,7 +1822,6 @@ qat-service-us-all: +@@ -1392,7 +1392,6 @@ qat-service-us-all: - adf-rl-all: quickassist-all + adf-rl-all: adf-rl-clean @$(MAKE) -C $(ADF_RL_DIR) - @$(INSTALL) -D -m 750 $(ADF_RL_DIR)/sla_mgr/build/linux_2.6/user_space/sla_mgr $(ICP_BUILD_OUTPUT)/$(ADF_RL_SLA_MGR_BIN) - ########################### - # ADF_CTL related targets # + adf-rl-clean: + @$(MAKE) clean -C $(ADF_RL_DIR)/sla_mgr/ diff --git a/quickassist/Makefile b/quickassist/Makefile -index f1c2c43..85be099 100755 +index bdcd856..5fc633f 100755 --- a/quickassist/Makefile +++ b/quickassist/Makefile -@@ -93,13 +93,12 @@ install_scripts: - #userspace common memory library - cmn_user: clean output_dir lac_lib_dir +@@ -97,6 +97,7 @@ cmn_user: clean output_dir lac_lib_dir + @cd $(CMN_MEM_PATH) && $(MAKE) ARCH=$(ICP_ARCH_USER) ICP_ENV_DIR=$(ICP_TOP_ENV) OS=linux ICP_OS?=linux_2.6 ICP_OS_LEVEL=user_space CPM_UPSTREAM=1 cm_user;\ echo ; echo 'Copying Common mem library'; -- cp /lib64/libusdm_drv_s.so /lib64/libusdm_drv.a $(ICP_BUILD_OUTPUT)/; + cp $(CMN_MEM_PATH)/libusdm_drv_s.so $(CMN_MEM_PATH)/libusdm_drv.a $(ICP_BUILD_OUTPUT)/; + cp /lib64/libudma.so* $(ICP_BUILD_OUTPUT)/; #common mem driver ko cmn_ko: clean output_dir - @echo ; echo 'Building usdm_drv.ko'; - @cd $(CMN_MEM_PATH) && $(MAKE) ICP_ENV_DIR=$(ICP_TOP_ENV) OS=linux ICP_OS?=linux_2.6 ICP_OS_LEVEL=kernel_space ICP_QDM_IOMMU=1 CPM_UPSTREAM=1 cm_kernel -- @cp $(CMN_MEM_PATH)/usdm_drv.ko $(ICP_BUILD_OUTPUT) - - - #userspace osal library diff --git a/quickassist/lookaside/access_layer/src/Makefile b/quickassist/lookaside/access_layer/src/Makefile -index 2b9b8be..c4ab694 100755 +index a2a6f20..59d7cba 100755 --- a/quickassist/lookaside/access_layer/src/Makefile +++ b/quickassist/lookaside/access_layer/src/Makefile @@ -104,7 +104,7 @@ ifeq ($(ICP_OS_LEVEL), user_space) @@ -56,7 +50,7 @@ index 2b9b8be..c4ab694 100755 ###########End of Define the output section####################### SUBDIRS=$(patsubst %/$(ICP_BUILD_OUTPUT_DIR)/,%,$(dir $(ADDITIONAL_KERNEL_LIBS))) diff --git a/quickassist/utilities/apps/Makefile b/quickassist/utilities/apps/Makefile -index 288e964..bec28ce 100644 +index f3f64a1..454ef8a 100644 --- a/quickassist/utilities/apps/Makefile +++ b/quickassist/utilities/apps/Makefile @@ -63,7 +63,6 @@ all: sla_mgr_build diff --git a/1007-Fix-uio-scan-logic.patch b/1027-Fix-uio-scan-logic.patch similarity index 87% rename from 1007-Fix-uio-scan-logic.patch rename to 1027-Fix-uio-scan-logic.patch index 053638cf7a4ffa6a33d69c4d0a1a91529f80a4ae..46a3af4d02ac9cfe7736b92ba457118d605b4c73 100644 --- a/1007-Fix-uio-scan-logic.patch +++ b/1027-Fix-uio-scan-logic.patch @@ -1,7 +1,7 @@ -From ae92b3af814ab918c7d591c687217f8c4d3a80b0 Mon Sep 17 00:00:00 2001 +From 8882577ae67f8b880c36b4e6349f69402885ad2e Mon Sep 17 00:00:00 2001 From: Xingrui Yi Date: Fri, 18 Aug 2023 15:59:17 +0800 -Subject: [PATCH] Fix uio scan logic +Subject: [PATCH 27/28] Fix uio scan logic add logic to scan uio in /dev @@ -15,11 +15,11 @@ Signed-off-by: Xingrui Yi 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/quickassist/lookaside/access_layer/src/qat_direct/src/adf_process_proxy.c b/quickassist/lookaside/access_layer/src/qat_direct/src/adf_process_proxy.c -index 941470a..ae9907f 100644 +index 5c6a1c1..97de7ce 100644 --- a/quickassist/lookaside/access_layer/src/qat_direct/src/adf_process_proxy.c +++ b/quickassist/lookaside/access_layer/src/qat_direct/src/adf_process_proxy.c -@@ -411,7 +411,12 @@ CpaStatus icp_adf_userProcessToStart(char const *const name_tml, char *name) - ADF_ERROR("Mutex unlock error\n"); +@@ -416,7 +416,12 @@ CpaStatus icp_adf_userProcessToStart(char const *const name_tml, char *name) + osalMutexUnlock(&processes_lock); return CPA_STATUS_FAIL; } - res = write(process_info_file, name_tml, name_len); @@ -33,10 +33,10 @@ index 941470a..ae9907f 100644 { close(process_info_file); diff --git a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user.h b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user.h -index 8caf6a7..49f17ad 100644 +index 7892e1b..23d0261 100644 --- a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user.h +++ b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user.h -@@ -46,5 +46,6 @@ int32_t adf_init_devices(void); +@@ -45,5 +45,6 @@ int32_t adf_init_devices(void); CpaStatus adf_proxy_get_devices(void); int32_t adf_cleanup_devices(void); int adf_proxy_poll_event(Cpa32U *dev_id, enum adf_event *event); @@ -44,14 +44,13 @@ index 8caf6a7..49f17ad 100644 #endif /* end of include guard: UIO_USER_H */ diff --git a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_device.c b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_device.c -index 2919910..a400ff0 100755 +index 1c3dc29..5f2cf3c 100755 --- a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_device.c +++ b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_device.c -@@ -839,3 +839,25 @@ CpaStatus adf_devmgrGetAccelHead(icp_accel_dev_t **pAccelDev) - *pAccelDev = (icp_accel_dev_t *)accel_tbl; +@@ -1077,6 +1077,28 @@ CpaStatus adf_devmgrGetAccelHead(icp_accel_dev_t **pAccelDev) return CPA_STATUS_SUCCESS; } -+ + +/* + * adf_get_tbl + * Returns accel table list for qat_dev_process write @@ -73,9 +72,12 @@ index 2919910..a400ff0 100755 + } + return tbl_list; +} -\ No newline at end of file ++ + /* + * icp_adf_isDeviceAvailable + * Returns true if at least one qat device is active diff --git a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_utils.c b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_utils.c -index 4237457..43e140d 100644 +index b21ca53..39f9ef0 100644 --- a/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_utils.c +++ b/quickassist/lookaside/access_layer/src/qat_direct/src/uio_user_utils.c @@ -41,6 +41,7 @@ @@ -115,10 +117,10 @@ index 4237457..43e140d 100644 } else diff --git a/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_common.h b/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_common.h -index 1d50fdf..e8f9c29 100644 +index 3e7d7f8..819ac72 100644 --- a/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/quickassist/qat/drivers/crypto/qat/qat_common/adf_cfg_common.h -@@ -143,6 +143,13 @@ struct adf_dev_miscellaneous_stats { +@@ -245,6 +245,13 @@ struct adf_dev_miscellaneous_stats { u64 misc_counter; }; @@ -133,5 +135,5 @@ index 1d50fdf..e8f9c29 100644 struct adf_pmisc_write_info { u32 accel_id; -- -2.31.1 +2.39.3 diff --git a/1028-In-case-libudma-may-does-not-exist-fix-it.patch b/1028-In-case-libudma-may-does-not-exist-fix-it.patch new file mode 100644 index 0000000000000000000000000000000000000000..16f9fb409f51b342e38189bbdc1d29b5b1a85abf --- /dev/null +++ b/1028-In-case-libudma-may-does-not-exist-fix-it.patch @@ -0,0 +1,43 @@ +From 8643645f100a026990c5b5b11f7c26d301b8e9e9 Mon Sep 17 00:00:00 2001 +From: Zelin Deng +Date: Mon, 17 Jun 2024 14:08:42 +0800 +Subject: [PATCH 28/28] In case libudma may does not exist, fix it + +Signed-off-by: Zelin Deng +--- + quickassist/Makefile | 2 +- + quickassist/lookaside/access_layer/src/Makefile | 4 ++++ + 2 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/quickassist/Makefile b/quickassist/Makefile +index 5fc633f..297a39a 100755 +--- a/quickassist/Makefile ++++ b/quickassist/Makefile +@@ -97,7 +97,7 @@ cmn_user: clean output_dir lac_lib_dir + @cd $(CMN_MEM_PATH) && $(MAKE) ARCH=$(ICP_ARCH_USER) ICP_ENV_DIR=$(ICP_TOP_ENV) OS=linux ICP_OS?=linux_2.6 ICP_OS_LEVEL=user_space CPM_UPSTREAM=1 cm_user;\ + echo ; echo 'Copying Common mem library'; + cp $(CMN_MEM_PATH)/libusdm_drv_s.so $(CMN_MEM_PATH)/libusdm_drv.a $(ICP_BUILD_OUTPUT)/; +- cp /lib64/libudma.so* $(ICP_BUILD_OUTPUT)/; ++ -cp /lib64/libudma.so* $(ICP_BUILD_OUTPUT)/ 2>/dev/null; + + #common mem driver ko + cmn_ko: clean output_dir +diff --git a/quickassist/lookaside/access_layer/src/Makefile b/quickassist/lookaside/access_layer/src/Makefile +index 59d7cba..58b3ee4 100755 +--- a/quickassist/lookaside/access_layer/src/Makefile ++++ b/quickassist/lookaside/access_layer/src/Makefile +@@ -104,7 +104,11 @@ ifeq ($(ICP_OS_LEVEL), user_space) + ADDITIONAL_KERNEL_LIBS += user/user.a + endif + ++ifeq ($(wildcard $(ICP_BUILD_OUTPUT)/libudma.so),) ++ADDITIONAL_LIBS+=$(ICP_BUILD_OUTPUT)/libusdm_drv_s.so ++else + ADDITIONAL_LIBS+=$(ICP_BUILD_OUTPUT)/libudma.so ++endif + + ###########End of Define the output section####################### + SUBDIRS=$(patsubst %/$(ICP_BUILD_OUTPUT_DIR)/,%,$(dir $(ADDITIONAL_KERNEL_LIBS))) +-- +2.39.3 + diff --git a/intel-QAT20-L.0.9.4-00004.tar.gz b/intel-QAT20-L.1.1.40-00018.tar.gz similarity index 32% rename from intel-QAT20-L.0.9.4-00004.tar.gz rename to intel-QAT20-L.1.1.40-00018.tar.gz index c5cfde7515a54a4da8ed6568441fef072d89b1a3..0cdc1c3b3be6d8054214dfab071eca8fdc1982b0 100644 Binary files a/intel-QAT20-L.0.9.4-00004.tar.gz and b/intel-QAT20-L.1.1.40-00018.tar.gz differ diff --git a/intel-QAT20.spec b/intel-QAT20.spec index 248784f0fd3237b8db6ff7f29e1fb42322f9b0ab..a933720c56d9fb83e0c4c3a92c752cac6aae0289 100644 --- a/intel-QAT20.spec +++ b/intel-QAT20.spec @@ -1,26 +1,48 @@ %define anolis_release 12 %define debug_package %{nil} -%define QAT_release 00004 +%define QAT_release 00018 Summary: QAT package(Library and sample application) Name: intel-QAT20 -Version: L.0.9.4 +Version: L.1.1.40 Release: %{QAT_release}.%{anolis_release}%{dist} Group: QAT/Base License: BSD and (BSD or GPLv2) Source: %{name}-%{version}-%{QAT_release}.tar.gz -Patch1000: 1000-bugfix-support-5.10.patch -Patch1001: 1001-bugfix-split-qat-vdcm-module.patch -Patch1002: 1002-bugfix-crash-of-rmmod-vdcm.patch -Patch1003: 1003-src-fix-warning-when-compiling.patch -Patch1004: 1004-build-split-kernel-space-and-user-space-building.patch -Patch1005: 1005-optimise-qat_service-scripts.patch -Patch1006: 1006-replace-usdm-with-udma.patch -Patch1007: 1007-Fix-uio-scan-logic.patch + + +Patch1001: 1001-Build-sample-code-with-PREBUILD_BINS-0.patch +Patch1002: 1002-Just-regard-statistic-is-false-rather-than-error.patch +Patch1003: 1003-crypto-qat-adf_get_etr_base-helper.patch +Patch1004: 1004-crypto-qat-move-PFVF-compat-checker-to-a-function.patch +Patch1005: 1005-crypto-qat-add-bank-save-and-restore-flows.patch +Patch1006: 1006-crypto-qat-add-interface-for-live-migration.patch +Patch1007: 1007-crypto-qat-implement-interface-for-live-migration.patch +Patch1008: 1008-vfio-qat-Add-vfio_pci-driver-for-Intel-QAT-VF-device.patch +Patch1009: 1009-add-dummy-helpers-and-c-files-to-make-build-successf.patch +Patch1010: 1010-make-qat-vfio-pci-drvier-to-be-built.patch +Patch1011: 1011-Use-self-defined-vfio-core-device-init-release-funct.patch +Patch1012: 1012-Remove-vfio-core-pci-init-release-definition-in-dumm.patch +Patch1013: 1013-Add-new-rl-full-support.patch +Patch1014: 1014-Uncomment-sla-in-vf-migs.patch +Patch1015: 1015-OOT-fw-only-accept-sla-req-init_cfg_sz-64.patch +Patch1016: 1016-Fix-typo-when-merging-code.patch +Patch1017: 1017-Move-migration-relevant-logic-to-qat_vf_vfio-module.patch +Patch1018: 1018-Add-telemetry-python-script-to-monitor-ring-pairs.patch +Patch1019: 1019-Add-per-rp-telemetry.patch +Patch1020: 1020-Build-patch-on-anck-5.10.patch +Patch1021: 1021-qat_4xxx-split-qat_4xxx-module-with-qat_vdcm-module.patch +Patch1022: 1022-src-fix-crash-when-rmmod-qat_vdcm.patch +Patch1023: 1023-Fix-null-pointer-issue-if-qat_vf_vfio_pci-is-used-in.patch +Patch1024: 1024-build-split-kernel-space-and-user-space-building.patch +Patch1025: 1025-optimise-qat_service-scripts.patch +Patch1026: 1026-replace-usdm-with-udma.patch +Patch1027: 1027-Fix-uio-scan-logic.patch +Patch1028: 1028-In-case-libudma-may-does-not-exist-fix-it.patch BuildRequires: gcc gcc-c++ make systemd-devel openssl-devel zlib-devel yasm -BuildRequires: udma-devel BuildRequires: libudev-devel >= 1.47 +BuildRequires: udma-devel BuildRequires: boost-devel Requires(post): %{_sbindir}/depmod @@ -46,7 +68,13 @@ that use the Intel QuickAssist APIs. %build unset ICP_ROOT ICP_ARCH_USER ICP_BUILDSYSTEM_PATH ICP_BUILD_OUTPUT ICP_ENV_DIR ICP_TOOLS_TARGET -%configure +ADDITIONAL_CONFIG_OPTIONS="" + +if [ $(echo `uname -r` | grep -i tinyos) ]; then + ADDITIONAL_CONFIG_OPTIONS="--enable-icp-sriov=host" +fi + +%configure ${ADDITIONAL_CONFIG_OPTIONS} # make only work in single thread make all-us @@ -71,6 +99,9 @@ SRCMODDIR=${RPM_BUILD_DIR}/%{name}-%{version}/quickassist/qat/${MODPATH} /usr/bin/install -D -m 750 build/adf_ctl %{buildroot}%{_sbindir} /usr/bin/install -D -m 750 build/vqat_ctl %{buildroot}%{_sbindir} /usr/bin/install -D -m 755 build/libqat_s.so %{buildroot}/%{_libdir} +/usr/bin/install -D -m 755 build/qat_4xxx.bin %{buildroot}/lib/firmware/qat_4xxx.bin +/usr/bin/install -D -m 755 build/qat_4xxx_mmp.bin %{buildroot}/lib/firmware/qat_4xxx_mmp.bin + /usr/bin/install -D -m 666 build/4xxx_template.conf %{buildroot}/usr/bin/QAT/build /usr/bin/install -D -m 666 build/4xxxvf_dev0.conf.vm %{buildroot}/usr/bin/QAT/build @@ -104,6 +135,8 @@ rm -rf $RPM_BUILD_ROOT /usr/bin/QAT/ %{_libdir}/libqat_s.so.0 %{_libdir}/libqat_s.so.%{version} +/lib/firmware/qat_4xxx.bin +/lib/firmware/qat_4xxx_mmp.bin %files devel %license LICENSE.BSD @@ -257,6 +290,9 @@ fi ${RM} -rf /etc/udev/rules.d/00-qat.rules /etc/default/qat %changelog +* Fri Jun 18 2024 Zelin Deng - 1.1.40-00018.1 +- Replace new oot package + * Thu Dec 14 2023 Xingrui Yi - 0.9.4-00004.12 - Change qat_4xxx fw file path