From d745d383939a21a149852157b6fad849ae5ee234 Mon Sep 17 00:00:00 2001 From: Jeremy Compostella Date: Wed, 2 Jul 2025 13:12:48 -0700 Subject: [PATCH] soc/intel/cmn/block/fast_spi: Add DMA support This commit introduces Direct Memory Access (DMA) support for the Intel Fast SPI block, enhancing data transfer capabilities from SPI flash memory. The primary motivation for this addition is to improve performance in multitasking environments by offloading read operations to DMA, thus freeing up CPU resources for other tasks. The traditional memory-mapped SPI flash read operations can be CPU-intensive and slow in scenarios where large data volumes are transferred. This feature is gated by a FAST_SPI_DMA configuration option. The DMA operations are integrated with existing SPI flash read functionalities, ensuring fallback to memory-mapped operations if DMA is unsupported or fails. The DMA code implementation uses mutex-based synchronization to ensure thread-safe DMA transactions. The boot_device_ro() function has been modified to check for DMA support and installs custom DMA operations when available. We conducted measurements on a 200 KB data transfer and noticed a 1.8% improvement with DMA compared to memcpy on memory-mapped SPINOR, with DMA taking 10.8 ms and memcpy taking 11 ms. Change-Id: I4b4ca9ff08e436ca627afa6b0d9bb00f3c450a5e Signed-off-by: Jeremy Compostella Reviewed-on: https://review.coreboot.org/c/coreboot/+/88277 Reviewed-by: Shuo Liu Reviewed-by: Kapil Porwal Tested-by: build bot (Jenkins) --- src/soc/intel/common/block/fast_spi/Kconfig | 10 + .../intel/common/block/fast_spi/Makefile.mk | 1 + .../common/block/fast_spi/fast_spi_dma.c | 405 ++++++++++++++++++ .../common/block/fast_spi/fast_spi_dma.h | 42 ++ .../intel/common/block/fast_spi/mmap_boot.c | 4 + 5 files changed, 462 insertions(+) create mode 100644 src/soc/intel/common/block/fast_spi/fast_spi_dma.c create mode 100644 src/soc/intel/common/block/fast_spi/fast_spi_dma.h diff --git a/src/soc/intel/common/block/fast_spi/Kconfig b/src/soc/intel/common/block/fast_spi/Kconfig index 3a541f861c..a983797989 100644 --- a/src/soc/intel/common/block/fast_spi/Kconfig +++ b/src/soc/intel/common/block/fast_spi/Kconfig @@ -60,3 +60,13 @@ config FAST_SPI_GENERATE_SSDT If this switch is selected, an entry in the SSDT will be generated for the controller to report the occupied resource which is not discoverable at OS runtime. + +config FAST_SPI_DMA + bool + help + Choose this option to allow the use of the SPI DMA controller, if + it's available. When enabled, the boot read-only region device + functions are enhanced with a DMA version of the readat() + callback. Although DMA transfers may not be significantly faster, + they can free up CPU resources for other meaningful tasks in a + multitasking environment (see COOP_MULTITASKING). diff --git a/src/soc/intel/common/block/fast_spi/Makefile.mk b/src/soc/intel/common/block/fast_spi/Makefile.mk index 6e274df69f..e2325d9f3e 100644 --- a/src/soc/intel/common/block/fast_spi/Makefile.mk +++ b/src/soc/intel/common/block/fast_spi/Makefile.mk @@ -10,6 +10,7 @@ romstage-$(CONFIG_SOC_INTEL_COMMON_BLOCK_FAST_SPI) += fast_spi_flash.c ramstage-$(CONFIG_SOC_INTEL_COMMON_BLOCK_FAST_SPI) += fast_spi.c ramstage-$(CONFIG_SOC_INTEL_COMMON_BLOCK_FAST_SPI) += fast_spi_flash.c +ramstage-$(CONFIG_FAST_SPI_DMA) += fast_spi_dma.c postcar-$(CONFIG_SOC_INTEL_COMMON_BLOCK_FAST_SPI) += fast_spi.c postcar-$(CONFIG_SOC_INTEL_COMMON_BLOCK_FAST_SPI) += fast_spi_flash.c diff --git a/src/soc/intel/common/block/fast_spi/fast_spi_dma.c b/src/soc/intel/common/block/fast_spi/fast_spi_dma.c new file mode 100644 index 0000000000..c21056aa18 --- /dev/null +++ b/src/soc/intel/common/block/fast_spi/fast_spi_dma.c @@ -0,0 +1,405 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include +#include +#include +#include +#include + +#define REGION_BIOS 1 + +/* BIOS DMA Fast SPI registers */ +enum fast_spi_dma_register { + FAST_SPI_DMA_DEST_ADDRESS_HIGH = 0xb0, + FAST_SPI_DMA_DEST_ADDRESS_LOW = 0xb4, + FAST_SPI_DMA_REGION = 0xb8, + FAST_SPI_DMA_CONTROL = 0xbc, + FAST_SPI_DMA_STATUS = 0xc0 +}; + +/* FAST_SPI_DMA_REGION */ +union fast_spi_dma_region { + uint32_t data; + struct { + uint32_t reserved: 16; + uint32_t offset_kb: 16; + } fields; +}; + +/* FAST_SPI_DMA_CONTROL */ +union fast_spi_dma_control { + uint32_t data; + struct { + uint32_t start: 1; + uint32_t reserved0: 5; + uint32_t reset_write_pointer: 1; + uint32_t reserved1: 1; + uint32_t flash_region: 4; + uint32_t reserved2: 3; + uint32_t lock: 1; + uint32_t size: 16; + } fields; +}; + +/* FAST_SPI_DMA_STATUS */ +union fast_spi_dma_status { + uint32_t data; + struct { + uint32_t reserved0: 2; + uint32_t complete: 1; + uint32_t error: 5; + uint32_t reserved1: 8; + uint32_t write_pointer: 16; + } fields; +}; + +#define FAST_SPI_DMA_BLOCK_ALIGN KiB +#define FAST_SPI_DMA_TIMEOUT_MSEC 1000 +#define FAST_SPI_DMA_DELAY_STEP_USEC 200 +#define FAST_SPI_DEV DEV_PTR(fast_spi) + +/* + * The bios_top variable stores the offset indicating the end of the BIOS region within + * the SPI flash memory map. It is calculated based on the size and arrangement of memory + * regions within the flash device. This value is used in DMA operations to determine the + * correct addressing and transfer boundaries. + */ +static uint32_t bios_top; + +/* + * Pointer to the operations structure for memory-mapped region devices. This pointer is + * used to store and access the original operations associated with the memory-mapped + * device, allowing for custom operations such as DMA transfers to be installed and + * utilized while preserving the original functionalities. + */ +static const struct region_device_ops *mmap_ops; + +/* + * The fast_spi_dma_mutex is used to ensure thread-safe access to DMA operations. This + * synchronization mechanism guarantees that only one thread can perform DMA operations + * at a time. + */ +static struct thread_mutex fast_spi_dma_mutex; + +static inline uint32_t fast_spi_read(enum fast_spi_dma_register reg) +{ + return pci_read_config32(FAST_SPI_DEV, reg); +} + +static inline void fast_spi_write(enum fast_spi_dma_register reg, uint32_t value) +{ + pci_write_config32(FAST_SPI_DEV, reg, value); +} + +/* + * Perform a DMA transfer using the Fast SPI interface. + * + * This function initiates a DMA transfer from the SPI flash to a specified buffer. It + * configures the necessary registers, handles potential errors, and ensures proper + * completion of the DMA operation. If the operation succeeds, it returns CB_SUCCESS; + * otherwise, it returns CB_ERR. + * + * @param buffer Pointer to the destination buffer where the data will be transferred. + * @param size Number of blocks to transfer, specified in 1KB units. + * @param offset Offset within the flash region from where the transfer begins. + * + * @return CB_SUCCESS if the DMA transfer completes successfully, + * CB_ERR if the operation encounters an error or times out. + */ +static enum cb_err fast_spi_do_dma_transfer(void *buffer, uint32_t size, uint32_t offset) +{ + enum cb_err ret = CB_ERR; + + union fast_spi_dma_control control = {fast_spi_read(FAST_SPI_DMA_CONTROL)}; + union fast_spi_dma_status status = {fast_spi_read(FAST_SPI_DMA_STATUS)}; + + /* Verify if the DMA is locked; if so, exit. FSP-S should lock the SPI DMA prior + to exiting. */ + if (control.fields.lock) { + printk(BIOS_WARNING, "Fast-SPI: DMA is locked\n"); + goto out; + } + + if (control.fields.start && !status.fields.complete) { + printk(BIOS_ERR, "Fast-SPI: DMA operation is still ongoing...\n"); + goto out; + } + + /* Clear any error bits in the status register. */ + if (status.fields.error) + fast_spi_write(FAST_SPI_DMA_STATUS, status.data); + + /* Configure the destination address for the DMA transfer. */ + fast_spi_write(FAST_SPI_DMA_DEST_ADDRESS_HIGH, (uint32_t)((uintptr_t)buffer >> 32)); + fast_spi_write(FAST_SPI_DMA_DEST_ADDRESS_LOW, (uint32_t)((uintptr_t)buffer)); + + /* Ensure the PCI command master bit is set. */ + uint16_t pci_command = pci_read_config16(FAST_SPI_DEV, PCI_COMMAND); + if (!(pci_command & PCI_COMMAND_MASTER)) + pci_write_config16(FAST_SPI_DEV, PCI_COMMAND, + pci_command | PCI_COMMAND_MASTER); + + /* Configure the region offset for the transfer. */ + union fast_spi_dma_region region = {fast_spi_read(FAST_SPI_DMA_REGION)}; + region.fields.offset_kb = offset - 1; + fast_spi_write(FAST_SPI_DMA_REGION, region.data); + + /* Set the buffer size and start the DMA transaction. */ + control.fields.size = size - 1; + control.fields.start = 1; + fast_spi_write(FAST_SPI_DMA_CONTROL, control.data); + + printk(BIOS_DEBUG, "Fast-SPI: Reading %d blocks via DMA\n", size); + + /* Initialize a stopwatch to track the timeout period. */ + struct stopwatch timer; + stopwatch_init_msecs_expire(&timer, FAST_SPI_DMA_TIMEOUT_MSEC); + + /* Poll for completion of the DMA operation. */ + do { + udelay(FAST_SPI_DMA_DELAY_STEP_USEC); + + control.data = fast_spi_read(FAST_SPI_DMA_CONTROL); + status.data = fast_spi_read(FAST_SPI_DMA_STATUS); + + if (status.fields.complete || status.fields.error || control.fields.lock) { + break; + } + } while (!stopwatch_expired(&timer)); + + if (status.fields.error) { + printk(BIOS_ERR, "Fast-SPI: error(s) detected, 0x%08x\n", status.data); + goto out; + } + + /* Check if the DMA has been locked since we started the DMA transaction; if it + is locked, exit with an error as we cannot guarantee that the data has been + loaded prior to locking. */ + if (control.fields.lock) { + printk(BIOS_ERR, "Fast-SPI: DMA has been locked\n"); + goto out; + } + + /* Check if the operation has timed out. */ + if (!status.fields.complete && stopwatch_expired(&timer)) { + printk(BIOS_ERR, "Fast-SPI: DMA operation timed out\n"); + goto out; + } + + printk(BIOS_DEBUG, "Fast-SPI: transfer completed in %" PRId64 " us\n", + stopwatch_duration_usecs(&timer)); + + /* Clear the complete status. */ + status.data = 0; + status.fields.complete = 1; + fast_spi_write(FAST_SPI_DMA_STATUS, status.data); + + /* Restore PCI command if it was changed. */ + if (!(pci_command & PCI_COMMAND_MASTER)) + pci_write_config16(FAST_SPI_DEV, PCI_COMMAND, pci_command); + + ret = CB_SUCCESS; + +out: + return ret; +} + +static enum cb_err fast_spi_dma_mmap_readat(const struct region_device *rd, void *b, + size_t offset, size_t size, ssize_t *transferred) +{ + ssize_t count = mmap_ops->readat(rd, b, offset, size); + if (count < 0) { + printk(BIOS_ERR, "Fast-SPI: Failed to read %zu bytes", size); + return CB_ERR; + } + *transferred += count; + if (count != size) { + printk(BIOS_ERR, "Fast-SPI: Partial read detected %zu/%zu bytes", + count, size); + return CB_ERR; + } + + return CB_SUCCESS; +} + +/* + * Perform DMA transfer from SPI flash to a buffer. + * + * This function is responsible for transferring data blocks from SPI flash memory to a + * specified buffer using DMA. It handles alignment of memory addresses, calculates the + * correct offset and block count, performs the DMA transfer, and falls back to + * memory-mapped operations if DMA transfer fails. Additionally, it ensures that leading + * and trailing bytes around the DMA blocks are correctly read and transferred. + * + * Address transformation: + * + * The address needs to be transformed from a memory map address into a DMA + * address. Consider the following diagram illustrating SPINOR addressing with a 32 MB + * SPI flash as an example: + * + * - On the left is memory addressing + * - On the right is DMA addressing + * + * BIOS end (32M) ---> +--------------+ | <- DMA start (0) + * ^ | | | + * | | | | + * offset ----------> | |
| | <- bios end - offset + * | | | | + * | | | | + * | | | | + * | | | | + * | | | | + * BIOS start (9M) -> | +--------------+ v <- DMA end (32 - 9 = 23M) + * | | DESC + other | + * bottom (0M) -----> | +--------------+ + * + * @param rd Pointer to the region device structure, representing the memory region. + * @param b Pointer to the destination buffer where data will be transferred. + * @param offset Address offset from where the transfer begins. + * @param size Total number of bytes to transfer. + * + * @return The total number of bytes transferred if successful, or -1 if the DMA + * transfer fails and fallback to memory map operation is required. + */ +static ssize_t fast_spi_dma_transfer(const struct region_device *rd, void *b, + size_t offset, size_t size) +{ + /* Calculate aligned offset and flash offset for DMA transfer. */ + uint32_t aligned_offset = ALIGN_UP(offset, FAST_SPI_DMA_BLOCK_ALIGN); + uint32_t flash_offset = (bios_top - aligned_offset) / FAST_SPI_DMA_BLOCK_ALIGN; + ssize_t early_bytes = aligned_offset - offset; + uint32_t block_count = (size - early_bytes) / FAST_SPI_DMA_BLOCK_ALIGN; + + /* Perform DMA transfer. */ + thread_mutex_lock(&fast_spi_dma_mutex); + enum cb_err ret = fast_spi_do_dma_transfer(b, block_count, flash_offset); + thread_mutex_unlock(&fast_spi_dma_mutex); + if (ret != CB_SUCCESS) { + printk(BIOS_ERR, "Fast-SPI: DMA transfer failed, falling back to memory map operation\n"); + return -1; + } + + /* Handle bytes preceding DMA blocks. */ + ssize_t transferred = block_count * FAST_SPI_DMA_BLOCK_ALIGN; + if (early_bytes) { + memmove((char *)b + early_bytes, b, transferred); + ret = fast_spi_dma_mmap_readat(rd, b, offset, early_bytes, &transferred); + if (ret != CB_SUCCESS) { + printk(BIOS_ERR, "Fast-SPI: Failed to read leading DMA block bytes"); + return -1; + } + } + + /* Handle bytes following DMA blocks. */ + size_t remaining = size - transferred; + if (remaining) { + ret = fast_spi_dma_mmap_readat(rd, (char *)b + transferred, + offset + transferred, remaining, &transferred); + if (ret != CB_SUCCESS) { + printk(BIOS_ERR, "Fast-SPI: Failed to read the remaining bytes"); + return transferred; + } + } + return transferred; +} + +/* + * Perform a read operation using DMA if applicable. + * + * This function attempts to perform a read operation using DMA for the specified region + * device if the size and alignment conditions are met. If the conditions are not met or + * if the DMA operation fails, it falls back to using the memory-mapped read operation. + * + * @param rd Pointer to the region device structure, representing the memory region. + * @param b Pointer to the destination buffer where data will be read. + * @param offset Memory map address offset from where the read begins. + * @param size Total number of bytes to read. + * + * @return The total number of bytes read if successful, or the result of the + * fallback memory-mapped read operation. + */ +static ssize_t fast_spi_dma_readat(const struct region_device *rd, void *b, + size_t offset, size_t size) +{ + /* Check if the size is greater than or equal to the DMA block alignment and if + the destination buffer is aligned accordingly. */ + if (size >= FAST_SPI_DMA_BLOCK_ALIGN && + IS_ALIGNED((uintptr_t)b, FAST_SPI_DMA_BLOCK_ALIGN)) { + ssize_t ret = fast_spi_dma_transfer(rd, b, offset, size); + if (ret != -1) + return ret; + } + + return mmap_ops->readat(rd, b, offset, size); +} + +/* + * Check if Fast SPI DMA is supported by the current device. + * + * This function determines whether Fast SPI DMA operations are supported by the current + * hardware configuration. It performs a one-time initialization to check the PCI vendor + * ID and the Fast SPI DMA control register. If the vendor ID is invalid or the control + * register does not indicate support for the BIOS region, the function returns false. + * Otherwise, it returns true. + * + * @return True if Fast SPI DMA is supported; false otherwise. + */ +bool fast_spi_dma_is_supported(void) +{ + static bool initialized, supported; + + if (initialized) + return supported; + initialized = true; + + if (pci_read_config32(FAST_SPI_DEV, PCI_VENDOR_ID) == 0xffffffff) { + printk(BIOS_ERR, "Fast-SPI: DMA is not supported\n"); + return false; + } + + union fast_spi_dma_control control = {fast_spi_read(FAST_SPI_DMA_CONTROL)}; + supported = control.data != 0xffffffff && + control.fields.flash_region == REGION_BIOS; + return supported; +} + +/* + * Install custom DMA operations for SPI flash memory access. + * + * This function sets up a specialized set of operations to access SPI flash memory + * regions using DMA transfers. The new operation set is derived from the original + * one. Only the original readat() callback is replaced by a DMA-specific version. The + * original operations are kept available for fallback use. The function is meant to run + * only once, ensuring the proper installation of the custom operations. + * + * @param mmap_xlate_rdev Pointer to the xlate_region_device structure, which contains + * information about the memory-mapped regions and operations. + */ +void spi_flash_dma_install_ops(struct xlate_region_device *mmap_xlate_rdev) +{ + static struct region_device_ops fast_spi_dma_rdev_ro_ops; + + /* Check if the operations have already been installed. */ + if (mmap_xlate_rdev->rdev.ops == &fast_spi_dma_rdev_ro_ops) + return; + + /* Calculate the top boundary of the BIOS region within the SPI flash. */ + for (size_t i = 0; i < mmap_xlate_rdev->window_count; i++) { + uint32_t begin = mmap_xlate_rdev->window_arr[i].sub_region.offset; + uint32_t end = begin + mmap_xlate_rdev->window_arr[i].sub_region.size; + if (bios_top < end) + bios_top = end; + } + + /* Store the original region device operations for fallback purposes. */ + mmap_ops = mmap_xlate_rdev->rdev.ops; + + /* Copy the original operations and replace the readat() callback with DMA-based + readat(). */ + memcpy(&fast_spi_dma_rdev_ro_ops, mmap_xlate_rdev->rdev.ops, + sizeof(fast_spi_dma_rdev_ro_ops)); + fast_spi_dma_rdev_ro_ops.readat = fast_spi_dma_readat; + mmap_xlate_rdev->rdev.ops = &fast_spi_dma_rdev_ro_ops; +} diff --git a/src/soc/intel/common/block/fast_spi/fast_spi_dma.h b/src/soc/intel/common/block/fast_spi/fast_spi_dma.h new file mode 100644 index 0000000000..3bb71ccbdc --- /dev/null +++ b/src/soc/intel/common/block/fast_spi/fast_spi_dma.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef SOC_INTEL_COMMON_BLOCK_FAST_SPI_DMA_H +#define SOC_INTEL_COMMON_BLOCK_FAST_SPI_DMA_H + +#include + +#if CONFIG(FAST_SPI_DMA) && ENV_RAMSTAGE +/* + * Check if Fast SPI DMA is supported by the current device. + * + * This function determines whether Fast SPI DMA operations are supported by the current + * hardware configuration. It performs a one-time initialization to check the PCI vendor + * ID and the Fast SPI DMA control register. If the vendor ID is invalid or the control + * register does not indicate support for the BIOS region, the function returns false. + * Otherwise, it returns true. + * + * @return True if Fast SPI DMA is supported; false otherwise. + */ +bool fast_spi_dma_is_supported(void); +#else +static inline bool fast_spi_dma_is_supported(void) +{ + return false; +} +#endif + +/* + * Install custom DMA operations for SPI flash memory access. + * + * This function sets up a specialized set of operations to access SPI flash memory + * regions using DMA transfers. The new operation set is derived from the original + * one. Only the original readat() callback is replaced by a DMA-specific version. The + * original operations are kept available for fallback use. The function is meant to run + * only once, ensuring the proper installation of the custom operations. + * + * @param mmap_xlate_rdev Pointer to the xlate_region_device structure, which contains + * information about the memory-mapped regions and operations. + */ +void spi_flash_dma_install_ops(struct xlate_region_device *mmap_xlate_rdev); + +#endif /* SOC_INTEL_COMMON_BLOCK_FAST_SPI_DMA_H */ diff --git a/src/soc/intel/common/block/fast_spi/mmap_boot.c b/src/soc/intel/common/block/fast_spi/mmap_boot.c index 2315c44bbd..af0a026b01 100644 --- a/src/soc/intel/common/block/fast_spi/mmap_boot.c +++ b/src/soc/intel/common/block/fast_spi/mmap_boot.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -143,6 +144,9 @@ const struct region_device *boot_device_ro(void) { bios_mmap_init(); + if (fast_spi_dma_is_supported()) + spi_flash_dma_install_ops(&real_dev); + return &real_dev.rdev; }