// SPDX-License-Identifier: GPL-2.0-only #include #include #include #include #include #include #include #include #include #include #include #include "e1000_regs.h" #include "e1000_defines.h" #include "e1000_82575.h" #define PCI_DEVICE_ID_INTEL_82576 0x10C9 #define IGB_MAX_CHUNK_SIZE 1024 #define MSIX_VECTOR 0 #define MSIX_VECTOR_MASK (1 << MSIX_VECTOR) #define RING_SIZE 4096 /* Number of descriptors in ring */ struct igb_tx_desc { union { struct { u64 buffer_addr; /* Address of descriptor's data buffer */ u32 cmd_type_len; /* Command/Type/Length */ u32 olinfo_status; /* Context/Buffer info */ } read; struct { u64 rsvd; /* Reserved */ u32 nxtseq_seed; /* Next sequence seed */ u32 status; /* Descriptor status */ } wb; }; }; struct igb_rx_desc { union { struct { u64 pkt_addr; /* Packet buffer address */ u64 hdr_addr; /* Header buffer address */ } read; struct { u16 pkt_info; /* RSS type, Packet type */ u16 hdr_info; /* Split Head, buf len */ u32 rss; /* RSS Hash */ u32 status_error; /* ext status/error */ u16 length; /* Packet length */ u16 vlan; /* VLAN tag */ } wb; /* writeback */ }; }; struct igb { void *bar0; u32 tx_tail; u32 rx_tail; struct igb_tx_desc tx_ring[RING_SIZE] __attribute__((aligned(128))); struct igb_rx_desc rx_ring[RING_SIZE] __attribute__((aligned(128))); }; static inline struct igb *to_igb_state(struct vfio_pci_device *device) { return (struct igb *)device->driver.region.vaddr; } static inline void igb_write32(struct igb *igb, u32 reg, u32 val) { writel(val, igb->bar0 + reg); } static inline u32 igb_read32(struct igb *igb, u32 reg) { return readl(igb->bar0 + reg); } static int igb_write_phy(struct igb *igb, u32 offset, u16 data) { u32 mdic; int i; /* * Write a PHY register over MDIO. * * A production driver would hold the SW/FW semaphore (SWSM.SWESMBI + the * SW_FW_SYNC PHY bit) across the MDIO transaction to serialize against the * device's management firmware. The selftest owns the assigned function * exclusively on a dedicated test device with no active manageability * contending for the PHY, so the sync is omitted; it should be added here * if this ever needs to run on a manageability-enabled NIC. */ mdic = (((u32)data) | (offset << E1000_MDIC_REG_SHIFT) | (1 << E1000_MDIC_PHY_SHIFT) | E1000_MDIC_OP_WRITE); igb_write32(igb, E1000_MDIC, mdic); for (i = 0; i < 1000; i++) { usleep(50); mdic = igb_read32(igb, E1000_MDIC); if (mdic & E1000_MDIC_READY) break; } if (!(mdic & E1000_MDIC_READY)) return -1; if (mdic & E1000_MDIC_ERROR) return -1; return 0; } /* * Configure the device for PHY internal loopback per 82576 datasheet * section 3.5.6.3.1. Force the PHY to 1Gb/s full duplex with loopback * enabled, then force the MAC link state to match. Internal loopback * wraps data at the end of the PHY datapath (section 3.5.6.3), so the * physical link state is irrelevant. * * Section 3.5.6.1 directs to "Use PHY Loopback instead of MAC Loopback * on the 82576", and section 3.5.6.2 states "MAC Loopback is not used * on this device." RCTL.LBM_MAC is still set elsewhere as a QEMU-only * accommodation; see the RCTL programming in the caller for the * rationale. */ static void igb_setup_loopback(struct igb *igb) { u32 ctrl; int ret; /* * Kick the autoneg machinery solely to bring STATUS.LU up under * QEMU's igb emulation: QEMU only updates STATUS.LU via its * autoneg-done timer, and without LU set its receive path * (e1000x_hw_rx_enabled) drops every loopback frame. On real * hardware autoneg cannot complete before the next PHY write * below clears the autoneg-enable bit, so this is effectively a * no-op there. */ (void)igb_write_phy(igb, MII_BMCR, BMCR_ANENABLE | BMCR_ANRESTART); /* PHY control: loopback + 1Gb/s full duplex, autoneg disabled. */ ret = igb_write_phy(igb, MII_BMCR, BMCR_LOOPBACK | BMCR_SPEED1000 | BMCR_FULLDPLX); VFIO_ASSERT_EQ(ret, 0, "Failed to write PHY control register"); /* * Brief delay before forcing the MAC, mirroring the kernel ethtool * selftest in igb_integrated_phy_loopback(). Not specified by the * datasheet, but empirically required by the kernel driver. */ usleep(50000); /* * Force the MAC to 1Gb/s full duplex with link up. Without forcing * the link state the descriptor engine does not run, since the chip * normally waits for a real negotiated link. */ ctrl = igb_read32(igb, E1000_CTRL); ctrl &= ~E1000_CTRL_SPD_SEL; ctrl |= E1000_CTRL_FRCSPD | E1000_CTRL_FRCDPX | E1000_CTRL_SPD_1000 | E1000_CTRL_FD | E1000_CTRL_SLU; igb_write32(igb, E1000_CTRL, ctrl); /* * Settling delay matching the kernel ethtool selftest's msleep(500) * at the tail of igb_integrated_phy_loopback(). Not specified by * the datasheet; empirical, and inherited from the kernel driver. */ usleep(500000); } static int igb_probe(struct vfio_pci_device *device) { if (!vfio_pci_device_match(device, PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82576)) return -EINVAL; return 0; } static void igb_reset(struct igb *igb) { int retries = 20; igb_write32(igb, E1000_CTRL, igb_read32(igb, E1000_CTRL) | E1000_CTRL_RST); /* * Must wait at least 1 millisecond after setting the reset bit before * checking if this device is ready to be used (82576 datasheet section * 4.2.1.6.1). The delay also ensures the reset has taken effect and * cleared EECD.AUTO_RD before it is polled below. */ usleep(1000); /* * Poll NVM Auto Read Done rather than CTRL.RST, matching * igb_get_auto_rd_done() in the igb driver: AUTO_RD implies both that * the reset completed and that the device finished re-reading its * configuration from NVM, which is what actually makes it usable. */ while (retries-- > 0 && !(igb_read32(igb, E1000_EECD) & E1000_EECD_AUTO_RD)) usleep(1000); /* * QEMU's igb emulation does not set E1000_EECD_AUTO_RD. If we timed out, * check if CTRL.RST is cleared, which is what QEMU uses to signal reset * completion. */ if (retries < 0) { VFIO_ASSERT_EQ(igb_read32(igb, E1000_CTRL) & E1000_CTRL_RST, 0, "Device reset did not complete (CTRL.RST not cleared)"); } igb_write32(igb, E1000_IMC, 0xFFFFFFFF); } /* * Program the device into a usable state. Split out of igb_init() so it * can be reused after a device reset to re-program the registers that * CTRL.RST clears. Expects bar0 to be mapped and MSI-X already enabled * via VFIO. */ static void igb_hw_init(struct vfio_pci_device *device) { struct igb *igb = to_igb_state(device); u64 iova_tx, iova_rx; u32 ctrl, rctl; u16 cmd_reg; int retries; iova_tx = to_iova(device, igb->tx_ring); iova_rx = to_iova(device, igb->rx_ring); /* Signal that the driver is loaded */ ctrl = igb_read32(igb, E1000_CTRL_EXT); ctrl |= E1000_CTRL_EXT_DRV_LOAD; ctrl &= ~E1000_CTRL_EXT_LINK_MODE_MASK; igb_write32(igb, E1000_CTRL_EXT, ctrl); /* Enable PCI Bus Master. */ cmd_reg = vfio_pci_config_readw(device, PCI_COMMAND); if ((cmd_reg & (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) != (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) { cmd_reg |= (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); vfio_pci_config_writew(device, PCI_COMMAND, cmd_reg); } /* Configure PHY internal loopback for testing. */ igb_setup_loopback(igb); /* * Disable DMA re-send on PCIe completion timeout (82576 datasheet * section 8.6.1, GCR.Completion_Timeout_Resend, bit 16). The * mix_and_match test intentionally submits descriptors targeting * unmapped IOVAs; with the default (set) value, the device keeps * retrying the failed read indefinitely, which keeps PCIe AER and * IOMMU error handling busy and interferes with reset recovery. */ ctrl = igb_read32(igb, E1000_GCR); ctrl &= ~E1000_GCR_CMPL_TMOUT_RESEND; igb_write32(igb, E1000_GCR, ctrl); /* Configure TX and RX descriptor rings */ igb_write32(igb, E1000_TDBAL(0), (u32)iova_tx); igb_write32(igb, E1000_TDBAH(0), (u32)(iova_tx >> 32)); igb_write32(igb, E1000_TDLEN(0), RING_SIZE * sizeof(struct igb_tx_desc)); igb_write32(igb, E1000_TDH(0), 0); igb_write32(igb, E1000_TDT(0), 0); igb_write32(igb, E1000_TXDCTL(0), E1000_TXDCTL_QUEUE_ENABLE); igb_write32(igb, E1000_RDBAL(0), (u32)iova_rx); igb_write32(igb, E1000_RDBAH(0), (u32)(iova_rx >> 32)); igb_write32(igb, E1000_RDLEN(0), RING_SIZE * sizeof(struct igb_rx_desc)); igb_write32(igb, E1000_RDH(0), 0); igb_write32(igb, E1000_RDT(0), 0); /* * Select the advanced one-buffer descriptor format. Per 82576 * datasheet section 7.1.5.2: "SRRCTL[n].DESCTYPE must be set to a * value other than 000b for the 82576 to write back the special * descriptors." struct igb_rx_desc matches the advanced one-buffer * writeback layout (section 7.1.5.2), so polling rx.wb.status_error * requires this format. Section 8.10.2 specifies DESCTYPE[27:25]. * * The direct write also zeroes SRRCTL.BSIZEPACKET, which is * intentional: per section 7.1.3.1 a zero BSIZEPACKET falls back to * the RCTL.BSIZE buffer size, whose reset default (00b) is 2048 * bytes -- ample for the loopback frames here. */ igb_write32(igb, E1000_SRRCTL(0), E1000_SRRCTL_DESCTYPE_ADV_ONEBUF); igb_write32(igb, E1000_RXDCTL(0), E1000_RXDCTL_QUEUE_ENABLE); /* * Enable Receiver and Transmitter. RCTL.LBM_MAC is set in addition * to PHY loopback as a QEMU-only accommodation: QEMU's emulated igb * does not honor PHY register 0 bit 14 (PHY internal loopback) and * relies on RCTL.LBM_MAC to wrap TX descriptors back to the RX * queue. Datasheet 8.10.1 (RCTL register) advises "When using the * internal PHY, LBM should remain set to 00b", so setting LBM_MAC * here deviates from datasheet guidance; empirically the bit has * no observable effect on real 82576 hardware because MAC loopback * is not implemented (datasheet 3.5.6.2). Setting both lets the * selftest work on both real hardware and QEMU without conditional * code paths. */ rctl = E1000_RCTL_EN | /* Receiver Enable */ E1000_RCTL_UPE | /* Unicast Promiscuous (for dummy MAC) */ E1000_RCTL_MPE | /* Multicast Promiscuous */ E1000_RCTL_BAM | /* Broadcast Accept Mode */ E1000_RCTL_LBM_MAC | /* MAC Loopback - for QEMU emulation only */ E1000_RCTL_SECRC; /* Strip CRC (needed for memcmp) */ igb_write32(igb, E1000_RCTL, rctl); igb_write32(igb, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP); /* * Wait for TX and RX queues to be enabled. Per the RXDCTL/TXDCTL * register definitions (8.10.10/8.12.13), the per-queue enable bit * "remains zero" until the global RCTL.RXEN/TCTL.TXEN are set, so * E1000_RCTL_EN and E1000_TCTL_EN must already be written above. */ retries = 2000; while (retries-- > 0) { if ((igb_read32(igb, E1000_TXDCTL(0)) & E1000_TXDCTL_QUEUE_ENABLE) && (igb_read32(igb, E1000_RXDCTL(0)) & E1000_RXDCTL_QUEUE_ENABLE)) break; usleep(10); } VFIO_ASSERT_GE(retries, 0); /* * Program MSI-X interrupt routing per 82576 datasheet: * * GPIE (section 7.3.2.11, Table 7-47): set Multiple_MSIX (bit 4) to * route interrupt causes through IVAR mapping, and EIAME (bit 30) * to apply EIAM on MSI-X assertion (without EIAME, EIAM only * applies on EICR read/write). * * EIAC (section 8.8.5): enable auto-clear of EICR for vector 0. * Without auto-clear the cause stays set after delivery and the * test can see spurious interrupts on the next memcpy batch. * * EIAM (section 8.8.6): enable auto-mask of EIMS for vector 0 on * MSI-X assertion (effective because EIAME is set). * * IVAR (section 7.3.1.2, register definition in 8.8.13): map RX * cause 0 to MSI-X vector 0 and mark the entry valid. */ igb_write32(igb, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME); igb_write32(igb, E1000_EIAC, MSIX_VECTOR_MASK); igb_write32(igb, E1000_EIAM, MSIX_VECTOR_MASK); /* Map vector 0 to interrupt cause 0 and mark it valid */ igb_write32(igb, E1000_IVAR0, E1000_IVAR_VALID); /* Enable interrupts on vector 0 */ igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); /* Initialize driver state and capability limits */ igb->tx_tail = 0; igb->rx_tail = 0; device->driver.max_memcpy_size = IGB_MAX_CHUNK_SIZE; device->driver.max_memcpy_count = RING_SIZE - 1; device->driver.msi = MSIX_VECTOR; } static void igb_init(struct vfio_pci_device *device) { struct igb *igb = to_igb_state(device); VFIO_ASSERT_GE(device->driver.region.size, sizeof(struct igb)); igb->bar0 = device->bars[0].vaddr; igb_reset(igb); /* * Enable MSI-X via VFIO before device-side register programming. * vfio_pci_msix_enable() only touches the VFIO IRQ machinery and the * PCI MSI-X capability via config space; it has no ordering * dependency on the device-side writes performed by igb_hw_init(). * Placing it here keeps igb_hw_init() reusable from the reset * recovery path (which calls vfio_pci_irq_reenable() instead). */ vfio_pci_msix_enable(device, MSIX_VECTOR, 1); igb_hw_init(device); } static void igb_remove(struct vfio_pci_device *device) { struct igb *igb = to_igb_state(device); igb_write32(igb, E1000_RCTL, 0); igb_write32(igb, E1000_TCTL, 0); igb_reset(igb); vfio_pci_msix_disable(device); } static void igb_irq_disable(struct igb *igb) { igb_write32(igb, E1000_EIMC, MSIX_VECTOR_MASK); } static void igb_irq_enable(struct igb *igb) { igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); } static void igb_irq_clear(struct igb *igb) { /* * Use write-to-clear (datasheet 7.3.4.2). In MSI-X mode with EIAC * programmed, section 8.8.5 explicitly states "If any bits are set * in EIAC, the EICR register should not be read", which rules out * the read-to-clear path in 7.3.4.3. Bits not in EIAC are still * cleared by writing 1. */ igb_write32(igb, E1000_EICR, 0xFFFFFFFF); } static void igb_memcpy_start(struct vfio_pci_device *device, iova_t src, iova_t dst, u64 size, u64 count) { struct igb *igb = to_igb_state(device); struct igb_rx_desc *rx; struct igb_tx_desc *tx; u32 i; VFIO_ASSERT_GE(size, 60, "IGB driver requires memcpy size to be at least 60 bytes (Ethernet minimum payload size)"); igb_irq_disable(igb); for (i = 0; i < count; i++) { tx = &igb->tx_ring[igb->tx_tail]; rx = &igb->rx_ring[igb->rx_tail]; memset(tx, 0, sizeof(struct igb_tx_desc)); memset(rx, 0, sizeof(struct igb_rx_desc)); rx->read.pkt_addr = cpu_to_le64(dst); rx->read.hdr_addr = cpu_to_le64(0); tx->read.buffer_addr = cpu_to_le64(src); /* * Build an advanced data descriptor per 82576 datasheet * section 7.2.2.3. DEXT marks the descriptor as advanced * (required by hardware); DTYP=data selects the data * descriptor; IFCS asks the MAC to append the Ethernet * FCS (without it the frame is dropped as malformed); * EOP marks end of packet. DTALEN is the buffer length * in bits 15:0 of cmd_type_len. */ tx->read.cmd_type_len = cpu_to_le32((uint32_t)size | E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_EOP); /* * PAYLEN (section 7.2.2.3.11) is the total payload size * in olinfo_status[31:14]. */ tx->read.olinfo_status = cpu_to_le32((uint32_t)size << E1000_ADVTXD_PAYLEN_SHIFT); igb->tx_tail = (igb->tx_tail + 1) % RING_SIZE; igb->rx_tail = (igb->rx_tail + 1) % RING_SIZE; } igb_write32(igb, E1000_RDT(0), igb->rx_tail); igb_write32(igb, E1000_TDT(0), igb->tx_tail); } /* * Reset the device via VFIO_DEVICE_RESET (PCIe FLR on the 82576) and * re-program it. VFIO_DEVICE_RESET tears down the kernel-side MSI-X * trigger but leaves user-side eventfds intact, so re-arm the trigger * via vfio_pci_irq_reenable() before reprogramming so any caller-cached * eventfd remains valid. * * FLR clears device-side state to power-on reset values (datasheet * 4.2.1.5.1: a PF FLR is "equivalent to a D0->D3->D0 transition"), so * EIMS and EICR come back as 0 from their register-defined initial * values, and igb_hw_init() resets tx_tail/rx_tail to 0. The next * igb_memcpy_start() will memset each descriptor it touches before * submission, so no explicit IMC/EICR writes or ring memsets are * needed here. */ static void igb_error_reset_and_reinit(struct vfio_pci_device *device) { vfio_pci_device_reset(device); vfio_pci_msix_reenable(device, MSIX_VECTOR, 1); igb_hw_init(device); } static int igb_memcpy_wait(struct vfio_pci_device *device) { struct igb *igb = to_igb_state(device); struct igb_rx_desc *rx; u32 status = 0; u32 prev_tail; int retries; prev_tail = (igb->rx_tail + RING_SIZE - 1) % RING_SIZE; rx = &igb->rx_ring[prev_tail]; /* * Real 82576 hardware processes the descriptor ring at line rate. * max_memcpy_size = (RING_SIZE - 1) * IGB_MAX_CHUNK_SIZE ~= 4 MB, * split into 4095 1 KB frames. At 1 Gb/s (~125 MB/s) the worst * valid memcpy takes ~32 ms on the wire, plus per-frame preamble, * SFD, IFG and FCS overhead (~3%) and descriptor fetch/writeback * latency. Wait up to ~200 ms before declaring the device hung; * ~6x the line-rate floor leaves comfortable headroom for host * scheduling jitter while keeping the intentional invalid-DMA * tests bounded. */ retries = 200; while (retries-- > 0) { status = le32_to_cpu(READ_ONCE(rx->wb.status_error)); if (status & 1) break; usleep(1000); } if (status & 1) /* * Ensure the test code doesn't speculatively read the DMA * destination buffer before we have verified that the * descriptor writeback is complete. */ rmb(); igb_irq_clear(igb); igb_irq_enable(igb); if (status & 1) return 0; /* * The descriptor never completed. On real 82576 hardware this * typically follows a DMA-read fault from one of the intentional * unmapped-IOVA tests; the fault leaves the descriptor engine * unable to service subsequent valid descriptors. CTRL.RST alone * reinitializes the queue registers but leaves the engine wedged * for the current process, so a broader VFIO_DEVICE_RESET (FLR) * is required. */ igb_error_reset_and_reinit(device); return -ETIMEDOUT; } static void igb_send_msi(struct vfio_pci_device *device) { struct igb *igb = to_igb_state(device); igb_write32(igb, E1000_EICS, MSIX_VECTOR_MASK); } const struct vfio_pci_driver_ops igb_ops = { .name = "igb", .probe = igb_probe, .init = igb_init, .remove = igb_remove, .memcpy_start = igb_memcpy_start, .memcpy_wait = igb_memcpy_wait, .send_msi = igb_send_msi, };