// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include "hw.h" struct gpu_device { enum gpu_arch arch; void *bar0; bool is_memory_clear_supported; const struct falcon *falcon; u32 pmc_enable_mask; bool fsp_dma_enabled; /* Pending memcpy parameters, set by memcpy_start() */ u64 memcpy_src; u64 memcpy_dst; u64 memcpy_size; }; static inline struct gpu_device *to_gpu_device(struct vfio_pci_device *device) { return device->driver.region.vaddr; } static enum gpu_arch nv_gpu_arch_lookup(u32 pmc_boot_0) { u32 arch = (pmc_boot_0 >> 24) & 0x1f; switch (arch) { case 0x0e: case 0x0f: case 0x10: return GPU_ARCH_KEPLER; case 0x11: return GPU_ARCH_MAXWELL_GEN1; case 0x12: return GPU_ARCH_MAXWELL_GEN2; case 0x13: /* P100 (impl 0) uses PMC reset; P4/P40 use engine reset */ if (((pmc_boot_0 >> 20) & 0xf) == 0) return GPU_ARCH_PASCAL; return GPU_ARCH_PASCAL_10X; case 0x14: return GPU_ARCH_VOLTA; case 0x16: return GPU_ARCH_TURING; case 0x17: return GPU_ARCH_AMPERE; case 0x18: return GPU_ARCH_HOPPER; case 0x19: return GPU_ARCH_ADA; default: return GPU_ARCH_UNKNOWN; } } static inline u32 gpu_read32(struct gpu_device *gpu, u32 offset) { return readl(gpu->bar0 + offset); } static inline void gpu_write32(struct gpu_device *gpu, u32 offset, u32 value) { writel(value, gpu->bar0 + offset); } static u64 get_elapsed_ms(struct timespec *start) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); return (now.tv_sec - start->tv_sec) * 1000 + (now.tv_nsec - start->tv_nsec) / 1000000; } static int gpu_poll_register(struct vfio_pci_device *device, const char *name, u32 offset, u32 expected, u32 mask, u32 timeout_ms) { struct gpu_device *gpu = to_gpu_device(device); struct timespec start; u64 elapsed_ms; u32 value; clock_gettime(CLOCK_MONOTONIC, &start); for (;;) { value = gpu_read32(gpu, offset); if ((value & mask) == expected) return 0; elapsed_ms = get_elapsed_ms(&start); if (elapsed_ms >= timeout_ms) break; usleep(1000); } dev_err(device, "Timeout polling %s (0x%x): value=0x%x expected=0x%x mask=0x%x after %lu ms\n", name, offset, value, expected, mask, elapsed_ms); return -ETIMEDOUT; } static int fsp_poll_queue(struct vfio_pci_device *device, const char *name, u32 head_reg, u32 tail_reg, bool wait_empty, u32 timeout_ms) { struct gpu_device *gpu = to_gpu_device(device); struct timespec start; u64 elapsed_ms; u32 head, tail; clock_gettime(CLOCK_MONOTONIC, &start); for (;;) { head = gpu_read32(gpu, head_reg); tail = gpu_read32(gpu, tail_reg); if (wait_empty ? (head == tail) : (head != tail)) return 0; elapsed_ms = get_elapsed_ms(&start); if (elapsed_ms >= timeout_ms) break; usleep(1000); } dev_err(device, "Timeout polling %s: head=0x%x tail=0x%x wait_empty=%d after %lu ms\n", name, head, tail, wait_empty, elapsed_ms); return -ETIMEDOUT; } static void fsp_emem_write(struct vfio_pci_device *device, u32 offset, const u32 *data, u32 count) { struct gpu_device *gpu = to_gpu_device(device); u32 i; /* Configure port with auto-increment for read and write */ gpu_write32(gpu, NV_FSP_EMEM_PORT2_CTRL, offset | NV_FALCON_EMEMC_AINCR | NV_FALCON_EMEMC_AINCW); for (i = 0; i < count; i++) gpu_write32(gpu, NV_FSP_EMEM_PORT2_DATA, data[i]); } static void fsp_emem_read(struct vfio_pci_device *device, u32 offset, u32 *data, u32 count) { struct gpu_device *gpu = to_gpu_device(device); u32 i; /* Configure port with auto-increment for read and write */ gpu_write32(gpu, NV_FSP_EMEM_PORT2_CTRL, offset | NV_FALCON_EMEMC_AINCR | NV_FALCON_EMEMC_AINCW); for (i = 0; i < count; i++) data[i] = gpu_read32(gpu, NV_FSP_EMEM_PORT2_DATA); } static int fsp_rpc_send_data(struct vfio_pci_device *device, const u32 *data, u32 count) { struct gpu_device *gpu = to_gpu_device(device); int ret; ret = fsp_poll_queue(device, "fsp_cmd_queue_empty", NV_FSP_QUEUE_HEAD, NV_FSP_QUEUE_TAIL, true, 1000); if (ret) return ret; fsp_emem_write(device, NV_FSP_RPC_EMEM_BASE, data, count); /* Update queue head/tail to signal data is ready */ gpu_write32(gpu, NV_FSP_QUEUE_TAIL, NV_FSP_RPC_EMEM_BASE + (count - 1) * 4); gpu_write32(gpu, NV_FSP_QUEUE_HEAD, NV_FSP_RPC_EMEM_BASE); return ret; } static int fsp_rpc_receive_data(struct vfio_pci_device *device, u32 *data, u32 max_count, u32 timeout_ms) { struct gpu_device *gpu = to_gpu_device(device); u32 head, tail; u32 msg_size_words; int ret; ret = fsp_poll_queue(device, "fsp_msg_queue_ready", NV_FSP_MSG_QUEUE_HEAD, NV_FSP_MSG_QUEUE_TAIL, false, timeout_ms); if (ret) return ret; head = gpu_read32(gpu, NV_FSP_MSG_QUEUE_HEAD); tail = gpu_read32(gpu, NV_FSP_MSG_QUEUE_TAIL); msg_size_words = (tail - head + 4) / 4; if (msg_size_words > max_count) msg_size_words = max_count; fsp_emem_read(device, NV_FSP_RPC_EMEM_BASE, data, msg_size_words); /* Reset message queue tail to acknowledge receipt */ gpu_write32(gpu, NV_FSP_MSG_QUEUE_TAIL, head); return msg_size_words; } static void fsp_reset_rpc_state(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); u32 head, tail; head = gpu_read32(gpu, NV_FSP_QUEUE_HEAD); tail = gpu_read32(gpu, NV_FSP_QUEUE_TAIL); if (head == tail) { head = gpu_read32(gpu, NV_FSP_MSG_QUEUE_HEAD); tail = gpu_read32(gpu, NV_FSP_MSG_QUEUE_TAIL); if (head == tail) return; } /* Best-effort drain; timeout is expected if no pending message. */ fsp_poll_queue(device, "fsp_msg_queue_drain", NV_FSP_MSG_QUEUE_HEAD, NV_FSP_MSG_QUEUE_TAIL, false, 5000); gpu_write32(gpu, NV_FSP_QUEUE_TAIL, NV_FSP_RPC_EMEM_BASE); gpu_write32(gpu, NV_FSP_QUEUE_HEAD, NV_FSP_RPC_EMEM_BASE); gpu_write32(gpu, NV_FSP_MSG_QUEUE_TAIL, NV_FSP_RPC_EMEM_BASE); gpu_write32(gpu, NV_FSP_MSG_QUEUE_HEAD, NV_FSP_RPC_EMEM_BASE); } static inline u32 mctp_header_build(u8 seid, u8 seq, bool som, bool eom) { u32 hdr = 0; hdr |= (seid & NV_MCTP_HDR_SEID_MASK) << NV_MCTP_HDR_SEID_SHIFT; hdr |= (seq & NV_MCTP_HDR_SEQ_MASK) << NV_MCTP_HDR_SEQ_SHIFT; if (som) hdr |= NV_MCTP_HDR_SOM_BIT; if (eom) hdr |= NV_MCTP_HDR_EOM_BIT; return hdr; } static inline u32 mctp_msg_header_build(u8 nvdm_type) { u32 hdr = 0; hdr |= (NV_MCTP_MSG_TYPE_VENDOR_DEFINED & NV_MCTP_MSG_TYPE_MASK) << NV_MCTP_MSG_TYPE_SHIFT; hdr |= (NV_MCTP_MSG_VENDOR_ID_NVIDIA & NV_MCTP_MSG_VENDOR_ID_MASK) << NV_MCTP_MSG_VENDOR_ID_SHIFT; hdr |= (nvdm_type & NV_MCTP_MSG_NVDM_TYPE_MASK) << NV_MCTP_MSG_NVDM_TYPE_SHIFT; return hdr; } static inline u8 mctp_msg_header_get_nvdm_type(u32 hdr) { return (hdr >> NV_MCTP_MSG_NVDM_TYPE_SHIFT) & NV_MCTP_MSG_NVDM_TYPE_MASK; } static int fsp_rpc_send_cmd(struct vfio_pci_device *device, u8 nvdm_type, const u32 *data, u32 data_count, u32 timeout_ms) { u32 max_packet_words = NV_FSP_RPC_MAX_PACKET_SIZE / 4; u32 packet[256]; u32 resp_buf[256]; u32 total_words; int resp_words; u8 resp_nvdm_type; int ret; total_words = 2 + data_count; if (total_words > max_packet_words) return -EINVAL; packet[0] = mctp_header_build(0, 0, true, true); packet[1] = mctp_msg_header_build(nvdm_type); if (data_count > 0) memcpy(&packet[2], data, data_count * sizeof(u32)); ret = fsp_rpc_send_data(device, packet, total_words); if (ret) return ret; resp_words = fsp_rpc_receive_data(device, resp_buf, 256, timeout_ms); if (resp_words < 0) return resp_words; if (resp_words < NV_FSP_RPC_MIN_RESPONSE_WORDS) return -EPROTO; resp_nvdm_type = mctp_msg_header_get_nvdm_type(resp_buf[1]); if (resp_nvdm_type != NV_NVDM_TYPE_RESPONSE) return -EPROTO; if (resp_buf[3] != nvdm_type) return -EPROTO; if (resp_buf[4] != 0) return -resp_buf[4]; return 0; } static int fsp_init(struct vfio_pci_device *device) { int ret; ret = gpu_poll_register(device, "fsp_boot_complete", NV_FSP_BOOT_COMPLETE_OFFSET, NV_FSP_BOOT_COMPLETE_SUCCESS, 0xffffffff, 5000); if (ret) return ret; fsp_reset_rpc_state(device); return ret; } static int fsp_fbdma_enable(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); u32 cmd_data = NV_FBDMA_SUBCMD_ENABLE; int ret = 0; if (gpu->fsp_dma_enabled) return ret; ret = fsp_rpc_send_cmd(device, NV_NVDM_TYPE_FBDMA, &cmd_data, 1, 5000); if (ret) return ret; gpu->fsp_dma_enabled = true; return ret; } static bool fsp_check_ofa_dma_support(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); u32 val = gpu_read32(gpu, NV_OFA_DMA_SUPPORT_CHECK_REG); return (val >> 16) != 0xbadf; } static u32 size_to_dma_encoding(u64 size) { VFIO_ASSERT_LE(size, NV_FALCON_DMA_MAX_TRANSFER_SIZE); VFIO_ASSERT_GE(size, NV_FALCON_DMA_MIN_TRANSFER_SIZE); VFIO_ASSERT_EQ(size & (size - 1), 0, "size must be power-of-2\n"); return ffs(size) - 3; } static void falcon_dmem_port_configure(struct vfio_pci_device *device, u32 offset, bool auto_inc_read, bool auto_inc_write) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon = gpu->falcon; u32 memc_value = offset; /* Set auto-increment flags */ if (auto_inc_read) memc_value |= NV_PPWR_FALCON_DMEMC_AINCR_TRUE; if (auto_inc_write) memc_value |= NV_PPWR_FALCON_DMEMC_AINCW_TRUE; gpu_write32(gpu, falcon->dmem_control_reg, memc_value); } static void falcon_select_core_falcon(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon = gpu->falcon; u32 core_select_reg = falcon->base_page + NV_FALCON_CORE_SELECT_OFFSET; u32 core_select; core_select = gpu_read32(gpu, core_select_reg); /* Clear bits 4:5 to select falcon core (not RISCV) */ core_select &= ~NV_FALCON_CORE_SELECT_MASK; gpu_write32(gpu, core_select_reg, core_select); } static int falcon_enable(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon = gpu->falcon; u32 mailbox_test_reg; u32 mailbox_val; if (falcon->no_outside_reset) return 0; /* Ada-specific: Check if falcon needs reset before enable */ if (gpu->arch == GPU_ARCH_ADA) { mailbox_test_reg = falcon->base_page + NV_FALCON_MAILBOX_TEST_OFFSET; mailbox_val = gpu_read32(gpu, mailbox_test_reg); if (mailbox_val == NV_FALCON_MAILBOX_RESET_MAGIC) gpu_write32(gpu, falcon->engine_reset, 1); } /* Enable the falcon based on control method */ if (gpu->pmc_enable_mask != 0) { u32 pmc_enable; /* Enable via PMC_ENABLE register */ pmc_enable = gpu_read32(gpu, NV_PMC_ENABLE); gpu_write32(gpu, NV_PMC_ENABLE, pmc_enable | gpu->pmc_enable_mask); } else { /* Enable by deasserting engine reset */ gpu_write32(gpu, falcon->engine_reset, 0); } if (gpu->arch < GPU_ARCH_HOPPER) { falcon_select_core_falcon(device); /* Wait for DMACTL to be ready (bits 1:2 should be 0) */ return gpu_poll_register(device, "falcon_dmactl", falcon->dmactl, 0, NV_FALCON_DMACTL_READY_MASK, 1000); } return 0; } static void falcon_disable(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon = gpu->falcon; u32 pmc_enable; if (falcon->no_outside_reset) return; if (gpu->pmc_enable_mask != 0) { /* Disable via PMC_ENABLE */ pmc_enable = gpu_read32(gpu, NV_PMC_ENABLE); gpu_write32(gpu, NV_PMC_ENABLE, pmc_enable & ~gpu->pmc_enable_mask); } else { /* Disable by asserting engine reset */ gpu_write32(gpu, falcon->engine_reset, 1); } } static int falcon_reset(struct vfio_pci_device *device) { falcon_disable(device); return falcon_enable(device); } static int nv_falcon_dma_init(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon; u32 transcfg; u32 dmactl; u32 ctl; int ret = 0; falcon = gpu->falcon; vfio_pci_cmd_set(device, PCI_COMMAND_MASTER); if (gpu->arch >= GPU_ARCH_HOPPER) { ret = fsp_init(device); if (ret) { dev_err(device, "Failed to init FSP: %d\n", ret); return ret; } ret = fsp_fbdma_enable(device); if (ret) { dev_err(device, "Failed to enable FSP FBDMA: %d\n", ret); return ret; } if (!fsp_check_ofa_dma_support(device)) { dev_err(device, "OFA DMA not supported with current firmware\n"); return -EOPNOTSUPP; } } if (gpu->is_memory_clear_supported) { /* For Turing+, wait for boot to complete first */ if (gpu->arch >= GPU_ARCH_TURING) { /* Wait for boot complete - Hopper+ uses FSP register */ if (gpu->arch >= GPU_ARCH_HOPPER) { ret = gpu_poll_register(device, "fsp_boot_complete", NV_FSP_BOOT_COMPLETE_OFFSET, NV_FSP_BOOT_COMPLETE_SUCCESS, 0xffffffff, 5000); } else { ret = gpu_poll_register(device, "boot_complete", NV_BOOT_COMPLETE_OFFSET, NV_BOOT_COMPLETE_SUCCESS, 0xffffffff, 5000); } if (ret) return ret; ret = gpu_poll_register(device, "memory_clear_finished", NV_MEM_CLEAR_OFFSET, 0x1, 0xffffffff, 5000); if (ret) return ret; } } ret = falcon_reset(device); if (ret) return ret; falcon_dmem_port_configure(device, 0, false, false); transcfg = gpu_read32(gpu, falcon->fbif_transcfg); transcfg &= ~NV_FBIF_TRANSCFG_TARGET_MASK; transcfg |= NV_FBIF_TRANSCFG_SYSMEM_DEFAULT; gpu_write32(gpu, falcon->fbif_transcfg, transcfg); gpu_write32(gpu, falcon->fbif_ctl2, 0x1); ctl = gpu_read32(gpu, falcon->fbif_ctl); ctl |= NV_FBIF_CTL_ALLOW_PHYS_MODE | NV_FBIF_CTL_ALLOW_FULL_PHYS_MODE; gpu_write32(gpu, falcon->fbif_ctl, ctl); dmactl = gpu_read32(gpu, falcon->dmactl); dmactl &= ~NV_FALCON_DMACTL_DMEM_SCRUBBING; gpu_write32(gpu, falcon->dmactl, dmactl); return ret; } static int nv_falcon_dma(struct vfio_pci_device *device, u64 address, u64 size, bool write) { struct gpu_device *gpu = to_gpu_device(device); const struct falcon *falcon = gpu->falcon; u32 dma_cmd; int ret; gpu_write32(gpu, NV_GPU_DMA_ADDR_TOP_BITS_REG, (address >> 47) & 0x1ffff); gpu_write32(gpu, falcon->base_page + NV_FALCON_DMA_ADDR_HIGH_OFFSET, (address >> 40) & 0x7f); gpu_write32(gpu, falcon->base_page + NV_FALCON_DMA_ADDR_LOW_OFFSET, (address >> 8) & 0xffffffff); gpu_write32(gpu, falcon->base_page + NV_FALCON_DMA_BLOCK_OFFSET, address & 0xff); gpu_write32(gpu, falcon->base_page + NV_FALCON_DMA_MEM_OFFSET, 0); dma_cmd = size_to_dma_encoding(size) << NV_FALCON_DMA_CMD_SIZE_SHIFT; /* Set direction: write (DMEM->mem) or read (mem->DMEM) */ if (write) dma_cmd |= NV_FALCON_DMA_CMD_WRITE_BIT; gpu_write32(gpu, falcon->base_page + NV_FALCON_DMA_CMD_OFFSET, dma_cmd); ret = gpu_poll_register(device, "dma_done", falcon->base_page + NV_FALCON_DMA_CMD_OFFSET, NV_FALCON_DMA_CMD_DONE_BIT, NV_FALCON_DMA_CMD_DONE_BIT, 1000); if (ret) dev_err(device, "Failed DMA %s (addr=0x%lx, size=%lu)\n", write ? "write" : "read", address, size); return ret; } static int nv_falcon_memcpy_chunk(struct vfio_pci_device *device, iova_t src, iova_t dst, u64 size) { int ret; ret = nv_falcon_dma(device, src, size, false); if (ret) return ret; return nv_falcon_dma(device, dst, size, true); } static int nv_falcon_probe(struct vfio_pci_device *device) { enum gpu_arch gpu_arch; u32 pmc_boot_0; void *bar0; int i; if (vfio_pci_config_readw(device, PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) return -ENODEV; if (vfio_pci_config_readw(device, PCI_CLASS_DEVICE) >> 8 != PCI_BASE_CLASS_DISPLAY) return -ENODEV; /* Get BAR0 pointer for reading GPU registers */ bar0 = device->bars[0].vaddr; if (!bar0) return -ENODEV; /* Read PMC_BOOT_0 register from BAR0 to identify GPU */ pmc_boot_0 = readl(bar0 + NV_PMC_BOOT_0); /* Look up GPU architecture to verify this is a supported GPU */ gpu_arch = nv_gpu_arch_lookup(pmc_boot_0); if (gpu_arch == GPU_ARCH_UNKNOWN) { dev_err(device, "Unsupported GPU architecture for PMC_BOOT_0: 0x%x\n", pmc_boot_0); return -ENODEV; } /* Check verified GPU map */ for (i = 0; i < VERIFIED_GPU_MAP_SIZE; i++) { if (verified_gpu_map[i] == pmc_boot_0) return 0; } dev_info(device, "Unvalidated GPU: PMC_BOOT_0: 0x%x, possibly not supported\n", pmc_boot_0); return 0; } static void nv_falcon_init(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); const struct gpu_properties *props; u32 pmc_boot_0; int ret; VFIO_ASSERT_GE(device->driver.region.size, sizeof(*gpu)); /* Read PMC_BOOT_0 register from BAR0 to identify GPU */ pmc_boot_0 = readl(device->bars[0].vaddr + NV_PMC_BOOT_0); /* Look up GPU architecture */ gpu->arch = nv_gpu_arch_lookup(pmc_boot_0); props = &gpu_properties_map[gpu->arch]; /* Populate GPU structure */ gpu->bar0 = device->bars[0].vaddr; gpu->is_memory_clear_supported = props->memory_clear_supported; gpu->falcon = &falcon_map[props->falcon_type]; gpu->pmc_enable_mask = props->pmc_enable_mask; /* Initialize falcon for DMA */ ret = nv_falcon_dma_init(device); VFIO_ASSERT_EQ(ret, 0, "Failed to initialize falcon DMA: %d\n", ret); device->driver.max_memcpy_size = NV_FALCON_DMA_MAX_TRANSFER_SIZE; device->driver.max_memcpy_count = NV_FALCON_DMA_MAX_TRANSFER_COUNT; } static void nv_falcon_remove(struct vfio_pci_device *device) { falcon_disable(device); vfio_pci_cmd_clear(device, PCI_COMMAND_MASTER); } /* * Falcon DMA can only process one transfer at a time, * so the actual work is deferred to memcpy_wait() to conform to the * memcpy_start()/memcpy_wait() contract. */ static void nv_falcon_memcpy_start(struct vfio_pci_device *device, iova_t src, iova_t dst, u64 size, u64 count) { struct gpu_device *gpu = to_gpu_device(device); VFIO_ASSERT_EQ(count, 1); VFIO_ASSERT_EQ(size & (NV_FALCON_DMA_MIN_TRANSFER_SIZE - 1), 0, "size 0x%lx must be %u-byte aligned\n", (unsigned long)size, NV_FALCON_DMA_MIN_TRANSFER_SIZE); gpu->memcpy_src = src; gpu->memcpy_dst = dst; gpu->memcpy_size = size; } /* * Return the largest power-of-2 bytes we can transfer from @addr * without crossing a DMA block boundary. */ static u64 dma_block_remain(u64 addr) { u64 offset = addr & (NV_FALCON_DMA_BLOCK_SIZE - 1); if (!offset) return NV_FALCON_DMA_BLOCK_SIZE; /* Lowest set bit of the offset is the largest aligned chunk */ return 1ULL << (ffs(offset) - 1); } static u64 rounddown_pow_of_two(u64 x) { return 1ULL << (63 - __builtin_clzll(x)); } static int nv_falcon_memcpy_wait(struct vfio_pci_device *device) { struct gpu_device *gpu = to_gpu_device(device); iova_t src = gpu->memcpy_src; iova_t dst = gpu->memcpy_dst; u64 remaining = gpu->memcpy_size; int ret = 0; /* * Falcon DMA supports power-of-2 transfer sizes in [4, 256] and * cannot cross 256-byte block boundaries. Decompose the request * into the largest valid chunk at each step. */ while (remaining) { u64 chunk = rounddown_pow_of_two(remaining); chunk = min(chunk, dma_block_remain(src)); chunk = min(chunk, dma_block_remain(dst)); ret = nv_falcon_memcpy_chunk(device, src, dst, chunk); if (ret) break; src += chunk; dst += chunk; remaining -= chunk; } return ret; } const struct vfio_pci_driver_ops nv_falcon_ops = { .name = "nv_falcon", .probe = nv_falcon_probe, .init = nv_falcon_init, .remove = nv_falcon_remove, .memcpy_start = nv_falcon_memcpy_start, .memcpy_wait = nv_falcon_memcpy_wait, };