diff options
Diffstat (limited to 'drivers/infiniband/hw')
32 files changed, 477 insertions, 264 deletions
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 365ec2767d25..572906347463 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -1024,6 +1024,9 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (rc) ibdev_err(&rdev->ibdev, "Failed to destroy HW QP"); + if (qp->dbr_obj) + kref_put(&qp->dbr_obj->usecnt, bnxt_re_dbr_kref_release); + if (rdma_is_kernel_res(&qp->ib_qp.res)) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); @@ -1136,23 +1139,42 @@ static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, return 0; } -static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, - struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, - struct bnxt_re_qp_req *ureq) +static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, + struct ib_umem *umem, + struct bnxt_qplib_sg_info *sginfo) { - struct bnxt_qplib_qp *qplib_qp; - int bytes = 0, psn_sz; - struct ib_umem *umem; - int psn_nume; + unsigned long page_size; - qplib_qp = &qp->qplib_qp; + if (!umem) + return -EINVAL; - bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); - /* Consider mapping PSN search memory only for RC QPs. */ - if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { + page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); + if (!page_size || page_size != SZ_4K) + return -EINVAL; + + sginfo->umem = umem; + sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); + sginfo->pgsize = page_size; + sginfo->pgshft = __builtin_ctz(page_size); + return 0; +} + +static int bnxt_re_get_psn_bytes(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx, + struct bnxt_qplib_qp *qplib_qp, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr) +{ + int psn_sz, psn_nume; + + if (rdev->dev_attr && + _is_host_msn_table(rdev->dev_attr->dev_cap_flags2)) + psn_sz = sizeof(struct sq_msn_search); + else psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? - sizeof(struct sq_psn_search_ext) : - sizeof(struct sq_psn_search); + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (!fixed_que_attr) { if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) { psn_nume = ureq->sq_slots; } else { @@ -1162,9 +1184,31 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, } if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2)) psn_nume = roundup_pow_of_two(psn_nume); - bytes += (psn_nume * psn_sz); + } else { + psn_nume = ureq->sq_npsn; } + return psn_nume * psn_sz; +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, + struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx, + struct bnxt_re_qp_req *ureq, + bool fixed_que_attr, + struct bnxt_re_dbr_obj *dbr_obj) +{ + struct bnxt_qplib_qp *qplib_qp; + struct ib_umem *umem; + int bytes; + int rc; + + qplib_qp = &qp->qplib_qp; + + bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) + bytes += bnxt_re_get_psn_bytes(rdev, cntx, qplib_qp, ureq, fixed_que_attr); + bytes = PAGE_ALIGN(bytes); umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes, IB_ACCESS_LOCAL_WRITE); @@ -1172,33 +1216,45 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, return PTR_ERR(umem); qp->sumem = umem; - qplib_qp->sq.sg_info.umem = umem; - qplib_qp->sq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT; - qplib_qp->qp_handle = ureq->qp_handle; + rc = bnxt_re_setup_sginfo(rdev, qp->sumem, &qplib_qp->sq.sg_info); + if (rc) + goto fail; + + if (qp->qplib_qp.srq) + goto done; - if (!qp->qplib_qp.srq) { - bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); - bytes = PAGE_ALIGN(bytes); - umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, - IB_ACCESS_LOCAL_WRITE); - if (IS_ERR(umem)) - goto rqfail; - qp->rumem = umem; - qplib_qp->rq.sg_info.umem = umem; - qplib_qp->rq.sg_info.pgsize = PAGE_SIZE; - qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT; + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes, + IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + goto fail; } - qplib_qp->dpi = &cntx->dpi; + qp->rumem = umem; + rc = bnxt_re_setup_sginfo(rdev, qp->rumem, &qplib_qp->rq.sg_info); + if (rc) + goto rqfail; + +done: + if (dbr_obj) + qplib_qp->dpi = &dbr_obj->dpi; + else + qplib_qp->dpi = &cntx->dpi; + qplib_qp->qp_handle = ureq->qp_handle; qplib_qp->is_user = true; return 0; + rqfail: + ib_umem_release(qp->rumem); + qp->rumem = NULL; + memset(&qplib_qp->rq.sg_info, 0, sizeof(qplib_qp->rq.sg_info)); +fail: ib_umem_release(qp->sumem); qp->sumem = NULL; memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info)); - - return PTR_ERR(umem); + return rc; } static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah @@ -1597,7 +1653,9 @@ out: return qptype; } -static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) +static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp, + bool fixed_que_attr, + struct bnxt_re_qp_req *req) { struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp; struct bnxt_qplib_q *sq = &qplib_qp->sq; @@ -1620,12 +1678,17 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) /* Update msn tbl size */ if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) { - if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); - else - qplib_qp->msn_tbl_sz = - roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2; + if (!fixed_que_attr) { + if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)); + else + qplib_qp->msn_tbl_sz = + roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) + / 2; + } else { + qplib_qp->msn_tbl_sz = req->sq_npsn; + } qplib_qp->msn = 0; } } @@ -1633,7 +1696,9 @@ static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp) static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct bnxt_re_ucontext *uctx, - struct bnxt_re_qp_req *ureq) + struct bnxt_re_qp_req *ureq, + struct bnxt_re_dbr_obj *dbr_obj, + bool fixed_que_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; @@ -1656,6 +1721,16 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, return qptype; qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx); + if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE && + (!ureq->sq_slots || ureq->sq_slots > BNXT_RE_MAX_SQ_SLOTS)) + return -EINVAL; + if (fixed_que_attr) { + if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE) + return -EOPNOTSUPP; + if (!ureq->sq_npsn || + ureq->sq_npsn > roundup_pow_of_two(ureq->sq_slots / 2)) + return -EINVAL; + } qplqp->dev_cap_flags = dev_attr->dev_cap_flags; qplqp->cctx = rdev->chip_ctx; if (init_attr->qp_type == IB_QPT_RC) { @@ -1699,12 +1774,13 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx); if (uctx) { /* This will update DPI and qp_handle */ - rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq); + rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq, fixed_que_attr, + dbr_obj); if (rc) return rc; } - bnxt_re_qp_calculate_msn_psn_size(qp); + bnxt_re_qp_calculate_msn_psn_size(qp, fixed_que_attr, ureq); rc = bnxt_re_setup_qp_hwqs(qp); if (rc) @@ -1835,8 +1911,11 @@ static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev) int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { + struct bnxt_re_dbr_obj *dbr_obj = NULL; struct bnxt_qplib_dev_attr *dev_attr; + struct uverbs_attr_bundle *attrs; struct bnxt_re_ucontext *uctx; + bool fixed_que_attr = false; struct bnxt_re_qp_req ureq; struct bnxt_re_dev *rdev; struct bnxt_re_pd *pd; @@ -1853,9 +1932,23 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (udata) { - rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0); + rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, + BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR); if (rc) return rc; + + attrs = rdma_udata_to_uverbs_attr_bundle(udata); + if (uverbs_attr_is_valid(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE)) { + dbr_obj = uverbs_attr_get_obj(attrs, + BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE); + if (IS_ERR(dbr_obj)) + return PTR_ERR(dbr_obj); + kref_get(&dbr_obj->usecnt); + qp->dbr_obj = dbr_obj; + } + if (ureq.comp_mask & BNXT_RE_QP_REQ_MASK_FIXED_QUE_ATTR) + fixed_que_attr = true; } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); @@ -1865,7 +1958,8 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, } qp->rdev = rdev; - rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq); + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq, + dbr_obj, fixed_que_attr); if (rc) goto fail; @@ -1935,6 +2029,8 @@ free_hwq: bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); bnxt_re_qp_free_umem(qp); fail: + if (dbr_obj) + kref_put(&dbr_obj->usecnt, bnxt_re_dbr_kref_release); return rc; } @@ -2029,11 +2125,11 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (ret) return ret; - if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) { - free_page((unsigned long)srq->uctx_srq_page); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) hash_del(&srq->hash_entry); - } bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) + free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); return ib_respond_empty_udata(udata); @@ -3332,11 +3428,11 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (ret) return ret; - if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) { - free_page((unsigned long)cq->uctx_cq_page); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) hash_del(&cq->hash_entry); - } bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) + free_page((unsigned long)cq->uctx_cq_page); bnxt_re_put_nq(rdev, nq); @@ -3345,26 +3441,6 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) return ib_respond_empty_udata(udata); } -static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev, - struct ib_umem *umem, - struct bnxt_qplib_sg_info *sginfo) -{ - unsigned long page_size; - - if (!umem) - return -EINVAL; - - page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0); - if (!page_size || page_size != SZ_4K) - return -EINVAL; - - sginfo->umem = umem; - sginfo->npages = ib_umem_num_dma_blocks(umem, page_size); - sginfo->pgsize = page_size; - sginfo->pgshft = __builtin_ctz(page_size); - return 0; -} - int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { @@ -4644,6 +4720,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) goto fail; } spin_lock_init(&uctx->sh_lock); + mutex_init(&uctx->wcdpi_lock); resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX; chip_met_rev_num = rdev->chip_ctx->chip_num; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 08f71a94d55d..acb40fc528d2 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -96,6 +96,7 @@ struct bnxt_re_qp { struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *dentry; + struct bnxt_re_dbr_obj *dbr_obj; /* doorbell region */ }; struct bnxt_re_cq { @@ -141,6 +142,7 @@ struct bnxt_re_ucontext { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_qplib_dpi wcdpi; + struct mutex wcdpi_lock; /* serialises WC DPI alloc/free */ void *shpg; spinlock_t sh_lock; /* protect shpg */ struct rdma_user_mmap_entry *shpage_mmap; @@ -167,7 +169,7 @@ struct bnxt_re_dbr_obj { struct bnxt_re_dev *rdev; struct bnxt_qplib_dpi dpi; struct bnxt_re_user_mmap_entry *entry; - atomic_t usecnt; /* QPs using this dbr */ + struct kref usecnt; /* 1 (uobject) + n (QPs using this dbr) */ }; struct bnxt_re_flow { @@ -308,4 +310,5 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); struct bnxt_re_user_mmap_entry* bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset, enum bnxt_re_mmap_flag mmap_flag, u64 *offset); +void bnxt_re_dbr_kref_release(struct kref *ref); #endif /* __BNXT_RE_IB_VERBS_H__ */ diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index a892f1172917..d25fdc458120 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -1093,8 +1093,6 @@ static int bnxt_re_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr) goto err; if (rdma_nl_put_driver_u32(msg, "element_size", mr_hwq->element_size)) goto err; - if (rdma_nl_put_driver_u64_hex(msg, "hwq", (unsigned long)mr_hwq)) - goto err; if (rdma_nl_put_driver_u64_hex(msg, "va", mr->qplib_mr.va)) goto err; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 95e0489c53c3..756f8b5f042a 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -764,9 +764,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; } @@ -774,6 +778,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, mutex_unlock(&res->dpi_tbl_lock); return 0; +fail_ioremap: + /* Roll back the bit we just claimed. */ + set_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = NULL; + mutex_unlock(&res->dpi_tbl_lock); + return -ENOMEM; + } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index 9fadd637cb5b..c4193ae75b54 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -369,6 +369,7 @@ int bnxt_qplib_destroy_flow(struct bnxt_qplib_res *res); #define BNXT_VAR_MAX_SLOT_ALIGN 256 #define BNXT_VAR_MAX_SGE 13 #define BNXT_RE_MAX_RQ_WQES 65536 +#define BNXT_RE_MAX_SQ_SLOTS 65536 #define BNXT_STATIC_MAX_SGE 6 diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index 3eaee7101615..ca6b1630cd83 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -76,8 +76,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * struct ib_ucontext *ib_uctx; struct bnxt_re_dev *rdev; u64 mmap_offset; + u32 dpi = 0; u32 length; - u32 dpi; u64 addr; int err; @@ -98,26 +98,39 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * switch (alloc_type) { case BNXT_RE_ALLOC_WC_PAGE: - if (cctx->modes.db_push) { + if (cctx->modes.db_push) { + mutex_lock(&uctx->wcdpi_lock); + /* already allocated — one WC page per context */ + if (uctx->wcdpi.dbr) { + mutex_unlock(&uctx->wcdpi_lock); + return -EEXIST; + } if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi, - uctx, BNXT_QPLIB_DPI_TYPE_WC)) + uctx, BNXT_QPLIB_DPI_TYPE_WC)) { + mutex_unlock(&uctx->wcdpi_lock); return -ENOMEM; + } length = PAGE_SIZE; dpi = uctx->wcdpi.dpi; addr = (u64)uctx->wcdpi.umdbr; mmap_flag = BNXT_RE_MMAP_WC_DB; + mutex_unlock(&uctx->wcdpi_lock); } else { return -EINVAL; } break; case BNXT_RE_ALLOC_DBR_BAR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_bar_addr; mmap_flag = BNXT_RE_MMAP_DBR_BAR; break; case BNXT_RE_ALLOC_DBR_PAGE: + if (!rdev->pacing.dbr_pacing) + return -EOPNOTSUPP; length = PAGE_SIZE; addr = (u64)rdev->pacing.dbr_page; mmap_flag = BNXT_RE_MMAP_DBR_PAGE; @@ -128,8 +141,15 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle * } entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset); - if (!entry) + if (!entry) { + if (mmap_flag == BNXT_RE_MMAP_WC_DB) { + mutex_lock(&uctx->wcdpi_lock); + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + mutex_unlock(&uctx->wcdpi_lock); + } return -ENOMEM; + } uobj->object = entry; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE); @@ -160,11 +180,16 @@ static int alloc_page_obj_cleanup(struct ib_uobject *uobject, switch (entry->mmap_flag) { case BNXT_RE_MMAP_WC_DB: - if (uctx && uctx->wcdpi.dbr) { + if (uctx) { struct bnxt_re_dev *rdev = uctx->rdev; - bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); - uctx->wcdpi.dbr = NULL; + mutex_lock(&uctx->wcdpi_lock); + if (uctx->wcdpi.dbr) { + bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + uctx->wcdpi.dbr = NULL; + } + mutex_unlock(&uctx->wcdpi_lock); } break; case BNXT_RE_MMAP_DBR_BAR: @@ -252,6 +277,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)cq->uctx_cq_page; + if (!addr) + return -EOPNOTSUPP; break; case BNXT_RE_SRQ_TOGGLE_MEM: srq = bnxt_re_search_for_srq(rdev, res_id); @@ -259,6 +286,8 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return -EINVAL; addr = (u64)srq->uctx_srq_page; + if (!addr) + return -EOPNOTSUPP; break; default: @@ -369,6 +398,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_DBR_ALLOC)(struct uverbs_attr_bundle *a } obj->rdev = rdev; + kref_init(&obj->usecnt); uobj->object = obj; uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_DBR_HANDLE); @@ -391,15 +421,33 @@ free_mem: return ret; } +void bnxt_re_dbr_kref_release(struct kref *ref) +{ + struct bnxt_re_dbr_obj *obj = + container_of(ref, struct bnxt_re_dbr_obj, usecnt); + struct bnxt_re_dev *rdev = obj->rdev; + + rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); + bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + kfree(obj); +} + static int bnxt_re_dbr_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { struct bnxt_re_dbr_obj *obj = uobject->object; - struct bnxt_re_dev *rdev = obj->rdev; - rdma_user_mmap_entry_remove(&obj->entry->rdma_entry); - bnxt_qplib_free_uc_dpi(&rdev->qplib_res, &obj->dpi); + /* If it is being destroyed explicitly while QPs still hold a + * reference (> 1), reject it with EBUSY. If no QP references + * or implicit teardown (process exit, driver removal), drop + * the uobject reference unconditionally. The object gets freed + * (bnxt_re_dbr_kref_release) when the usecnt goes to zero. + */ + if (why == RDMA_REMOVE_DESTROY && kref_read(&obj->usecnt) > 1) + return -EBUSY; + + kref_put(&obj->usecnt, bnxt_re_dbr_kref_release); return 0; } @@ -459,11 +507,26 @@ DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR, DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_DEFAULT_DBR, &UVERBS_METHOD(BNXT_RE_METHOD_GET_DEFAULT_DBR)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + bnxt_re_qp_create, + UVERBS_OBJECT_QP, + UVERBS_METHOD_QP_CREATE, + UVERBS_ATTR_IDR(BNXT_RE_CREATE_QP_ATTR_DBR_HANDLE, + BNXT_RE_OBJECT_DBR, + UVERBS_ACCESS_READ, + UA_OPTIONAL)); + +const struct uapi_definition bnxt_re_create_qp_defs[] = { + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_QP, &bnxt_re_qp_create), + {}, +}; + const struct uapi_definition bnxt_re_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DBR), UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_DEFAULT_DBR), + UAPI_DEF_CHAIN(bnxt_re_create_qp_defs), {} }; diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 8b5a5b32b0fa..b7fd8b1fbbbd 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -1206,6 +1206,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) { struct hfi1_devdata *dd; + struct ib_device *ibdev; int ret, nports; /* extra is * number of ports */ @@ -1227,7 +1228,17 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, "Could not allocate unit ID: error %d\n", -ret); goto bail; } - rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit); + + /* + * FIXME: rvt and its users want to touch the ibdev before + * registration and have things like the name work. We don't have the + * infrastructure in the core to support this directly today, hack it + * to work by setting the name manually here. + */ + ibdev = &dd->verbs_dev.rdi.ibdev; + dev_set_name(&ibdev->dev, "%s_%d", class_name(), dd->unit); + strscpy(ibdev->name, dev_name(&ibdev->dev), IB_DEVICE_NAME_MAX); + /* * If the BIOS does not have the NUMA node information set, select * NUMA 0 so we get consistent performance. diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 621568e11405..f7e220ab4219 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -174,9 +174,9 @@ static int hns_roce_create_cqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_CQC, hr_cq->cqn); if (ret) - ibdev_err(ibdev, - "failed to send create cmd for CQ(0x%lx), ret = %d.\n", - hr_cq->cqn, ret); + ibdev_err_ratelimited(ibdev, + "failed to send create cmd for CQ(0x%lx), ret = %d.\n", + hr_cq->cqn, ret); hns_roce_free_cmd_mailbox(hr_dev, mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index b869cdc54118..db32c5897640 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -26,17 +26,6 @@ static const struct file_operations hns_debugfs_seqfile_fops = { .llseek = seq_lseek }; -static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq, - const char *name, struct dentry *parent, - int (*read_fn)(struct seq_file *, void *), - void *data) -{ - debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops); - - seq->read = read_fn; - seq->data = data; -} - static const char * const sw_stat_info[] = { [HNS_ROCE_DFX_AEQE_CNT] = "aeqe", [HNS_ROCE_DFX_CEQE_CNT] = "ceqe", @@ -76,10 +65,12 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev, { struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root; - dbgfs->root = debugfs_create_dir("sw_stat", parent); + dbgfs->sw_stat.read = sw_stat_debugfs_show; + dbgfs->sw_stat.data = hr_dev; - init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root, - sw_stat_debugfs_show, hr_dev); + dbgfs->root = debugfs_create_dir("sw_stat", parent); + debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat, + &hns_debugfs_seqfile_fops); } /* debugfs for device */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index e7c9e30ad2d8..ccb40f8a48b7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev, bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); switch (bt_num) { case 3: - index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx; index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num * chunk_ba_num + - l1_idx * chunk_ba_num + l2_idx; + index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num + + (u64)l1_idx * chunk_ba_num + l2_idx; break; case 2: index->l0 = l0_idx; - index->buf = l0_idx * chunk_ba_num + l1_idx; + index->buf = (u64)l0_idx * chunk_ba_num + l1_idx; break; case 1: index->buf = l0_idx; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index fa36700d0db2..60478382e82f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6192,9 +6192,9 @@ static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); hns_roce_free_cmd_mailbox(hr_dev, mailbox); if (ret) - ibdev_err(&hr_dev->ib_dev, - "failed to handle cmd of modifying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to handle cmd of modifying SRQ, ret = %d.\n", + ret); } out: @@ -6220,9 +6220,9 @@ static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd of querying SRQ, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd of querying SRQ, ret = %d.\n", + ret); goto out; } @@ -6328,9 +6328,9 @@ static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, key_to_hw_index(key)); if (ret) { - ibdev_err(&hr_dev->ib_dev, - "failed to process cmd when querying MPT, ret = %d.\n", - ret); + ibdev_err_ratelimited(&hr_dev->ib_dev, + "failed to process cmd when querying MPT, ret = %d.\n", + ret); goto err_mailbox; } @@ -7586,8 +7586,8 @@ static int __init hns_roce_hw_v2_init(void) static void __exit hns_roce_hw_v2_exit(void) { - hns_roce_dealloc_bond_grp(); hnae3_unregister_client(&hns_roce_hw_v2_client); + hns_roce_dealloc_bond_grp(); hns_roce_cleanup_debugfs(); } diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 0dbe99aab6ad..1e5471035454 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -1113,7 +1113,7 @@ static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) unsigned long flags; spin_lock_irqsave(&hr_cq->lock, flags); - if (cq->comp_handler) { + if (cq->comp_handler && hr_cq->ib_cq.poll_ctx != IB_POLL_DIRECT) { if (!hr_cq->is_armed) { hr_cq->is_armed = 1; list_add_tail(&hr_cq->node, cq_list); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 25bfd3970f5b..c13d29cd0897 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -173,7 +173,7 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx & (hr_dev->caps.num_mtpts - 1)); if (ret) { - dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + dev_err_ratelimited(dev, "failed to create mpt, ret = %d.\n", ret); goto err_page; } @@ -319,7 +319,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT, mtpt_idx); if (ret) - ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); + ibdev_warn_ratelimited(ib_dev, "failed to destroy MPT, ret = %d.\n", ret); mr->enabled = 0; mr->iova = virt_addr; @@ -350,7 +350,7 @@ struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT, mtpt_idx); if (ret) { - ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret); + ibdev_err_ratelimited(ib_dev, "failed to create MPT, ret = %d.\n", ret); goto free_cmd_mbox; } diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 8b94cbdfa54d..8feb5851067d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -103,7 +103,7 @@ static int hns_roce_create_srqc(struct hns_roce_dev *hr_dev, ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_SRQ, srq->srqn); if (ret) - ibdev_err(ibdev, "failed to config SRQC, ret = %d.\n", ret); + ibdev_err_ratelimited(ibdev, "failed to config SRQC, ret = %d.\n", ret); err_mbox: hns_roce_free_cmd_mailbox(hr_dev, mailbox); diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index f9be467d137f..c345cc654256 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -235,8 +235,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { if (cqp_request->waiting) { - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } else if (cqp_request->callback_fcn) { cqp_request->callback_fcn(cqp_request); } @@ -1107,9 +1106,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitqueue of the cqp_requests and add them to the list */ + /* init the completion of the cqp_requests and add them to the list */ for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); + init_completion(&cqp->cqp_requests[i].comp); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } init_waitqueue_head(&cqp->remove_wq); diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 3d49bd57bae7..8c17a201c1fd 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -161,13 +161,12 @@ struct irdma_cqp_compl_info { struct irdma_cqp_request { struct cqp_cmds_info info; - wait_queue_head_t waitq; + struct completion comp; struct list_head list; refcount_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; bool pending:1; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index 495e5daff4b4..8e9e159f19ff 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, if (cqp_request) { cqp_request->dynamic = true; if (wait) - init_waitqueue_head(&cqp_request->waitq); + init_completion(&cqp_request->comp); } } if (!cqp_request) { @@ -480,7 +480,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp, if (cqp_request->dynamic) { kfree(cqp_request); } else { - WRITE_ONCE(cqp_request->request_done, false); + reinit_completion(&cqp_request->comp); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; cqp_request->pending = false; @@ -515,8 +515,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp, { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } wait_event_timeout(cqp->remove_wq, refcount_read(&cqp_request->refcnt) == 1, 1000); @@ -609,9 +608,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); - if (wait_event_timeout(cqp_request->waitq, - READ_ONCE(cqp_request->request_done), - msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) + if (wait_for_completion_timeout(&cqp_request->comp, + msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) break; if (cqp_request->pending) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 8cd427532805..7da7a7e8b30c 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2572,7 +2572,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, } cqmr_shadow = &iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; - cqmr->split = true; } else { info.shadow_area_pa = cqmr->shadow; } @@ -2781,10 +2780,11 @@ static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, * irdma_copy_user_pgaddrs - copy user page address to pble's os locally * @iwmr: iwmr for IB's user page addresses * @pbl: ple pointer to save 1 level or 0 level pble + * @pbl_len: Max number of PBL entries to populate * @level: indicated level 0, 1 or 2 */ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, - enum irdma_pble_level level) + u32 pbl_len, enum irdma_pble_level level) { struct ib_umem *region = iwmr->region; struct irdma_pbl *iwpbl = &iwmr->iwpbl; @@ -2792,7 +2792,9 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, struct irdma_pble_info *pinfo; struct ib_block_iter biter; u32 idx = 0; - u32 pbl_cnt = 0; + + if (!pbl_len) + return; pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; @@ -2801,7 +2803,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { *pbl = rdma_block_iter_dma_address(&biter); - if (++pbl_cnt == palloc->total_cnt) + if (!--pbl_len) break; pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); } @@ -2877,6 +2879,7 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, u64 *pbl; int status; enum irdma_pble_level level = PBLE_LEVEL_1; + u32 pbl_len; if (lvl) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, @@ -2884,16 +2887,18 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, if (status) return status; + pbl_len = palloc->total_cnt; iwpbl->pbl_allocated = true; level = palloc->level; pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf; pbl = pinfo->addr; } else { + pbl_len = IRDMA_MAX_SAVED_PHY_PGADDR; pbl = iwmr->pgaddrmem; } - irdma_copy_user_pgaddrs(iwmr, pbl, level); + irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); if (lvl) iwmr->pgaddrmem[0] = *pbl; @@ -2974,7 +2979,8 @@ static int irdma_handle_q_mem(struct irdma_device *iwdev, case IRDMA_MEMREG_TYPE_CQ: hmc_p = &cqmr->cq_pbl; - if (!cqmr->split) + if (!(iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & + IRDMA_FEATURE_CQ_RESIZE)) cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; if (lvl) @@ -3319,6 +3325,7 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, int err; lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + iwmr->access = access; err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index aabbb3442098..289ebc9b23ca 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -65,7 +65,6 @@ struct irdma_hmc_pble { struct irdma_cq_mr { struct irdma_hmc_pble cq_pbl; dma_addr_t shadow; - bool split; }; struct irdma_srq_mr { diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index afc2fc124fee..307ae01bf26f 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -600,8 +600,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_EDR; + ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { props->gid_tbl_len = 16; diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 5a799d6df93e..898c8363422a 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -135,6 +135,7 @@ struct mlx4_ib_mr { dma_addr_t page_map; u32 npages; u32 max_pages; + int access_flags; struct mlx4_mr mmr; struct ib_umem *umem; size_t page_map_size; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 6747bca30677..67b0e8954b5e 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -181,6 +181,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_mr; + mr->access_flags = access_flags; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; mr->ibmr.page_size = 1U << shift; @@ -241,6 +242,8 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, if (err) goto release_mpt_entry; + } else { + mr_access_flags = mmr->access_flags; } if (flags & IB_MR_REREG_TRANS) { @@ -282,8 +285,10 @@ struct ib_mr *mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, * return a failure. But dereg_mr will free the resources. */ err = mlx4_mr_hw_write_mpt(dev->dev, &mmr->mmr, pmpt_entry); - if (!err && flags & IB_MR_REREG_ACCESS) - mmr->mmr.access = mr_access_flags; + if (!err && flags & IB_MR_REREG_ACCESS) { + mmr->access_flags = mr_access_flags; + mmr->mmr.access = convert_access(mr_access_flags); + } release_mpt_entry: mlx4_mr_hw_put_mpt(dev->dev, pmpt_entry); diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c index b8fa4ecfc961..e688ad66a895 100644 --- a/drivers/infiniband/hw/mlx4/sysfs.c +++ b/drivers/infiniband/hw/mlx4/sysfs.c @@ -636,12 +636,6 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) p->port_num = port_num; p->slave = slave; - ret = kobject_init_and_add(&p->kobj, &port_type, - kobject_get(dev->dev_ports_parent[slave]), - "%d", port_num); - if (ret) - goto err_alloc; - p->pkey_group.name = "pkey_idx"; p->pkey_group.attrs = alloc_group_attrs(show_port_pkey, @@ -649,13 +643,9 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) dev->dev->caps.pkey_table_len[port_num]); if (!p->pkey_group.attrs) { ret = -ENOMEM; - goto err_alloc; + goto err_free_port; } - ret = sysfs_create_group(&p->kobj, &p->pkey_group); - if (ret) - goto err_free_pkey; - p->gid_group.name = "gid_idx"; p->gid_group.attrs = alloc_group_attrs(show_port_gid_idx, NULL, 1); if (!p->gid_group.attrs) { @@ -663,28 +653,47 @@ static int add_port(struct mlx4_ib_dev *dev, int port_num, int slave) goto err_free_pkey; } + ret = kobject_init_and_add(&p->kobj, &port_type, + kobject_get(dev->dev_ports_parent[slave]), + "%d", port_num); + if (ret) + goto err_put; + + ret = sysfs_create_group(&p->kobj, &p->pkey_group); + if (ret) + goto err_del; + ret = sysfs_create_group(&p->kobj, &p->gid_group); if (ret) - goto err_free_gid; + goto err_remove_pkey; ret = add_vf_smi_entries(p); if (ret) - goto err_free_gid; + goto err_remove_gid; list_add_tail(&p->kobj.entry, &dev->pkeys.pkey_port_list[slave]); return 0; -err_free_gid: - kfree(p->gid_group.attrs[0]); - kfree(p->gid_group.attrs); +err_remove_gid: + sysfs_remove_group(&p->kobj, &p->gid_group); + +err_remove_pkey: + sysfs_remove_group(&p->kobj, &p->pkey_group); + +err_del: + kobject_del(&p->kobj); + +err_put: + kobject_put(dev->dev_ports_parent[slave]); + kobject_put(&p->kobj); + return ret; err_free_pkey: for (i = 0; i < dev->dev->caps.pkey_table_len[port_num]; ++i) kfree(p->pkey_group.attrs[i]); kfree(p->pkey_group.attrs); -err_alloc: - kobject_put(dev->dev_ports_parent[slave]); +err_free_port: kfree(p); return ret; } diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 645ebcc0832d..c2ae5a140471 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -1913,6 +1913,17 @@ sub_bytes: return err; } +static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1) +{ + struct devx_event_subscription *s; + + list_for_each_entry(s, list, event_list) + if (s->xa_key_level1 == key_level1) + return true; + + return false; +} + static void subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, u32 key_level1, @@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( event_sub = kzalloc_obj(*event_sub); if (!event_sub) { + if (!devx_key_in_sub_list(&sub_list, key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + key_level1, + obj, + obj_id); err = -ENOMEM; goto err; } + event_sub->ev_file = ev_file; + event_sub->xa_key_level1 = key_level1; list_add_tail(&event_sub->event_list, &sub_list); uverbs_uobject_get(&ev_file->uobj); if (use_eventfd) { @@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( } event_sub->cookie = cookie; - event_sub->ev_file = ev_file; - /* May be needed upon cleanup the devx object/subscription */ - event_sub->xa_key_level1 = key_level1; event_sub->xa_key_level2 = obj_id; INIT_LIST_HEAD(&event_sub->obj_list); } @@ -2225,10 +2240,11 @@ err: list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { list_del(&event_sub->event_list); - subscribe_event_xa_dealloc(devx_event_table, - event_sub->xa_key_level1, - obj, - obj_id); + if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1)) + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); if (event_sub->eventfd) eventfd_ctx_put(event_sub->eventfd); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 61078281953d..1144bfee5804 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -2069,9 +2069,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn, if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) || (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) && !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - return err; + return 0; + + err = mlx5_ib_enable_lb(dev, true, false); + if (err) + mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid); - return mlx5_ib_enable_lb(dev, true, false); + return err; } static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn, @@ -2658,7 +2662,7 @@ err: if (!dyn_uar) return err; - mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid); + mlx5_cmd_uar_dealloc(dev->mdev, uar_index, context->devx_uid); free_bfreg: mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx); @@ -4488,6 +4492,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) dev->port[i].roce.last_port_state = IB_PORT_DOWN; } + mutex_init(&dev->lb.mutex); + err = mlx5r_cmd_query_special_mkeys(dev); if (err) return err; @@ -4788,11 +4794,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (err) return err; - if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && - (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) || - MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) - mutex_init(&dev->lb.mutex); - if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { err = mlx5_ib_init_var_region(dev); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e156dc4d7529..0a2b8ede0d81 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -331,6 +331,10 @@ struct mlx5_ib_flow_db { #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4 #define MLX5_IB_WR_UMR IB_WR_RESERVED1 +/* + * A valid pdn is required when flags include MLX5_IB_UPD_XLT_ENABLE, + * MLX5_IB_UPD_XLT_PD or MLX5_IB_UPD_XLT_ACCESS. + */ #define MLX5_IB_UPD_XLT_ZAP BIT(0) #define MLX5_IB_UPD_XLT_ENABLE BIT(1) #define MLX5_IB_UPD_XLT_ATOMIC BIT(2) @@ -1209,6 +1213,11 @@ static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) return container_of(ibpd, struct mlx5_ib_pd, ibpd); } +static inline u32 mlx5_mr_pdn(struct mlx5_ib_mr *mr) +{ + return to_mpd(mr->ibmr.pd)->pdn; +} + static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq) { return container_of(ibsrq, struct mlx5_ib_srq, ibsrq); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index fb40b44496f4..4118cd2d15fd 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -31,6 +31,7 @@ * SOFTWARE. */ +#include <linux/bitfield.h> #include <linux/kref.h> #include <linux/random.h> #include <linux/debugfs.h> @@ -163,9 +164,8 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev, #define MLX5_FRMR_POOLS_KEY_VENDOR_KEY_SUPPORTED \ MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT 16 -#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK 0xFF0000 -#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK 0xFFFF +#define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16) +#define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0) static struct mlx5_ib_mr * _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, @@ -194,7 +194,8 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem, ph ^= MLX5_IB_NO_PH; mr->ibmr.frmr.key.kernel_vendor_key = - st_index | (ph << MLX5_FRMR_POOLS_KERNEL_KEY_PH_SHIFT); + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) | + FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph); err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr); if (err) { kfree(mr); @@ -271,9 +272,10 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, get_mkc_octo_size(access_mode, key->num_dma_blocks)); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - st_index = key->kernel_vendor_key & - MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK; - ph = key->kernel_vendor_key & MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK; + st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, + key->kernel_vendor_key); + ph = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, + key->kernel_vendor_key); if (ph) { /* Normalize ph: swap MLX5_IB_NO_PH for 0 */ if (ph == MLX5_IB_NO_PH) @@ -294,7 +296,7 @@ static int mlx5r_create_mkeys(struct ib_device *device, struct ib_frmr_key *key, free_in: kfree(in); if (err) - for (; i > 0; i--) + for (i--; i >= 0; i--) mlx5_core_destroy_mkey(dev->mdev, handles[i]); return err; } @@ -781,7 +783,8 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, * configured properly but left disabled. It is safe to go ahead * and configure it again via UMR while enabling it. */ - err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); + err = mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE, + to_mpd(pd)->pdn); if (err) { mlx5_ib_dereg_mr(&mr->ibmr, NULL); return ERR_PTR(err); @@ -890,7 +893,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) if (!umem_dmabuf->sgt || !mr) return; - mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); + /* MLX5_IB_UPD_XLT_ZAP does not change the pdn */ + mlx5r_umr_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP, 0); ib_umem_dmabuf_unmap_pages(umem_dmabuf); } @@ -1128,10 +1132,8 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, if (err) return err; - if (flags & IB_MR_REREG_PD) { - mr->ibmr.pd = pd; + if (flags & IB_MR_REREG_PD) upd_flags |= MLX5_IB_UPD_XLT_PD; - } if (flags & IB_MR_REREG_ACCESS) { mr->access_flags = access_flags; upd_flags |= MLX5_IB_UPD_XLT_ACCESS; @@ -1141,7 +1143,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); mr->umem = new_umem; - err = mlx5r_umr_update_mr_pas(mr, upd_flags); + err = mlx5r_umr_update_mr_pas(mr, upd_flags, to_mpd(pd)->pdn); if (err) { /* * The MR is revoked at this point so there is no issue to free @@ -1188,6 +1190,21 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, if (!(flags & IB_MR_REREG_PD)) new_pd = ib_mr->pd; + if (mr->is_odp_implicit && !(flags & IB_MR_REREG_TRANS)) { + if (!(new_access_flags & IB_ACCESS_ON_DEMAND)) + return ERR_PTR(-EOPNOTSUPP); + + /* + * Due to all the child mkeys we cannot actually change an + * implicit MR in place. If the user did not specify a new + * translation then force the fixed implicit MR values. + */ + start = 0; + iova = 0; + length = U64_MAX; + flags |= IB_MR_REREG_TRANS; + } + if (!(flags & IB_MR_REREG_TRANS)) { struct ib_umem *umem; @@ -1202,7 +1219,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, } /* DM or ODP MR's don't have a normal umem so we can't re-use it */ if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) - goto recreate; + return ERR_PTR(-EOPNOTSUPP); /* * Only one active MR can refer to a umem at one time, revoke @@ -1381,9 +1398,12 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) bool is_odp = is_odp_mr(mr); int ret; - if (mr->ibmr.frmr.pool && !mlx5_umr_revoke_mr_with_lock(mr) && - !ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr)) - return 0; + if (mr->ibmr.frmr.pool) { + if (!mlx5_umr_revoke_mr_with_lock(mr)) { + ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr); + return 0; + } + } if (is_odp) mutex_lock(&to_ib_umem_odp(mr->umem)->umem_mutex); @@ -1404,6 +1424,10 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr) dma_resv_unlock( to_ib_umem_dmabuf(mr->umem)->attach->dmabuf->resv); } + + if (mr->ibmr.frmr.pool && !ret) + ib_frmr_pool_drop(&mr->ibmr); + return ret; } diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1119ce163ea7..42235b9dd6ab 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -833,12 +833,14 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, u32 *bytes_mapped, u32 flags) { struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); + struct mlx5_ib_dev *dev = mr_to_mdev(mr); int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT; unsigned int old_page_shift = mr->page_shift; unsigned int page_shift; unsigned long page_size; u32 xlt_flags = 0; + u32 pdn = 0; int err; if (flags & MLX5_PF_FLAGS_ENABLE) @@ -857,8 +859,12 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = -EINVAL; } else { page_shift = order_base_2(page_size); + if (mr->data_direct) + pdn = dev->ddr.pdn; + else + pdn = mlx5_mr_pdn(mr); if (page_shift != mr->page_shift && mr->dmabuf_faulted) { - err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, + err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, page_shift); } else { mr->page_shift = page_shift; @@ -866,8 +872,8 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, err = mlx5r_umr_update_data_direct_ksm_pas( mr, xlt_flags); else - err = mlx5r_umr_update_mr_pas(mr, - xlt_flags); + err = mlx5r_umr_update_mr_pas(mr, xlt_flags, + pdn); } } dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8fd05532c09c..3dc0c85587ff 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -451,16 +451,13 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, if (ucmd) { qp->rq.wqe_cnt = ucmd->rq_wqe_count; - if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift)) - return -EINVAL; qp->rq.wqe_shift = ucmd->rq_wqe_shift; - if ((1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) < - wq_sig) + if (check_shl_overflow(1, qp->rq.wqe_shift, &wqe_size)) + return -EINVAL; + if (wqe_size / sizeof(struct mlx5_wqe_data_seg) < wq_sig) return -EINVAL; qp->rq.max_gs = - (1 << qp->rq.wqe_shift) / - sizeof(struct mlx5_wqe_data_seg) - + wqe_size / sizeof(struct mlx5_wqe_data_seg) - wq_sig; qp->rq.max_post = qp->rq.wqe_cnt; } else { @@ -3109,12 +3106,14 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, int err; if (params->is_rss_raw) { + rdma_restrack_no_track(&qp->ibqp.res); err = create_rss_raw_qp_tir(dev, pd, qp, params); goto out; } switch (qp->type) { case MLX5_IB_QPT_DCT: + rdma_restrack_no_track(&qp->ibqp.res); err = create_dct(dev, pd, qp, params); break; case MLX5_IB_QPT_DCI: diff --git a/drivers/infiniband/hw/mlx5/restrack.c b/drivers/infiniband/hw/mlx5/restrack.c index 67841922c7b8..00a9bcb2603f 100644 --- a/drivers/infiniband/hw/mlx5/restrack.c +++ b/drivers/infiniband/hw/mlx5/restrack.c @@ -178,9 +178,6 @@ static int fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp) ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "REG_UMR"); break; - case MLX5_IB_QPT_DCT: - ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCT"); - break; case MLX5_IB_QPT_DCI: ret = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUBTYPE, "DCI"); break; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index f2139474be37..f3f428f5e1b6 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -603,11 +603,11 @@ mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev, struct mlx5_mkey_seg *mkey_seg, - struct mlx5_ib_mr *mr, + struct mlx5_ib_mr *mr, u32 pdn, unsigned int page_shift) { mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags); - MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); + MLX5_SET(mkc, mkey_seg, pd, pdn); MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length); MLX5_SET(mkc, mkey_seg, log_page_size, page_shift); @@ -670,23 +670,22 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev, wqe->data_seg.byte_count = cpu_to_be32(sg->length); } -static void -_mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe, - struct ib_sge *sg, unsigned int flags, - unsigned int page_shift, bool dd) +static void _mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, + struct mlx5r_umr_wqe *wqe, struct ib_sge *sg, + unsigned int flags, u32 pdn, + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift); - if (dd) /* Use the data direct internal kernel PD */ - MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg); } -static int -_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, - size_t start_block, size_t nblocks) +static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, + u32 pdn, bool dd, size_t start_block, + size_t nblocks) { size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -720,7 +719,7 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, orig_sg_length = sg.length; - _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd); + _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, pdn, mr->page_shift); /* Set initial translation offset to start_block */ offset = (u64)start_block * ent_size; @@ -811,7 +810,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + true, start_block, nblocks); } int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, @@ -821,12 +821,13 @@ int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, } int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks) + u32 pdn, size_t start_block, size_t nblocks) { if (WARN_ON(mr->umem->is_odp)) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks); + return _mlx5r_umr_update_mr_pas(mr, flags, pdn, false, start_block, + nblocks); } /* @@ -834,9 +835,9 @@ int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP * flag may be used. */ -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn) { - return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, 0, 0); } static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) @@ -861,6 +862,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, size_t orig_sg_length; size_t pages_iter; struct ib_sge sg; + u32 pdn = mlx5_mr_pdn(mr); int err = 0; void *xlt; @@ -895,7 +897,8 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, } mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg); - mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift); + mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, pdn, + page_shift); mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); for (pages_mapped = 0; @@ -915,7 +918,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, */ err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); if (err) - return err; + break; dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); @@ -937,8 +940,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, * pinned and the HW can switch from 4K to huge-page alignment). */ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd) + unsigned int page_shift) { struct mlx5_ib_dev *dev = mr_to_mdev(mr); struct mlx5r_umr_wqe wqe = {}; @@ -953,16 +955,8 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, /* Fill mkey segment with the new page size, keep the rest unchanged */ MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift); - if (dd) - MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn); - else - MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); - MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova); MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length); - MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); - MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, - mlx5_mkey_variant(mr->mmkey.key)); err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); if (!err) @@ -971,17 +965,18 @@ int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, return err; } -static inline int -_mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks, bool dd) +static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, + unsigned int flags, u32 pdn, + size_t start_block, + size_t nblocks, bool dd) { if (dd) return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, start_block, nblocks); else - return mlx5r_umr_update_mr_pas_range(mr, flags, start_block, - nblocks); + return mlx5r_umr_update_mr_pas_range(mr, flags, pdn, + start_block, nblocks); } /** @@ -995,11 +990,9 @@ _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, * Return: On success, returns the number of entries that were zapped. * On error, returns a negative error code. */ -static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, - unsigned int flags, - unsigned int page_shift, - size_t *nblocks, - bool dd) +static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, unsigned int flags, + unsigned int page_shift, size_t *nblocks, + u32 pdn, bool dd) { unsigned int old_page_shift = mr->page_shift; struct mlx5_ib_dev *dev = mr_to_mdev(mr); @@ -1039,7 +1032,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, */ if (*nblocks) mr->page_shift = max_page_shift; - err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd); + err = _mlx5r_dmabuf_umr_update_pas(mr, flags, pdn, 0, *nblocks, dd); if (err) { mr->page_shift = old_page_shift; return err; @@ -1049,7 +1042,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * non-present. */ if (*nblocks) { - err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd); + err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift); if (err) { mr->page_shift = old_page_shift; return err; @@ -1064,6 +1057,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * entries accordingly * @mr: The memory region to update * @xlt_flags: Translation table update flags + * @pdn: Protection domain number * @page_shift: The new (optimized) page shift to use * * This function updates the page size and mkey translation entries for a DMABUF @@ -1083,7 +1077,7 @@ static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, * * Returns 0 on success or a negative error code on failure. */ -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift) { unsigned int old_page_shift = mr->page_shift; @@ -1092,7 +1086,7 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, int err; err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks, - mr->data_direct); + pdn, mr->data_direct); if (err) return err; @@ -1105,20 +1099,17 @@ int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, * the page size in the mkey yet. */ err = _mlx5r_dmabuf_umr_update_pas( - mr, - xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, - zapped_blocks, - total_blocks - zapped_blocks, + mr, xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, pdn, + zapped_blocks, total_blocks - zapped_blocks, mr->data_direct); if (err) goto err; } - err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift, - mr->data_direct); + err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift); if (err) goto err; - err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, + err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, pdn, 0, zapped_blocks, mr->data_direct); if (err) goto err; diff --git a/drivers/infiniband/hw/mlx5/umr.h b/drivers/infiniband/hw/mlx5/umr.h index 7eeaf6a94c97..99192ec67957 100644 --- a/drivers/infiniband/hw/mlx5/umr.h +++ b/drivers/infiniband/hw/mlx5/umr.h @@ -101,14 +101,13 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, size_t nblocks); int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags); int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, - size_t start_block, size_t nblocks); -int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags); + u32 pdn, size_t start_block, size_t nblocks); +int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, u32 pdn); int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags); int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, - unsigned int page_shift, - bool dd); -int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, + unsigned int page_shift); +int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, u32 pdn, unsigned int page_shift); #endif /* _MLX5_IB_UMR_H */ |