git: 78ae174e420a - main - mlx5: support DevX UMEM page size bitmap
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 07 Oct 2026 17:10:02 UTC
The branch main has been updated by kib:
URL: https://cgit.FreeBSD.org/src/commit/?id=78ae174e420a2eaa340b41247a260bc067fae0c6
commit 78ae174e420a2eaa340b41247a260bc067fae0c6
Author: Ariel Ehrenberg <aehrenberg@nvidia.com>
AuthorDate: 2026-10-06 10:45:06 +0000
Commit: Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-10-07 17:09:41 +0000
mlx5: support DevX UMEM page size bitmap
rdma-core's mlx5dv_devx_umem_reg_ex() passes the page sizes the caller
can use in MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP. Add the attribute and
choose the UMEM page size, page offset and MTT count the way Linux commit
7610ab57de56 ("RDMA/mlx5: Allow larger pages in DevX umem") does.
Linux looks at the DMA list only. The UMEM is mapped at an IOVA equal to
the DMA address of its first byte, ib_umem_find_best_pgsz() picks the
largest page size of the bitmap that this mapping allows, and
devx_umem_find_best_pgsize() halves it while the first DMA address is not
aligned to it or the length is not a multiple of it, stopping at
PAGE_SIZE, even if that is below every size in the bitmap.
The FreeBSD OFED layer has none of the ib_umem helpers this relies on,
so local copies of the current Linux ones are added.
ib_umem_find_best_pgoff() and ib_umem_dma_offset() come from Linux
commit b045db62f6f6 ("RDMA/mlx5: Use ib_umem_find_best_pgoff() for
SRQ"), ib_umem_num_dma_blocks() from Linux commit a665aca89a41
("RDMA/umem: Split ib_umem_num_pages() into ib_umem_num_dma_blocks()").
The ib_umem_find_best_pgsz() logic includes Linux commit 486055f5e09d
("RDMA/core: Fix best page size finding when it can cross SG entries"),
without which no page size above 4K would ever be found, as FreeBSD's
ib_umem_get() maps every page as an SG entry of its own, and Linux
commit 09ea6837a043 ("RDMA/umem: Be careful about boundary conditions in
ib_umem_find_best_pgsz()").
Without the attribute the bitmap defaults to GENMASK_ULL(63, 12), as in
Linux. The attribute is read with uverbs_copy_from(), FreeBSD's
uverbs_get_const_default() works on an s64 and rejects any value with bit
63 set, the default included. The page size is kept in a u64, Linux
returns it as an unsigned int and so fails registrations that would use
pages of 4G or more.
Reviewed by: kib
Sponsored by: Nvidia networking
MFC after: 1 month
---
sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c | 152 ++++++++++++++++++++--
sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h | 1 +
2 files changed, 144 insertions(+), 9 deletions(-)
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c
index 5f4aa160cfcf..2a3c60b2ec6d 100644
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c
@@ -40,6 +40,8 @@
#include <sys/priv.h>
+#include <linux/bitops.h>
+#include <linux/overflow.h>
#include <linux/xarray.h>
#include <linux/rculist.h>
#include <linux/srcu.h>
@@ -2082,21 +2084,153 @@ err:
return err;
}
+/*
+ * A DevX UMEM is mapped at an IOVA equal to the DMA address of its first
+ * byte, the UMEM VA plays no part in choosing its page size or MTT list.
+ */
+static u64 devx_umem_start_dma_addr(struct ib_umem *umem)
+{
+ return sg_dma_address(umem->sg_head.sgl) + ib_umem_offset(umem);
+}
+
+static u64 devx_umem_dma_offset(struct ib_umem *umem, u64 pgsz)
+{
+ return devx_umem_start_dma_addr(umem) & (pgsz - 1);
+}
+
+static size_t devx_umem_num_dma_blocks(struct ib_umem *umem, u64 pgsz)
+{
+ u64 iova = devx_umem_start_dma_addr(umem);
+
+ return (ALIGN(iova + umem->length, pgsz) - ALIGN_DOWN(iova, pgsz)) /
+ pgsz;
+}
+
+static u64 devx_umem_find_best_pgoff(struct ib_umem *umem, u64 pgsz_bitmap)
+{
+ u64 iova = devx_umem_start_dma_addr(umem);
+ u64 pgoff = ib_umem_offset(umem);
+ u64 curr_base = ~0ULL;
+ u64 curr_len = 0;
+ u64 mask = 0;
+ struct scatterlist *sg;
+ unsigned int bits;
+ u64 last_iova;
+ u64 end;
+ int i;
+
+ /* The best result is the smallest page size that results in the minimum
+ * number of required pages. Compute the largest page size that could
+ * work based on IOVA bits that don't change.
+ */
+ if (check_add_overflow(umem->length - 1, iova, &last_iova))
+ return 0;
+ bits = fls64(iova ^ last_iova);
+ if (bits < 64)
+ mask = pgsz_bitmap & GENMASK_ULL(63, bits);
+
+ for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) {
+ /* If the current entry is physically contiguous with the
+ * previous one, no need to take its start addresses into
+ * consideration.
+ */
+ if (check_add_overflow(curr_base, curr_len, &end) ||
+ end != sg_dma_address(sg)) {
+ curr_base = sg_dma_address(sg);
+ curr_len = 0;
+
+ /* Reduce max page size if IOVA/DMA bits differ */
+ mask |= (curr_base + pgoff) ^ iova;
+
+ /* The alignment of any IOVA matching a discontinuity
+ * point in the DMA list sets the maximum possible page
+ * size as this must be a starting point of a new page
+ * that needs to be aligned.
+ */
+ if (i != 0)
+ mask |= iova;
+ }
+
+ curr_len += sg_dma_len(sg);
+ iova += sg_dma_len(sg) - pgoff;
+
+ pgoff = 0;
+ }
+
+ /* The mask accumulates 1's in each position where the IOVA and DMA
+ * address differ, thus the length of trailing 0 is the largest page
+ * size that can pass the IOVA through to the DMA address.
+ */
+ if (mask)
+ pgsz_bitmap &= GENMASK_ULL(__ffs64(mask), 0);
+ return pgsz_bitmap ? rounddown_pow_of_two(pgsz_bitmap) : 0;
+}
+
+static u64 devx_umem_find_best_pgsize(struct ib_umem *umem, u64 pgsz_bitmap)
+{
+ u64 page_size;
+
+ /* Don't bother checking larger page sizes as offset must be zero and
+ * total DEVX umem length must be equal to total umem length.
+ */
+ pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
+ PAGE_SHIFT),
+ MLX5_ADAPTER_PAGE_SHIFT);
+ if (!pgsz_bitmap)
+ return 0;
+
+ page_size = devx_umem_find_best_pgoff(umem, pgsz_bitmap);
+ if (!page_size)
+ return 0;
+
+ /* If the page_size is less than the CPU page size then we can use the
+ * offset and create a umem which is a subset of the page list.
+ * For larger page sizes we can't be sure the DMA list reflects the
+ * VA so we must ensure that the umem extent is exactly equal to the
+ * page list. Reduce the page size until one of these cases is true.
+ */
+ while ((devx_umem_dma_offset(umem, page_size) != 0 ||
+ (umem->length % page_size) != 0) &&
+ page_size > PAGE_SIZE)
+ page_size /= 2;
+
+ return page_size;
+}
+
static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
struct uverbs_attr_bundle *attrs,
struct devx_umem *obj)
{
+ u64 pgsz_bitmap;
+ u64 page_size;
u64 addr;
size_t size;
u32 access;
- int npages;
int err;
- u32 page_mask;
if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
return -EFAULT;
+ /*
+ * If the user does not pass in pgsz_bitmap then the user promises not
+ * to use umem_offset!=0 in any commands that allocate on top of the
+ * umem.
+ *
+ * If the user wants to use a umem_offset then it must pass in
+ * pgsz_bitmap which guides the maximum page size and thus maximum
+ * object alignment inside the umem. See the PRM.
+ *
+ * Users are not allowed to use IOVA here, mkeys are not supported on
+ * umem.
+ */
+ pgsz_bitmap = GENMASK_ULL(63, min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT));
+ if (uverbs_attr_is_valid(attrs,
+ MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP) &&
+ uverbs_copy_from(&pgsz_bitmap, attrs,
+ MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP))
+ return -EFAULT;
+
err = uverbs_get_flags32(&access, attrs,
MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
IB_ACCESS_LOCAL_WRITE |
@@ -2113,17 +2247,15 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
if (IS_ERR(obj->umem))
return PTR_ERR(obj->umem);
- mlx5_ib_cont_pages(obj->umem, obj->umem->address,
- MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
- &obj->page_shift, &obj->ncont, NULL);
-
- if (!npages) {
+ page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap);
+ if (!page_size) {
ib_umem_release(obj->umem);
return -EINVAL;
}
- page_mask = (1 << obj->page_shift) - 1;
- obj->page_offset = obj->umem->address & page_mask;
+ obj->page_shift = order_base_2(page_size);
+ obj->page_offset = devx_umem_dma_offset(obj->umem, page_size);
+ obj->ncont = devx_umem_num_dma_blocks(obj->umem, page_size);
return 0;
}
@@ -2733,6 +2865,8 @@ DECLARE_UVERBS_NAMED_METHOD(
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
enum ib_access_flags),
+ UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
+ u64),
UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
diff --git a/sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 24f3388c3182..f55d4f2837d2 100644
--- a/sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -154,6 +154,7 @@ enum mlx5_ib_devx_umem_reg_attrs {
MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
+ MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
};
enum mlx5_ib_devx_umem_dereg_attrs {