git: f8f5b459d21e - main - Update user access region, UAR, APIs in the core in mlx5core.
Hans Petter Selasky
hselasky at FreeBSD.org
Fri Jan 8 12:34:52 UTC 2021
The branch main has been updated by hselasky:
URL: https://cgit.FreeBSD.org/src/commit/?id=f8f5b459d21ec9dd1ca5d9de319d8b440fef84a8
commit f8f5b459d21ec9dd1ca5d9de319d8b440fef84a8
Author: Hans Petter Selasky <hselasky at FreeBSD.org>
AuthorDate: 2021-01-08 12:30:26 +0000
Commit: Hans Petter Selasky <hselasky at FreeBSD.org>
CommitDate: 2021-01-08 12:33:46 +0000
Update user access region, UAR, APIs in the core in mlx5core.
This change include several changes as listed below all related to UAR.
UAR is a special PCI memory area where the so-called doorbell register and
blue flame register live. Blue flame is a feature for sending small packets
more efficiently via a PCI memory page, instead of using PCI DMA.
- All structures and functions named xxx_uuars were renamed into xxx_bfreg.
- Remove partially implemented Blueflame support from mlx5en(4) and mlx5ib.
- Implement blue flame register allocator.
- Use blue flame register allocator in mlx5ib.
- A common UAR page is now allocated by the core to support doorbell register
writes for all of mlx5en and mlx5ib, instead of allocating one UAR per
sendqueue.
- Add support for DEVX query UAR.
- Add support for 4K UAR for libmlx5.
Linux commits:
7c043e908a74ae0a935037cdd984d0cb89b2b970
2f5ff26478adaff5ed9b7ad4079d6a710b5f27e7
0b80c14f009758cefeed0edff4f9141957964211
30aa60b3bd12bd79b5324b7b595bd3446ab24b52
5fe9dec0d045437e48f112b8fa705197bd7bc3c0
0118717583cda6f4f36092853ad0345e8150b286
a6d51b68611e98f05042ada662aed5dbe3279c1e
MFC after: 1 week
Sponsored by: Mellanox Technologies // NVIDIA Networking
---
sys/dev/mlx5/cq.h | 2 +-
sys/dev/mlx5/device.h | 35 ++--
sys/dev/mlx5/driver.h | 89 ++++----
sys/dev/mlx5/mlx5_core/mlx5_cq.c | 1 +
sys/dev/mlx5/mlx5_core/mlx5_eq.c | 15 +-
sys/dev/mlx5/mlx5_core/mlx5_main.c | 61 +++---
sys/dev/mlx5/mlx5_core/mlx5_uar.c | 340 +++++++++++++++++++++----------
sys/dev/mlx5/mlx5_en/en.h | 19 +-
sys/dev/mlx5/mlx5_en/en_rl.h | 1 -
sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 44 ++--
sys/dev/mlx5/mlx5_en/mlx5_en_rl.c | 16 +-
sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 2 +-
sys/dev/mlx5/mlx5_ib/mlx5_ib.h | 67 +++++-
sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c | 26 ++-
sys/dev/mlx5/mlx5_ib/mlx5_ib_gsi.c | 2 +-
sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c | 342 ++++++++++++++++++++++---------
sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c | 369 ++++++++++++++++------------------
sys/dev/mlx5/mlx5_ifc.h | 7 +-
sys/ofed/include/uapi/rdma/mlx5-abi.h | 36 +++-
19 files changed, 871 insertions(+), 603 deletions(-)
diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h
index 55386422efc2..d5e167498fd2 100644
--- a/sys/dev/mlx5/cq.h
+++ b/sys/dev/mlx5/cq.h
@@ -42,7 +42,7 @@ struct mlx5_core_cq {
int irqn;
void (*comp) (struct mlx5_core_cq *, struct mlx5_eqe *);
void (*event) (struct mlx5_core_cq *, int);
- struct mlx5_uar *uar;
+ struct mlx5_uars_page *uar;
u32 cons_index;
unsigned arm_sn;
struct mlx5_rsc_debug *dbg;
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index 3431193cdafd..69057d5f2470 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -245,10 +245,22 @@ enum {
};
enum {
- MLX5_BF_REGS_PER_PAGE = 4,
- MLX5_MAX_UAR_PAGES = 1 << 8,
- MLX5_NON_FP_BF_REGS_PER_PAGE = 2,
- MLX5_MAX_UUARS = MLX5_MAX_UAR_PAGES * MLX5_NON_FP_BF_REGS_PER_PAGE,
+ MLX5_ADAPTER_PAGE_SHIFT = 12,
+ MLX5_ADAPTER_PAGE_SIZE = 1 << MLX5_ADAPTER_PAGE_SHIFT,
+};
+
+enum {
+ MLX5_BFREGS_PER_UAR = 4,
+ MLX5_MAX_UARS = 1 << 8,
+ MLX5_NON_FP_BFREGS_PER_UAR = 2,
+ MLX5_FP_BFREGS_PER_UAR = MLX5_BFREGS_PER_UAR -
+ MLX5_NON_FP_BFREGS_PER_UAR,
+ MLX5_MAX_BFREGS = MLX5_MAX_UARS *
+ MLX5_NON_FP_BFREGS_PER_UAR,
+ MLX5_UARS_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
+ MLX5_NON_FP_BFREGS_IN_PAGE = MLX5_NON_FP_BFREGS_PER_UAR * MLX5_UARS_IN_PAGE,
+ MLX5_MIN_DYN_BFREGS = 512,
+ MLX5_MAX_DYN_BFREGS = 1024,
};
enum {
@@ -407,11 +419,6 @@ enum {
MLX5_MAX_PAGE_SHIFT = 31
};
-enum {
- MLX5_ADAPTER_PAGE_SHIFT = 12,
- MLX5_ADAPTER_PAGE_SIZE = 1 << MLX5_ADAPTER_PAGE_SHIFT,
-};
-
enum {
MLX5_CAP_OFF_CMDIF_CSUM = 46,
};
@@ -1146,16 +1153,6 @@ enum {
MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP = 0x2,
};
-enum {
- MLX5_NUM_UUARS_PER_PAGE = MLX5_NON_FP_BF_REGS_PER_PAGE,
- MLX5_DEF_TOT_UUARS = 8 * MLX5_NUM_UUARS_PER_PAGE,
-};
-
-enum {
- NUM_DRIVER_UARS = 4,
- NUM_LOW_LAT_UUARS = 4,
-};
-
enum {
MLX5_CAP_PORT_TYPE_IB = 0x0,
MLX5_CAP_PORT_TYPE_ETH = 0x1,
diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
index ec9c4f618f67..5d4f58d7e1fd 100644
--- a/sys/dev/mlx5/driver.h
+++ b/sys/dev/mlx5/driver.h
@@ -219,38 +219,6 @@ enum {
#define MLX5_PROT_MASK(link_mode) (1 << link_mode)
-struct mlx5_uuar_info {
- struct mlx5_uar *uars;
- int num_uars;
- int num_low_latency_uuars;
- unsigned long *bitmap;
- unsigned int *count;
- struct mlx5_bf *bfs;
-
- /*
- * protect uuar allocation data structs
- */
- struct mutex lock;
- u32 ver;
-};
-
-struct mlx5_bf {
- void __iomem *reg;
- void __iomem *regreg;
- int buf_size;
- struct mlx5_uar *uar;
- unsigned long offset;
- int need_lock;
- /* protect blue flame buffer selection when needed
- */
- spinlock_t lock;
-
- /* serialize 64 bit writes when done as two 32 bit accesses
- */
- spinlock_t lock32;
- int uuarn;
-};
-
struct mlx5_cmd_first {
__be32 data[4];
};
@@ -464,6 +432,39 @@ struct mlx5_core_rsc_common {
struct completion free;
};
+struct mlx5_uars_page {
+ void __iomem *map;
+ bool wc;
+ u32 index;
+ struct list_head list;
+ unsigned int bfregs;
+ unsigned long *reg_bitmap; /* for non fast path bf regs */
+ unsigned long *fp_bitmap;
+ unsigned int reg_avail;
+ unsigned int fp_avail;
+ struct kref ref_count;
+ struct mlx5_core_dev *mdev;
+};
+
+struct mlx5_bfreg_head {
+ /* protect blue flame registers allocations */
+ struct mutex lock;
+ struct list_head list;
+};
+
+struct mlx5_bfreg_data {
+ struct mlx5_bfreg_head reg_head;
+ struct mlx5_bfreg_head wc_head;
+};
+
+struct mlx5_sq_bfreg {
+ void __iomem *map;
+ struct mlx5_uars_page *up;
+ bool wc;
+ u32 index;
+ unsigned int offset;
+};
+
struct mlx5_core_srq {
struct mlx5_core_rsc_common common; /* must be first */
u32 srqn;
@@ -489,13 +490,6 @@ struct mlx5_eq_table {
spinlock_t lock;
};
-struct mlx5_uar {
- u32 index;
- void __iomem *bf_map;
- void __iomem *map;
-};
-
-
struct mlx5_core_health {
struct mlx5_health_buffer __iomem *health;
__be32 __iomem *health_counter;
@@ -578,12 +572,9 @@ struct mlx5_priv {
char name[MLX5_MAX_NAME_LEN];
struct mlx5_eq_table eq_table;
struct msix_entry *msix_arr;
- struct mlx5_uuar_info uuari;
MLX5_DECLARE_DOORBELL_LOCK(cq_uar_lock);
int disable_irqs;
- struct io_mapping *bf_mapping;
-
/* pages stuff */
struct workqueue_struct *pg_wq;
struct rb_root page_root;
@@ -632,6 +623,9 @@ struct mlx5_priv {
struct mlx5_pme_stats pme_stats;
struct mlx5_eswitch *eswitch;
+
+ struct mlx5_bfreg_data bfregs;
+ struct mlx5_uars_page *uar;
};
enum mlx5_device_state {
@@ -976,10 +970,11 @@ int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size);
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn);
-int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
-int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
-int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar);
-void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar);
+int mlx5_alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
+ bool map_wc, bool fast_path);
+void mlx5_free_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg);
+struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev);
+void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up);
void mlx5_health_cleanup(struct mlx5_core_dev *dev);
int mlx5_health_init(struct mlx5_core_dev *dev);
void mlx5_start_health_poll(struct mlx5_core_dev *dev);
@@ -1049,7 +1044,7 @@ struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn);
void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vector, enum mlx5_cmd_mode mode);
void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type);
int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
- int nent, u64 mask, struct mlx5_uar *uar);
+ int nent, u64 mask);
int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
int mlx5_start_eqs(struct mlx5_core_dev *dev);
int mlx5_stop_eqs(struct mlx5_core_dev *dev);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cq.c b/sys/dev/mlx5/mlx5_core/mlx5_cq.c
index 929fd1d6bc31..248d8c9d75c8 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_cq.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_cq.c
@@ -149,6 +149,7 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
goto err_cmd;
cq->pid = curthread->td_proc->p_pid;
+ cq->uar = dev->priv.uar;
return 0;
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
index 9e744e132c36..2dc134d0f9c4 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
@@ -419,7 +419,7 @@ static void init_eq_buf(struct mlx5_eq *eq)
}
int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
- int nent, u64 mask, struct mlx5_uar *uar)
+ int nent, u64 mask)
{
u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
struct mlx5_priv *priv = &dev->priv;
@@ -454,7 +454,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
- MLX5_SET(eqc, eqc, uar_page, uar->index);
+ MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
MLX5_SET(eqc, eqc, intr, vecidx);
MLX5_SET(eqc, eqc, log_page_size,
eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
@@ -466,7 +466,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
eq->irqn = vecidx;
eq->dev = dev;
- eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
+ eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
err = request_irq(priv->msix_arr[vecidx].vector, mlx5_msix_handler, 0,
"mlx5_core", eq);
if (err)
@@ -569,8 +569,7 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
}
err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
- MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
- &dev->priv.uuari.uars[0]);
+ MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD);
if (err) {
mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
return err;
@@ -579,8 +578,7 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
mlx5_cmd_use_events(dev);
err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
- MLX5_NUM_ASYNC_EQE, async_event_mask,
- &dev->priv.uuari.uars[0]);
+ MLX5_NUM_ASYNC_EQE, async_event_mask);
if (err) {
mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
goto err1;
@@ -589,8 +587,7 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
err = mlx5_create_map_eq(dev, &table->pages_eq,
MLX5_EQ_VEC_PAGES,
/* TODO: sriov max_vf + */ 1,
- 1 << MLX5_EVENT_TYPE_PAGE_REQUEST,
- &dev->priv.uuari.uars[0]);
+ 1 << MLX5_EVENT_TYPE_PAGE_REQUEST);
if (err) {
mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
goto err2;
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
index 843227f928fa..27326d692261 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2019, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -455,6 +455,12 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
/* disable cmdif checksum */
MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
+ /* Enable 4K UAR only when HCA supports it and page size is bigger
+ * than 4K.
+ */
+ if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096)
+ MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1);
+
/* enable drain sigerr */
MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1);
@@ -650,8 +656,7 @@ static int alloc_comp_eqs(struct mlx5_core_dev *dev)
eq = kzalloc(sizeof(*eq), GFP_KERNEL);
err = mlx5_create_map_eq(dev, eq,
- i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
- &dev->priv.uuari.uars[0]);
+ i + MLX5_EQ_VEC_COMP_BASE, nent, 0);
if (err) {
kfree(eq);
goto clean;
@@ -670,22 +675,6 @@ clean:
return err;
}
-static int map_bf_area(struct mlx5_core_dev *dev)
-{
- resource_size_t bf_start = pci_resource_start(dev->pdev, 0);
- resource_size_t bf_len = pci_resource_len(dev->pdev, 0);
-
- dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len);
-
- return dev->priv.bf_mapping ? 0 : -ENOMEM;
-}
-
-static void unmap_bf_area(struct mlx5_core_dev *dev)
-{
- if (dev->priv.bf_mapping)
- io_mapping_free(dev->priv.bf_mapping);
-}
-
static inline int fw_initializing(struct mlx5_core_dev *dev)
{
return ioread32be(&dev->iseg->initializing) >> 31;
@@ -1117,22 +1106,23 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
goto err_stop_poll;
}
- err = mlx5_enable_msix(dev);
- if (err) {
- mlx5_core_err(dev, "enable msix failed\n");
+ dev->priv.uar = mlx5_get_uars_page(dev);
+ if (IS_ERR(dev->priv.uar)) {
+ mlx5_core_err(dev, "Failed allocating uar, aborting\n");
+ err = PTR_ERR(dev->priv.uar);
goto err_cleanup_once;
}
- err = mlx5_alloc_uuars(dev, &priv->uuari);
+ err = mlx5_enable_msix(dev);
if (err) {
- mlx5_core_err(dev, "Failed allocating uar, aborting\n");
- goto err_disable_msix;
+ mlx5_core_err(dev, "enable msix failed\n");
+ goto err_cleanup_uar;
}
err = mlx5_start_eqs(dev);
if (err) {
mlx5_core_err(dev, "Failed to start pages and async EQs\n");
- goto err_free_uar;
+ goto err_disable_msix;
}
err = alloc_comp_eqs(dev);
@@ -1141,9 +1131,6 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
goto err_stop_eqs;
}
- if (map_bf_area(dev))
- mlx5_core_err(dev, "Failed to map blue flame area\n");
-
err = mlx5_init_fs(dev);
if (err) {
mlx5_core_err(dev, "flow steering init %d\n", err);
@@ -1185,17 +1172,16 @@ err_fs:
err_free_comp_eqs:
free_comp_eqs(dev);
- unmap_bf_area(dev);
err_stop_eqs:
mlx5_stop_eqs(dev);
-err_free_uar:
- mlx5_free_uuars(dev, &priv->uuari);
-
err_disable_msix:
mlx5_disable_msix(dev);
+err_cleanup_uar:
+ mlx5_put_uars_page(dev, dev->priv.uar);
+
err_cleanup_once:
if (boot)
mlx5_cleanup_once(dev);
@@ -1248,12 +1234,11 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
mlx5_fpga_device_stop(dev);
mlx5_mpfs_destroy(dev);
mlx5_cleanup_fs(dev);
- unmap_bf_area(dev);
mlx5_wait_for_reclaim_vfs_pages(dev);
free_comp_eqs(dev);
mlx5_stop_eqs(dev);
- mlx5_free_uuars(dev, &priv->uuari);
mlx5_disable_msix(dev);
+ mlx5_put_uars_page(dev, dev->priv.uar);
if (cleanup)
mlx5_cleanup_once(dev);
mlx5_stop_health_poll(dev, cleanup);
@@ -1586,6 +1571,12 @@ static int init_one(struct pci_dev *pdev,
spin_lock_init(&priv->ctx_lock);
mutex_init(&dev->pci_status_mutex);
mutex_init(&dev->intf_state_mutex);
+
+ mutex_init(&priv->bfregs.reg_head.lock);
+ mutex_init(&priv->bfregs.wc_head.lock);
+ INIT_LIST_HEAD(&priv->bfregs.reg_head.list);
+ INIT_LIST_HEAD(&priv->bfregs.wc_head.list);
+
mtx_init(&dev->dump_lock, "mlx5dmp", NULL, MTX_DEF | MTX_NEW);
err = mlx5_pci_init(dev, priv);
if (err) {
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_uar.c b/sys/dev/mlx5/mlx5_core/mlx5_uar.c
index 43a95d64bc88..1a662ade0aee 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_uar.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_uar.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2020, Mellanox Technologies. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -33,171 +33,291 @@
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
{
- u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {0};
u32 out[MLX5_ST_SZ_DW(alloc_uar_out)] = {0};
+ u32 in[MLX5_ST_SZ_DW(alloc_uar_in)] = {0};
int err;
MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
-
err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
- if (err)
- return err;
-
- *uarn = MLX5_GET(alloc_uar_out, out, uar);
-
- return 0;
+ if (!err)
+ *uarn = MLX5_GET(alloc_uar_out, out, uar);
+ return err;
}
EXPORT_SYMBOL(mlx5_cmd_alloc_uar);
int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
{
- u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {0};
u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)] = {0};
+ u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)] = {0};
MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);
MLX5_SET(dealloc_uar_in, in, uar, uarn);
-
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_cmd_free_uar);
-static int need_uuar_lock(int uuarn)
+static int uars_per_sys_page(struct mlx5_core_dev *mdev)
{
- int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE;
-
- if (uuarn == 0 || tot_uuars - NUM_LOW_LAT_UUARS)
- return 0;
+ if (MLX5_CAP_GEN(mdev, uar_4k))
+ return MLX5_CAP_GEN(mdev, num_of_uars_per_page);
return 1;
}
-int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari)
+static u64 uar2pfn(struct mlx5_core_dev *mdev, u32 index)
{
- int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE;
- struct mlx5_bf *bf;
- phys_addr_t addr;
- int err;
- int i;
+ u32 system_page_index;
- uuari->num_uars = NUM_DRIVER_UARS;
- uuari->num_low_latency_uuars = NUM_LOW_LAT_UUARS;
+ if (MLX5_CAP_GEN(mdev, uar_4k))
+ system_page_index = index >> (PAGE_SHIFT - MLX5_ADAPTER_PAGE_SHIFT);
+ else
+ system_page_index = index;
- mutex_init(&uuari->lock);
- uuari->uars = kcalloc(uuari->num_uars, sizeof(*uuari->uars), GFP_KERNEL);
+ return (pci_resource_start(mdev->pdev, 0) >> PAGE_SHIFT) + system_page_index;
+}
- uuari->bfs = kcalloc(tot_uuars, sizeof(*uuari->bfs), GFP_KERNEL);
+static void up_rel_func(struct kref *kref)
+{
+ struct mlx5_uars_page *up = container_of(kref, struct mlx5_uars_page, ref_count);
+
+ list_del(&up->list);
+ iounmap(up->map);
+ if (mlx5_cmd_free_uar(up->mdev, up->index))
+ mlx5_core_warn(up->mdev, "failed to free uar index %d\n", up->index);
+ bitmap_free(up->reg_bitmap);
+ bitmap_free(up->fp_bitmap);
+ kfree(up);
+}
- uuari->bitmap = kcalloc(BITS_TO_LONGS(tot_uuars), sizeof(*uuari->bitmap),
- GFP_KERNEL);
+static struct mlx5_uars_page *alloc_uars_page(struct mlx5_core_dev *mdev,
+ bool map_wc)
+{
+ struct mlx5_uars_page *up;
+ int err = -ENOMEM;
+ phys_addr_t pfn;
+ int bfregs;
+ int i;
- uuari->count = kcalloc(tot_uuars, sizeof(*uuari->count), GFP_KERNEL);
+ bfregs = uars_per_sys_page(mdev) * MLX5_BFREGS_PER_UAR;
+ up = kzalloc(sizeof(*up), GFP_KERNEL);
+ if (!up)
+ return ERR_PTR(err);
- for (i = 0; i < uuari->num_uars; i++) {
- err = mlx5_cmd_alloc_uar(dev, &uuari->uars[i].index);
- if (err)
- goto out_count;
+ up->mdev = mdev;
+ up->reg_bitmap = bitmap_zalloc(bfregs, GFP_KERNEL);
+ if (!up->reg_bitmap)
+ goto error1;
- addr = pci_resource_start(dev->pdev, 0) +
- ((phys_addr_t)(uuari->uars[i].index) << PAGE_SHIFT);
- uuari->uars[i].map = ioremap(addr, PAGE_SIZE);
- if (!uuari->uars[i].map) {
- mlx5_cmd_free_uar(dev, uuari->uars[i].index);
- err = -ENOMEM;
- goto out_count;
- }
- mlx5_core_dbg(dev, "allocated uar index 0x%x, mmaped at %p\n",
- uuari->uars[i].index, uuari->uars[i].map);
- }
+ up->fp_bitmap = bitmap_zalloc(bfregs, GFP_KERNEL);
+ if (!up->fp_bitmap)
+ goto error1;
- for (i = 0; i < tot_uuars; i++) {
- bf = &uuari->bfs[i];
-
- bf->buf_size = (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) / 2;
- bf->uar = &uuari->uars[i / MLX5_BF_REGS_PER_PAGE];
- bf->regreg = uuari->uars[i / MLX5_BF_REGS_PER_PAGE].map;
- bf->reg = NULL; /* Add WC support */
- bf->offset = (i % MLX5_BF_REGS_PER_PAGE) *
- (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) +
- MLX5_BF_OFFSET;
- bf->need_lock = need_uuar_lock(i);
- spin_lock_init(&bf->lock);
- spin_lock_init(&bf->lock32);
- bf->uuarn = i;
- }
+ for (i = 0; i < bfregs; i++)
+ if ((i % MLX5_BFREGS_PER_UAR) < MLX5_NON_FP_BFREGS_PER_UAR)
+ set_bit(i, up->reg_bitmap);
+ else
+ set_bit(i, up->fp_bitmap);
- return 0;
+ up->bfregs = bfregs;
+ up->fp_avail = bfregs * MLX5_FP_BFREGS_PER_UAR / MLX5_BFREGS_PER_UAR;
+ up->reg_avail = bfregs * MLX5_NON_FP_BFREGS_PER_UAR / MLX5_BFREGS_PER_UAR;
-out_count:
- for (i--; i >= 0; i--) {
- iounmap(uuari->uars[i].map);
- mlx5_cmd_free_uar(dev, uuari->uars[i].index);
+ err = mlx5_cmd_alloc_uar(mdev, &up->index);
+ if (err) {
+ mlx5_core_warn(mdev, "mlx5_cmd_alloc_uar() failed, %d\n", err);
+ goto error1;
}
- kfree(uuari->count);
- kfree(uuari->bitmap);
+ pfn = uar2pfn(mdev, up->index);
+ if (map_wc) {
+ up->map = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE);
+ if (!up->map) {
+ err = -EAGAIN;
+ goto error2;
+ }
+ } else {
+ up->map = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
+ if (!up->map) {
+ err = -ENOMEM;
+ goto error2;
+ }
+ }
+ kref_init(&up->ref_count);
+ mlx5_core_dbg(mdev, "allocated UAR page: index %d, total bfregs %d\n",
+ up->index, up->bfregs);
+ return up;
+
+error2:
+ if (mlx5_cmd_free_uar(mdev, up->index))
+ mlx5_core_warn(mdev, "failed to free uar index %d\n", up->index);
+error1:
+ bitmap_free(up->fp_bitmap);
+ bitmap_free(up->reg_bitmap);
+ kfree(up);
+ return ERR_PTR(err);
+}
- kfree(uuari->bfs);
+struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev)
+{
+ struct mlx5_uars_page *ret;
+
+ mutex_lock(&mdev->priv.bfregs.reg_head.lock);
+ if (!list_empty(&mdev->priv.bfregs.reg_head.list)) {
+ ret = list_first_entry(&mdev->priv.bfregs.reg_head.list,
+ struct mlx5_uars_page, list);
+ kref_get(&ret->ref_count);
+ goto out;
+ }
+ ret = alloc_uars_page(mdev, false);
+ if (IS_ERR(ret))
+ goto out;
+ list_add(&ret->list, &mdev->priv.bfregs.reg_head.list);
+out:
+ mutex_unlock(&mdev->priv.bfregs.reg_head.lock);
+
+ return ret;
+}
+EXPORT_SYMBOL(mlx5_get_uars_page);
- kfree(uuari->uars);
- return err;
+void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up)
+{
+ mutex_lock(&mdev->priv.bfregs.reg_head.lock);
+ kref_put(&up->ref_count, up_rel_func);
+ mutex_unlock(&mdev->priv.bfregs.reg_head.lock);
}
+EXPORT_SYMBOL(mlx5_put_uars_page);
-int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari)
+static unsigned long map_offset(struct mlx5_core_dev *mdev, int dbi)
{
- int i = uuari->num_uars;
+ /* return the offset in bytes from the start of the page to the
+ * blue flame area of the UAR
+ */
+ return dbi / MLX5_BFREGS_PER_UAR * MLX5_ADAPTER_PAGE_SIZE +
+ (dbi % MLX5_BFREGS_PER_UAR) *
+ (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) + MLX5_BF_OFFSET;
+}
- for (i--; i >= 0; i--) {
- iounmap(uuari->uars[i].map);
- mlx5_cmd_free_uar(dev, uuari->uars[i].index);
+static int alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
+ bool map_wc, bool fast_path)
+{
+ struct mlx5_bfreg_data *bfregs;
+ struct mlx5_uars_page *up;
+ struct list_head *head;
+ unsigned long *bitmap;
+ unsigned int *avail;
+ struct mutex *lock; /* pointer to right mutex */
+ int dbi;
+
+ bfregs = &mdev->priv.bfregs;
+ if (map_wc) {
+ head = &bfregs->wc_head.list;
+ lock = &bfregs->wc_head.lock;
+ } else {
+ head = &bfregs->reg_head.list;
+ lock = &bfregs->reg_head.lock;
}
-
- kfree(uuari->count);
- kfree(uuari->bitmap);
- kfree(uuari->bfs);
- kfree(uuari->uars);
+ mutex_lock(lock);
+ if (list_empty(head)) {
+ up = alloc_uars_page(mdev, map_wc);
+ if (IS_ERR(up)) {
+ mutex_unlock(lock);
+ return PTR_ERR(up);
+ }
+ list_add(&up->list, head);
+ } else {
+ up = list_entry(head->next, struct mlx5_uars_page, list);
+ kref_get(&up->ref_count);
+ }
+ if (fast_path) {
+ bitmap = up->fp_bitmap;
+ avail = &up->fp_avail;
+ } else {
+ bitmap = up->reg_bitmap;
+ avail = &up->reg_avail;
+ }
+ dbi = find_first_bit(bitmap, up->bfregs);
+ clear_bit(dbi, bitmap);
+ (*avail)--;
+ if (!(*avail))
+ list_del(&up->list);
+
+ bfreg->map = up->map + map_offset(mdev, dbi);
+ bfreg->up = up;
+ bfreg->wc = map_wc;
+ bfreg->index = up->index + dbi / MLX5_BFREGS_PER_UAR;
+ mutex_unlock(lock);
return 0;
}
-int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar)
+int mlx5_alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg,
+ bool map_wc, bool fast_path)
{
- phys_addr_t pfn;
- phys_addr_t uar_bar_start;
int err;
- err = mlx5_cmd_alloc_uar(mdev, &uar->index);
- if (err) {
- mlx5_core_warn(mdev, "mlx5_cmd_alloc_uar() failed, %d\n", err);
- return err;
- }
+ err = alloc_bfreg(mdev, bfreg, map_wc, fast_path);
+ if (!err)
+ return 0;
- uar_bar_start = pci_resource_start(mdev->pdev, 0);
- pfn = (uar_bar_start >> PAGE_SHIFT) + uar->index;
- uar->map = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
- if (!uar->map) {
- mlx5_core_warn(mdev, "ioremap() failed, %d\n", err);
- err = -ENOMEM;
- goto err_free_uar;
- }
+ if (err == -EAGAIN && map_wc)
+ return alloc_bfreg(mdev, bfreg, false, fast_path);
- if (mdev->priv.bf_mapping)
- uar->bf_map = io_mapping_map_wc(mdev->priv.bf_mapping,
- uar->index << PAGE_SHIFT,
- PAGE_SIZE);
+ return err;
+}
+EXPORT_SYMBOL(mlx5_alloc_bfreg);
- return 0;
+static unsigned int addr_to_dbi_in_syspage(struct mlx5_core_dev *dev,
+ struct mlx5_uars_page *up,
+ struct mlx5_sq_bfreg *bfreg)
+{
+ unsigned int uar_idx;
+ unsigned int bfreg_idx;
+ unsigned int bf_reg_size;
-err_free_uar:
- mlx5_cmd_free_uar(mdev, uar->index);
+ bf_reg_size = 1 << MLX5_CAP_GEN(dev, log_bf_reg_size);
- return err;
+ uar_idx = (bfreg->map - up->map) >> MLX5_ADAPTER_PAGE_SHIFT;
+ bfreg_idx = (((uintptr_t)bfreg->map % MLX5_ADAPTER_PAGE_SIZE) - MLX5_BF_OFFSET) / bf_reg_size;
+
+ return uar_idx * MLX5_BFREGS_PER_UAR + bfreg_idx;
}
-EXPORT_SYMBOL(mlx5_alloc_map_uar);
-void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar)
+void mlx5_free_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg)
{
- io_mapping_unmap(uar->bf_map);
- iounmap(uar->map);
- mlx5_cmd_free_uar(mdev, uar->index);
+ struct mlx5_bfreg_data *bfregs;
+ struct mlx5_uars_page *up;
+ struct mutex *lock; /* pointer to right mutex */
+ unsigned int dbi;
+ bool fp;
+ unsigned int *avail;
+ unsigned long *bitmap;
+ struct list_head *head;
+
+ bfregs = &mdev->priv.bfregs;
+ if (bfreg->wc) {
+ head = &bfregs->wc_head.list;
+ lock = &bfregs->wc_head.lock;
+ } else {
+ head = &bfregs->reg_head.list;
+ lock = &bfregs->reg_head.lock;
+ }
+ up = bfreg->up;
+ dbi = addr_to_dbi_in_syspage(mdev, up, bfreg);
+ fp = (dbi % MLX5_BFREGS_PER_UAR) >= MLX5_NON_FP_BFREGS_PER_UAR;
+ if (fp) {
+ avail = &up->fp_avail;
+ bitmap = up->fp_bitmap;
+ } else {
+ avail = &up->reg_avail;
+ bitmap = up->reg_bitmap;
+ }
+ mutex_lock(lock);
+ (*avail)++;
+ set_bit(dbi, bitmap);
+ if (*avail == 1)
+ list_add_tail(&up->list, head);
+
+ kref_put(&up->ref_count, up_rel_func);
+ mutex_unlock(lock);
}
-EXPORT_SYMBOL(mlx5_unmap_free_uar);
+EXPORT_SYMBOL(mlx5_free_bfreg);
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 4c3d696ed41c..b7c05264008a 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -827,7 +827,7 @@ struct mlx5e_sq {
/* read only */
struct mlx5_wq_cyc wq;
- struct mlx5_uar uar;
+ void __iomem *uar_map;
struct ifnet *ifp;
u32 sqn;
u32 bf_buf_size;
@@ -1001,7 +1001,6 @@ struct mlx5e_priv {
#define PRIV_LOCKED(priv) sx_xlocked(&(priv)->state_lock)
#define PRIV_ASSERT_LOCKED(priv) sx_assert(&(priv)->state_lock, SA_XLOCKED)
struct sx state_lock; /* Protects Interface state */
- struct mlx5_uar cq_uar;
u32 pdn;
u32 tdn;
struct mlx5_core_mr mr;
@@ -1055,6 +1054,8 @@ struct mlx5e_priv {
struct mlx5e_dcbx dcbx;
bool sw_is_port_buf_owner;
+ struct mlx5_sq_bfreg bfreg;
+
struct pfil_head *pfil;
struct mlx5e_channel channel[];
};
@@ -1127,7 +1128,7 @@ int mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv);
void mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv);
static inline void
-mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe, int bf_sz)
+mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe)
{
u16 ofst = MLX5_BF_OFFSET + sq->bf_offset;
@@ -1142,16 +1143,8 @@ mlx5e_tx_notify_hw(struct mlx5e_sq *sq, u32 *wqe, int bf_sz)
*/
wmb();
- if (bf_sz) {
- __iowrite64_copy(sq->uar.bf_map + ofst, wqe, bf_sz);
-
- /* flush the write-combining mapped buffer */
- wmb();
-
- } else {
- mlx5_write64(wqe, sq->uar.map + ofst,
- MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
- }
+ mlx5_write64(wqe, sq->uar_map + ofst,
+ MLX5_GET_DOORBELL_LOCK(&sq->priv->doorbell_lock));
sq->bf_offset ^= sq->bf_buf_size;
}
diff --git a/sys/dev/mlx5/mlx5_en/en_rl.h b/sys/dev/mlx5/mlx5_en/en_rl.h
index 6ad2f26e5a12..f30e8ba8cc07 100644
--- a/sys/dev/mlx5/mlx5_en/en_rl.h
+++ b/sys/dev/mlx5/mlx5_en/en_rl.h
@@ -156,7 +156,6 @@ struct mlx5e_rl_priv_data {
struct mlx5e_rl_channel_param chan_param;
struct mlx5e_rl_params param;
struct mlx5e_rl_stats stats;
- struct mlx5_uar sq_uar;
struct mlx5e_rl_worker *workers;
struct mlx5e_priv *priv;
uint64_t *rate_limit_table;
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 1e6562fed667..9fc3af186c0d 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -1660,14 +1660,12 @@ mlx5e_create_sq(struct mlx5e_channel *c,
&sq->dma_tag)))
goto done;
*** 1680 LINES SKIPPED ***
More information about the dev-commits-src-all
mailing list