svn commit: r312882 - in head/sys/dev/mlx5: . mlx5_core
Hans Petter Selasky
hselasky at FreeBSD.org
Fri Jan 27 11:46:57 UTC 2017
Author: hselasky
Date: Fri Jan 27 11:46:55 2017
New Revision: 312882
URL: https://svnweb.freebsd.org/changeset/base/312882
Log:
Use the busdma API to allocate all DMA-able memory.
The MLX5 driver has four different types of DMA allocations which are
now allocated using busdma:
1) The 4K firmware DMA-able blocks. One busdma object per 4K allocation.
2) Data for firmware commands use the 4K firmware blocks split into four 1K blocks.
3) The 4K firmware blocks are also used for doorbell pages.
4) The RQ-, SQ- and CQ- DMA rings. One busdma object per allocation.
After this patch the mlx5en driver can be used with DMAR enabled in
the FreeBSD kernel.
MFC after: 1 week
Sponsored by: Mellanox Technologies
Modified:
head/sys/dev/mlx5/device.h
head/sys/dev/mlx5/driver.h
head/sys/dev/mlx5/mlx5_core/mlx5_alloc.c
head/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
head/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c
Modified: head/sys/dev/mlx5/device.h
==============================================================================
--- head/sys/dev/mlx5/device.h Fri Jan 27 11:29:33 2017 (r312881)
+++ head/sys/dev/mlx5/device.h Fri Jan 27 11:46:55 2017 (r312882)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -103,6 +103,7 @@ __mlx5_mask(typ, fld))
enum {
MLX5_MAX_COMMANDS = 32,
MLX5_CMD_DATA_BLOCK_SIZE = 512,
+ MLX5_CMD_MBOX_SIZE = 1024,
MLX5_PCI_CMD_XPORT = 7,
MLX5_MKEY_BSF_OCTO_SIZE = 4,
MLX5_MAX_PSVS = 4,
@@ -523,6 +524,11 @@ struct mlx5_cmd_prot_block {
u8 sig;
};
+#define MLX5_NUM_CMDS_IN_ADAPTER_PAGE \
+ (MLX5_ADAPTER_PAGE_SIZE / MLX5_CMD_MBOX_SIZE)
+CTASSERT(MLX5_CMD_MBOX_SIZE >= sizeof(struct mlx5_cmd_prot_block));
+CTASSERT(MLX5_CMD_MBOX_SIZE <= MLX5_ADAPTER_PAGE_SIZE);
+
enum {
MLX5_CQE_SYND_FLUSHED_IN_ERROR = 5,
};
Modified: head/sys/dev/mlx5/driver.h
==============================================================================
--- head/sys/dev/mlx5/driver.h Fri Jan 27 11:29:33 2017 (r312881)
+++ head/sys/dev/mlx5/driver.h Fri Jan 27 11:46:55 2017 (r312882)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -258,13 +258,26 @@ struct mlx5_cmd_first {
__be32 data[4];
};
-struct mlx5_cmd_msg {
- struct list_head list;
- struct cache_ent *cache;
- u32 len;
- struct mlx5_cmd_first first;
- struct mlx5_cmd_mailbox *next;
+struct cache_ent;
+struct mlx5_fw_page {
+ union {
+ struct rb_node rb_node;
+ struct list_head list;
+ };
+ struct mlx5_cmd_first first;
+ struct mlx5_core_dev *dev;
+ bus_dmamap_t dma_map;
+ bus_addr_t dma_addr;
+ void *virt_addr;
+ struct cache_ent *cache;
+ u32 numpages;
+ u16 load_done;
+#define MLX5_LOAD_ST_NONE 0
+#define MLX5_LOAD_ST_SUCCESS 1
+#define MLX5_LOAD_ST_FAILURE 2
+ u16 func_id;
};
+#define mlx5_cmd_msg mlx5_fw_page
struct mlx5_cmd_debug {
struct dentry *dbg_root;
@@ -304,9 +317,16 @@ struct mlx5_cmd_stats {
};
struct mlx5_cmd {
- void *cmd_alloc_buf;
- dma_addr_t alloc_dma;
- int alloc_size;
+ struct mlx5_fw_page *cmd_page;
+ bus_dma_tag_t dma_tag;
+ struct sx dma_sx;
+ struct mtx dma_mtx;
+#define MLX5_DMA_OWNED(dev) mtx_owned(&(dev)->cmd.dma_mtx)
+#define MLX5_DMA_LOCK(dev) mtx_lock(&(dev)->cmd.dma_mtx)
+#define MLX5_DMA_UNLOCK(dev) mtx_unlock(&(dev)->cmd.dma_mtx)
+ struct cv dma_cv;
+#define MLX5_DMA_DONE(dev) cv_broadcast(&(dev)->cmd.dma_cv)
+#define MLX5_DMA_WAIT(dev) cv_wait(&(dev)->cmd.dma_cv, &(dev)->cmd.dma_mtx)
void *cmd_buf;
dma_addr_t dma;
u16 cmdif_rev;
@@ -331,7 +351,6 @@ struct mlx5_cmd {
struct semaphore pages_sem;
int mode;
struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS];
- struct pci_pool *pool;
struct mlx5_cmd_debug dbg;
struct cmd_msg_cache cache;
int checksum_disabled;
@@ -345,24 +364,18 @@ struct mlx5_port_caps {
u8 ext_port_cap;
};
-struct mlx5_cmd_mailbox {
- void *buf;
- dma_addr_t dma;
- struct mlx5_cmd_mailbox *next;
-};
-
-struct mlx5_buf_list {
- void *buf;
- dma_addr_t map;
-};
-
struct mlx5_buf {
- struct mlx5_buf_list direct;
- struct mlx5_buf_list *page_list;
- int nbufs;
+ bus_dma_tag_t dma_tag;
+ bus_dmamap_t dma_map;
+ struct mlx5_core_dev *dev;
+ struct {
+ void *buf;
+ } direct;
+ u64 *page_list;
int npages;
int size;
u8 page_shift;
+ u8 load_done;
};
struct mlx5_eq {
@@ -521,7 +534,6 @@ struct mlx5_priv {
struct rb_root page_root;
s64 fw_pages;
atomic_t reg_pages;
- struct list_head free_list;
s64 pages_per_func[MLX5_MAX_NUMBER_OF_VFS];
struct mlx5_core_health health;
@@ -664,7 +676,7 @@ struct mlx5_vport_counters {
};
enum {
- MLX5_DB_PER_PAGE = PAGE_SIZE / L1_CACHE_BYTES,
+ MLX5_DB_PER_PAGE = MLX5_ADAPTER_PAGE_SIZE / L1_CACHE_BYTES,
};
struct mlx5_core_dct {
@@ -688,6 +700,7 @@ enum {
struct mlx5_db_pgdir {
struct list_head list;
DECLARE_BITMAP(bitmap, MLX5_DB_PER_PAGE);
+ struct mlx5_fw_page *fw_page;
__be32 *db_page;
dma_addr_t db_dma;
};
@@ -697,6 +710,7 @@ typedef void (*mlx5_cmd_cbk_t)(int statu
struct mlx5_cmd_work_ent {
struct mlx5_cmd_msg *in;
struct mlx5_cmd_msg *out;
+ int uin_size;
void *uout;
int uout_size;
mlx5_cmd_cbk_t callback;
@@ -721,13 +735,10 @@ struct mlx5_pas {
u8 log_sz;
};
-static inline void *mlx5_buf_offset(struct mlx5_buf *buf, int offset)
+static inline void *
+mlx5_buf_offset(struct mlx5_buf *buf, int offset)
{
- if (likely(BITS_PER_LONG == 64 || buf->nbufs == 1))
- return buf->direct.buf + offset;
- else
- return buf->page_list[offset >> PAGE_SHIFT].buf +
- (offset & (PAGE_SIZE - 1));
+ return ((char *)buf->direct.buf + offset);
}
@@ -816,8 +827,9 @@ void mlx5_health_cleanup(void);
void __init mlx5_health_init(void);
void mlx5_start_health_poll(struct mlx5_core_dev *dev);
void mlx5_stop_health_poll(struct mlx5_core_dev *dev);
-int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int max_direct,
- struct mlx5_buf *buf, int node);
+
+#define mlx5_buf_alloc_node(dev, size, direct, buf, node) \
+ mlx5_buf_alloc(dev, size, direct, buf)
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, int max_direct,
struct mlx5_buf *buf);
void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf);
@@ -845,6 +857,12 @@ int mlx5_core_alloc_pd(struct mlx5_core_
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn);
int mlx5_core_mad_ifc(struct mlx5_core_dev *dev, void *inb, void *outb,
u16 opmod, u8 port);
+void mlx5_fwp_flush(struct mlx5_fw_page *fwp);
+void mlx5_fwp_invalidate(struct mlx5_fw_page *fwp);
+struct mlx5_fw_page *mlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num);
+void mlx5_fwp_free(struct mlx5_fw_page *fwp);
+u64 mlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset);
+void *mlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset);
void mlx5_pagealloc_init(struct mlx5_core_dev *dev);
void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev);
int mlx5_pagealloc_start(struct mlx5_core_dev *dev);
Modified: head/sys/dev/mlx5/mlx5_core/mlx5_alloc.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_alloc.c Fri Jan 27 11:29:33 2017 (r312881)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_alloc.c Fri Jan 27 11:46:55 2017 (r312882)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -40,106 +40,110 @@
* multiple pages, so we don't require too much contiguous memory.
*/
-static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
- size_t size, dma_addr_t *dma_handle,
- int node)
+static void
+mlx5_buf_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
- void *cpu_handle;
-
- cpu_handle = dma_zalloc_coherent(&dev->pdev->dev, size,
- dma_handle, GFP_KERNEL);
- return cpu_handle;
-}
-
-int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int max_direct,
- struct mlx5_buf *buf, int node)
-{
- dma_addr_t t;
-
- buf->size = size;
- if (size <= max_direct) {
- buf->nbufs = 1;
- buf->npages = 1;
- buf->page_shift = (u8)get_order(size) + PAGE_SHIFT;
- buf->direct.buf = mlx5_dma_zalloc_coherent_node(dev, size,
- &t, node);
- if (!buf->direct.buf)
- return -ENOMEM;
-
- buf->direct.map = t;
-
- while (t & ((1 << buf->page_shift) - 1)) {
- --buf->page_shift;
- buf->npages *= 2;
+ struct mlx5_buf *buf;
+ uint8_t owned;
+ int x;
+
+ buf = (struct mlx5_buf *)arg;
+ owned = MLX5_DMA_OWNED(buf->dev);
+
+ if (!owned)
+ MLX5_DMA_LOCK(buf->dev);
+
+ if (error == 0) {
+ for (x = 0; x != nseg; x++) {
+ buf->page_list[x] = segs[x].ds_addr;
+ KASSERT(segs[x].ds_len == PAGE_SIZE, ("Invalid segment size"));
}
+ buf->load_done = MLX5_LOAD_ST_SUCCESS;
} else {
- int i;
-
- buf->direct.buf = NULL;
- buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
- buf->npages = buf->nbufs;
- buf->page_shift = PAGE_SHIFT;
- buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
- GFP_KERNEL);
-
- for (i = 0; i < buf->nbufs; i++) {
- buf->page_list[i].buf =
- mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
- &t, node);
+ buf->load_done = MLX5_LOAD_ST_FAILURE;
+ }
+ MLX5_DMA_DONE(buf->dev);
- buf->page_list[i].map = t;
- }
+ if (!owned)
+ MLX5_DMA_UNLOCK(buf->dev);
+}
- if (BITS_PER_LONG == 64) {
- struct page **pages;
+int
+mlx5_buf_alloc(struct mlx5_core_dev *dev, int size,
+ int max_direct, struct mlx5_buf *buf)
+{
+ int err;
- pages = kmalloc(sizeof(*pages) * (buf->nbufs + 1),
- GFP_KERNEL);
- for (i = 0; i < buf->nbufs; i++)
- pages[i] = virt_to_page(buf->page_list[i].buf);
- pages[buf->nbufs] = pages[0];
- buf->direct.buf = vmap(pages, buf->nbufs + 1, VM_MAP,
- PAGE_KERNEL);
- kfree(pages);
- if (!buf->direct.buf)
- goto err_free;
- }
+ buf->npages = howmany(size, PAGE_SIZE);
+ buf->page_shift = PAGE_SHIFT;
+ buf->load_done = MLX5_LOAD_ST_NONE;
+ buf->dev = dev;
+ buf->page_list = kcalloc(buf->npages, sizeof(*buf->page_list),
+ GFP_KERNEL);
+
+ err = -bus_dma_tag_create(
+ bus_get_dma_tag(dev->pdev->dev.bsddev),
+ PAGE_SIZE, /* alignment */
+ 0, /* no boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ PAGE_SIZE * buf->npages, /* maxsize */
+ buf->npages, /* nsegments */
+ PAGE_SIZE, /* maxsegsize */
+ 0, /* flags */
+ NULL, NULL, /* lockfunc, lockfuncarg */
+ &buf->dma_tag);
+
+ if (err != 0)
+ goto err_dma_tag;
+
+ /* allocate memory */
+ err = -bus_dmamem_alloc(buf->dma_tag, &buf->direct.buf,
+ BUS_DMA_WAITOK | BUS_DMA_COHERENT, &buf->dma_map);
+ if (err != 0)
+ goto err_dma_alloc;
+
+ /* load memory into DMA */
+ MLX5_DMA_LOCK(dev);
+ err = bus_dmamap_load(
+ buf->dma_tag, buf->dma_map, buf->direct.buf,
+ PAGE_SIZE * buf->npages, &mlx5_buf_load_mem_cb,
+ buf, BUS_DMA_WAITOK | BUS_DMA_COHERENT);
+
+ while (buf->load_done == MLX5_LOAD_ST_NONE)
+ MLX5_DMA_WAIT(dev);
+ MLX5_DMA_UNLOCK(dev);
+
+ /* check for error */
+ if (buf->load_done != MLX5_LOAD_ST_SUCCESS) {
+ err = -ENOMEM;
+ goto err_dma_load;
}
- return 0;
-
-err_free:
- mlx5_buf_free(dev, buf);
+ /* clean memory */
+ memset(buf->direct.buf, 0, PAGE_SIZE * buf->npages);
- return -ENOMEM;
+ /* flush memory to RAM */
+ bus_dmamap_sync(buf->dev->cmd.dma_tag, buf->dma_map, BUS_DMASYNC_PREWRITE);
+ return (0);
+
+err_dma_load:
+ bus_dmamem_free(buf->dma_tag, buf->direct.buf, buf->dma_map);
+err_dma_alloc:
+ bus_dma_tag_destroy(buf->dma_tag);
+err_dma_tag:
+ kfree(buf->page_list);
+ return (err);
}
-int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, int max_direct,
- struct mlx5_buf *buf)
-{
- return mlx5_buf_alloc_node(dev, size, max_direct,
- buf, dev->priv.numa_node);
-}
-EXPORT_SYMBOL_GPL(mlx5_buf_alloc);
-
-
void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf)
{
- if (buf->nbufs == 1)
- dma_free_coherent(&dev->pdev->dev, buf->size, buf->direct.buf,
- buf->direct.map);
- else {
- int i;
- if (BITS_PER_LONG == 64 && buf->direct.buf)
- vunmap(buf->direct.buf);
-
- for (i = 0; i < buf->nbufs; i++)
- if (buf->page_list[i].buf)
- dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
- buf->page_list[i].buf,
- buf->page_list[i].map);
- kfree(buf->page_list);
- }
+
+ bus_dmamap_unload(buf->dma_tag, buf->dma_map);
+ bus_dmamem_free(buf->dma_tag, buf->direct.buf, buf->dma_map);
+ bus_dma_tag_destroy(buf->dma_tag);
+ kfree(buf->page_list);
}
EXPORT_SYMBOL_GPL(mlx5_buf_free);
@@ -152,8 +156,17 @@ static struct mlx5_db_pgdir *mlx5_alloc_
bitmap_fill(pgdir->bitmap, MLX5_DB_PER_PAGE);
- pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
- &pgdir->db_dma, node);
+ pgdir->fw_page = mlx5_fwp_alloc(dev, GFP_KERNEL, 1);
+ if (pgdir->fw_page != NULL) {
+ pgdir->db_page = pgdir->fw_page->virt_addr;
+ pgdir->db_dma = pgdir->fw_page->dma_addr;
+
+ /* clean allocated memory */
+ memset(pgdir->db_page, 0, MLX5_ADAPTER_PAGE_SIZE);
+
+ /* flush memory to RAM */
+ mlx5_fwp_flush(pgdir->fw_page);
+ }
if (!pgdir->db_page) {
kfree(pgdir);
return NULL;
@@ -228,8 +241,7 @@ void mlx5_db_free(struct mlx5_core_dev *
__set_bit(db->index, db->u.pgdir->bitmap);
if (bitmap_full(db->u.pgdir->bitmap, MLX5_DB_PER_PAGE)) {
- dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
- db->u.pgdir->db_page, db->u.pgdir->db_dma);
+ mlx5_fwp_free(db->u.pgdir->fw_page);
list_del(&db->u.pgdir->list);
kfree(db->u.pgdir);
}
@@ -238,19 +250,12 @@ void mlx5_db_free(struct mlx5_core_dev *
}
EXPORT_SYMBOL_GPL(mlx5_db_free);
-
-void mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas)
+void
+mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas)
{
- u64 addr;
int i;
- for (i = 0; i < buf->npages; i++) {
- if (buf->nbufs == 1)
- addr = buf->direct.map + ((u64)i << buf->page_shift);
- else
- addr = buf->page_list[i].map;
-
- pas[i] = cpu_to_be64(addr);
- }
+ for (i = 0; i != buf->npages; i++)
+ pas[i] = cpu_to_be64(buf->page_list[i]);
}
EXPORT_SYMBOL_GPL(mlx5_fill_page_array);
Modified: head/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_cmd.c Fri Jan 27 11:29:33 2017 (r312881)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_cmd.c Fri Jan 27 11:46:55 2017 (r312882)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
+ * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -77,6 +77,7 @@ enum {
static struct mlx5_cmd_work_ent *alloc_cmd(struct mlx5_cmd *cmd,
struct mlx5_cmd_msg *in,
+ int uin_size,
struct mlx5_cmd_msg *out,
void *uout, int uout_size,
mlx5_cmd_cbk_t cbk,
@@ -90,6 +91,7 @@ static struct mlx5_cmd_work_ent *alloc_c
return ERR_PTR(-ENOMEM);
ent->in = in;
+ ent->uin_size = uin_size;
ent->out = out;
ent->uout = uout;
ent->uout_size = uout_size;
@@ -192,14 +194,26 @@ static void calc_block_sig(struct mlx5_c
}
}
-static void calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum)
+static void
+calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum)
{
- struct mlx5_cmd_mailbox *next = msg->next;
+ size_t i;
- while (next) {
- calc_block_sig(next->buf, token, csum);
- next = next->next;
+ for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) {
+ struct mlx5_cmd_prot_block *block;
+
+ block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE);
+
+ /* compute signature */
+ calc_block_sig(block, token, csum);
+
+ /* check for last block */
+ if (block->next == 0)
+ break;
}
+
+ /* make sure data gets written to RAM */
+ mlx5_fwp_flush(msg);
}
static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
@@ -235,10 +249,11 @@ static void free_cmd(struct mlx5_cmd_wor
kfree(ent);
}
-
-static int verify_signature(struct mlx5_cmd_work_ent *ent)
+static int
+verify_signature(struct mlx5_cmd_work_ent *ent)
{
- struct mlx5_cmd_mailbox *next = ent->out->next;
+ struct mlx5_cmd_msg *msg = ent->out;
+ size_t i;
int err;
u8 sig;
@@ -246,15 +261,21 @@ static int verify_signature(struct mlx5_
if (sig != 0xff)
return -EINVAL;
- while (next) {
- err = verify_block_sig(next->buf);
- if (err)
- return err;
+ for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) {
+ struct mlx5_cmd_prot_block *block;
- next = next->next;
- }
+ block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE);
- return 0;
+ /* compute signature */
+ err = verify_block_sig(block);
+ if (err != 0)
+ return (err);
+
+ /* check for last block */
+ if (block->next == 0)
+ break;
+ }
+ return (0);
}
static void dump_buf(void *buf, int size, int data_only, int offset)
@@ -681,9 +702,10 @@ static void dump_command(struct mlx5_cor
{
u16 op = be16_to_cpu(((struct mlx5_inbox_hdr *)(ent->lay->in))->opcode);
struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
- struct mlx5_cmd_mailbox *next = msg->next;
+ size_t i;
int data_only;
- u32 offset = 0;
+ int offset = 0;
+ int msg_len = input ? ent->uin_size : ent->uout_size;
int dump_len;
data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
@@ -711,17 +733,28 @@ static void dump_command(struct mlx5_cor
offset += sizeof(*ent->lay);
}
- while (next && offset < msg->len) {
+ for (i = 0; i != (msg->numpages * MLX5_NUM_CMDS_IN_ADAPTER_PAGE); i++) {
+ struct mlx5_cmd_prot_block *block;
+
+ block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE);
+
if (data_only) {
- dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
- dump_buf(next->buf, dump_len, 1, offset);
+ if (offset >= msg_len)
+ break;
+ dump_len = min_t(int,
+ MLX5_CMD_DATA_BLOCK_SIZE, msg_len - offset);
+
+ dump_buf(block->data, dump_len, 1, offset);
offset += MLX5_CMD_DATA_BLOCK_SIZE;
} else {
mlx5_core_dbg(dev, "command block:\n");
- dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset);
- offset += sizeof(struct mlx5_cmd_prot_block);
+ dump_buf(block, sizeof(*block), 0, offset);
+ offset += sizeof(*block);
}
- next = next->next;
+
+ /* check for last block */
+ if (block->next == 0)
+ break;
}
if (data_only)
@@ -982,12 +1015,12 @@ static void cmd_work_handler(struct work
memset(lay, 0, sizeof(*lay));
memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
ent->op = be32_to_cpu(lay->in[0]) >> 16;
- if (ent->in->next)
- lay->in_ptr = cpu_to_be64(ent->in->next->dma);
- lay->inlen = cpu_to_be32(ent->in->len);
- if (ent->out->next)
- lay->out_ptr = cpu_to_be64(ent->out->next->dma);
- lay->outlen = cpu_to_be32(ent->out->len);
+ if (ent->in->numpages != 0)
+ lay->in_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->in, 0));
+ if (ent->out->numpages != 0)
+ lay->out_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->out, 0));
+ lay->inlen = cpu_to_be32(ent->uin_size);
+ lay->outlen = cpu_to_be32(ent->uout_size);
lay->type = MLX5_PCI_CMD_XPORT;
lay->token = ent->token;
lay->status_own = CMD_OWNER_HW;
@@ -997,14 +1030,14 @@ static void cmd_work_handler(struct work
ent->busy = 0;
/* ring doorbell after the descriptor is valid */
mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
- wmb();
+ /* make sure data is written to RAM */
+ mlx5_fwp_flush(cmd->cmd_page);
iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
mmiowb();
/* if not in polling don't use ent after this point*/
if (cmd->mode == CMD_MODE_POLLING) {
poll_timeout(ent);
/* make sure we read the descriptor after ownership is SW */
- rmb();
mlx5_cmd_comp_handler(dev, 1U << ent->idx);
}
}
@@ -1078,6 +1111,7 @@ static int wait_func(struct mlx5_core_de
* 2. page queue commands do not support asynchrous completion
*/
static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
+ int uin_size,
struct mlx5_cmd_msg *out, void *uout, int uout_size,
mlx5_cmd_cbk_t callback,
void *context, int page_queue, u8 *status)
@@ -1092,8 +1126,8 @@ static int mlx5_cmd_invoke(struct mlx5_c
if (callback && page_queue)
return -EINVAL;
- ent = alloc_cmd(cmd, in, out, uout, uout_size, callback, context,
- page_queue);
+ ent = alloc_cmd(cmd, in, uin_size, out, uout, uout_size, callback,
+ context, page_queue);
if (IS_ERR(ent))
return PTR_ERR(ent);
@@ -1138,159 +1172,98 @@ out:
return err;
}
-static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size)
+static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, size_t size)
{
- struct mlx5_cmd_prot_block *block;
- struct mlx5_cmd_mailbox *next;
- int copy;
-
- if (!to || !from)
- return -ENOMEM;
-
- copy = min_t(int, size, sizeof(to->first.data));
- memcpy(to->first.data, from, copy);
- size -= copy;
- from += copy;
-
- next = to->next;
- while (size) {
- if (!next) {
- /* this is a BUG */
- return -ENOMEM;
- }
+ size_t delta;
+ size_t i;
- copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
- block = next->buf;
- memcpy(block->data, from, copy);
- from += copy;
- size -= copy;
- next = next->next;
- }
+ if (to == NULL || from == NULL)
+ return (-ENOMEM);
- return 0;
-}
+ delta = min_t(size_t, size, sizeof(to->first.data));
+ memcpy(to->first.data, from, delta);
+ from = (char *)from + delta;
+ size -= delta;
-static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
-{
- struct mlx5_cmd_prot_block *block;
- struct mlx5_cmd_mailbox *next;
- int copy;
-
- if (!to || !from)
- return -ENOMEM;
-
- copy = min_t(int, size, sizeof(from->first.data));
- memcpy(to, from->first.data, copy);
- size -= copy;
- to += copy;
-
- next = from->next;
- while (size) {
- if (!next) {
- /* this is a BUG */
- return -ENOMEM;
- }
+ for (i = 0; size != 0; i++) {
+ struct mlx5_cmd_prot_block *block;
- copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
- block = next->buf;
+ block = mlx5_fwp_get_virt(to, i * MLX5_CMD_MBOX_SIZE);
- memcpy(to, block->data, copy);
- to += copy;
- size -= copy;
- next = next->next;
+ delta = min_t(size_t, size, MLX5_CMD_DATA_BLOCK_SIZE);
+ memcpy(block->data, from, delta);
+ from = (char *)from + delta;
+ size -= delta;
}
-
- return 0;
+ return (0);
}
-static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
- gfp_t flags)
+static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
{
- struct mlx5_cmd_mailbox *mailbox;
+ size_t delta;
+ size_t i;
- mailbox = kmalloc(sizeof(*mailbox), flags);
- if (!mailbox)
- return ERR_PTR(-ENOMEM);
+ if (to == NULL || from == NULL)
+ return (-ENOMEM);
- mailbox->buf = pci_pool_alloc(dev->cmd.pool, flags,
- &mailbox->dma);
- if (!mailbox->buf) {
- mlx5_core_dbg(dev, "failed allocation\n");
- kfree(mailbox);
- return ERR_PTR(-ENOMEM);
- }
- memset(mailbox->buf, 0, sizeof(struct mlx5_cmd_prot_block));
- mailbox->next = NULL;
+ delta = min_t(size_t, size, sizeof(from->first.data));
+ memcpy(to, from->first.data, delta);
+ to = (char *)to + delta;
+ size -= delta;
- return mailbox;
-}
+ for (i = 0; size != 0; i++) {
+ struct mlx5_cmd_prot_block *block;
-static void free_cmd_box(struct mlx5_core_dev *dev,
- struct mlx5_cmd_mailbox *mailbox)
-{
- pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
- kfree(mailbox);
+ block = mlx5_fwp_get_virt(from, i * MLX5_CMD_MBOX_SIZE);
+
+ delta = min_t(size_t, size, MLX5_CMD_DATA_BLOCK_SIZE);
+ memcpy(to, block->data, delta);
+ to = (char *)to + delta;
+ size -= delta;
+ }
+ return (0);
}
-static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
- gfp_t flags, int size)
+static struct mlx5_cmd_msg *
+mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev, gfp_t flags, size_t size)
{
- struct mlx5_cmd_mailbox *tmp, *head = NULL;
- struct mlx5_cmd_prot_block *block;
struct mlx5_cmd_msg *msg;
- int blen;
- int err;
- int n;
- int i;
+ size_t blen;
+ size_t n;
+ size_t i;
- msg = kzalloc(sizeof(*msg), flags);
- if (!msg)
- return ERR_PTR(-ENOMEM);
+ blen = size - min_t(size_t, sizeof(msg->first.data), size);
+ n = howmany(blen, MLX5_CMD_DATA_BLOCK_SIZE);
- blen = size - min_t(int, sizeof(msg->first.data), size);
- n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE;
+ msg = mlx5_fwp_alloc(dev, flags, howmany(n, MLX5_NUM_CMDS_IN_ADAPTER_PAGE));
+ if (msg == NULL)
+ return (ERR_PTR(-ENOMEM));
- for (i = 0; i < n; i++) {
- tmp = alloc_cmd_box(dev, flags);
- if (IS_ERR(tmp)) {
- mlx5_core_warn(dev, "failed allocating block\n");
- err = PTR_ERR(tmp);
- goto err_alloc;
- }
+ for (i = 0; i != n; i++) {
+ struct mlx5_cmd_prot_block *block;
- block = tmp->buf;
- tmp->next = head;
- block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
- block->block_num = cpu_to_be32(n - i - 1);
- head = tmp;
- }
- msg->next = head;
- msg->len = size;
- return msg;
+ block = mlx5_fwp_get_virt(msg, i * MLX5_CMD_MBOX_SIZE);
-err_alloc:
- while (head) {
- tmp = head->next;
- free_cmd_box(dev, head);
- head = tmp;
+ memset(block, 0, MLX5_CMD_MBOX_SIZE);
+
+ if (i != (n - 1)) {
+ u64 dma = mlx5_fwp_get_dma(msg, (i + 1) * MLX5_CMD_MBOX_SIZE);
+ block->next = cpu_to_be64(dma);
+ }
+ block->block_num = cpu_to_be32(i);
}
- kfree(msg);
- return ERR_PTR(err);
+ /* make sure initial data is written to RAM */
+ mlx5_fwp_flush(msg);
+
+ return (msg);
}
-static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
- struct mlx5_cmd_msg *msg)
+static void
+mlx5_free_cmd_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
{
- struct mlx5_cmd_mailbox *head = msg->next;
- struct mlx5_cmd_mailbox *next;
- while (head) {
- next = head->next;
- free_cmd_box(dev, head);
- head = next;
- }
- kfree(msg);
+ mlx5_fwp_free(msg);
}
static void set_wqname(struct mlx5_core_dev *dev)
@@ -1356,6 +1329,9 @@ void mlx5_cmd_comp_handler(struct mlx5_c
struct mlx5_cmd_work_ent *ent;
int i;
+ /* make sure data gets read from RAM */
+ mlx5_fwp_invalidate(cmd->cmd_page);
+
while (vector != 0) {
i = ffs(vector) - 1;
vector &= ~(1U << i);
@@ -1363,6 +1339,8 @@ void mlx5_cmd_comp_handler(struct mlx5_c
ent->ts2 = ktime_get_ns();
memcpy(ent->out->first.data, ent->lay->out,
sizeof(ent->lay->out));
+ /* make sure data gets read from RAM */
+ mlx5_fwp_invalidate(ent->out);
dump_command(dev, ent, 0);
if (!ent->ret) {
if (!cmd->checksum_disabled)
@@ -1432,10 +1410,6 @@ static struct mlx5_cmd_msg *alloc_msg(st
if (!list_empty(&ent->head)) {
msg = list_entry(ent->head.next, struct mlx5_cmd_msg,
list);
- /* For cached lists, we must explicitly state what is
- * the real size
- */
- msg->len = in_size;
list_del(&msg->list);
}
spin_unlock_irq(&ent->lock);
@@ -1485,8 +1459,8 @@ static int cmd_exec_helper(struct mlx5_c
goto out_in;
}
- err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
- pages_queue, &status);
+ err = mlx5_cmd_invoke(dev, inb, in_size, outb, out, out_size, callback,
+ context, pages_queue, &status);
if (err) {
if (err == -ETIMEDOUT)
return err;
@@ -1583,44 +1557,67 @@ ex_err:
return err;
}
-static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
+static int
+alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
{
- struct device *ddev = &dev->pdev->dev;
- cmd->cmd_alloc_buf = dma_zalloc_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE,
- &cmd->alloc_dma, GFP_KERNEL);
- if (!cmd->cmd_alloc_buf)
- return -ENOMEM;
-
- /* make sure it is aligned to 4K */
- if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
- cmd->cmd_buf = cmd->cmd_alloc_buf;
- cmd->dma = cmd->alloc_dma;
- cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
- return 0;
+ int err;
+
+ sx_init(&cmd->dma_sx, "MLX5-DMA-SX");
+ mtx_init(&cmd->dma_mtx, "MLX5-DMA-MTX", NULL, MTX_DEF);
+ cv_init(&cmd->dma_cv, "MLX5-DMA-CV");
+
+ /*
+ * Create global DMA descriptor tag for allocating
+ * 4K firmware pages:
+ */
+ err = -bus_dma_tag_create(
+ bus_get_dma_tag(dev->pdev->dev.bsddev),
+ MLX5_ADAPTER_PAGE_SIZE, /* alignment */
+ 0, /* no boundary */
+ BUS_SPACE_MAXADDR, /* lowaddr */
+ BUS_SPACE_MAXADDR, /* highaddr */
+ NULL, NULL, /* filter, filterarg */
+ MLX5_ADAPTER_PAGE_SIZE, /* maxsize */
+ 1, /* nsegments */
+ MLX5_ADAPTER_PAGE_SIZE, /* maxsegsize */
+ 0, /* flags */
+ NULL, NULL, /* lockfunc, lockfuncarg */
+ &cmd->dma_tag);
+ if (err != 0)
+ goto failure_destroy_sx;
+
+ cmd->cmd_page = mlx5_fwp_alloc(dev, GFP_KERNEL, 1);
+ if (cmd->cmd_page == NULL) {
+ err = -ENOMEM;
+ goto failure_alloc_page;
}
+ cmd->dma = mlx5_fwp_get_dma(cmd->cmd_page, 0);
+ cmd->cmd_buf = mlx5_fwp_get_virt(cmd->cmd_page, 0);
+ return (0);
- dma_free_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, cmd->alloc_dma);
- cmd->cmd_alloc_buf = dma_zalloc_coherent(ddev, 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
- &cmd->alloc_dma, GFP_KERNEL);
- if (!cmd->cmd_alloc_buf)
- return -ENOMEM;
-
- cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
- cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
- cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list