svn commit: r216909 - in projects/ofed/base/contrib/ofed:
libibverbs/man libmlx4/fixes
Jeff Roberson
jeff at FreeBSD.org
Mon Jan 3 05:35:16 UTC 2011
Author: jeff
Date: Mon Jan 3 05:35:15 2011
New Revision: 216909
URL: http://svn.freebsd.org/changeset/base/216909
Log:
- Add files missing from the last merge.
Sponsored by: Isilon Systems, iX Systems, and Panasas.
Added:
projects/ofed/base/contrib/ofed/libibverbs/man/ibv_create_xrc_rcv_qp.3
projects/ofed/base/contrib/ofed/libibverbs/man/ibv_modify_xrc_rcv_qp.3
projects/ofed/base/contrib/ofed/libibverbs/man/ibv_open_xrc_domain.3
projects/ofed/base/contrib/ofed/libibverbs/man/ibv_query_xrc_rcv_qp.3
projects/ofed/base/contrib/ofed/libibverbs/man/ibv_reg_xrc_rcv_qp.3
projects/ofed/base/contrib/ofed/libibverbs/man/verbs.7
projects/ofed/base/contrib/ofed/libmlx4/fixes/fix_inline_size.patch
Added: projects/ofed/base/contrib/ofed/libibverbs/man/ibv_create_xrc_rcv_qp.3
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/ibv_create_xrc_rcv_qp.3 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,70 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_CREATE_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_create_xrc_rcv_qp \- create an XRC queue pair (QP) for serving as a receive-side only QP
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr " "*init_attr" ,
+.BI " uint32_t " "*xrc_rcv_qpn" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_create_xrc_rcv_qp()
+creates an XRC queue pair (QP) for serving as a receive-side only QP and returns its number through the pointer
+.I xrc_rcv_qpn\fR.
+This QP number should be passed to the remote node (sender).
+The remote node will use
+.I xrc_rcv_qpn
+in
+.B ibv_post_send()
+when sending to an XRC SRQ on this host in the same xrc domain as the XRC receive QP.
+This QP is created in kernel space, and persists until the last process registered for the QP
+calls
+.B ibv_unreg_xrc_rcv_qp()
+(at which time the QP is destroyed).
+.PP
+The process which creates this QP is automatically registered for it, and should also call
+.B ibv_unreg_xrc_rcv_qp()
+at some point, to unregister.
+
+Processes which wish to receive on an XRC SRQ via this QP should call
+.B ibv_reg_xrc_rcv_qp()
+for this QP, to guarantee that the QP will not be destroyed while they are still using it for receiving on the XRC SRQ.
+.PP
+The argument
+.I qp_init_attr
+is an ibv_qp_init_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_init_attr {
+.in +8
+void *qp_context; /* value is being ignored */
+struct ibv_cq *send_cq; /* value is being ignored */
+struct ibv_cq *recv_cq; /* value is being ignored */
+struct ibv_srq *srq; /* value is being ignored */
+struct ibv_qp_cap cap; /* value is being ignored */
+enum ibv_qp_type qp_type; /* value is being ignored */
+int sq_sig_all; /* value is being ignored */
+struct ibv_xrc_domain *xrc_domain; /* XRC domain the QP will be associated with */
+.in -8
+};
+.fi
+.PP
+Most of the attributes in
+.I qp_init_attr
+are being ignored because this QP is a receive only QP and all RR are being posted to an SRQ.
+.SH "RETURN VALUE"
+.B ibv_create_xrc_rcv_qp()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "SEE ALSO"
+.BR ibv_open_xrc_domain (3),
+.BR ibv_modify_xrc_rcv_qp (3),
+.BR ibv_query_xrc_rcv_qp (3),
+.BR ibv_reg_xrc_rcv_qp (3),
+.BR ibv_unreg_xrc_rcv_qp (3),
+.BR ibv_post_send (3)
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
Added: projects/ofed/base/contrib/ofed/libibverbs/man/ibv_modify_xrc_rcv_qp.3
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/ibv_modify_xrc_rcv_qp.3 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,141 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_MODIFY_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_modify_xrc_rcv_qp \- modify the attributes of an XRC receive queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" ,
+.BI " struct ibv_qp_attr " "*attr" ", int " "attr_mask" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_modify_qp()
+modifies the attributes of an XRC receive QP with the number
+.I xrc_qp_num
+which is associated with the XRC domain
+.I xrc_domain
+with the attributes in
+.I attr
+according to the mask
+.I attr_mask
+and move the QP state through the following transitions: Reset -> Init -> RTR.
+.I attr_mask
+should indicate all of the attributes which will be used in this QP transition and the following masks (at least) should be set:
+.PP
+.nf
+Next state Required attributes
+\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-
+Init \fB IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR
+ \fB IBV_QP_ACCESS_FLAGS \fR
+RTR \fB IBV_QP_STATE, IBV_QP_AV, IBV_QP_PATH_MTU, \fR
+ \fB IBV_QP_DEST_QPN, IBV_QP_RQ_PSN, \fR
+ \fB IBV_QP_MAX_DEST_RD_ATOMIC, IBV_QP_MIN_RNR_TIMER \fR
+.fi
+.PP
+The user can add optional attributes as well.
+.PP
+The argument \fIattr\fR is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_attr {
+.in +8
+enum ibv_qp_state qp_state; /* Move the QP to this state */
+enum ibv_qp_state cur_qp_state; /* Assume this is the current QP state */
+enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */
+enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */
+uint32_t qkey; /* Q_Key for the QP (valid only for UD QPs) */
+uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */
+uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */
+uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */
+int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */
+struct ibv_qp_cap cap; /* QP capabilities (valid if HCA supports QP resizing) */
+struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */
+struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */
+uint16_t pkey_index; /* Primary P_Key index */
+uint16_t alt_pkey_index; /* Alternate P_Key index */
+uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification (Valid only if qp_state is SQD) */
+uint8_t sq_draining; /* Is the QP draining? Irrelevant for ibv_modify_qp() */
+uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */
+uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */
+uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */
+uint8_t port_num; /* Primary port number */
+uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */
+uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */
+uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */
+uint8_t alt_port_num; /* Alternate port number */
+uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */
+.in -8
+};
+.fi
+.PP
+For details on struct ibv_qp_cap see the description of
+.B ibv_create_qp()\fR.
+For details on struct ibv_ah_attr see the description of
+.B ibv_create_ah()\fR.
+.PP
+The argument
+.I attr_mask
+specifies the QP attributes to be modified.
+The argument is either 0 or the bitwise OR of one or more of the following flags:
+.PP
+.TP
+.B IBV_QP_STATE \fR Modify qp_state
+.TP
+.B IBV_QP_CUR_STATE \fR Set cur_qp_state
+.TP
+.B IBV_QP_EN_SQD_ASYNC_NOTIFY \fR Set en_sqd_async_notify
+.TP
+.B IBV_QP_ACCESS_FLAGS \fR Set qp_access_flags
+.TP
+.B IBV_QP_PKEY_INDEX \fR Set pkey_index
+.TP
+.B IBV_QP_PORT \fR Set port_num
+.TP
+.B IBV_QP_QKEY \fR Set qkey
+.TP
+.B IBV_QP_AV \fR Set ah_attr
+.TP
+.B IBV_QP_PATH_MTU \fR Set path_mtu
+.TP
+.B IBV_QP_TIMEOUT \fR Set timeout
+.TP
+.B IBV_QP_RETRY_CNT \fR Set retry_cnt
+.TP
+.B IBV_QP_RNR_RETRY \fR Set rnr_retry
+.TP
+.B IBV_QP_RQ_PSN \fR Set rq_psn
+.TP
+.B IBV_QP_MAX_QP_RD_ATOMIC \fR Set max_rd_atomic
+.TP
+.B IBV_QP_ALT_PATH \fR Set the alternative path via: alt_ah_attr, alt_pkey_index, alt_port_num, alt_timeout
+.TP
+.B IBV_QP_MIN_RNR_TIMER \fR Set min_rnr_timer
+.TP
+.B IBV_QP_SQ_PSN \fR Set sq_psn
+.TP
+.B IBV_QP_MAX_DEST_RD_ATOMIC \fR Set max_dest_rd_atomic
+.TP
+.B IBV_QP_PATH_MIG_STATE \fR Set path_mig_state
+.TP
+.B IBV_QP_CAP \fR Set cap
+.TP
+.B IBV_QP_DEST_QPN \fR Set dest_qp_num
+.SH "RETURN VALUE"
+.B ibv_modify_xrc_rcv_qp()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+If any of the modify attributes or the modify mask are invalid, none
+of the attributes will be modified (including the QP state).
+.PP
+Not all devices support alternate paths. To check if a device supports it, check if the
+.B IBV_DEVICE_AUTO_PATH_MIG
+bit is set in the device capabilities flags.
+.SH "SEE ALSO"
+.BR ibv_open_xrc_domain (3),
+.BR ibv_create_xrc_rcv_qp (3),
+.BR ibv_query_xrc_rcv_qp (3)
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
Added: projects/ofed/base/contrib/ofed/libibverbs/man/ibv_open_xrc_domain.3
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/ibv_open_xrc_domain.3 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,80 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_OPEN_XRC_DOMAIN 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_open_xrc_domain, ibv_close_xrc_domain \- open or close an eXtended Reliable Connection (XRC) domain
+.SH "SYNOPSIS"
+.nf
+.B #include <fcntl.h>
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_xrc_domain *ibv_open_xrc_domain(struct ibv_context " "*context" ","
+.BI " int " "fd" ", int " "oflag" );
+.nl
+.BI "int ibv_close_xrc_domain(struct ibv_xrc_domain " "*d" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_open_xrc_domain()
+open an XRC domain for the InfiniBand device context
+.I context
+or return a reference to an opened one\fR.
+.I fd
+is the file descriptor to be associated with the XRC domain.
+The argument
+.I oflag
+describes the desired file creation attributes; it is either 0 or the bitwise OR of one or more of the following flags:
+.PP
+.TP
+.B O_CREAT
+If a domain belonging to device named by context is already associated with the inode, this flag has
+no effect, except as noted under
+.BR O_EXCL
+below. Otherwise, a new XRC domain is created and is associated with inode specified by
+.IR fd\fR.
+.TP
+.B O_EXCL
+If
+.BR O_EXCL
+and
+.BR O_CREAT
+are set, open will fail if a domain associated with the inode exists.
+The check for the existence of the domain and creation
+of the domain if it does not exist is atomic with respect to other
+processes executing open with
+.IR fd
+naming the same inode.
+.PP
+If
+.I fd
+equals -1, no inode is is associated with the domain, and the only valid value for
+.I oflag
+is
+.B O_CREAT\fR.
+.PP
+.B ibv_close_xrc_domain()
+closes the XRC domain
+.I d\fR.
+If this is the last reference, the XRC domain will be destroyed.
+.SH "RETURN VALUE"
+.B ibv_open_xrc_domain()
+returns a pointer to an opened XRC, or NULL if the request fails.
+.PP
+.B ibv_close_xrc_domain()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+Not all devices support XRC. To check if a device supports it, check if the
+.B IBV_DEVICE_XRC
+bit is set in the device capabilities flags.
+.PP
+.B ibv_close_xrc_domain()
+may fail if any QP or SRQ are still associated with the XRC domain being closed.
+.SH "SEE ALSO"
+.BR ibv_create_xrc_srq (3),
+.BR ibv_create_qp (3),
+.BR ibv_create_xrc_rcv_qp (3),
+.BR ibv_modify_xrc_rcv_qp (3),
+.BR ibv_query_xrc_rcv_qp (3),
+.BR ibv_reg_xrc_rcv_qp (3)
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
Added: projects/ofed/base/contrib/ofed/libibverbs/man/ibv_query_xrc_rcv_qp.3
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/ibv_query_xrc_rcv_qp.3 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,89 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_QUERY_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_query_xrc_rcv_qp \- get the attributes of an XRC receive queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" ,
+.BI " struct ibv_qp_attr " "*attr" ", int " "attr_mask" ,
+.BI " struct ibv_qp_init_attr " "*init_attr" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_query_xrc_rcv_qp()
+gets the attributes specified in
+.I attr_mask
+for the XRC receive QP with the number
+.I xrc_qp_num
+which is associated with the XRC domain
+.I xrc_domain
+and returns them through the pointers
+.I attr
+and
+.I init_attr\fR.
+The argument
+.I attr
+is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_attr {
+.in +8
+enum ibv_qp_state qp_state; /* Current QP state */
+enum ibv_qp_state cur_qp_state; /* Current QP state - irrelevant for ibv_query_qp */
+enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */
+enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */
+uint32_t qkey; /* Q_Key of the QP (valid only for UD QPs) */
+uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */
+uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */
+uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */
+int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */
+struct ibv_qp_cap cap; /* QP capabilities */
+struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */
+struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */
+uint16_t pkey_index; /* Primary P_Key index */
+uint16_t alt_pkey_index; /* Alternate P_Key index */
+uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification - irrelevant for ibv_query_qp */
+uint8_t sq_draining; /* Is the QP draining? (Valid only if qp_state is SQD) */
+uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */
+uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */
+uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */
+uint8_t port_num; /* Primary port number */
+uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */
+uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */
+uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */
+uint8_t alt_port_num; /* Alternate port number */
+uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */
+.in -8
+};
+.fi
+.PP
+For details on struct ibv_qp_cap see the description of
+.B ibv_create_qp()\fR.
+For details on struct ibv_ah_attr see the description of
+.B ibv_create_ah()\fR.
+.SH "RETURN VALUE"
+.B ibv_query_xrc_rcv_qp()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+The argument
+.I attr_mask
+is a hint that specifies the minimum list of attributes to retrieve.
+Some InfiniBand devices may return extra attributes not requested, for
+example if the value can be returned cheaply.
+.PP
+Attribute values are valid if they have been set using
+.B ibv_modify_xrc_rcv_qp()\fR.
+The exact list of valid attributes depends on the QP state.
+.PP
+Multiple calls to
+.B ibv_query_xrc_rcv_qp()
+may yield some differences in the values returned for the following attributes: qp_state, path_mig_state, sq_draining, ah_attr (if APM is enabled).
+.SH "SEE ALSO"
+.BR ibv_open_xrc_domain (3),
+.BR ibv_create_xrc_rcv_qp (3),
+.BR ibv_modify_xrc_rcv_qp (3)
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
Added: projects/ofed/base/contrib/ofed/libibverbs/man/ibv_reg_xrc_rcv_qp.3
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/ibv_reg_xrc_rcv_qp.3 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,57 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_REG_XRC_RCV_QP 3 2008-10-02 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_reg_xrc_rcv_qp, ibv_unreg_xrc_rcv_qp \- register and unregister a user process with an XRC receive queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" ");
+.nl
+.BI "int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" ");
+.fi
+.SH "DESCRIPTION"
+.B ibv_reg_xrc_rcv_qp()
+registers a user process with the XRC receive QP (created via
+.B ibv_create_xrc_rcv_qp()
+) whose number is
+.I xrc_qp_num\fR,
+and which is associated with the XRC domain
+.I xrc_domain\fR.
+.PP
+.B ibv_unreg_xrc_rcv_qp()
+unregisters a user process from the XRC receive QP number
+.I xrc_qp_num\fR,
+which is associated with the XRC domain
+.I xrc_domain\fR.
+When the number of user processes registered with this XRC receive QP drops to zero, the QP is destroyed.
+.SH "RETURN VALUE"
+.B ibv_reg_xrc_rcv_qp()
+and
+.B ibv_unreg_xrc_rcv_qp()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "NOTES"
+.B ibv_reg_xrc_rcv_qp()
+and
+.B ibv_unreg_xrc_rcv_qp()
+may fail if the number
+.I xrc_qp_num
+is not a number of a valid XRC receive QP (the QP is not allocated or it is the number of a non-XRC QP), or
+the XRC receive QP was created with an XRC domain other than
+.I xrc_domain\fR.
+
+If a process is still registered with any XRC RCV QPs belonging to some domain,
+.B ibv_close_xrc_domain()
+will return failure if called for that domain in that process.
+
+.B ibv_create_xrc_rcv_qp()
+performs an implicit registration for the creating process; when that process is finished with the XRC RCV QP, it should call
+.B ibv_unreg_xrc_rcv_qp()
+for that QP. Note that if no other processes are registered with the QP at this time, its registration count will drop to zero and it will be destroyed.
+.SH "SEE ALSO"
+.BR ibv_open_xrc_domain (3),
+.BR ibv_create_xrc_rcv_qp (3)
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
Added: projects/ofed/base/contrib/ofed/libibverbs/man/verbs.7
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libibverbs/man/verbs.7 Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,228 @@
+.\" -*- nroff -*-
+.\"
+.TH VERBS 7 2008-02-25 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+verbs \- Infiniband verbs library
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.fi
+.SH "DESCRIPTION"
+This library is an implementation of the verbs based on the Infiniband specification volume 1.2 chapter 11. It handles the control path of creating, modifying, querying and destroying resources such as Protection Domains (PD), Completion Queues (CQ), Queue-Pairs (QP), Shared Receive Queues (SRQ), Address Handles (AH), Memory Regions (MR). It also handles sending and receiving data posted to QPs and SRQs, getting completions from CQs using polling and completions events.
+
+The control path is implemented through system calls to the uverbs kernel module which further calls the low level HW driver. The data path is implemented through calls made to low level HW library which in most cases interacts directly with the HW providing kernel and network stack bypass (saving context/mode switches) along with zero copy and an asynchronous I/O model.
+
+
+Typically, under network and RDMA programming, there are operations which involve interaction with remote peers (such as address resolution and connection establishment) and remote entities (such as route resolution and joining a multicast group under IB), where a resource managed through IB verbs such as QP or AH would be eventually created or effected from this interaction. In such cases, applications whose addressing semantics is based on IP can use librdmacm (see rdma_cm(7)) which works in conjunction with libibverbs.
+
+This library is thread safe library and verbs can be called from every thread in the process (the same resource can even be handled from different threads, for example: ibv_poll_cq can be called from more than one thread).
+
+However, it is up to the user to stop working with a resource after it was destroyed (by the same thread or by any other thread), this may result a segmentation fault.
+
+If fork (or any other system call that perform fork directly or indirectly) is being used, please see ibv_fork_init(3).
+
+.LP
+The following shall be declared as functions and may also be defined
+as macros. Function prototypes shall be provided.
+.RS
+.nf
+
+\fB
+.B Library functions
+
+int ibv_fork_init(void);
+
+.B Device functions
+
+struct ibv_device **ibv_get_device_list(int *num_devices);
+void ibv_free_device_list(struct ibv_device **list);
+const char *ibv_get_device_name(struct ibv_device *device);
+uint64_t ibv_get_device_guid(struct ibv_device *device);
+
+.B Context functions
+
+struct ibv_context *ibv_open_device(struct ibv_device *device);
+int ibv_close_device(struct ibv_context *context);
+
+.B Queries
+
+int ibv_query_device(struct ibv_context *context,
+ struct ibv_device_attr *device_attr);
+int ibv_query_port(struct ibv_context *context, uint8_t port_num,
+ struct ibv_port_attr *port_attr);
+int ibv_query_pkey(struct ibv_context *context, uint8_t port_num,
+ int index, uint16_t *pkey);
+int ibv_query_gid(struct ibv_context *context, uint8_t port_num,
+ int index, union ibv_gid *gid);
+
+.B Asynchronous events
+
+int ibv_get_async_event(struct ibv_context *context,
+ struct ibv_async_event *event);
+void ibv_ack_async_event(struct ibv_async_event *event);
+
+.B Protection Domains
+
+struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);
+int ibv_dealloc_pd(struct ibv_pd *pd);
+
+.B Memory Regions
+
+struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
+ size_t length, enum ibv_access_flags access);
+int ibv_dereg_mr(struct ibv_mr *mr);
+
+.B Address Handles
+
+struct ibv_ah *ibv_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
+int ibv_init_ah_from_wc(struct ibv_context *context, uint8_t port_num,
+ struct ibv_wc *wc, struct ibv_grh *grh,
+ struct ibv_ah_attr *ah_attr);
+struct ibv_ah *ibv_create_ah_from_wc(struct ibv_pd *pd, struct ibv_wc *wc,
+ struct ibv_grh *grh, uint8_t port_num);
+int ibv_destroy_ah(struct ibv_ah *ah);
+
+.B Completion event channels
+
+struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);
+int ibv_destroy_comp_channel(struct ibv_comp_channel *channel);
+
+.B Completion Queues Control
+
+struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
+ void *cq_context,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
+int ibv_destroy_cq(struct ibv_cq *cq);
+int ibv_resize_cq(struct ibv_cq *cq, int cqe);
+
+.B Reading Completions from CQ
+
+int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc);
+
+.B Requesting / Managing CQ events
+
+int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only);
+int ibv_get_cq_event(struct ibv_comp_channel *channel,
+ struct ibv_cq **cq, void **cq_context);
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents);
+
+.B Shared Receive Queue control
+
+struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
+ struct ibv_srq_init_attr *srq_init_attr);
+int ibv_destroy_srq(struct ibv_srq *srq);
+int ibv_modify_srq(struct ibv_srq *srq,
+ struct ibv_srq_attr *srq_attr,
+ enum ibv_srq_attr_mask srq_attr_mask);
+int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
+
+.B eXtended Reliable Connection control
+
+struct ibv_xrc_domain *ibv_open_xrc_domain(struct ibv_context *context,
+ int fd, int oflag);
+int ibv_close_xrc_domain(struct ibv_xrc_domain *d);
+struct ibv_srq *ibv_create_xrc_srq(struct ibv_pd *pd,
+ struct ibv_xrc_domain *xrc_domain,
+ struct ibv_cq *xrc_cq,
+ struct ibv_srq_init_attr *srq_init_attr);
+int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
+ uint32_t *xrc_rcv_qpn);
+int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num,
+ struct ibv_qp_attr *attr, int attr_mask);
+int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num,
+ struct ibv_qp_attr *attr, int attr_mask,
+ struct ibv_qp_init_attr *init_attr);
+int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num);
+int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num);
+
+.B Queue Pair control
+
+struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr);
+int ibv_destroy_qp(struct ibv_qp *qp);
+int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ enum ibv_qp_attr_mask attr_mask);
+int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ enum ibv_qp_attr_mask attr_mask,
+ struct ibv_qp_init_attr *init_attr);
+
+.B posting Work Requests to QPs/SRQs
+int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
+ struct ibv_send_wr **bad_wr);
+int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr);
+int ibv_post_srq_recv(struct ibv_srq *srq,
+ struct ibv_recv_wr *recv_wr,
+ struct ibv_recv_wr **bad_recv_wr);
+
+.B Multicast group
+
+int ibv_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
+int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
+
+.B General functions
+
+int ibv_rate_to_mult(enum ibv_rate rate);
+enum ibv_rate mult_to_ibv_rate(int mult);
+\fP
+.SH "SEE ALSO"
+.LP
+\fIibv_fork_init\fP(),
+\fIibv_get_device_list\fP(),
+\fIibv_free_device_list\fP(),
+\fIibv_get_device_name\fP(),
+\fIibv_get_device_guid\fP(),
+\fIibv_open_device\fP(),
+\fIibv_close_device\fP(),
+\fIibv_query_device\fP(),
+\fIibv_query_port\fP(),
+\fIibv_query_pkey\fP(),
+\fIibv_query_gid\fP(),
+\fIibv_get_async_event\fP(),
+\fIibv_ack_async_event\fP(),
+\fIibv_alloc_pd\fP(),
+\fIibv_dealloc_pd\fP(),
+\fIibv_reg_mr\fP(),
+\fIibv_dereg_mr\fP(),
+\fIibv_create_ah\fP(),
+\fIibv_init_ah_from_wc\fP(),
+\fIibv_create_ah_from_wc\fP(),
+\fIibv_destroy_ah\fP(),
+\fIibv_create_comp_channel\fP(),
+\fIibv_destroy_comp_channel\fP(),
+\fIibv_create_cq\fP(),
+\fIibv_destroy_cq\fP(),
+\fIibv_resize_cq\fP(),
+\fIibv_poll_cq\fP(),
+\fIibv_req_notify_cq\fP(),
+\fIibv_get_cq_event\fP(),
+\fIibv_ack_cq_events\fP(),
+\fIibv_create_srq\fP(),
+\fIibv_destroy_srq\fP(),
+\fIibv_modify_srq\fP(),
+\fIibv_query_srq\fP(),
+\fIibv_open_xrc_domain\fP(),
+\fIibv_close_xrc_domain\fP(),
+\fIibv_create_xrc_srq\fP(),
+\fIibv_create_xrc_rcv_qp\fP(),
+\fIibv_modify_xrc_rcv_qp\fP(),
+\fIibv_query_xrc_rcv_qp\fP(),
+\fIibv_reg_xrc_rcv_qp\fP(),
+\fIibv_unreg_xrc_rcv_qp\fP(),
+\fIibv_post_srq_recv\fP(),
+\fIibv_create_qp\fP(),
+\fIibv_destroy_qp\fP(),
+\fIibv_modify_qp\fP(),
+\fIibv_query_qp\fP(),
+\fIibv_post_send\fP(),
+\fIibv_post_recv\fP(),
+\fIibv_attach_mcast\fP(),
+\fIibv_detach_mcast\fP(),
+\fIibv_rate_to_mult\fP(),
+\fImult_to_ibv_rate\fP()
+.SH "AUTHORS"
+.TP
+Dotan Barak <dotanb at mellanox.co.il>
+.TP
+Or Gerlitz <ogerlitz at voltaire.com>
Added: projects/ofed/base/contrib/ofed/libmlx4/fixes/fix_inline_size.patch
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/ofed/base/contrib/ofed/libmlx4/fixes/fix_inline_size.patch Mon Jan 3 05:35:15 2011 (r216909)
@@ -0,0 +1,126 @@
+From 6f9c7b5c83a832884b3e47a6ce52145991073586 Mon Sep 17 00:00:00 2001
+From: Eli Cohen <eli at mellanox.co.il>
+Date: Sun, 19 Sep 2010 11:28:40 +0200
+Subject: [PATCH] libmlx4: fix possible inline size
+
+The current driver checks required inline size by making sure it does not
+exceed 1024. This is wrong since the whole WQE is limited to 1008 bytes.
+Moreover, a more careful claculation is required to avoid cases where the
+application requests inline support in a certain size that when used later
+could cause connections to stall due to bad WQEs. This patch takes into account
+the size of the WQE, the segements used to create a WQE and the overhead
+incured by the inline segments themselves.
+
+Signed-off-by: Eli Cohen <eli at mellanox.co.il>
+---
+ src/verbs.c | 45 ++++++++++++++++++++++++++++++++++++++++-----
+ 1 files changed, 40 insertions(+), 5 deletions(-)
+
+Index: libmlx4/src/verbs.c
+===================================================================
+--- libmlx4.orig/src/verbs.c 2010-09-29 11:10:17.691587848 +0200
++++ libmlx4/src/verbs.c 2010-09-29 11:16:11.031586721 +0200
+@@ -402,6 +402,44 @@ int mlx4_destroy_srq(struct ibv_srq *ibs
+ return 0;
+ }
+
++static int verify_sizes(struct ibv_qp_init_attr *attr, struct mlx4_context *context)
++{
++ int size;
++ int nsegs;
++
++ if (attr->cap.max_send_wr > context->max_qp_wr ||
++ attr->cap.max_recv_wr > context->max_qp_wr ||
++ attr->cap.max_send_sge > context->max_sge ||
++ attr->cap.max_recv_sge > context->max_sge)
++ return -1;
++
++ if (attr->cap.max_inline_data) {
++ nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
++ size = MLX4_MAX_WQE_SIZE - nsegs * sizeof (struct mlx4_wqe_inline_seg);
++ switch (attr->qp_type) {
++ case IBV_QPT_UD:
++ size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
++ sizeof (struct mlx4_wqe_datagram_seg));
++ break;
++
++ case IBV_QPT_RC:
++ case IBV_QPT_UC:
++ case IBV_QPT_XRC:
++ size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
++ sizeof (struct mlx4_wqe_raddr_seg));
++ break;
++
++ default:
++ return 0;
++ }
++
++ if (attr->cap.max_inline_data > size)
++ return -1;
++ }
++
++ return 0;
++}
++
+ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+ {
+ struct mlx4_create_qp cmd;
+@@ -412,11 +450,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv
+
+
+ /* Sanity check QP size before proceeding */
+- if (attr->cap.max_send_wr > context->max_qp_wr ||
+- attr->cap.max_recv_wr > context->max_qp_wr ||
+- attr->cap.max_send_sge > context->max_sge ||
+- attr->cap.max_recv_sge > context->max_sge ||
+- attr->cap.max_inline_data > 1024)
++ if (verify_sizes(attr, context))
+ return NULL;
+
+ qp = malloc(sizeof *qp);
+Index: libmlx4/src/mlx4.h
+===================================================================
+--- libmlx4.orig/src/mlx4.h 2010-09-29 11:10:17.691587848 +0200
++++ libmlx4/src/mlx4.h 2010-09-29 11:11:35.559586971 +0200
+@@ -159,6 +159,10 @@ enum {
+ MLX4_CQE_OPCODE_RESIZE = 0x16,
+ };
+
++enum {
++ MLX4_MAX_WQE_SIZE = 1008
++};
++
+ struct mlx4_device {
+ struct ibv_device ibv_dev;
+ int page_size;
+@@ -410,6 +414,7 @@ int mlx4_post_recv(struct ibv_qp *ibqp,
+ struct ibv_recv_wr **bad_wr);
+ void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
+ struct mlx4_qp *qp);
++int num_inline_segs(int data, enum ibv_qp_type type);
+ int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
+ enum ibv_qp_type type, struct mlx4_qp *qp);
+ void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
+Index: libmlx4/src/qp.c
+===================================================================
+--- libmlx4.orig/src/qp.c 2010-09-29 11:10:17.691587848 +0200
++++ libmlx4/src/qp.c 2010-09-29 11:12:14.931587492 +0200
+@@ -505,7 +505,7 @@ out:
+ return ret;
+ }
+
+-static int num_inline_segs(int data, enum ibv_qp_type type)
++int num_inline_segs(int data, enum ibv_qp_type type)
+ {
+ /*
+ * Inline data segments are not allowed to cross 64 byte
+@@ -634,7 +634,8 @@ void mlx4_set_sq_sizes(struct mlx4_qp *q
+ int wqe_size;
+ struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);
+
+- wqe_size = (1 << qp->sq.wqe_shift) - sizeof (struct mlx4_wqe_ctrl_seg);
++ wqe_size = min((1 << qp->sq.wqe_shift), MLX4_MAX_WQE_SIZE) -
++ sizeof (struct mlx4_wqe_ctrl_seg);
+ switch (type) {
+ case IBV_QPT_UD:
+ wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);
More information about the svn-src-projects
mailing list