1commit b7c5820ce48a4fc3c8464872f0a554df6e7bbb47 2Author: Dotan Barak <dotanb@mellanox.co.il> 3Date: Mon Feb 25 16:50:38 2008 +0200 4 5 Add XRC (eXtended Reliable Connection) support to all of the relevant man pages 6 and add new man pages to new XRC verbs. 7 8 Signed-off-by: Dotan Barak <dotanb@dev.mellanox.co.il> 9 Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> 10 11diff --git a/man/ibv_create_qp.3 b/man/ibv_create_qp.3 12index abd5449..fb6f041 100644 13Index: libibverbs/man/ibv_create_qp.3 14=================================================================== 15--- libibverbs.orig/man/ibv_create_qp.3 2008-06-05 15:21:54.000000000 +0300 16+++ libibverbs/man/ibv_create_qp.3 2008-06-05 16:25:21.000000000 +0300 17@@ -28,8 +28,9 @@ struct ibv_cq *send_cq; 18 struct ibv_cq *recv_cq; /* CQ to be associated with the Receive Queue (RQ) */ 19 struct ibv_srq *srq; /* SRQ handle if QP is to be associated with an SRQ, otherwise NULL */ 20 struct ibv_qp_cap cap; /* QP capabilities */ 21-enum ibv_qp_type qp_type; /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, or IBV_QPT_UD */ 22+enum ibv_qp_type qp_type; /* QP Transport Service Type: IBV_QPT_RC, IBV_QPT_UC, IBV_QPT_UD or IBV_QPT_XRC */ 23 int sq_sig_all; /* If set, each Work Request (WR) submitted to the SQ generates a completion entry */ 24+struct ibv_xrc_domain *xrc_domain; /* XRC domain the QP will be associated with (valid only for IBV_QPT_XRC QP), otherwise NULL */ 25 .in -8 26 }; 27 .sp 28Index: libibverbs/man/ibv_create_srq.3 29=================================================================== 30--- libibverbs.orig/man/ibv_create_srq.3 2008-06-05 15:21:54.000000000 +0300 31+++ libibverbs/man/ibv_create_srq.3 2008-06-05 16:25:21.000000000 +0300 32@@ -10,12 +10,26 @@ ibv_create_srq, ibv_destroy_srq \- creat 33 .BI "struct ibv_srq *ibv_create_srq(struct ibv_pd " "*pd" ", struct " 34 .BI " ibv_srq_init_attr " "*srq_init_attr" ); 35 .sp 36+.BI "struct ibv_srq *ibv_create_xrc_srq(struct ibv_pd " "*pd" ", 37+.BI " struct ibv_xrc_domain " "*xrc_domain" ", 38+.BI " struct ibv_cq " "*xrc_cq" ", 39+.BI " struct ibv_srq_init_attr " "*srq_init_attr" ); 40+.sp 41 .BI "int ibv_destroy_srq(struct ibv_srq " "*srq" ); 42 .fi 43 .SH "DESCRIPTION" 44 .B ibv_create_srq() 45 creates a shared receive queue (SRQ) associated with the protection domain 46 .I pd\fR. 47+.PP 48+.B ibv_create_xrc_srq() 49+creates an XRC shared receive queue (SRQ) associated with the protection domain 50+.I pd\fR, 51+the XRC domain 52+.I xrc_domain 53+and the CQ which will hold the XRC completion 54+.I xrc_cq\fR. 55+.PP 56 The argument 57 .I srq_init_attr 58 is an ibv_srq_init_attr struct, as defined in <infiniband/verbs.h>. 59Index: libibverbs/man/ibv_create_xrc_rcv_qp.3 60=================================================================== 61--- /dev/null 1970-01-01 00:00:00.000000000 +0000 62+++ libibverbs/man/ibv_create_xrc_rcv_qp.3 2008-06-05 16:25:21.000000000 +0300 63@@ -0,0 +1,70 @@ 64+.\" -*- nroff -*- 65+.\" 66+.TH IBV_CREATE_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual" 67+.SH "NAME" 68+ibv_create_xrc_rcv_qp \- create an XRC queue pair (QP) for serving as a receive-side only QP 69+.SH "SYNOPSIS" 70+.nf 71+.B #include <infiniband/verbs.h> 72+.sp 73+.BI "int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr " "*init_attr" , 74+.BI " uint32_t " "*xrc_rcv_qpn" ); 75+.fi 76+.SH "DESCRIPTION" 77+.B ibv_create_xrc_rcv_qp() 78+creates an XRC queue pair (QP) for serving as a receive-side only QP and returns its number through the pointer 79+.I xrc_rcv_qpn\fR. 80+This QP number should be passed to the remote node (sender). 81+The remote node will use 82+.I xrc_rcv_qpn 83+in 84+.B ibv_post_send() 85+when sending to an XRC SRQ on this host in the same xrc domain as the XRC receive QP. 86+This QP is created in kernel space, and persists until the last process registered for the QP 87+calls 88+.B ibv_unreg_xrc_rcv_qp() 89+(at which time the QP is destroyed). 90+.PP 91+The process which creates this QP is automatically registered for it, and should also call 92+.B ibv_unreg_xrc_rcv_qp() 93+at some point, to unregister. 94+ 95+Processes which wish to receive on an XRC SRQ via this QP should call 96+.B ibv_reg_xrc_rcv_qp() 97+for this QP, to guarantee that the QP will not be destroyed while they are still using it for receiving on the XRC SRQ. 98+.PP 99+The argument 100+.I qp_init_attr 101+is an ibv_qp_init_attr struct, as defined in <infiniband/verbs.h>. 102+.PP 103+.nf 104+struct ibv_qp_init_attr { 105+.in +8 106+void *qp_context; /* value is being ignored */ 107+struct ibv_cq *send_cq; /* value is being ignored */ 108+struct ibv_cq *recv_cq; /* value is being ignored */ 109+struct ibv_srq *srq; /* value is being ignored */ 110+struct ibv_qp_cap cap; /* value is being ignored */ 111+enum ibv_qp_type qp_type; /* value is being ignored */ 112+int sq_sig_all; /* value is being ignored */ 113+struct ibv_xrc_domain *xrc_domain; /* XRC domain the QP will be associated with */ 114+.in -8 115+}; 116+.fi 117+.PP 118+Most of the attributes in 119+.I qp_init_attr 120+are being ignored because this QP is a receive only QP and all RR are being posted to an SRQ. 121+.SH "RETURN VALUE" 122+.B ibv_create_xrc_rcv_qp() 123+returns 0 on success, or the value of errno on failure (which indicates the failure reason). 124+.SH "SEE ALSO" 125+.BR ibv_open_xrc_domain (3), 126+.BR ibv_modify_xrc_rcv_qp (3), 127+.BR ibv_query_xrc_rcv_qp (3), 128+.BR ibv_reg_xrc_rcv_qp (3), 129+.BR ibv_unreg_xrc_rcv_qp (3), 130+.BR ibv_post_send (3) 131+.SH "AUTHORS" 132+.TP 133+Dotan Barak <dotanb@mellanox.co.il> 134Index: libibverbs/man/ibv_modify_xrc_rcv_qp.3 135=================================================================== 136--- /dev/null 1970-01-01 00:00:00.000000000 +0000 137+++ libibverbs/man/ibv_modify_xrc_rcv_qp.3 2008-06-05 16:25:21.000000000 +0300 138@@ -0,0 +1,141 @@ 139+.\" -*- nroff -*- 140+.\" 141+.TH IBV_MODIFY_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual" 142+.SH "NAME" 143+ibv_modify_xrc_rcv_qp \- modify the attributes of an XRC receive queue pair (QP) 144+.SH "SYNOPSIS" 145+.nf 146+.B #include <infiniband/verbs.h> 147+.sp 148+.BI "int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" , 149+.BI " struct ibv_qp_attr " "*attr" ", int " "attr_mask" ); 150+.fi 151+.SH "DESCRIPTION" 152+.B ibv_modify_qp() 153+modifies the attributes of an XRC receive QP with the number 154+.I xrc_qp_num 155+which is associated with the XRC domain 156+.I xrc_domain 157+with the attributes in 158+.I attr 159+according to the mask 160+.I attr_mask 161+and move the QP state through the following transitions: Reset -> Init -> RTR. 162+.I attr_mask 163+should indicate all of the attributes which will be used in this QP transition and the following masks (at least) should be set: 164+.PP 165+.nf 166+Next state Required attributes 167+\-\-\-\-\-\-\-\-\-\- \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- 168+Init \fB IBV_QP_STATE, IBV_QP_PKEY_INDEX, IBV_QP_PORT, \fR 169+ \fB IBV_QP_ACCESS_FLAGS \fR 170+RTR \fB IBV_QP_STATE, IBV_QP_AV, IBV_QP_PATH_MTU, \fR 171+ \fB IBV_QP_DEST_QPN, IBV_QP_RQ_PSN, \fR 172+ \fB IBV_QP_MAX_DEST_RD_ATOMIC, IBV_QP_MIN_RNR_TIMER \fR 173+.fi 174+.PP 175+The user can add optional attributes as well. 176+.PP 177+The argument \fIattr\fR is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>. 178+.PP 179+.nf 180+struct ibv_qp_attr { 181+.in +8 182+enum ibv_qp_state qp_state; /* Move the QP to this state */ 183+enum ibv_qp_state cur_qp_state; /* Assume this is the current QP state */ 184+enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */ 185+enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */ 186+uint32_t qkey; /* Q_Key for the QP (valid only for UD QPs) */ 187+uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */ 188+uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */ 189+uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */ 190+int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */ 191+struct ibv_qp_cap cap; /* QP capabilities (valid if HCA supports QP resizing) */ 192+struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */ 193+struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */ 194+uint16_t pkey_index; /* Primary P_Key index */ 195+uint16_t alt_pkey_index; /* Alternate P_Key index */ 196+uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification (Valid only if qp_state is SQD) */ 197+uint8_t sq_draining; /* Is the QP draining? Irrelevant for ibv_modify_qp() */ 198+uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */ 199+uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */ 200+uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */ 201+uint8_t port_num; /* Primary port number */ 202+uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */ 203+uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */ 204+uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */ 205+uint8_t alt_port_num; /* Alternate port number */ 206+uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */ 207+.in -8 208+}; 209+.fi 210+.PP 211+For details on struct ibv_qp_cap see the description of 212+.B ibv_create_qp()\fR. 213+For details on struct ibv_ah_attr see the description of 214+.B ibv_create_ah()\fR. 215+.PP 216+The argument 217+.I attr_mask 218+specifies the QP attributes to be modified. 219+The argument is either 0 or the bitwise OR of one or more of the following flags: 220+.PP 221+.TP 222+.B IBV_QP_STATE \fR Modify qp_state 223+.TP 224+.B IBV_QP_CUR_STATE \fR Set cur_qp_state 225+.TP 226+.B IBV_QP_EN_SQD_ASYNC_NOTIFY \fR Set en_sqd_async_notify 227+.TP 228+.B IBV_QP_ACCESS_FLAGS \fR Set qp_access_flags 229+.TP 230+.B IBV_QP_PKEY_INDEX \fR Set pkey_index 231+.TP 232+.B IBV_QP_PORT \fR Set port_num 233+.TP 234+.B IBV_QP_QKEY \fR Set qkey 235+.TP 236+.B IBV_QP_AV \fR Set ah_attr 237+.TP 238+.B IBV_QP_PATH_MTU \fR Set path_mtu 239+.TP 240+.B IBV_QP_TIMEOUT \fR Set timeout 241+.TP 242+.B IBV_QP_RETRY_CNT \fR Set retry_cnt 243+.TP 244+.B IBV_QP_RNR_RETRY \fR Set rnr_retry 245+.TP 246+.B IBV_QP_RQ_PSN \fR Set rq_psn 247+.TP 248+.B IBV_QP_MAX_QP_RD_ATOMIC \fR Set max_rd_atomic 249+.TP 250+.B IBV_QP_ALT_PATH \fR Set the alternative path via: alt_ah_attr, alt_pkey_index, alt_port_num, alt_timeout 251+.TP 252+.B IBV_QP_MIN_RNR_TIMER \fR Set min_rnr_timer 253+.TP 254+.B IBV_QP_SQ_PSN \fR Set sq_psn 255+.TP 256+.B IBV_QP_MAX_DEST_RD_ATOMIC \fR Set max_dest_rd_atomic 257+.TP 258+.B IBV_QP_PATH_MIG_STATE \fR Set path_mig_state 259+.TP 260+.B IBV_QP_CAP \fR Set cap 261+.TP 262+.B IBV_QP_DEST_QPN \fR Set dest_qp_num 263+.SH "RETURN VALUE" 264+.B ibv_modify_xrc_rcv_qp() 265+returns 0 on success, or the value of errno on failure (which indicates the failure reason). 266+.SH "NOTES" 267+If any of the modify attributes or the modify mask are invalid, none 268+of the attributes will be modified (including the QP state). 269+.PP 270+Not all devices support alternate paths. To check if a device supports it, check if the 271+.B IBV_DEVICE_AUTO_PATH_MIG 272+bit is set in the device capabilities flags. 273+.SH "SEE ALSO" 274+.BR ibv_open_xrc_domain (3), 275+.BR ibv_create_xrc_rcv_qp (3), 276+.BR ibv_query_xrc_rcv_qp (3) 277+.SH "AUTHORS" 278+.TP 279+Dotan Barak <dotanb@mellanox.co.il> 280Index: libibverbs/man/ibv_open_xrc_domain.3 281=================================================================== 282--- /dev/null 1970-01-01 00:00:00.000000000 +0000 283+++ libibverbs/man/ibv_open_xrc_domain.3 2008-06-05 16:25:21.000000000 +0300 284@@ -0,0 +1,80 @@ 285+.\" -*- nroff -*- 286+.\" 287+.TH IBV_OPEN_XRC_DOMAIN 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual" 288+.SH "NAME" 289+ibv_open_xrc_domain, ibv_close_xrc_domain \- open or close an eXtended Reliable Connection (XRC) domain 290+.SH "SYNOPSIS" 291+.nf 292+.B #include <fcntl.h> 293+.B #include <infiniband/verbs.h> 294+.sp 295+.BI "struct ibv_xrc_domain *ibv_open_xrc_domain(struct ibv_context " "*context" "," 296+.BI " int " "fd" ", int " "oflag" ); 297+.nl 298+.BI "int ibv_close_xrc_domain(struct ibv_xrc_domain " "*d" ); 299+.fi 300+.SH "DESCRIPTION" 301+.B ibv_open_xrc_domain() 302+open an XRC domain for the InfiniBand device context 303+.I context 304+or return a reference to an opened one\fR. 305+.I fd 306+is the file descriptor to be associated with the XRC domain. 307+The argument 308+.I oflag 309+describes the desired file creation attributes; it is either 0 or the bitwise OR of one or more of the following flags: 310+.PP 311+.TP 312+.B O_CREAT 313+If a domain belonging to device named by context is already associated with the inode, this flag has 314+no effect, except as noted under 315+.BR O_EXCL 316+below. Otherwise, a new XRC domain is created and is associated with inode specified by 317+.IR fd\fR. 318+.TP 319+.B O_EXCL 320+If 321+.BR O_EXCL 322+and 323+.BR O_CREAT 324+are set, open will fail if a domain associated with the inode exists. 325+The check for the existence of the domain and creation 326+of the domain if it does not exist is atomic with respect to other 327+processes executing open with 328+.IR fd 329+naming the same inode. 330+.PP 331+If 332+.I fd 333+equals -1, no inode is is associated with the domain, and the only valid value for 334+.I oflag 335+is 336+.B O_CREAT\fR. 337+.PP 338+.B ibv_close_xrc_domain() 339+closes the XRC domain 340+.I d\fR. 341+If this is the last reference, the XRC domain will be destroyed. 342+.SH "RETURN VALUE" 343+.B ibv_open_xrc_domain() 344+returns a pointer to an opened XRC, or NULL if the request fails. 345+.PP 346+.B ibv_close_xrc_domain() 347+returns 0 on success, or the value of errno on failure (which indicates the failure reason). 348+.SH "NOTES" 349+Not all devices support XRC. To check if a device supports it, check if the 350+.B IBV_DEVICE_XRC 351+bit is set in the device capabilities flags. 352+.PP 353+.B ibv_close_xrc_domain() 354+may fail if any QP or SRQ are still associated with the XRC domain being closed. 355+.SH "SEE ALSO" 356+.BR ibv_create_xrc_srq (3), 357+.BR ibv_create_qp (3), 358+.BR ibv_create_xrc_rcv_qp (3), 359+.BR ibv_modify_xrc_rcv_qp (3), 360+.BR ibv_query_xrc_rcv_qp (3), 361+.BR ibv_reg_xrc_rcv_qp (3) 362+.SH "AUTHORS" 363+.TP 364+Dotan Barak <dotanb@mellanox.co.il> 365Index: libibverbs/man/ibv_post_send.3 366=================================================================== 367--- libibverbs.orig/man/ibv_post_send.3 2008-06-05 15:21:55.000000000 +0300 368+++ libibverbs/man/ibv_post_send.3 2008-06-05 16:25:21.000000000 +0300 369@@ -60,6 +60,7 @@ uint32_t remote_qkey; /* Q_Key 370 } ud; 371 .in -8 372 } wr; 373+uint32_t xrc_remote_srq_num; /* SRQ number of the destination XRC */ 374 .in -8 375 }; 376 .sp 377@@ -76,15 +77,15 @@ uint32_t lkey; 378 Each QP Transport Service Type supports a specific set of opcodes, as shown in the following table: 379 .PP 380 .nf 381-OPCODE | IBV_QPT_UD | IBV_QPT_UC | IBV_QPT_RC 382-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\- 383-IBV_WR_SEND | X | X | X 384-IBV_WR_SEND_WITH_IMM | X | X | X 385-IBV_WR_RDMA_WRITE | | X | X 386-IBV_WR_RDMA_WRITE_WITH_IMM | | X | X 387-IBV_WR_RDMA_READ | | | X 388-IBV_WR_ATOMIC_CMP_AND_SWP | | | X 389-IBV_WR_ATOMIC_FETCH_AND_ADD | | | X 390+OPCODE | IBV_QPT_UD | IBV_QPT_UC | IBV_QPT_RC | IBV_QPT_XRC 391+\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\-+\-\-\-\-\-\-\-\-\-\-\-\- 392+IBV_WR_SEND | X | X | X | X 393+IBV_WR_SEND_WITH_IMM | X | X | X | X 394+IBV_WR_RDMA_WRITE | | X | X | X 395+IBV_WR_RDMA_WRITE_WITH_IMM | | X | X | X 396+IBV_WR_RDMA_READ | | | X | X 397+IBV_WR_ATOMIC_CMP_AND_SWP | | | X | X 398+IBV_WR_ATOMIC_FETCH_AND_ADD | | | X | X 399 .fi 400 .PP 401 The attribute send_flags describes the properties of the \s-1WR\s0. It is either 0 or the bitwise \s-1OR\s0 of one or more of the following flags: 402@@ -114,6 +115,7 @@ IBV_SEND_INLINE flag was set, the buffer 403 after the call returns. 404 .SH "SEE ALSO" 405 .BR ibv_create_qp (3), 406+.BR ibv_create_xrc_rcv_qp (3), 407 .BR ibv_create_ah (3), 408 .BR ibv_post_recv (3), 409 .BR ibv_post_srq_recv (3), 410Index: libibverbs/man/ibv_query_xrc_rcv_qp.3 411=================================================================== 412--- /dev/null 1970-01-01 00:00:00.000000000 +0000 413+++ libibverbs/man/ibv_query_xrc_rcv_qp.3 2008-06-05 16:25:21.000000000 +0300 414@@ -0,0 +1,89 @@ 415+.\" -*- nroff -*- 416+.\" 417+.TH IBV_QUERY_XRC_RCV_QP 3 2008-02-10 libibverbs "Libibverbs Programmer's Manual" 418+.SH "NAME" 419+ibv_query_xrc_rcv_qp \- get the attributes of an XRC receive queue pair (QP) 420+.SH "SYNOPSIS" 421+.nf 422+.B #include <infiniband/verbs.h> 423+.sp 424+.BI "int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" , 425+.BI " struct ibv_qp_attr " "*attr" ", int " "attr_mask" , 426+.BI " struct ibv_qp_init_attr " "*init_attr" ); 427+.fi 428+.SH "DESCRIPTION" 429+.B ibv_query_xrc_rcv_qp() 430+gets the attributes specified in 431+.I attr_mask 432+for the XRC receive QP with the number 433+.I xrc_qp_num 434+which is associated with the XRC domain 435+.I xrc_domain 436+and returns them through the pointers 437+.I attr 438+and 439+.I init_attr\fR. 440+The argument 441+.I attr 442+is an ibv_qp_attr struct, as defined in <infiniband/verbs.h>. 443+.PP 444+.nf 445+struct ibv_qp_attr { 446+.in +8 447+enum ibv_qp_state qp_state; /* Current QP state */ 448+enum ibv_qp_state cur_qp_state; /* Current QP state - irrelevant for ibv_query_qp */ 449+enum ibv_mtu path_mtu; /* Path MTU (valid only for RC/UC QPs) */ 450+enum ibv_mig_state path_mig_state; /* Path migration state (valid if HCA supports APM) */ 451+uint32_t qkey; /* Q_Key of the QP (valid only for UD QPs) */ 452+uint32_t rq_psn; /* PSN for receive queue (valid only for RC/UC QPs) */ 453+uint32_t sq_psn; /* PSN for send queue (valid only for RC/UC QPs) */ 454+uint32_t dest_qp_num; /* Destination QP number (valid only for RC/UC QPs) */ 455+int qp_access_flags; /* Mask of enabled remote access operations (valid only for RC/UC QPs) */ 456+struct ibv_qp_cap cap; /* QP capabilities */ 457+struct ibv_ah_attr ah_attr; /* Primary path address vector (valid only for RC/UC QPs) */ 458+struct ibv_ah_attr alt_ah_attr; /* Alternate path address vector (valid only for RC/UC QPs) */ 459+uint16_t pkey_index; /* Primary P_Key index */ 460+uint16_t alt_pkey_index; /* Alternate P_Key index */ 461+uint8_t en_sqd_async_notify; /* Enable SQD.drained async notification - irrelevant for ibv_query_qp */ 462+uint8_t sq_draining; /* Is the QP draining? (Valid only if qp_state is SQD) */ 463+uint8_t max_rd_atomic; /* Number of outstanding RDMA reads & atomic operations on the destination QP (valid only for RC QPs) */ 464+uint8_t max_dest_rd_atomic; /* Number of responder resources for handling incoming RDMA reads & atomic operations (valid only for RC QPs) */ 465+uint8_t min_rnr_timer; /* Minimum RNR NAK timer (valid only for RC QPs) */ 466+uint8_t port_num; /* Primary port number */ 467+uint8_t timeout; /* Local ack timeout for primary path (valid only for RC QPs) */ 468+uint8_t retry_cnt; /* Retry count (valid only for RC QPs) */ 469+uint8_t rnr_retry; /* RNR retry (valid only for RC QPs) */ 470+uint8_t alt_port_num; /* Alternate port number */ 471+uint8_t alt_timeout; /* Local ack timeout for alternate path (valid only for RC QPs) */ 472+.in -8 473+}; 474+.fi 475+.PP 476+For details on struct ibv_qp_cap see the description of 477+.B ibv_create_qp()\fR. 478+For details on struct ibv_ah_attr see the description of 479+.B ibv_create_ah()\fR. 480+.SH "RETURN VALUE" 481+.B ibv_query_xrc_rcv_qp() 482+returns 0 on success, or the value of errno on failure (which indicates the failure reason). 483+.SH "NOTES" 484+The argument 485+.I attr_mask 486+is a hint that specifies the minimum list of attributes to retrieve. 487+Some InfiniBand devices may return extra attributes not requested, for 488+example if the value can be returned cheaply. 489+.PP 490+Attribute values are valid if they have been set using 491+.B ibv_modify_xrc_rcv_qp()\fR. 492+The exact list of valid attributes depends on the QP state. 493+.PP 494+Multiple calls to 495+.B ibv_query_xrc_rcv_qp() 496+may yield some differences in the values returned for the following attributes: qp_state, path_mig_state, sq_draining, ah_attr (if APM is enabled). 497+.SH "SEE ALSO" 498+.BR ibv_open_xrc_domain (3), 499+.BR ibv_create_xrc_rcv_qp (3), 500+.BR ibv_modify_xrc_rcv_qp (3) 501+.SH "AUTHORS" 502+.TP 503+Dotan Barak <dotanb@mellanox.co.il> 504Index: libibverbs/man/ibv_reg_xrc_rcv_qp.3 505=================================================================== 506--- /dev/null 1970-01-01 00:00:00.000000000 +0000 507+++ libibverbs/man/ibv_reg_xrc_rcv_qp.3 2008-06-05 16:25:21.000000000 +0300 508@@ -0,0 +1,57 @@ 509+.\" -*- nroff -*- 510+.\" 511+.TH IBV_REG_XRC_RCV_QP 3 2008-10-02 libibverbs "Libibverbs Programmer's Manual" 512+.SH "NAME" 513+ibv_reg_xrc_rcv_qp, ibv_unreg_xrc_rcv_qp \- register and unregister a user process with an XRC receive queue pair (QP) 514+.SH "SYNOPSIS" 515+.nf 516+.B #include <infiniband/verbs.h> 517+.sp 518+.BI "int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" "); 519+.nl 520+.BI "int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain " "*xrc_domain" ", uint32_t " "xrc_qp_num" "); 521+.fi 522+.SH "DESCRIPTION" 523+.B ibv_reg_xrc_rcv_qp() 524+registers a user process with the XRC receive QP (created via 525+.B ibv_create_xrc_rcv_qp() 526+) whose number is 527+.I xrc_qp_num\fR, 528+and which is associated with the XRC domain 529+.I xrc_domain\fR. 530+.PP 531+.B ibv_unreg_xrc_rcv_qp() 532+unregisters a user process from the XRC receive QP number 533+.I xrc_qp_num\fR, 534+which is associated with the XRC domain 535+.I xrc_domain\fR. 536+When the number of user processes registered with this XRC receive QP drops to zero, the QP is destroyed. 537+.SH "RETURN VALUE" 538+.B ibv_reg_xrc_rcv_qp() 539+and 540+.B ibv_unreg_xrc_rcv_qp() 541+returns 0 on success, or the value of errno on failure (which indicates the failure reason). 542+.SH "NOTES" 543+.B ibv_reg_xrc_rcv_qp() 544+and 545+.B ibv_unreg_xrc_rcv_qp() 546+may fail if the number 547+.I xrc_qp_num 548+is not a number of a valid XRC receive QP (the QP is not allocated or it is the number of a non-XRC QP), or 549+the XRC receive QP was created with an XRC domain other than 550+.I xrc_domain\fR. 551+ 552+If a process is still registered with any XRC RCV QPs belonging to some domain, 553+.B ibv_close_xrc_domain() 554+will return failure if called for that domain in that process. 555+ 556+.B ibv_create_xrc_rcv_qp() 557+performs an implicit registration for the creating process; when that process is finished with the XRC RCV QP, it should call 558+.B ibv_unreg_xrc_rcv_qp() 559+for that QP. Note that if no other processes are registered with the QP at this time, its registration count will drop to zero and it will be destroyed. 560+.SH "SEE ALSO" 561+.BR ibv_open_xrc_domain (3), 562+.BR ibv_create_xrc_rcv_qp (3) 563+.SH "AUTHORS" 564+.TP 565+Dotan Barak <dotanb@mellanox.co.il> 566Index: libibverbs/man/verbs.7 567=================================================================== 568--- libibverbs.orig/man/verbs.7 2008-06-05 15:24:00.000000000 +0300 569+++ libibverbs/man/verbs.7 2008-06-05 16:25:21.000000000 +0300 570@@ -1,6 +1,6 @@ 571 .\" -*- nroff -*- 572 .\" 573-.TH VERBS 7 2008-01-17 libibverbs "Libibverbs Programmer's Manual" 574+.TH VERBS 7 2008-02-25 libibverbs "Libibverbs Programmer's Manual" 575 .SH "NAME" 576 verbs \- Infiniband verbs library 577 .SH "SYNOPSIS" 578@@ -8,7 +8,7 @@ verbs \- Infiniband verbs library 579 .B #include <infiniband/verbs.h> 580 .fi 581 .SH "DESCRIPTION" 582-This library is an implementation of the verbs according to the Infiniband specification volume 1.2. It handles the control path of creating, modifying, querying and destroying resources such as Protection Domains (PD), Completion Queues (CQ), Queue-Pairs (QP), Shared Receive Queues (SRQ), Address Handles (AH), Memory Regions (MR). It also handles sending and receiving data posted to QPs and SRQs, getting completions from CQs using polling and completions events. 583+This library is an implementation of the verbs based on the Infiniband specification volume 1.2 chapter 11. It handles the control path of creating, modifying, querying and destroying resources such as Protection Domains (PD), Completion Queues (CQ), Queue-Pairs (QP), Shared Receive Queues (SRQ), Address Handles (AH), Memory Regions (MR). It also handles sending and receiving data posted to QPs and SRQs, getting completions from CQs using polling and completions events. 584 585 The control path is implemented through system calls to the uverbs kernel module which further calls the low level HW driver. The data path is implemented through calls made to low level HW library which in most cases interacts directly with the HW providing kernel and network stack bypass (saving context/mode switches) along with zero copy and an asynchronous I/O model. 586 587@@ -117,6 +117,25 @@ int ibv_modify_srq(struct ibv_srq *srq, 588 enum ibv_srq_attr_mask srq_attr_mask); 589 int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); 590 591+.B eXtended Reliable Connection control 592+ 593+struct ibv_xrc_domain *ibv_open_xrc_domain(struct ibv_context *context, 594+ int fd, int oflag); 595+int ibv_close_xrc_domain(struct ibv_xrc_domain *d); 596+struct ibv_srq *ibv_create_xrc_srq(struct ibv_pd *pd, 597+ struct ibv_xrc_domain *xrc_domain, 598+ struct ibv_cq *xrc_cq, 599+ struct ibv_srq_init_attr *srq_init_attr); 600+int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, 601+ uint32_t *xrc_rcv_qpn); 602+int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num, 603+ struct ibv_qp_attr *attr, int attr_mask); 604+int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num, 605+ struct ibv_qp_attr *attr, int attr_mask, 606+ struct ibv_qp_init_attr *init_attr); 607+int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num); 608+int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num); 609+ 610 .B Queue Pair control 611 612 struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, 613@@ -183,6 +202,14 @@ enum ibv_rate mult_to_ibv_rate(int mult) 614 \fIibv_destroy_srq\fP(), 615 \fIibv_modify_srq\fP(), 616 \fIibv_query_srq\fP(), 617+\fIibv_open_xrc_domain\fP(), 618+\fIibv_close_xrc_domain\fP(), 619+\fIibv_create_xrc_srq\fP(), 620+\fIibv_create_xrc_rcv_qp\fP(), 621+\fIibv_modify_xrc_rcv_qp\fP(), 622+\fIibv_query_xrc_rcv_qp\fP(), 623+\fIibv_reg_xrc_rcv_qp\fP(), 624+\fIibv_unreg_xrc_rcv_qp\fP(), 625 \fIibv_post_srq_recv\fP(), 626 \fIibv_create_qp\fP(), 627 \fIibv_destroy_qp\fP(), 628Index: libibverbs/Makefile.am 629=================================================================== 630--- libibverbs.orig/Makefile.am 2008-06-05 15:24:00.000000000 +0300 631+++ libibverbs/Makefile.am 2008-06-05 16:39:27.000000000 +0300 632@@ -44,15 +44,18 @@ man_MANS = man/ibv_asyncwatch.1 man/ibv_ 633 man/ibv_srq_pingpong.1 man/ibv_alloc_pd.3 man/ibv_attach_mcast.3 \ 634 man/ibv_create_ah.3 man/ibv_create_ah_from_wc.3 \ 635 man/ibv_create_comp_channel.3 man/ibv_create_cq.3 \ 636- man/ibv_create_qp.3 man/ibv_create_srq.3 man/ibv_event_type_str.3 \ 637+ man/ibv_create_qp.3 man/ibv_create_srq.3 \ 638+ man/ibv_create_xrc_rcv_qp.3 man/ibv_event_type_str.3 \ 639 man/ibv_fork_init.3 man/ibv_get_async_event.3 \ 640 man/ibv_get_cq_event.3 man/ibv_get_device_guid.3 \ 641 man/ibv_get_device_list.3 man/ibv_get_device_name.3 \ 642- man/ibv_modify_qp.3 man/ibv_modify_srq.3 man/ibv_open_device.3 \ 643+ man/ibv_modify_qp.3 man/ibv_modify_srq.3 man/ibv_modify_xrc_rcv_qp.3 \ 644+ man/ibv_open_device.3 man/ibv_open_xrc_domain.3 \ 645 man/ibv_poll_cq.3 man/ibv_post_recv.3 man/ibv_post_send.3 \ 646 man/ibv_post_srq_recv.3 man/ibv_query_device.3 man/ibv_query_gid.3 \ 647 man/ibv_query_pkey.3 man/ibv_query_port.3 man/ibv_query_qp.3 \ 648- man/ibv_query_srq.3 man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 \ 649+ man/ibv_query_srq.3 man/ibv_query_xrc_rcv_qp.3 \ 650+ man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 man/ibv_reg_xrc_rcv_qp.3 \ 651 man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/verbs.7 652 653 DEBIAN = debian/changelog debian/compat debian/control debian/copyright \ 654@@ -74,6 +77,8 @@ install-data-hook: 655 $(RM) ibv_ack_async_event.3 && \ 656 $(RM) ibv_ack_cq_events.3 && \ 657 $(RM) ibv_close_device.3 && \ 658+ $(RM) ibv_close_xrc_domain.3 && \ 659+ $(RM) ibv_create_xrc_srq.3 && \ 660 $(RM) ibv_dealloc_pd.3 && \ 661 $(RM) ibv_dereg_mr.3 && \ 662 $(RM) ibv_destroy_ah.3 && \ 663@@ -84,12 +89,15 @@ install-data-hook: 664 $(RM) ibv_detach_mcast.3 && \ 665 $(RM) ibv_free_device_list.3 && \ 666 $(RM) ibv_init_ah_from_wc.3 && \ 667+ $(RM) ibv_unreg_xrc_rcv_qp.3 && \ 668 $(RM) mult_to_ibv_rate.3 && \ 669 $(RM) ibv_node_type_str.3 && \ 670 $(RM) ibv_port_state_str.3 && \ 671 $(LN_S) ibv_get_async_event.3 ibv_ack_async_event.3 && \ 672 $(LN_S) ibv_get_cq_event.3 ibv_ack_cq_events.3 && \ 673 $(LN_S) ibv_open_device.3 ibv_close_device.3 && \ 674+ $(LN_S) ibv_open_xrc_domain.3 ibv_close_xrc_domain.3 && \ 675+ $(LN_S) ibv_create_srq.3 ibv_create_xrc_srq.3 && \ 676 $(LN_S) ibv_alloc_pd.3 ibv_dealloc_pd.3 && \ 677 $(LN_S) ibv_reg_mr.3 ibv_dereg_mr.3 && \ 678 $(LN_S) ibv_create_ah.3 ibv_destroy_ah.3 && \ 679@@ -100,6 +108,7 @@ install-data-hook: 680 $(LN_S) ibv_attach_mcast.3 ibv_detach_mcast.3 && \ 681 $(LN_S) ibv_get_device_list.3 ibv_free_device_list.3 && \ 682 $(LN_S) ibv_create_ah_from_wc.3 ibv_init_ah_from_wc.3 && \ 683+ $(LN_S) ibv_reg_xrc_rcv_qp.3 ibv_unreg_xrc_rcv_qp.3 && \ 684 $(LN_S) ibv_rate_to_mult.3 mult_to_ibv_rate.3 && \ 685 $(LN_S) ibv_event_type_str.3 ibv_node_type_str.3 && \ 686 $(LN_S) ibv_event_type_str.3 ibv_port_state_str.3 687