1Added support for XRC receive-only QPs. 2(OFED 1.3 libibverbs commit 6e99cddf835d4715ea7ca3641944e6285f27f2df) 3 4V2: 51. checkpatch.pl cleanups 62. Fixed u64 alignment problems in kern-abi.h 73. eliminated unneeded default_symvers 84. Added ibv_xrc_rcv_xxx lines to libibverbs.map IBVERBS_1.1 9 10Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il> 11--- 12 include/infiniband/driver.h | 12 ++- 13 include/infiniband/kern-abi.h | 99 +++++++++++++++++++- 14 include/infiniband/verbs.h | 123 +++++++++++++++++++++++ 15 src/cmd.c | 215 +++++++++++++++++++++++++++++++++++++++++ 16 src/device.c | 52 +++++----- 17 src/libibverbs.map | 10 ++ 18 src/verbs.c | 59 +++++++++++ 19 7 files changed, 543 insertions(+), 27 deletions(-) 20 21Index: libibverbs/include/infiniband/driver.h 22=================================================================== 23--- libibverbs.orig/include/infiniband/driver.h 2009-11-01 15:18:20.624171000 +0200 24+++ libibverbs/include/infiniband/driver.h 2009-11-01 15:18:24.572283000 +0200 25@@ -144,7 +144,17 @@ int ibv_cmd_open_xrc_domain(struct ibv_c 26 struct ibv_open_xrc_domain_resp *resp, 27 size_t resp_size); 28 int ibv_cmd_close_xrc_domain(struct ibv_xrc_domain *d); 29- 30+int ibv_cmd_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, 31+ uint32_t *xrc_rcv_qpn); 32+int ibv_cmd_modify_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_rcv_qpn, 33+ struct ibv_qp_attr *attr, int attr_mask); 34+int ibv_cmd_query_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_rcv_qpn, 35+ struct ibv_qp_attr *attr, int attr_mask, 36+ struct ibv_qp_init_attr *init_attr); 37+int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, 38+ uint32_t xrc_qp_num); 39+int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, 40+ uint32_t xrc_qp_num); 41 42 /* 43 * sysfs helper functions 44Index: libibverbs/include/infiniband/kern-abi.h 45=================================================================== 46--- libibverbs.orig/include/infiniband/kern-abi.h 2009-11-01 15:18:20.629168000 +0200 47+++ libibverbs/include/infiniband/kern-abi.h 2009-11-01 15:18:24.577283000 +0200 48@@ -88,7 +88,12 @@ enum { 49 IB_USER_VERBS_CMD_POST_SRQ_RECV, 50 IB_USER_VERBS_CMD_CREATE_XRC_SRQ, 51 IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN, 52- IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN 53+ IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN, 54+ IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP, 55+ IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP, 56+ IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP, 57+ IB_USER_VERBS_CMD_REG_XRC_RCV_QP, 58+ IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP, 59 }; 60 61 /* 62@@ -570,6 +575,93 @@ struct ibv_destroy_qp_resp { 63 __u32 events_reported; 64 }; 65 66+struct ibv_create_xrc_rcv_qp { 67+ __u32 command; 68+ __u16 in_words; 69+ __u16 out_words; 70+ __u64 response; 71+ __u64 user_handle; 72+ __u32 xrc_domain_handle; 73+ __u32 max_send_wr; 74+ __u32 max_recv_wr; 75+ __u32 max_send_sge; 76+ __u32 max_recv_sge; 77+ __u32 max_inline_data; 78+ __u8 sq_sig_all; 79+ __u8 qp_type; 80+ __u8 reserved[6]; 81+ __u64 driver_data[0]; 82+}; 83+ 84+struct ibv_create_xrc_rcv_qp_resp { 85+ __u32 qpn; 86+ __u32 reserved; 87+}; 88+ 89+struct ibv_modify_xrc_rcv_qp { 90+ __u32 command; 91+ __u16 in_words; 92+ __u16 out_words; 93+ __u32 xrc_domain_handle; 94+ __u32 qp_num; 95+ struct ibv_qp_dest dest; 96+ struct ibv_qp_dest alt_dest; 97+ __u32 attr_mask; 98+ __u32 qkey; 99+ __u32 rq_psn; 100+ __u32 sq_psn; 101+ __u32 dest_qp_num; 102+ __u32 qp_access_flags; 103+ __u16 pkey_index; 104+ __u16 alt_pkey_index; 105+ __u8 qp_state; 106+ __u8 cur_qp_state; 107+ __u8 path_mtu; 108+ __u8 path_mig_state; 109+ __u8 en_sqd_async_notify; 110+ __u8 max_rd_atomic; 111+ __u8 max_dest_rd_atomic; 112+ __u8 min_rnr_timer; 113+ __u8 port_num; 114+ __u8 timeout; 115+ __u8 retry_cnt; 116+ __u8 rnr_retry; 117+ __u8 alt_port_num; 118+ __u8 alt_timeout; 119+ __u8 reserved[6]; 120+ __u64 driver_data[0]; 121+}; 122+ 123+struct ibv_query_xrc_rcv_qp { 124+ __u32 command; 125+ __u16 in_words; 126+ __u16 out_words; 127+ __u64 response; 128+ __u32 xrc_domain_handle; 129+ __u32 qp_num; 130+ __u32 attr_mask; 131+ __u32 reserved; 132+ __u64 driver_data[0]; 133+}; 134+ 135+struct ibv_reg_xrc_rcv_qp { 136+ __u32 command; 137+ __u16 in_words; 138+ __u16 out_words; 139+ __u32 xrc_domain_handle; 140+ __u32 qp_num; 141+ __u64 driver_data[0]; 142+}; 143+ 144+struct ibv_unreg_xrc_rcv_qp { 145+ __u32 command; 146+ __u16 in_words; 147+ __u16 out_words; 148+ __u32 xrc_domain_handle; 149+ __u32 qp_num; 150+ __u64 driver_data[0]; 151+}; 152+ 153 struct ibv_kern_send_wr { 154 __u64 wr_id; 155 __u32 num_sge; 156@@ -848,6 +940,11 @@ enum { 157 IB_USER_VERBS_CMD_CREATE_XRC_SRQ_V2 = -1, 158 IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN_V2 = -1, 159 IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN_V2 = -1, 160+ IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP_V2 = -1, 161+ IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP_V2 = -1, 162+ IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1, 163+ IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1, 164+ IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1, 165 }; 166 167 struct ibv_destroy_cq_v1 { 168Index: libibverbs/include/infiniband/verbs.h 169=================================================================== 170--- libibverbs.orig/include/infiniband/verbs.h 2009-11-01 15:18:20.635171000 +0200 171+++ libibverbs/include/infiniband/verbs.h 2009-11-01 15:18:24.585280000 +0200 172@@ -205,12 +205,17 @@ enum ibv_event_type { 173 IBV_EVENT_CLIENT_REREGISTER 174 }; 175 176+enum ibv_event_flags { 177+ IBV_XRC_QP_EVENT_FLAG = 0x80000000, 178+}; 179+ 180 struct ibv_async_event { 181 union { 182 struct ibv_cq *cq; 183 struct ibv_qp *qp; 184 struct ibv_srq *srq; 185 int port_num; 186+ uint32_t xrc_qp_num; 187 } element; 188 enum ibv_event_type event_type; 189 }; 190@@ -648,6 +653,22 @@ struct ibv_more_ops { 191 struct ibv_xrc_domain * (*open_xrc_domain)(struct ibv_context *context, 192 int fd, int oflag); 193 int (*close_xrc_domain)(struct ibv_xrc_domain *d); 194+ int (*create_xrc_rcv_qp)(struct ibv_qp_init_attr *init_attr, 195+ uint32_t *xrc_qp_num); 196+ int (*modify_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain, 197+ uint32_t xrc_qp_num, 198+ struct ibv_qp_attr *attr, 199+ int attr_mask); 200+ int (*query_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain, 201+ uint32_t xrc_qp_num, 202+ struct ibv_qp_attr *attr, 203+ int attr_mask, 204+ struct ibv_qp_init_attr *init_attr); 205+ int (*reg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain, 206+ uint32_t xrc_qp_num); 207+ int (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain, 208+ uint32_t xrc_qp_num); 209+ 210 }; 211 212 struct ibv_context_ops { 213@@ -1174,6 +1195,108 @@ struct ibv_xrc_domain *ibv_open_xrc_doma 214 */ 215 int ibv_close_xrc_domain(struct ibv_xrc_domain *d); 216 217+/** 218+ * ibv_create_xrc_rcv_qp - creates an XRC QP for serving as a receive-side-only QP, 219+ * 220+ * This QP is created in kernel space, and persists until the last process 221+ * registered for the QP calls ibv_unreg_xrc_rcv_qp() (at which time the QP 222+ * is destroyed). 223+ * 224+ * @init_attr: init attributes to use for QP. xrc domain MUST be included here. 225+ * All other fields are ignored. 226+ * 227+ * @xrc_rcv_qpn: qp_num of created QP (if success). To be passed to the 228+ * remote node (sender). The remote node will use xrc_rcv_qpn 229+ * in ibv_post_send when sending to XRC SRQ's on this host 230+ * in the same xrc domain. 231+ * 232+ * RETURNS: success (0), or a (negative) error value. 233+ * 234+ * NOTE: this verb also registers the calling user-process with the QP at its 235+ * creation time (implicit call to ibv_reg_xrc_rcv_qp), to avoid race 236+ * conditions. The creating process will need to call ibv_unreg_xrc_qp() 237+ * for the QP to release it from this process. 238+ */ 239+int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, 240+ uint32_t *xrc_rcv_qpn); 241+ 242+/** 243+ * ibv_modify_xrc_rcv_qp - modifies an xrc_rcv qp. 244+ * 245+ * @xrc_domain: xrc domain the QP belongs to (for verification). 246+ * @xrc_qp_num: The (24 bit) number of the XRC QP. 247+ * @attr: modify-qp attributes. The following fields must be specified: 248+ * for RESET_2_INIT: qp_state, pkey_index , port, qp_access_flags 249+ * for INIT_2_RTR: qp_state, path_mtu, dest_qp_num, rq_psn, 250+ * max_dest_rd_atomic, min_rnr_timer, ah_attr 251+ * The QP need not be brought to RTS for the QP to operate as a 252+ * receive-only QP. 253+ * @attr_mask: bitmap indicating which attributes are provided in the attr 254+ * struct. Used for validity checking. 255+ * The following bits must be set: 256+ * for RESET_2_INIT: IBV_QP_PKEY_INDEX, IBV_QP_PORT, 257+ * IBV_QP_ACCESS_FLAGS, IBV_QP_STATE 258+ * for INIT_2_RTR: IBV_QP_AV, IBV_QP_PATH_MTU, IBV_QP_DEST_QPN, 259+ * IBV_QP_RQ_PSN, IBV_QP_MAX_DEST_RD_ATOMIC, 260+ * IBV_QP_MIN_RNR_TIMER, IBV_QP_STATE 261+ * 262+ * RETURNS: success (0), or a (positive) error value. 263+ * 264+ */ 265+int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, 266+ uint32_t xrc_qp_num, 267+ struct ibv_qp_attr *attr, int attr_mask); 268+ 269+/** 270+ * ibv_query_xrc_rcv_qp - queries an xrc_rcv qp. 271+ * 272+ * @xrc_domain: xrc domain the QP belongs to (for verification). 273+ * @xrc_qp_num: The (24 bit) number of the XRC QP. 274+ * @attr: for returning qp attributes. 275+ * @attr_mask: bitmap indicating which attributes to return. 276+ * @init_attr: for returning the init attributes 277+ * 278+ * RETURNS: success (0), or a (positive) error value. 279+ * 280+ */ 281+int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num, 282+ struct ibv_qp_attr *attr, int attr_mask, 283+ struct ibv_qp_init_attr *init_attr); 284+ 285+/** 286+ * ibv_reg_xrc_rcv_qp: registers a user process with an XRC QP which serves as 287+ * a receive-side only QP. 288+ * 289+ * @xrc_domain: xrc domain the QP belongs to (for verification). 290+ * @xrc_qp_num: The (24 bit) number of the XRC QP. 291+ * 292+ * RETURNS: success (0), 293+ * or error (EINVAL), if: 294+ * 1. There is no such QP_num allocated. 295+ * 2. The QP is allocated, but is not an receive XRC QP 296+ * 3. The XRC QP does not belong to the given domain. 297+ */ 298+int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num); 299+ 300+/** 301+ * ibv_unreg_xrc_rcv_qp: detaches a user process from an XRC QP serving as 302+ * a receive-side only QP. If as a result, there are no remaining 303+ * userspace processes registered for this XRC QP, it is destroyed. 304+ * 305+ * @xrc_domain: xrc domain the QP belongs to (for verification). 306+ * @xrc_qp_num: The (24 bit) number of the XRC QP. 307+ * 308+ * RETURNS: success (0), 309+ * or error (EINVAL), if: 310+ * 1. There is no such QP_num allocated. 311+ * 2. The QP is allocated, but is not an XRC QP 312+ * 3. The XRC QP does not belong to the given domain. 313+ * NOTE: There is no reason to return a special code if the QP is destroyed. 314+ * The unregister simply succeeds. 315+ */ 316+int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, 317+ uint32_t xrc_qp_num); 318+ 319 END_C_DECLS 320 321 # undef __attribute_const 322Index: libibverbs/src/cmd.c 323=================================================================== 324--- libibverbs.orig/src/cmd.c 2009-11-01 15:18:20.643167000 +0200 325+++ libibverbs/src/cmd.c 2009-11-01 15:18:24.592284000 +0200 326@@ -828,6 +828,188 @@ int ibv_cmd_modify_qp(struct ibv_qp *qp, 327 return 0; 328 } 329 330+int ibv_cmd_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, 331+ uint32_t *xrc_rcv_qpn) 332+{ 333+ struct ibv_create_xrc_rcv_qp cmd; 334+ struct ibv_create_xrc_rcv_qp_resp resp; 335+ 336+ if (abi_ver < 6) 337+ return ENOSYS; 338+ 339+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_XRC_RCV_QP, &resp, 340+ sizeof resp); 341+ 342+ cmd.xrc_domain_handle = init_attr->xrc_domain->handle; 343+ cmd.max_send_wr = init_attr->cap.max_send_wr; 344+ cmd.max_recv_wr = init_attr->cap.max_recv_wr; 345+ cmd.max_send_sge = init_attr->cap.max_send_sge; 346+ cmd.max_recv_sge = init_attr->cap.max_recv_sge; 347+ cmd.max_inline_data = init_attr->cap.max_inline_data; 348+ cmd.sq_sig_all = init_attr->sq_sig_all; 349+ cmd.qp_type = init_attr->qp_type; 350+ cmd.reserved[0] = cmd.reserved[1] = 0; 351+ 352+ if (write(init_attr->xrc_domain->context->cmd_fd, &cmd, sizeof cmd) != 353+ sizeof cmd) 354+ return errno; 355+ 356+ *xrc_rcv_qpn = resp.qpn; 357+ 358+ return 0; 359+} 360+ 361+int ibv_cmd_modify_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num, 362+ struct ibv_qp_attr *attr, int attr_mask) 363+{ 364+ struct ibv_modify_xrc_rcv_qp cmd; 365+ 366+ if (abi_ver < 6) 367+ return ENOSYS; 368+ 369+ IBV_INIT_CMD(&cmd, sizeof cmd, MODIFY_XRC_RCV_QP); 370+ 371+ cmd.xrc_domain_handle = d->handle; 372+ cmd.qp_num = xrc_qp_num; 373+ cmd.attr_mask = attr_mask; 374+ cmd.qkey = attr->qkey; 375+ cmd.rq_psn = attr->rq_psn; 376+ cmd.sq_psn = attr->sq_psn; 377+ cmd.dest_qp_num = attr->dest_qp_num; 378+ cmd.qp_access_flags = attr->qp_access_flags; 379+ cmd.pkey_index = attr->pkey_index; 380+ cmd.alt_pkey_index = attr->alt_pkey_index; 381+ cmd.qp_state = attr->qp_state; 382+ cmd.cur_qp_state = attr->cur_qp_state; 383+ cmd.path_mtu = attr->path_mtu; 384+ cmd.path_mig_state = attr->path_mig_state; 385+ cmd.en_sqd_async_notify = attr->en_sqd_async_notify; 386+ cmd.max_rd_atomic = attr->max_rd_atomic; 387+ cmd.max_dest_rd_atomic = attr->max_dest_rd_atomic; 388+ cmd.min_rnr_timer = attr->min_rnr_timer; 389+ cmd.port_num = attr->port_num; 390+ cmd.timeout = attr->timeout; 391+ cmd.retry_cnt = attr->retry_cnt; 392+ cmd.rnr_retry = attr->rnr_retry; 393+ cmd.alt_port_num = attr->alt_port_num; 394+ cmd.alt_timeout = attr->alt_timeout; 395+ 396+ memcpy(cmd.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 397+ cmd.dest.flow_label = attr->ah_attr.grh.flow_label; 398+ cmd.dest.dlid = attr->ah_attr.dlid; 399+ cmd.dest.reserved = 0; 400+ cmd.dest.sgid_index = attr->ah_attr.grh.sgid_index; 401+ cmd.dest.hop_limit = attr->ah_attr.grh.hop_limit; 402+ cmd.dest.traffic_class = attr->ah_attr.grh.traffic_class; 403+ cmd.dest.sl = attr->ah_attr.sl; 404+ cmd.dest.src_path_bits = attr->ah_attr.src_path_bits; 405+ cmd.dest.static_rate = attr->ah_attr.static_rate; 406+ cmd.dest.is_global = attr->ah_attr.is_global; 407+ cmd.dest.port_num = attr->ah_attr.port_num; 408+ 409+ memcpy(cmd.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 410+ cmd.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 411+ cmd.alt_dest.dlid = attr->alt_ah_attr.dlid; 412+ cmd.alt_dest.reserved = 0; 413+ cmd.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 414+ cmd.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 415+ cmd.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 416+ cmd.alt_dest.sl = attr->alt_ah_attr.sl; 417+ cmd.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 418+ cmd.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 419+ cmd.alt_dest.is_global = attr->alt_ah_attr.is_global; 420+ cmd.alt_dest.port_num = attr->alt_ah_attr.port_num; 421+ 422+ cmd.reserved[0] = cmd.reserved[1] = 0; 423+ 424+ if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 425+ return errno; 426+ 427+ return 0; 428+} 429+ 430+int ibv_cmd_query_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num, 431+ struct ibv_qp_attr *attr, int attr_mask, 432+ struct ibv_qp_init_attr *init_attr) 433+{ 434+ struct ibv_query_xrc_rcv_qp cmd; 435+ struct ibv_query_qp_resp resp; 436+ 437+ if (abi_ver < 6) 438+ return ENOSYS; 439+ 440+ IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_XRC_RCV_QP, &resp, 441+ sizeof resp); 442+ cmd.xrc_domain_handle = d->handle; 443+ cmd.qp_num = xrc_qp_num; 444+ cmd.attr_mask = attr_mask; 445+ 446+ if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 447+ return errno; 448+ 449+ VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 450+ 451+ attr->qkey = resp.qkey; 452+ attr->rq_psn = resp.rq_psn; 453+ attr->sq_psn = resp.sq_psn; 454+ attr->dest_qp_num = resp.dest_qp_num; 455+ attr->qp_access_flags = resp.qp_access_flags; 456+ attr->pkey_index = resp.pkey_index; 457+ attr->alt_pkey_index = resp.alt_pkey_index; 458+ attr->qp_state = resp.qp_state; 459+ attr->cur_qp_state = resp.cur_qp_state; 460+ attr->path_mtu = resp.path_mtu; 461+ attr->path_mig_state = resp.path_mig_state; 462+ attr->sq_draining = resp.sq_draining; 463+ attr->max_rd_atomic = resp.max_rd_atomic; 464+ attr->max_dest_rd_atomic = resp.max_dest_rd_atomic; 465+ attr->min_rnr_timer = resp.min_rnr_timer; 466+ attr->port_num = resp.port_num; 467+ attr->timeout = resp.timeout; 468+ attr->retry_cnt = resp.retry_cnt; 469+ attr->rnr_retry = resp.rnr_retry; 470+ attr->alt_port_num = resp.alt_port_num; 471+ attr->alt_timeout = resp.alt_timeout; 472+ attr->cap.max_send_wr = resp.max_send_wr; 473+ attr->cap.max_recv_wr = resp.max_recv_wr; 474+ attr->cap.max_send_sge = resp.max_send_sge; 475+ attr->cap.max_recv_sge = resp.max_recv_sge; 476+ attr->cap.max_inline_data = resp.max_inline_data; 477+ 478+ memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16); 479+ attr->ah_attr.grh.flow_label = resp.dest.flow_label; 480+ attr->ah_attr.dlid = resp.dest.dlid; 481+ attr->ah_attr.grh.sgid_index = resp.dest.sgid_index; 482+ attr->ah_attr.grh.hop_limit = resp.dest.hop_limit; 483+ attr->ah_attr.grh.traffic_class = resp.dest.traffic_class; 484+ attr->ah_attr.sl = resp.dest.sl; 485+ attr->ah_attr.src_path_bits = resp.dest.src_path_bits; 486+ attr->ah_attr.static_rate = resp.dest.static_rate; 487+ attr->ah_attr.is_global = resp.dest.is_global; 488+ attr->ah_attr.port_num = resp.dest.port_num; 489+ 490+ memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16); 491+ attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label; 492+ attr->alt_ah_attr.dlid = resp.alt_dest.dlid; 493+ attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index; 494+ attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit; 495+ attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class; 496+ attr->alt_ah_attr.sl = resp.alt_dest.sl; 497+ attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits; 498+ attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate; 499+ attr->alt_ah_attr.is_global = resp.alt_dest.is_global; 500+ attr->alt_ah_attr.port_num = resp.alt_dest.port_num; 501+ 502+ init_attr->cap.max_send_wr = resp.max_send_wr; 503+ init_attr->cap.max_recv_wr = resp.max_recv_wr; 504+ init_attr->cap.max_send_sge = resp.max_send_sge; 505+ init_attr->cap.max_recv_sge = resp.max_recv_sge; 506+ init_attr->cap.max_inline_data = resp.max_inline_data; 507+ init_attr->sq_sig_all = resp.sq_sig_all; 508+ 509+ return 0; 510+} 511+ 512 static int ibv_cmd_destroy_qp_v1(struct ibv_qp *qp) 513 { 514 struct ibv_destroy_qp_v1 cmd; 515@@ -1192,3 +1374,36 @@ int ibv_cmd_close_xrc_domain(struct ibv_ 516 return 0; 517 } 518 519+int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num) 520+{ 521+ struct ibv_reg_xrc_rcv_qp cmd; 522+ 523+ if (abi_ver < 6) 524+ return ENOSYS; 525+ 526+ IBV_INIT_CMD(&cmd, sizeof cmd, REG_XRC_RCV_QP); 527+ cmd.xrc_domain_handle = d->handle; 528+ cmd.qp_num = xrc_qp_num; 529+ 530+ if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 531+ return errno; 532+ return 0; 533+} 534+ 535+int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num) 536+{ 537+ struct ibv_unreg_xrc_rcv_qp cmd; 538+ 539+ if (abi_ver < 6) 540+ return ENOSYS; 541+ 542+ IBV_INIT_CMD(&cmd, sizeof cmd, UNREG_XRC_RCV_QP); 543+ cmd.xrc_domain_handle = d->handle; 544+ cmd.qp_num = xrc_qp_num; 545+ 546+ if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 547+ return errno; 548+ return 0; 549+} 550+ 551+ 552Index: libibverbs/src/device.c 553=================================================================== 554--- libibverbs.orig/src/device.c 2009-11-01 15:18:17.794116000 +0200 555+++ libibverbs/src/device.c 2009-11-01 15:18:24.597279000 +0200 556@@ -191,31 +191,33 @@ int __ibv_get_async_event(struct ibv_con 557 558 event->event_type = ev.event_type; 559 560- switch (event->event_type) { 561- case IBV_EVENT_CQ_ERR: 562- event->element.cq = (void *) (uintptr_t) ev.element; 563- break; 564- 565- case IBV_EVENT_QP_FATAL: 566- case IBV_EVENT_QP_REQ_ERR: 567- case IBV_EVENT_QP_ACCESS_ERR: 568- case IBV_EVENT_COMM_EST: 569- case IBV_EVENT_SQ_DRAINED: 570- case IBV_EVENT_PATH_MIG: 571- case IBV_EVENT_PATH_MIG_ERR: 572- case IBV_EVENT_QP_LAST_WQE_REACHED: 573- event->element.qp = (void *) (uintptr_t) ev.element; 574- break; 575- 576- case IBV_EVENT_SRQ_ERR: 577- case IBV_EVENT_SRQ_LIMIT_REACHED: 578- event->element.srq = (void *) (uintptr_t) ev.element; 579- break; 580- 581- default: 582- event->element.port_num = ev.element; 583- break; 584- } 585+ if (event->event_type & IBV_XRC_QP_EVENT_FLAG) { 586+ event->element.xrc_qp_num = ev.element; 587+ } else 588+ switch (event->event_type) { 589+ case IBV_EVENT_CQ_ERR: 590+ event->element.cq = (void *) (uintptr_t) ev.element; 591+ break; 592+ 593+ case IBV_EVENT_QP_FATAL: 594+ case IBV_EVENT_QP_REQ_ERR: 595+ case IBV_EVENT_QP_ACCESS_ERR: 596+ case IBV_EVENT_COMM_EST: 597+ case IBV_EVENT_SQ_DRAINED: 598+ case IBV_EVENT_PATH_MIG: 599+ case IBV_EVENT_PATH_MIG_ERR: 600+ case IBV_EVENT_QP_LAST_WQE_REACHED: 601+ event->element.qp = (void *) (uintptr_t) ev.element; 602+ break; 603+ 604+ case IBV_EVENT_SRQ_ERR: 605+ case IBV_EVENT_SRQ_LIMIT_REACHED: 606+ event->element.srq = (void *) (uintptr_t) ev.element; 607+ break; 608+ default: 609+ event->element.port_num = ev.element; 610+ break; 611+ } 612 613 if (context->ops.async_event) 614 context->ops.async_event(event); 615Index: libibverbs/src/libibverbs.map 616=================================================================== 617--- libibverbs.orig/src/libibverbs.map 2009-11-01 15:18:20.646169000 +0200 618+++ libibverbs/src/libibverbs.map 2009-11-01 15:18:24.600279000 +0200 619@@ -97,6 +97,16 @@ IBVERBS_1.1 { 620 ibv_cmd_open_xrc_domain; 621 ibv_close_xrc_domain; 622 ibv_cmd_close_xrc_domain; 623+ ibv_create_xrc_rcv_qp; 624+ ibv_cmd_create_xrc_rcv_qp; 625+ ibv_modify_xrc_rcv_qp; 626+ ibv_cmd_modify_xrc_rcv_qp; 627+ ibv_query_xrc_rcv_qp; 628+ ibv_cmd_query_xrc_rcv_qp; 629+ ibv_reg_xrc_rcv_qp; 630+ ibv_cmd_reg_xrc_rcv_qp; 631+ ibv_unreg_xrc_rcv_qp; 632+ ibv_cmd_unreg_xrc_rcv_qp; 633 634 ibv_node_type_str; 635 ibv_port_state_str; 636Index: libibverbs/src/verbs.c 637=================================================================== 638--- libibverbs.orig/src/verbs.c 2009-11-01 15:18:20.650169000 +0200 639+++ libibverbs/src/verbs.c 2009-11-01 15:18:24.604279000 +0200 640@@ -597,3 +597,62 @@ int ibv_close_xrc_domain(struct ibv_xrc_ 641 642 return d->context->more_ops->close_xrc_domain(d); 643 } 644+ 645+int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr, 646+ uint32_t *xrc_rcv_qpn) 647+{ 648+ struct ibv_context *c; 649+ if (!init_attr || !(init_attr->xrc_domain)) 650+ return EINVAL; 651+ 652+ c = init_attr->xrc_domain->context; 653+ if (!c->more_ops) 654+ return ENOSYS; 655+ 656+ return c->more_ops->create_xrc_rcv_qp(init_attr, 657+ xrc_rcv_qpn); 658+} 659+ 660+int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *d, 661+ uint32_t xrc_rcv_qpn, 662+ struct ibv_qp_attr *attr, 663+ int attr_mask) 664+{ 665+ if (!d || !attr) 666+ return EINVAL; 667+ 668+ if (!d->context->more_ops) 669+ return ENOSYS; 670+ 671+ return d->context->more_ops->modify_xrc_rcv_qp(d, xrc_rcv_qpn, attr, 672+ attr_mask); 673+} 674+ 675+int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *d, 676+ uint32_t xrc_rcv_qpn, 677+ struct ibv_qp_attr *attr, 678+ int attr_mask, 679+ struct ibv_qp_init_attr *init_attr) 680+{ 681+ if (!d) 682+ return EINVAL; 683+ 684+ if (!d->context->more_ops) 685+ return ENOSYS; 686+ 687+ return d->context->more_ops->query_xrc_rcv_qp(d, xrc_rcv_qpn, attr, 688+ attr_mask, init_attr); 689+} 690+ 691+int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *d, 692+ uint32_t xrc_rcv_qpn) 693+{ 694+ return d->context->more_ops->reg_xrc_rcv_qp(d, xrc_rcv_qpn); 695+} 696+ 697+int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *d, 698+ uint32_t xrc_rcv_qpn) 699+{ 700+ return d->context->more_ops->unreg_xrc_rcv_qp(d, xrc_rcv_qpn); 701+} 702+ 703