1/* 2 * Copyright (c) 2004-2006 Voltaire Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#if HAVE_CONFIG_H 35# include <config.h> 36#endif /* HAVE_CONFIG_H */ 37 38#include <stdio.h> 39#include <stdlib.h> 40#include <unistd.h> 41#include <pthread.h> 42#include <sys/time.h> 43#include <string.h> 44#include <errno.h> 45 46#include <infiniband/umad.h> 47#include "mad.h" 48 49#define MAX_CLASS 256 50 51struct ibmad_port { 52 int port_id; /* file descriptor returned by umad_open() */ 53 int class_agents[MAX_CLASS]; /* class2agent mapper */ 54}; 55 56int ibdebug; 57 58static int mad_portid = -1; 59static int iberrs; 60 61static int madrpc_retries = MAD_DEF_RETRIES; 62static int def_madrpc_timeout = MAD_DEF_TIMEOUT_MS; 63static void *save_mad; 64static int save_mad_len = 256; 65 66#undef DEBUG 67#define DEBUG if (ibdebug) IBWARN 68#define ERRS if (iberrs || ibdebug) IBWARN 69 70#define MAD_TID(mad) (*((uint64_t *)((char *)(mad) + 8))) 71 72void 73madrpc_show_errors(int set) 74{ 75 iberrs = set; 76} 77 78void 79madrpc_save_mad(void *madbuf, int len) 80{ 81 save_mad = madbuf; 82 save_mad_len = len; 83} 84 85int 86madrpc_set_retries(int retries) 87{ 88 if (retries > 0) 89 madrpc_retries = retries; 90 return madrpc_retries; 91} 92 93int 94madrpc_set_timeout(int timeout) 95{ 96 def_madrpc_timeout = timeout; 97 return 0; 98} 99 100int 101madrpc_def_timeout(void) 102{ 103 return def_madrpc_timeout; 104} 105 106int 107madrpc_portid(void) 108{ 109 return mad_portid; 110} 111 112static int 113_do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, 114 int timeout) 115{ 116 uint32_t trid; /* only low 32 bits */ 117 int retries; 118 int length, status; 119 120 if (!timeout) 121 timeout = def_madrpc_timeout; 122 123 if (ibdebug > 1) { 124 IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len); 125 xdump(stderr, "send buf\n", sndbuf, umad_size() + len); 126 } 127 128 if (save_mad) { 129 memcpy(save_mad, umad_get_mad(sndbuf), 130 save_mad_len < len ? save_mad_len : len); 131 save_mad = 0; 132 } 133 134 trid = mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F); 135 136 for (retries = 0; retries < madrpc_retries; retries++) { 137 if (retries) { 138 ERRS("retry %d (timeout %d ms)", retries, timeout); 139 } 140 141 length = len; 142 if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) { 143 IBWARN("send failed; %s", strerror(errno)); 144 return -1; 145 } 146 147 /* Use same timeout on receive side just in case */ 148 /* send packet is lost somewhere. */ 149 do { 150 if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) { 151 IBWARN("recv failed: %s", strerror(errno)); 152 return -1; 153 } 154 155 if (ibdebug > 1) { 156 IBWARN("rcv buf:"); 157 xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE); 158 } 159 } while ((uint32_t)mad_get_field64(umad_get_mad(rcvbuf), 0, IB_MAD_TRID_F) != trid); 160 161 status = umad_status(rcvbuf); 162 if (!status) 163 return length; /* done */ 164 if (status == ENOMEM) 165 return length; 166 } 167 168 ERRS("timeout after %d retries, %d ms", retries, timeout * retries); 169 return -1; 170} 171 172void * 173mad_rpc(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport, void *payload, 174 void *rcvdata) 175{ 176 const struct ibmad_port *p = port_id; 177 int status, len; 178 uint8_t sndbuf[1024], rcvbuf[1024], *mad; 179 180 len = 0; 181 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); 182 183 if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0) 184 return 0; 185 186 if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf, 187 p->class_agents[rpc->mgtclass], 188 len, rpc->timeout)) < 0) { 189 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); 190 return 0; 191 } 192 193 mad = umad_get_mad(rcvbuf); 194 195 if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) { 196 ERRS("MAD completed with error status 0x%x; dport (%s)", 197 status, portid2str(dport)); 198 return 0; 199 } 200 201 if (ibdebug) { 202 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz); 203 xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz); 204 } 205 206 if (rcvdata) 207 memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz); 208 209 return rcvdata; 210} 211 212void * 213mad_rpc_rmpp(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport, 214 ib_rmpp_hdr_t *rmpp, void *data) 215{ 216 const struct ibmad_port *p = port_id; 217 int status, len; 218 uint8_t sndbuf[1024], rcvbuf[1024], *mad; 219 220 memset(sndbuf, 0, umad_size() + IB_MAD_SIZE); 221 222 DEBUG("rmpp %p data %p", rmpp, data); 223 224 if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0) 225 return 0; 226 227 if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf, 228 p->class_agents[rpc->mgtclass], 229 len, rpc->timeout)) < 0) { 230 IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport)); 231 return 0; 232 } 233 234 mad = umad_get_mad(rcvbuf); 235 236 if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) { 237 ERRS("MAD completed with error status 0x%x; dport (%s)", 238 status, portid2str(dport)); 239 return 0; 240 } 241 242 if (ibdebug) { 243 IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz); 244 xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs, 245 rpc->datasz); 246 } 247 248 if (rmpp) { 249 rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F); 250 if ((rmpp->flags & 0x3) && 251 mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) { 252 IBWARN("bad rmpp version"); 253 return 0; 254 } 255 rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F); 256 rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F); 257 DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status); 258 rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F); 259 rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F); 260 } 261 262 if (data) 263 memcpy(data, mad + rpc->dataoffs, rpc->datasz); 264 265 rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F); 266 267 return data; 268} 269 270void * 271madrpc(ib_rpc_t *rpc, ib_portid_t *dport, void *payload, void *rcvdata) 272{ 273 struct ibmad_port port; 274 275 port.port_id = mad_portid; 276 port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass); 277 return mad_rpc(&port, rpc, dport, payload, rcvdata); 278} 279 280void * 281madrpc_rmpp(ib_rpc_t *rpc, ib_portid_t *dport, ib_rmpp_hdr_t *rmpp, void *data) 282{ 283 struct ibmad_port port; 284 285 port.port_id = mad_portid; 286 port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass); 287 return mad_rpc_rmpp(&port, rpc, dport, rmpp, data); 288} 289 290static pthread_mutex_t rpclock = PTHREAD_MUTEX_INITIALIZER; 291 292void 293madrpc_lock(void) 294{ 295 pthread_mutex_lock(&rpclock); 296} 297 298void 299madrpc_unlock(void) 300{ 301 pthread_mutex_unlock(&rpclock); 302} 303 304void 305madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes) 306{ 307 if (umad_init() < 0) 308 IBPANIC("can't init UMAD library"); 309 310 if ((mad_portid = umad_open_port(dev_name, dev_port)) < 0) 311 IBPANIC("can't open UMAD port (%s:%d)", dev_name, dev_port); 312 313 if (num_classes >= MAX_CLASS) 314 IBPANIC("too many classes %d requested", num_classes); 315 316 while (num_classes--) { 317 int rmpp_version = 0; 318 int mgmt = *mgmt_classes++; 319 320 if (mgmt == IB_SA_CLASS) 321 rmpp_version = 1; 322 if (mad_register_client(mgmt, rmpp_version) < 0) 323 IBPANIC("client_register for mgmt class %d failed", mgmt); 324 } 325} 326 327void * 328mad_rpc_open_port(char *dev_name, int dev_port, 329 int *mgmt_classes, int num_classes) 330{ 331 struct ibmad_port *p; 332 int port_id; 333 334 if (num_classes >= MAX_CLASS) { 335 IBWARN("too many classes %d requested", num_classes); 336 errno = EINVAL; 337 return NULL; 338 } 339 340 if (umad_init() < 0) { 341 IBWARN("can't init UMAD library"); 342 errno = ENODEV; 343 return NULL; 344 } 345 346 p = malloc(sizeof(*p)); 347 if (!p) { 348 errno = ENOMEM; 349 return NULL; 350 } 351 memset(p, 0, sizeof(*p)); 352 353 if ((port_id = umad_open_port(dev_name, dev_port)) < 0) { 354 IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port); 355 if (!errno) 356 errno = EIO; 357 free(p); 358 return NULL; 359 } 360 361 while (num_classes--) { 362 int rmpp_version = 0; 363 int mgmt = *mgmt_classes++; 364 int agent; 365 366 if (mgmt == IB_SA_CLASS) 367 rmpp_version = 1; 368 if (mgmt < 0 || mgmt >= MAX_CLASS || 369 (agent = mad_register_port_client(port_id, mgmt, 370 rmpp_version)) < 0) { 371 IBWARN("client_register for mgmt %d failed", mgmt); 372 if(!errno) 373 errno = EINVAL; 374 umad_close_port(port_id); 375 free(p); 376 return NULL; 377 } 378 p->class_agents[mgmt] = agent; 379 } 380 381 p->port_id = port_id; 382 return p; 383} 384 385void 386mad_rpc_close_port(void *port_id) 387{ 388 struct ibmad_port *p = port_id; 389 390 umad_close_port(p->port_id); 391 free(p); 392} 393 394uint8_t * 395sa_call(void *rcvbuf, ib_portid_t *portid, ib_sa_call_t *sa, unsigned timeout) 396{ 397 struct ibmad_port port; 398 399 port.port_id = mad_portid; 400 port.class_agents[IB_SA_CLASS] = mad_class_agent(IB_SA_CLASS); 401 return sa_rpc_call(&port, rcvbuf, portid, sa, timeout); 402} 403