1/* 2 * IBM eServer eHCA Infiniband device driver for Linux on POWER 3 * 4 * Completion queue handling 5 * 6 * Authors: Waleri Fomin <fomin@de.ibm.com> 7 * Khadija Souissi <souissi@de.ibm.com> 8 * Reinhard Ernst <rernst@de.ibm.com> 9 * Heiko J Schick <schickhj@de.ibm.com> 10 * Hoang-Nam Nguyen <hnguyen@de.ibm.com> 11 * 12 * 13 * Copyright (c) 2005 IBM Corporation 14 * 15 * All rights reserved. 16 * 17 * This source code is distributed under a dual license of GPL v2.0 and OpenIB 18 * BSD. 19 * 20 * OpenIB BSD License 21 * 22 * Redistribution and use in source and binary forms, with or without 23 * modification, are permitted provided that the following conditions are met: 24 * 25 * Redistributions of source code must retain the above copyright notice, this 26 * list of conditions and the following disclaimer. 27 * 28 * Redistributions in binary form must reproduce the above copyright notice, 29 * this list of conditions and the following disclaimer in the documentation 30 * and/or other materials 31 * provided with the distribution. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 34 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 37 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 38 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 39 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 40 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 41 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 43 * POSSIBILITY OF SUCH DAMAGE. 44 */ 45 46#include <asm/current.h> 47 48#include "ehca_iverbs.h" 49#include "ehca_classes.h" 50#include "ehca_irq.h" 51#include "hcp_if.h" 52 53static struct kmem_cache *cq_cache; 54 55int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp) 56{ 57 unsigned int qp_num = qp->real_qp_num; 58 unsigned int key = qp_num & (QP_HASHTAB_LEN-1); 59 unsigned long spl_flags; 60 61 spin_lock_irqsave(&cq->spinlock, spl_flags); 62 hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]); 63 spin_unlock_irqrestore(&cq->spinlock, spl_flags); 64 65 ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x", 66 cq->cq_number, qp_num); 67 68 return 0; 69} 70 71int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num) 72{ 73 int ret = -EINVAL; 74 unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); 75 struct hlist_node *iter; 76 struct ehca_qp *qp; 77 unsigned long spl_flags; 78 79 spin_lock_irqsave(&cq->spinlock, spl_flags); 80 hlist_for_each(iter, &cq->qp_hashtab[key]) { 81 qp = hlist_entry(iter, struct ehca_qp, list_entries); 82 if (qp->real_qp_num == real_qp_num) { 83 hlist_del(iter); 84 ehca_dbg(cq->ib_cq.device, 85 "removed qp from cq .cq_num=%x real_qp_num=%x", 86 cq->cq_number, real_qp_num); 87 ret = 0; 88 break; 89 } 90 } 91 spin_unlock_irqrestore(&cq->spinlock, spl_flags); 92 if (ret) 93 ehca_err(cq->ib_cq.device, 94 "qp not found cq_num=%x real_qp_num=%x", 95 cq->cq_number, real_qp_num); 96 97 return ret; 98} 99 100struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num) 101{ 102 struct ehca_qp *ret = NULL; 103 unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1); 104 struct hlist_node *iter; 105 struct ehca_qp *qp; 106 hlist_for_each(iter, &cq->qp_hashtab[key]) { 107 qp = hlist_entry(iter, struct ehca_qp, list_entries); 108 if (qp->real_qp_num == real_qp_num) { 109 ret = qp; 110 break; 111 } 112 } 113 return ret; 114} 115 116struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector, 117 struct ib_ucontext *context, 118 struct ib_udata *udata) 119{ 120 static const u32 additional_cqe = 20; 121 struct ib_cq *cq; 122 struct ehca_cq *my_cq; 123 struct ehca_shca *shca = 124 container_of(device, struct ehca_shca, ib_device); 125 struct ipz_adapter_handle adapter_handle; 126 struct ehca_alloc_cq_parms param; /* h_call's out parameters */ 127 struct h_galpa gal; 128 void *vpage; 129 u32 counter; 130 u64 rpage, cqx_fec, h_ret; 131 int ipz_rc, ret, i; 132 unsigned long flags; 133 134 if (cqe >= 0xFFFFFFFF - 64 - additional_cqe) 135 return ERR_PTR(-EINVAL); 136 137 my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL); 138 if (!my_cq) { 139 ehca_err(device, "Out of memory for ehca_cq struct device=%p", 140 device); 141 return ERR_PTR(-ENOMEM); 142 } 143 144 memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms)); 145 146 spin_lock_init(&my_cq->spinlock); 147 spin_lock_init(&my_cq->cb_lock); 148 spin_lock_init(&my_cq->task_lock); 149 init_waitqueue_head(&my_cq->wait_completion); 150 my_cq->ownpid = current->tgid; 151 152 cq = &my_cq->ib_cq; 153 154 adapter_handle = shca->ipz_hca_handle; 155 param.eq_handle = shca->eq.ipz_eq_handle; 156 157 do { 158 if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) { 159 cq = ERR_PTR(-ENOMEM); 160 ehca_err(device, "Can't reserve idr nr. device=%p", 161 device); 162 goto create_cq_exit1; 163 } 164 165 spin_lock_irqsave(&ehca_cq_idr_lock, flags); 166 ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token); 167 spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); 168 169 } while (ret == -EAGAIN); 170 171 if (ret) { 172 cq = ERR_PTR(-ENOMEM); 173 ehca_err(device, "Can't allocate new idr entry. device=%p", 174 device); 175 goto create_cq_exit1; 176 } 177 178 /* 179 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer 180 * for receiving errors CQEs. 181 */ 182 param.nr_cqe = cqe + additional_cqe; 183 h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m); 184 185 if (h_ret != H_SUCCESS) { 186 ehca_err(device, "hipz_h_alloc_resource_cq() failed " 187 "h_ret=%lx device=%p", h_ret, device); 188 cq = ERR_PTR(ehca2ib_return_code(h_ret)); 189 goto create_cq_exit2; 190 } 191 192 ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages, 193 EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0); 194 if (!ipz_rc) { 195 ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p", 196 ipz_rc, device); 197 cq = ERR_PTR(-EINVAL); 198 goto create_cq_exit3; 199 } 200 201 for (counter = 0; counter < param.act_pages; counter++) { 202 vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); 203 if (!vpage) { 204 ehca_err(device, "ipz_qpageit_get_inc() " 205 "returns NULL device=%p", device); 206 cq = ERR_PTR(-EAGAIN); 207 goto create_cq_exit4; 208 } 209 rpage = virt_to_abs(vpage); 210 211 h_ret = hipz_h_register_rpage_cq(adapter_handle, 212 my_cq->ipz_cq_handle, 213 &my_cq->pf, 214 0, 215 0, 216 rpage, 217 1, 218 my_cq->galpas. 219 kernel); 220 221 if (h_ret < H_SUCCESS) { 222 ehca_err(device, "hipz_h_register_rpage_cq() failed " 223 "ehca_cq=%p cq_num=%x h_ret=%lx counter=%i " 224 "act_pages=%i", my_cq, my_cq->cq_number, 225 h_ret, counter, param.act_pages); 226 cq = ERR_PTR(-EINVAL); 227 goto create_cq_exit4; 228 } 229 230 if (counter == (param.act_pages - 1)) { 231 vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue); 232 if ((h_ret != H_SUCCESS) || vpage) { 233 ehca_err(device, "Registration of pages not " 234 "complete ehca_cq=%p cq_num=%x " 235 "h_ret=%lx", my_cq, my_cq->cq_number, 236 h_ret); 237 cq = ERR_PTR(-EAGAIN); 238 goto create_cq_exit4; 239 } 240 } else { 241 if (h_ret != H_PAGE_REGISTERED) { 242 ehca_err(device, "Registration of page failed " 243 "ehca_cq=%p cq_num=%x h_ret=%lx" 244 "counter=%i act_pages=%i", 245 my_cq, my_cq->cq_number, 246 h_ret, counter, param.act_pages); 247 cq = ERR_PTR(-ENOMEM); 248 goto create_cq_exit4; 249 } 250 } 251 } 252 253 ipz_qeit_reset(&my_cq->ipz_queue); 254 255 gal = my_cq->galpas.kernel; 256 cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec)); 257 ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx", 258 my_cq, my_cq->cq_number, cqx_fec); 259 260 my_cq->ib_cq.cqe = my_cq->nr_of_entries = 261 param.act_nr_of_entries - additional_cqe; 262 my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff; 263 264 for (i = 0; i < QP_HASHTAB_LEN; i++) 265 INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]); 266 267 if (context) { 268 struct ipz_queue *ipz_queue = &my_cq->ipz_queue; 269 struct ehca_create_cq_resp resp; 270 memset(&resp, 0, sizeof(resp)); 271 resp.cq_number = my_cq->cq_number; 272 resp.token = my_cq->token; 273 resp.ipz_queue.qe_size = ipz_queue->qe_size; 274 resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg; 275 resp.ipz_queue.queue_length = ipz_queue->queue_length; 276 resp.ipz_queue.pagesize = ipz_queue->pagesize; 277 resp.ipz_queue.toggle_state = ipz_queue->toggle_state; 278 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) { 279 ehca_err(device, "Copy to udata failed."); 280 goto create_cq_exit4; 281 } 282 } 283 284 return cq; 285 286create_cq_exit4: 287 ipz_queue_dtor(&my_cq->ipz_queue); 288 289create_cq_exit3: 290 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); 291 if (h_ret != H_SUCCESS) 292 ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p " 293 "cq_num=%x h_ret=%lx", my_cq, my_cq->cq_number, h_ret); 294 295create_cq_exit2: 296 spin_lock_irqsave(&ehca_cq_idr_lock, flags); 297 idr_remove(&ehca_cq_idr, my_cq->token); 298 spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); 299 300create_cq_exit1: 301 kmem_cache_free(cq_cache, my_cq); 302 303 return cq; 304} 305 306static int get_cq_nr_events(struct ehca_cq *my_cq) 307{ 308 int ret; 309 unsigned long flags; 310 spin_lock_irqsave(&ehca_cq_idr_lock, flags); 311 ret = my_cq->nr_events; 312 spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); 313 return ret; 314} 315 316int ehca_destroy_cq(struct ib_cq *cq) 317{ 318 u64 h_ret; 319 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); 320 int cq_num = my_cq->cq_number; 321 struct ib_device *device = cq->device; 322 struct ehca_shca *shca = container_of(device, struct ehca_shca, 323 ib_device); 324 struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle; 325 u32 cur_pid = current->tgid; 326 unsigned long flags; 327 328 if (cq->uobject) { 329 if (my_cq->mm_count_galpa || my_cq->mm_count_queue) { 330 ehca_err(device, "Resources still referenced in " 331 "user space cq_num=%x", my_cq->cq_number); 332 return -EINVAL; 333 } 334 if (my_cq->ownpid != cur_pid) { 335 ehca_err(device, "Invalid caller pid=%x ownpid=%x " 336 "cq_num=%x", 337 cur_pid, my_cq->ownpid, my_cq->cq_number); 338 return -EINVAL; 339 } 340 } 341 342 spin_lock_irqsave(&ehca_cq_idr_lock, flags); 343 while (my_cq->nr_events) { 344 spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); 345 wait_event(my_cq->wait_completion, !get_cq_nr_events(my_cq)); 346 spin_lock_irqsave(&ehca_cq_idr_lock, flags); 347 /* recheck nr_events to assure no cqe has just arrived */ 348 } 349 350 idr_remove(&ehca_cq_idr, my_cq->token); 351 spin_unlock_irqrestore(&ehca_cq_idr_lock, flags); 352 353 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0); 354 if (h_ret == H_R_STATE) { 355 /* cq in err: read err data and destroy it forcibly */ 356 ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err " 357 "state. Try to delete it forcibly.", 358 my_cq, cq_num, my_cq->ipz_cq_handle.handle); 359 ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle); 360 h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1); 361 if (h_ret == H_SUCCESS) 362 ehca_dbg(device, "cq_num=%x deleted successfully.", 363 cq_num); 364 } 365 if (h_ret != H_SUCCESS) { 366 ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lx " 367 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num); 368 return ehca2ib_return_code(h_ret); 369 } 370 ipz_queue_dtor(&my_cq->ipz_queue); 371 kmem_cache_free(cq_cache, my_cq); 372 373 return 0; 374} 375 376int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) 377{ 378 struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq); 379 u32 cur_pid = current->tgid; 380 381 if (cq->uobject && my_cq->ownpid != cur_pid) { 382 ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x", 383 cur_pid, my_cq->ownpid); 384 return -EINVAL; 385 } 386 387 /* TODO: proper resize needs to be done */ 388 ehca_err(cq->device, "not implemented yet"); 389 390 return -EFAULT; 391} 392 393int ehca_init_cq_cache(void) 394{ 395 cq_cache = kmem_cache_create("ehca_cache_cq", 396 sizeof(struct ehca_cq), 0, 397 SLAB_HWCACHE_ALIGN, 398 NULL, NULL); 399 if (!cq_cache) 400 return -ENOMEM; 401 return 0; 402} 403 404void ehca_cleanup_cq_cache(void) 405{ 406 if (cq_cache) 407 kmem_cache_destroy(cq_cache); 408} 409