1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2018-2020 Broadcom. 4 */ 5 6#include <linux/delay.h> 7#include <linux/fs.h> 8#include <linux/hash.h> 9#include <linux/interrupt.h> 10#include <linux/list.h> 11#include <linux/module.h> 12#include <linux/poll.h> 13#include <linux/sizes.h> 14#include <linux/spinlock.h> 15#include <linux/timer.h> 16 17#include "bcm_vk.h" 18#include "bcm_vk_msg.h" 19#include "bcm_vk_sg.h" 20 21/* functions to manipulate the transport id in msg block */ 22#define BCM_VK_MSG_Q_SHIFT 4 23#define BCM_VK_MSG_Q_MASK 0xF 24#define BCM_VK_MSG_ID_MASK 0xFFF 25 26#define BCM_VK_DMA_DRAIN_MAX_MS 2000 27 28/* number x q_size will be the max number of msg processed per loop */ 29#define BCM_VK_MSG_PROC_MAX_LOOP 2 30 31/* module parameter */ 32static bool hb_mon = true; 33module_param(hb_mon, bool, 0444); 34MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n"); 35static int batch_log = 1; 36module_param(batch_log, int, 0444); 37MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n"); 38 39static bool hb_mon_is_on(void) 40{ 41 return hb_mon; 42} 43 44static u32 get_q_num(const struct vk_msg_blk *msg) 45{ 46 u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK; 47 48 if (q_num >= VK_MSGQ_PER_CHAN_MAX) 49 q_num = VK_MSGQ_NUM_DEFAULT; 50 return q_num; 51} 52 53static void set_q_num(struct vk_msg_blk *msg, u32 q_num) 54{ 55 u32 trans_q; 56 57 if (q_num >= VK_MSGQ_PER_CHAN_MAX) 58 trans_q = VK_MSGQ_NUM_DEFAULT; 59 else 60 trans_q = q_num; 61 62 msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q; 63} 64 65static u32 get_msg_id(const struct vk_msg_blk *msg) 66{ 67 return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK); 68} 69 70static void set_msg_id(struct vk_msg_blk *msg, u32 val) 71{ 72 msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg); 73} 74 75static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc) 76{ 77 return ((idx + inc) & qinfo->q_mask); 78} 79 80static 81struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo, 82 u32 idx) 83{ 84 return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx); 85} 86 87static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq, 88 const struct bcm_vk_sync_qinfo *qinfo) 89{ 90 u32 wr_idx, rd_idx; 91 92 wr_idx = readl_relaxed(&msgq->wr_idx); 93 rd_idx = readl_relaxed(&msgq->rd_idx); 94 95 return ((wr_idx - rd_idx) & qinfo->q_mask); 96} 97 98static 99u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, 100 const struct bcm_vk_sync_qinfo *qinfo) 101{ 102 return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1); 103} 104 105/* number of retries when enqueue message fails before returning EAGAIN */ 106#define BCM_VK_H2VK_ENQ_RETRY 10 107#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50 108 109bool bcm_vk_drv_access_ok(struct bcm_vk *vk) 110{ 111 return (!!atomic_read(&vk->msgq_inited)); 112} 113 114void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) 115{ 116 struct bcm_vk_alert *alert = &vk->host_alert; 117 unsigned long flags; 118 119 /* use irqsave version as this maybe called inside timer interrupt */ 120 spin_lock_irqsave(&vk->host_alert_lock, flags); 121 alert->notfs |= bit_mask; 122 spin_unlock_irqrestore(&vk->host_alert_lock, flags); 123 124 if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0) 125 queue_work(vk->wq_thread, &vk->wq_work); 126} 127 128/* 129 * Heartbeat related defines 130 * The heartbeat from host is a last resort. If stuck condition happens 131 * on the card, firmware is supposed to detect it. Therefore, the heartbeat 132 * values used will be more relaxed on the driver, which need to be bigger 133 * than the watchdog timeout on the card. The watchdog timeout on the card 134 * is 20s, with a jitter of 2s => 22s. We use a value of 27s here. 135 */ 136#define BCM_VK_HB_TIMER_S 3 137#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) 138#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) 139 140static void bcm_vk_hb_poll(struct work_struct *work) 141{ 142 u32 uptime_s; 143 struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, 144 work); 145 struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl); 146 147 if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) { 148 /* read uptime from register and compare */ 149 uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME); 150 151 if (uptime_s == hb->last_uptime) 152 hb->lost_cnt++; 153 else /* reset to avoid accumulation */ 154 hb->lost_cnt = 0; 155 156 dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n", 157 hb->last_uptime, uptime_s, hb->lost_cnt); 158 159 /* 160 * if the interface goes down without any activity, a value 161 * of 0xFFFFFFFF will be continuously read, and the detection 162 * will be happened eventually. 163 */ 164 hb->last_uptime = uptime_s; 165 } else { 166 /* reset heart beat lost cnt */ 167 hb->lost_cnt = 0; 168 } 169 170 /* next, check if heartbeat exceeds limit */ 171 if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) { 172 dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n", 173 BCM_VK_HB_LOST_MAX, 174 BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S); 175 176 bcm_vk_blk_drv_access(vk); 177 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); 178 } 179 /* re-arm timer */ 180 schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); 181} 182 183void bcm_vk_hb_init(struct bcm_vk *vk) 184{ 185 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; 186 187 INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll); 188 schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE); 189} 190 191void bcm_vk_hb_deinit(struct bcm_vk *vk) 192{ 193 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; 194 195 cancel_delayed_work_sync(&hb->work); 196} 197 198static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk, 199 unsigned int start, 200 unsigned int nbits) 201{ 202 spin_lock(&vk->msg_id_lock); 203 bitmap_clear(vk->bmap, start, nbits); 204 spin_unlock(&vk->msg_id_lock); 205} 206 207/* 208 * allocate a ctx per file struct 209 */ 210static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) 211{ 212 u32 i; 213 struct bcm_vk_ctx *ctx = NULL; 214 u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT); 215 216 spin_lock(&vk->ctx_lock); 217 218 /* check if it is in reset, if so, don't allow */ 219 if (vk->reset_pid) { 220 dev_err(&vk->pdev->dev, 221 "No context allowed during reset by pid %d\n", 222 vk->reset_pid); 223 224 goto in_reset_exit; 225 } 226 227 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { 228 if (!vk->ctx[i].in_use) { 229 vk->ctx[i].in_use = true; 230 ctx = &vk->ctx[i]; 231 break; 232 } 233 } 234 235 if (!ctx) { 236 dev_err(&vk->pdev->dev, "All context in use\n"); 237 238 goto all_in_use_exit; 239 } 240 241 /* set the pid and insert it to hash table */ 242 ctx->pid = pid; 243 ctx->hash_idx = hash_idx; 244 list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head); 245 246 /* increase kref */ 247 kref_get(&vk->kref); 248 249 /* clear counter */ 250 atomic_set(&ctx->pend_cnt, 0); 251 atomic_set(&ctx->dma_cnt, 0); 252 init_waitqueue_head(&ctx->rd_wq); 253 254all_in_use_exit: 255in_reset_exit: 256 spin_unlock(&vk->ctx_lock); 257 258 return ctx; 259} 260 261static u16 bcm_vk_get_msg_id(struct bcm_vk *vk) 262{ 263 u16 rc = VK_MSG_ID_OVERFLOW; 264 u16 test_bit_count = 0; 265 266 spin_lock(&vk->msg_id_lock); 267 while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) { 268 /* 269 * first time come in this loop, msg_id will be 0 270 * and the first one tested will be 1. We skip 271 * VK_SIMPLEX_MSG_ID (0) for one way host2vk 272 * communication 273 */ 274 vk->msg_id++; 275 if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE) 276 vk->msg_id = 1; 277 278 if (test_bit(vk->msg_id, vk->bmap)) { 279 test_bit_count++; 280 continue; 281 } 282 rc = vk->msg_id; 283 bitmap_set(vk->bmap, vk->msg_id, 1); 284 break; 285 } 286 spin_unlock(&vk->msg_id_lock); 287 288 return rc; 289} 290 291static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) 292{ 293 u32 idx; 294 u32 hash_idx; 295 pid_t pid; 296 struct bcm_vk_ctx *entry; 297 int count = 0; 298 299 if (!ctx) { 300 dev_err(&vk->pdev->dev, "NULL context detected\n"); 301 return -EINVAL; 302 } 303 idx = ctx->idx; 304 pid = ctx->pid; 305 306 spin_lock(&vk->ctx_lock); 307 308 if (!vk->ctx[idx].in_use) { 309 dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx); 310 } else { 311 vk->ctx[idx].in_use = false; 312 vk->ctx[idx].miscdev = NULL; 313 314 /* Remove it from hash list and see if it is the last one. */ 315 list_del(&ctx->node); 316 hash_idx = ctx->hash_idx; 317 list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) { 318 if (entry->pid == pid) 319 count++; 320 } 321 } 322 323 spin_unlock(&vk->ctx_lock); 324 325 return count; 326} 327 328static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry) 329{ 330 int proc_cnt; 331 332 bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt); 333 if (proc_cnt) 334 atomic_dec(&entry->ctx->dma_cnt); 335 336 kfree(entry->to_h_msg); 337 kfree(entry); 338} 339 340static void bcm_vk_drain_all_pend(struct device *dev, 341 struct bcm_vk_msg_chan *chan, 342 struct bcm_vk_ctx *ctx) 343{ 344 u32 num; 345 struct bcm_vk_wkent *entry, *tmp; 346 struct bcm_vk *vk; 347 struct list_head del_q; 348 349 if (ctx) 350 vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); 351 352 INIT_LIST_HEAD(&del_q); 353 spin_lock(&chan->pendq_lock); 354 for (num = 0; num < chan->q_nr; num++) { 355 list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { 356 if ((!ctx) || (entry->ctx->idx == ctx->idx)) { 357 list_move_tail(&entry->node, &del_q); 358 } 359 } 360 } 361 spin_unlock(&chan->pendq_lock); 362 363 /* batch clean up */ 364 num = 0; 365 list_for_each_entry_safe(entry, tmp, &del_q, node) { 366 list_del(&entry->node); 367 num++; 368 if (ctx) { 369 struct vk_msg_blk *msg; 370 int bit_set; 371 bool responded; 372 u32 msg_id; 373 374 /* if it is specific ctx, log for any stuck */ 375 msg = entry->to_v_msg; 376 msg_id = get_msg_id(msg); 377 bit_set = test_bit(msg_id, vk->bmap); 378 responded = entry->to_h_msg ? true : false; 379 if (num <= batch_log) 380 dev_info(dev, 381 "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n", 382 msg->function_id, msg->size, 383 msg_id, entry->seq_num, 384 msg->context_id, entry->ctx->idx, 385 msg->cmd, msg->arg, 386 responded ? "T" : "F", bit_set); 387 if (responded) 388 atomic_dec(&ctx->pend_cnt); 389 else if (bit_set) 390 bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); 391 } 392 bcm_vk_free_wkent(dev, entry); 393 } 394 if (num && ctx) 395 dev_info(dev, "Total drained items %d [fd-%d]\n", 396 num, ctx->idx); 397} 398 399void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk) 400{ 401 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); 402 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); 403} 404 405/* 406 * Function to sync up the messages queue info that is provided by BAR1 407 */ 408int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) 409{ 410 struct bcm_vk_msgq __iomem *msgq; 411 struct device *dev = &vk->pdev->dev; 412 u32 msgq_off; 413 u32 num_q; 414 struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan, 415 &vk->to_h_msg_chan}; 416 struct bcm_vk_msg_chan *chan; 417 int i, j; 418 int ret = 0; 419 420 /* 421 * If the driver is loaded at startup where vk OS is not up yet, 422 * the msgq-info may not be available until a later time. In 423 * this case, we skip and the sync function is supposed to be 424 * called again. 425 */ 426 if (!bcm_vk_msgq_marker_valid(vk)) { 427 dev_info(dev, "BAR1 msgq marker not initialized.\n"); 428 return -EAGAIN; 429 } 430 431 msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF); 432 433 /* each side is always half the total */ 434 num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2; 435 if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) { 436 dev_err(dev, 437 "Advertised msgq %d error - max %d allowed\n", 438 num_q, VK_MSGQ_PER_CHAN_MAX); 439 return -EINVAL; 440 } 441 442 vk->to_v_msg_chan.q_nr = num_q; 443 vk->to_h_msg_chan.q_nr = num_q; 444 445 /* first msgq location */ 446 msgq = vk->bar[BAR_1] + msgq_off; 447 448 /* 449 * if this function is called when it is already inited, 450 * something is wrong 451 */ 452 if (bcm_vk_drv_access_ok(vk) && !force_sync) { 453 dev_err(dev, "Msgq info already in sync\n"); 454 return -EPERM; 455 } 456 457 for (i = 0; i < ARRAY_SIZE(chan_list); i++) { 458 chan = chan_list[i]; 459 memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo)); 460 461 for (j = 0; j < num_q; j++) { 462 struct bcm_vk_sync_qinfo *qinfo; 463 u32 msgq_start; 464 u32 msgq_size; 465 u32 msgq_nxt; 466 u32 msgq_db_offset, q_db_offset; 467 468 chan->msgq[j] = msgq; 469 msgq_start = readl_relaxed(&msgq->start); 470 msgq_size = readl_relaxed(&msgq->size); 471 msgq_nxt = readl_relaxed(&msgq->nxt); 472 msgq_db_offset = readl_relaxed(&msgq->db_offset); 473 q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1)); 474 if (q_db_offset == (~msgq_db_offset >> DB_SHIFT)) 475 msgq_db_offset = q_db_offset; 476 else 477 /* fall back to default */ 478 msgq_db_offset = VK_BAR0_Q_DB_BASE(j); 479 480 dev_info(dev, 481 "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n", 482 j, 483 readw_relaxed(&msgq->type), 484 readw_relaxed(&msgq->num), 485 msgq_start, 486 msgq_db_offset, 487 readl_relaxed(&msgq->rd_idx), 488 readl_relaxed(&msgq->wr_idx), 489 msgq_size, 490 msgq_nxt); 491 492 qinfo = &chan->sync_qinfo[j]; 493 /* formulate and record static info */ 494 qinfo->q_start = vk->bar[BAR_1] + msgq_start; 495 qinfo->q_size = msgq_size; 496 /* set low threshold as 50% or 1/2 */ 497 qinfo->q_low = qinfo->q_size >> 1; 498 qinfo->q_mask = qinfo->q_size - 1; 499 qinfo->q_db_offset = msgq_db_offset; 500 501 msgq++; 502 } 503 } 504 atomic_set(&vk->msgq_inited, 1); 505 506 return ret; 507} 508 509static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan) 510{ 511 u32 i; 512 513 mutex_init(&chan->msgq_mutex); 514 spin_lock_init(&chan->pendq_lock); 515 for (i = 0; i < VK_MSGQ_MAX_NR; i++) 516 INIT_LIST_HEAD(&chan->pendq[i]); 517 518 return 0; 519} 520 521static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num, 522 struct bcm_vk_wkent *entry) 523{ 524 struct bcm_vk_ctx *ctx; 525 526 spin_lock(&chan->pendq_lock); 527 list_add_tail(&entry->node, &chan->pendq[q_num]); 528 if (entry->to_h_msg) { 529 ctx = entry->ctx; 530 atomic_inc(&ctx->pend_cnt); 531 wake_up_interruptible(&ctx->rd_wq); 532 } 533 spin_unlock(&chan->pendq_lock); 534} 535 536static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk, 537 struct bcm_vk_wkent *entry, 538 struct _vk_data *data, 539 unsigned int num_planes) 540{ 541 unsigned int i; 542 unsigned int item_cnt = 0; 543 struct device *dev = &vk->pdev->dev; 544 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; 545 struct vk_msg_blk *msg = &entry->to_v_msg[0]; 546 struct bcm_vk_msgq __iomem *msgq; 547 struct bcm_vk_sync_qinfo *qinfo; 548 u32 ib_sgl_size = 0; 549 u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks]; 550 u32 avail; 551 u32 q_num; 552 553 /* check if high watermark is hit, and if so, skip */ 554 q_num = get_q_num(msg); 555 msgq = chan->msgq[q_num]; 556 qinfo = &chan->sync_qinfo[q_num]; 557 avail = msgq_avail_space(msgq, qinfo); 558 if (avail < qinfo->q_low) { 559 dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n", 560 avail, qinfo->q_size); 561 return 0; 562 } 563 564 for (i = 0; i < num_planes; i++) { 565 if (data[i].address && 566 (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) { 567 item_cnt++; 568 memcpy(buf, entry->dma[i].sglist, data[i].size); 569 ib_sgl_size += data[i].size; 570 buf += data[i].size; 571 } 572 } 573 574 dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n", 575 item_cnt, ib_sgl_size, vk->ib_sgl_size); 576 577 /* round up size */ 578 ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1) 579 >> VK_MSGQ_BLK_SZ_SHIFT; 580 581 return ib_sgl_size; 582} 583 584void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val) 585{ 586 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; 587 struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num]; 588 589 vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset); 590} 591 592static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry) 593{ 594 static u32 seq_num; 595 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; 596 struct device *dev = &vk->pdev->dev; 597 struct vk_msg_blk *src = &entry->to_v_msg[0]; 598 599 struct vk_msg_blk __iomem *dst; 600 struct bcm_vk_msgq __iomem *msgq; 601 struct bcm_vk_sync_qinfo *qinfo; 602 u32 q_num = get_q_num(src); 603 u32 wr_idx; /* local copy */ 604 u32 i; 605 u32 avail; 606 u32 retry; 607 608 if (entry->to_v_blks != src->size + 1) { 609 dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n", 610 entry->to_v_blks, 611 src->size + 1, 612 get_msg_id(src), 613 src->function_id, 614 src->context_id); 615 return -EMSGSIZE; 616 } 617 618 msgq = chan->msgq[q_num]; 619 qinfo = &chan->sync_qinfo[q_num]; 620 621 mutex_lock(&chan->msgq_mutex); 622 623 avail = msgq_avail_space(msgq, qinfo); 624 625 /* if not enough space, return EAGAIN and let app handles it */ 626 retry = 0; 627 while ((avail < entry->to_v_blks) && 628 (retry++ < BCM_VK_H2VK_ENQ_RETRY)) { 629 mutex_unlock(&chan->msgq_mutex); 630 631 msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS); 632 mutex_lock(&chan->msgq_mutex); 633 avail = msgq_avail_space(msgq, qinfo); 634 } 635 if (retry > BCM_VK_H2VK_ENQ_RETRY) { 636 mutex_unlock(&chan->msgq_mutex); 637 return -EAGAIN; 638 } 639 640 /* at this point, mutex is taken and there is enough space */ 641 entry->seq_num = seq_num++; /* update debug seq number */ 642 wr_idx = readl_relaxed(&msgq->wr_idx); 643 644 if (wr_idx >= qinfo->q_size) { 645 dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!", 646 wr_idx, qinfo->q_size); 647 bcm_vk_blk_drv_access(vk); 648 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN); 649 goto idx_err; 650 } 651 652 dst = msgq_blk_addr(qinfo, wr_idx); 653 for (i = 0; i < entry->to_v_blks; i++) { 654 memcpy_toio(dst, src, sizeof(*dst)); 655 656 src++; 657 wr_idx = msgq_inc(qinfo, wr_idx, 1); 658 dst = msgq_blk_addr(qinfo, wr_idx); 659 } 660 661 /* flush the write pointer */ 662 writel(wr_idx, &msgq->wr_idx); 663 664 /* log new info for debugging */ 665 dev_dbg(dev, 666 "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n", 667 readl_relaxed(&msgq->num), 668 readl_relaxed(&msgq->rd_idx), 669 wr_idx, 670 entry->to_v_blks, 671 msgq_occupied(msgq, qinfo), 672 msgq_avail_space(msgq, qinfo), 673 readl_relaxed(&msgq->size)); 674 /* 675 * press door bell based on queue number. 1 is added to the wr_idx 676 * to avoid the value of 0 appearing on the VK side to distinguish 677 * from initial value. 678 */ 679 bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1); 680idx_err: 681 mutex_unlock(&chan->msgq_mutex); 682 return 0; 683} 684 685int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, 686 const pid_t pid, const u32 q_num) 687{ 688 int rc = 0; 689 struct bcm_vk_wkent *entry; 690 struct device *dev = &vk->pdev->dev; 691 692 /* 693 * check if the marker is still good. Sometimes, the PCIe interface may 694 * have gone done, and if so and we ship down thing based on broken 695 * values, kernel may panic. 696 */ 697 if (!bcm_vk_msgq_marker_valid(vk)) { 698 dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n", 699 vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY)); 700 return -EINVAL; 701 } 702 703 entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL); 704 if (!entry) 705 return -ENOMEM; 706 entry->to_v_blks = 1; /* always 1 block */ 707 708 /* fill up necessary data */ 709 entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN; 710 set_q_num(&entry->to_v_msg[0], q_num); 711 set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID); 712 713 entry->to_v_msg[0].cmd = shut_type; 714 entry->to_v_msg[0].arg = pid; 715 716 rc = bcm_to_v_msg_enqueue(vk, entry); 717 if (rc) 718 dev_err(dev, 719 "Sending shutdown message to q %d for pid %d fails.\n", 720 get_q_num(&entry->to_v_msg[0]), pid); 721 722 kfree(entry); 723 724 return rc; 725} 726 727static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid, 728 const u32 q_num) 729{ 730 int rc = 0; 731 struct device *dev = &vk->pdev->dev; 732 733 /* 734 * don't send down or do anything if message queue is not initialized 735 * and if it is the reset session, clear it. 736 */ 737 if (!bcm_vk_drv_access_ok(vk)) { 738 if (vk->reset_pid == pid) 739 vk->reset_pid = 0; 740 return -EPERM; 741 } 742 743 dev_dbg(dev, "No more sessions, shut down pid %d\n", pid); 744 745 /* only need to do it if it is not the reset process */ 746 if (vk->reset_pid != pid) 747 rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); 748 else 749 /* put reset_pid to 0 if it is exiting last session */ 750 vk->reset_pid = 0; 751 752 return rc; 753} 754 755static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk, 756 struct bcm_vk_msg_chan *chan, 757 u16 q_num, 758 u16 msg_id) 759{ 760 struct bcm_vk_wkent *entry = NULL, *iter; 761 762 spin_lock(&chan->pendq_lock); 763 list_for_each_entry(iter, &chan->pendq[q_num], node) { 764 if (get_msg_id(&iter->to_v_msg[0]) == msg_id) { 765 list_del(&iter->node); 766 entry = iter; 767 bcm_vk_msgid_bitmap_clear(vk, msg_id, 1); 768 break; 769 } 770 } 771 spin_unlock(&chan->pendq_lock); 772 return entry; 773} 774 775s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk) 776{ 777 struct device *dev = &vk->pdev->dev; 778 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; 779 struct vk_msg_blk *data; 780 struct vk_msg_blk __iomem *src; 781 struct vk_msg_blk *dst; 782 struct bcm_vk_msgq __iomem *msgq; 783 struct bcm_vk_sync_qinfo *qinfo; 784 struct bcm_vk_wkent *entry; 785 u32 rd_idx, wr_idx; 786 u32 q_num, msg_id, j; 787 u32 num_blks; 788 s32 total = 0; 789 int cnt = 0; 790 int msg_processed = 0; 791 int max_msg_to_process; 792 bool exit_loop; 793 794 /* 795 * drain all the messages from the queues, and find its pending 796 * entry in the to_v queue, based on msg_id & q_num, and move the 797 * entry to the to_h pending queue, waiting for user space 798 * program to extract 799 */ 800 mutex_lock(&chan->msgq_mutex); 801 802 for (q_num = 0; q_num < chan->q_nr; q_num++) { 803 msgq = chan->msgq[q_num]; 804 qinfo = &chan->sync_qinfo[q_num]; 805 max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size; 806 807 rd_idx = readl_relaxed(&msgq->rd_idx); 808 wr_idx = readl_relaxed(&msgq->wr_idx); 809 msg_processed = 0; 810 exit_loop = false; 811 while ((rd_idx != wr_idx) && !exit_loop) { 812 u8 src_size; 813 814 /* 815 * Make a local copy and get pointer to src blk 816 * The rd_idx is masked before getting the pointer to 817 * avoid out of bound access in case the interface goes 818 * down. It will end up pointing to the last block in 819 * the buffer, but subsequent src->size check would be 820 * able to catch this. 821 */ 822 src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask); 823 src_size = readb(&src->size); 824 825 if ((rd_idx >= qinfo->q_size) || 826 (src_size > (qinfo->q_size - 1))) { 827 dev_crit(dev, 828 "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!", 829 rd_idx, src_size, qinfo->q_size); 830 bcm_vk_blk_drv_access(vk); 831 bcm_vk_set_host_alert(vk, 832 ERR_LOG_HOST_PCIE_DWN); 833 goto idx_err; 834 } 835 836 num_blks = src_size + 1; 837 data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL); 838 if (data) { 839 /* copy messages and linearize it */ 840 dst = data; 841 for (j = 0; j < num_blks; j++) { 842 memcpy_fromio(dst, src, sizeof(*dst)); 843 844 dst++; 845 rd_idx = msgq_inc(qinfo, rd_idx, 1); 846 src = msgq_blk_addr(qinfo, rd_idx); 847 } 848 total++; 849 } else { 850 /* 851 * if we could not allocate memory in kernel, 852 * that is fatal. 853 */ 854 dev_crit(dev, "Kernel mem allocation failure.\n"); 855 total = -ENOMEM; 856 goto idx_err; 857 } 858 859 /* flush rd pointer after a message is dequeued */ 860 writel(rd_idx, &msgq->rd_idx); 861 862 /* log new info for debugging */ 863 dev_dbg(dev, 864 "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n", 865 readl_relaxed(&msgq->num), 866 rd_idx, 867 wr_idx, 868 num_blks, 869 msgq_occupied(msgq, qinfo), 870 msgq_avail_space(msgq, qinfo), 871 readl_relaxed(&msgq->size)); 872 873 /* 874 * No need to search if it is an autonomous one-way 875 * message from driver, as these messages do not bear 876 * a to_v pending item. Currently, only the shutdown 877 * message falls into this category. 878 */ 879 if (data->function_id == VK_FID_SHUTDOWN) { 880 kfree(data); 881 continue; 882 } 883 884 msg_id = get_msg_id(data); 885 /* lookup original message in to_v direction */ 886 entry = bcm_vk_dequeue_pending(vk, 887 &vk->to_v_msg_chan, 888 q_num, 889 msg_id); 890 891 /* 892 * if there is message to does not have prior send, 893 * this is the location to add here 894 */ 895 if (entry) { 896 entry->to_h_blks = num_blks; 897 entry->to_h_msg = data; 898 bcm_vk_append_pendq(&vk->to_h_msg_chan, 899 q_num, entry); 900 901 } else { 902 if (cnt++ < batch_log) 903 dev_info(dev, 904 "Could not find MsgId[0x%x] for resp func %d bmap %d\n", 905 msg_id, data->function_id, 906 test_bit(msg_id, vk->bmap)); 907 kfree(data); 908 } 909 /* Fetch wr_idx to handle more back-to-back events */ 910 wr_idx = readl(&msgq->wr_idx); 911 912 /* 913 * cap the max so that even we try to handle more back-to-back events, 914 * so that it won't hold CPU too long or in case rd/wr idexes are 915 * corrupted which triggers infinite looping. 916 */ 917 if (++msg_processed >= max_msg_to_process) { 918 dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n", 919 q_num, max_msg_to_process); 920 exit_loop = true; 921 } 922 } 923 } 924idx_err: 925 mutex_unlock(&chan->msgq_mutex); 926 dev_dbg(dev, "total %d drained from queues\n", total); 927 928 return total; 929} 930 931/* 932 * init routine for all required data structures 933 */ 934static int bcm_vk_data_init(struct bcm_vk *vk) 935{ 936 int i; 937 938 spin_lock_init(&vk->ctx_lock); 939 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { 940 vk->ctx[i].in_use = false; 941 vk->ctx[i].idx = i; /* self identity */ 942 vk->ctx[i].miscdev = NULL; 943 } 944 spin_lock_init(&vk->msg_id_lock); 945 spin_lock_init(&vk->host_alert_lock); 946 vk->msg_id = 0; 947 948 /* initialize hash table */ 949 for (i = 0; i < VK_PID_HT_SZ; i++) 950 INIT_LIST_HEAD(&vk->pid_ht[i].head); 951 952 return 0; 953} 954 955irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id) 956{ 957 struct bcm_vk *vk = dev_id; 958 959 if (!bcm_vk_drv_access_ok(vk)) { 960 dev_err(&vk->pdev->dev, 961 "Interrupt %d received when msgq not inited\n", irq); 962 goto skip_schedule_work; 963 } 964 965 queue_work(vk->wq_thread, &vk->wq_work); 966 967skip_schedule_work: 968 return IRQ_HANDLED; 969} 970 971int bcm_vk_open(struct inode *inode, struct file *p_file) 972{ 973 struct bcm_vk_ctx *ctx; 974 struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data; 975 struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev); 976 struct device *dev = &vk->pdev->dev; 977 int rc = 0; 978 979 /* get a context and set it up for file */ 980 ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current)); 981 if (!ctx) { 982 dev_err(dev, "Error allocating context\n"); 983 rc = -ENOMEM; 984 } else { 985 /* 986 * set up context and replace private data with context for 987 * other methods to use. Reason for the context is because 988 * it is allowed for multiple sessions to open the sysfs, and 989 * for each file open, when upper layer query the response, 990 * only those that are tied to a specific open should be 991 * returned. The context->idx will be used for such binding 992 */ 993 ctx->miscdev = miscdev; 994 p_file->private_data = ctx; 995 dev_dbg(dev, "ctx_returned with idx %d, pid %d\n", 996 ctx->idx, ctx->pid); 997 } 998 return rc; 999} 1000 1001ssize_t bcm_vk_read(struct file *p_file, 1002 char __user *buf, 1003 size_t count, 1004 loff_t *f_pos) 1005{ 1006 ssize_t rc = -ENOMSG; 1007 struct bcm_vk_ctx *ctx = p_file->private_data; 1008 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, 1009 miscdev); 1010 struct device *dev = &vk->pdev->dev; 1011 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; 1012 struct bcm_vk_wkent *entry = NULL, *iter; 1013 u32 q_num; 1014 u32 rsp_length; 1015 1016 if (!bcm_vk_drv_access_ok(vk)) 1017 return -EPERM; 1018 1019 dev_dbg(dev, "Buf count %zu\n", count); 1020 1021 /* 1022 * search through the pendq on the to_h chan, and return only those 1023 * that belongs to the same context. Search is always from the high to 1024 * the low priority queues 1025 */ 1026 spin_lock(&chan->pendq_lock); 1027 for (q_num = 0; q_num < chan->q_nr; q_num++) { 1028 list_for_each_entry(iter, &chan->pendq[q_num], node) { 1029 if (iter->ctx->idx == ctx->idx) { 1030 if (count >= 1031 (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) { 1032 list_del(&iter->node); 1033 atomic_dec(&ctx->pend_cnt); 1034 entry = iter; 1035 } else { 1036 /* buffer not big enough */ 1037 rc = -EMSGSIZE; 1038 } 1039 goto read_loop_exit; 1040 } 1041 } 1042 } 1043read_loop_exit: 1044 spin_unlock(&chan->pendq_lock); 1045 1046 if (entry) { 1047 /* retrieve the passed down msg_id */ 1048 set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id); 1049 rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE; 1050 if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0) 1051 rc = rsp_length; 1052 1053 bcm_vk_free_wkent(dev, entry); 1054 } else if (rc == -EMSGSIZE) { 1055 struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; 1056 1057 /* 1058 * in this case, return just the first block, so 1059 * that app knows what size it is looking for. 1060 */ 1061 set_msg_id(&tmp_msg, entry->usr_msg_id); 1062 tmp_msg.size = entry->to_h_blks - 1; 1063 if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { 1064 dev_err(dev, "Error return 1st block in -EMSGSIZE\n"); 1065 rc = -EFAULT; 1066 } 1067 } 1068 return rc; 1069} 1070 1071ssize_t bcm_vk_write(struct file *p_file, 1072 const char __user *buf, 1073 size_t count, 1074 loff_t *f_pos) 1075{ 1076 ssize_t rc; 1077 struct bcm_vk_ctx *ctx = p_file->private_data; 1078 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, 1079 miscdev); 1080 struct bcm_vk_msgq __iomem *msgq; 1081 struct device *dev = &vk->pdev->dev; 1082 struct bcm_vk_wkent *entry; 1083 u32 sgl_extra_blks; 1084 u32 q_num; 1085 u32 msg_size; 1086 u32 msgq_size; 1087 1088 if (!bcm_vk_drv_access_ok(vk)) 1089 return -EPERM; 1090 1091 dev_dbg(dev, "Msg count %zu\n", count); 1092 1093 /* first, do sanity check where count should be multiple of basic blk */ 1094 if (count & (VK_MSGQ_BLK_SIZE - 1)) { 1095 dev_err(dev, "Failure with size %zu not multiple of %zu\n", 1096 count, VK_MSGQ_BLK_SIZE); 1097 rc = -EINVAL; 1098 goto write_err; 1099 } 1100 1101 /* allocate the work entry + buffer for size count and inband sgl */ 1102 entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size, 1103 GFP_KERNEL); 1104 if (!entry) { 1105 rc = -ENOMEM; 1106 goto write_err; 1107 } 1108 1109 /* now copy msg from user space, and then formulate the work entry */ 1110 if (copy_from_user(&entry->to_v_msg[0], buf, count)) { 1111 rc = -EFAULT; 1112 goto write_free_ent; 1113 } 1114 1115 entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT; 1116 entry->ctx = ctx; 1117 1118 /* do a check on the blk size which could not exceed queue space */ 1119 q_num = get_q_num(&entry->to_v_msg[0]); 1120 msgq = vk->to_v_msg_chan.msgq[q_num]; 1121 msgq_size = readl_relaxed(&msgq->size); 1122 if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT) 1123 > (msgq_size - 1)) { 1124 dev_err(dev, "Blk size %d exceed max queue size allowed %d\n", 1125 entry->to_v_blks, msgq_size - 1); 1126 rc = -EINVAL; 1127 goto write_free_ent; 1128 } 1129 1130 /* Use internal message id */ 1131 entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]); 1132 rc = bcm_vk_get_msg_id(vk); 1133 if (rc == VK_MSG_ID_OVERFLOW) { 1134 dev_err(dev, "msg_id overflow\n"); 1135 rc = -EOVERFLOW; 1136 goto write_free_ent; 1137 } 1138 set_msg_id(&entry->to_v_msg[0], rc); 1139 ctx->q_num = q_num; 1140 1141 dev_dbg(dev, 1142 "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n", 1143 ctx->q_num, ctx->idx, entry->usr_msg_id, 1144 get_msg_id(&entry->to_v_msg[0])); 1145 1146 if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) { 1147 /* Convert any pointers to sg list */ 1148 unsigned int num_planes; 1149 int dir; 1150 struct _vk_data *data; 1151 1152 /* 1153 * check if we are in reset, if so, no buffer transfer is 1154 * allowed and return error. 1155 */ 1156 if (vk->reset_pid) { 1157 dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n", 1158 ctx->pid); 1159 rc = -EACCES; 1160 goto write_free_msgid; 1161 } 1162 1163 num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK; 1164 if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD) 1165 dir = DMA_FROM_DEVICE; 1166 else 1167 dir = DMA_TO_DEVICE; 1168 1169 /* Calculate vk_data location */ 1170 /* Go to end of the message */ 1171 msg_size = entry->to_v_msg[0].size; 1172 if (msg_size > entry->to_v_blks) { 1173 rc = -EMSGSIZE; 1174 goto write_free_msgid; 1175 } 1176 1177 data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1]; 1178 1179 /* Now back up to the start of the pointers */ 1180 data -= num_planes; 1181 1182 /* Convert user addresses to DMA SG List */ 1183 rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes); 1184 if (rc) 1185 goto write_free_msgid; 1186 1187 atomic_inc(&ctx->dma_cnt); 1188 /* try to embed inband sgl */ 1189 sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data, 1190 num_planes); 1191 entry->to_v_blks += sgl_extra_blks; 1192 entry->to_v_msg[0].size += sgl_extra_blks; 1193 } else if (entry->to_v_msg[0].function_id == VK_FID_INIT && 1194 entry->to_v_msg[0].context_id == VK_NEW_CTX) { 1195 /* 1196 * Init happens in 2 stages, only the first stage contains the 1197 * pid that needs translating. 1198 */ 1199 pid_t org_pid, pid; 1200 1201 /* 1202 * translate the pid into the unique host space as user 1203 * may run sessions inside containers or process 1204 * namespaces. 1205 */ 1206#define VK_MSG_PID_MASK 0xffffff00 1207#define VK_MSG_PID_SH 8 1208 org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK) 1209 >> VK_MSG_PID_SH; 1210 1211 pid = task_tgid_nr(current); 1212 entry->to_v_msg[0].arg = 1213 (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) | 1214 (pid << VK_MSG_PID_SH); 1215 if (org_pid != pid) 1216 dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n", 1217 org_pid, org_pid, pid, pid); 1218 } 1219 1220 /* 1221 * store work entry to pending queue until a response is received. 1222 * This needs to be done before enqueuing the message 1223 */ 1224 bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry); 1225 1226 rc = bcm_to_v_msg_enqueue(vk, entry); 1227 if (rc) { 1228 dev_err(dev, "Fail to enqueue msg to to_v queue\n"); 1229 1230 /* remove message from pending list */ 1231 entry = bcm_vk_dequeue_pending 1232 (vk, 1233 &vk->to_v_msg_chan, 1234 q_num, 1235 get_msg_id(&entry->to_v_msg[0])); 1236 goto write_free_ent; 1237 } 1238 1239 return count; 1240 1241write_free_msgid: 1242 bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1); 1243write_free_ent: 1244 kfree(entry); 1245write_err: 1246 return rc; 1247} 1248 1249__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait) 1250{ 1251 __poll_t ret = 0; 1252 int cnt; 1253 struct bcm_vk_ctx *ctx = p_file->private_data; 1254 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); 1255 struct device *dev = &vk->pdev->dev; 1256 1257 poll_wait(p_file, &ctx->rd_wq, wait); 1258 1259 cnt = atomic_read(&ctx->pend_cnt); 1260 if (cnt) { 1261 ret = (__force __poll_t)(POLLIN | POLLRDNORM); 1262 if (cnt < 0) { 1263 dev_err(dev, "Error cnt %d, setting back to 0", cnt); 1264 atomic_set(&ctx->pend_cnt, 0); 1265 } 1266 } 1267 1268 return ret; 1269} 1270 1271int bcm_vk_release(struct inode *inode, struct file *p_file) 1272{ 1273 int ret; 1274 struct bcm_vk_ctx *ctx = p_file->private_data; 1275 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); 1276 struct device *dev = &vk->pdev->dev; 1277 pid_t pid = ctx->pid; 1278 int dma_cnt; 1279 unsigned long timeout, start_time; 1280 1281 /* 1282 * if there are outstanding DMA transactions, need to delay long enough 1283 * to ensure that the card side would have stopped touching the host buffer 1284 * and its SGL list. A race condition could happen if the host app is killed 1285 * abruptly, eg kill -9, while some DMA transfer orders are still inflight. 1286 * Nothing could be done except for a delay as host side is running in a 1287 * completely async fashion. 1288 */ 1289 start_time = jiffies; 1290 timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS); 1291 do { 1292 if (time_after(jiffies, timeout)) { 1293 dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n", 1294 dma_cnt, ctx->idx, pid); 1295 break; 1296 } 1297 dma_cnt = atomic_read(&ctx->dma_cnt); 1298 cpu_relax(); 1299 cond_resched(); 1300 } while (dma_cnt); 1301 dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n", 1302 ctx->idx, pid, jiffies_to_msecs(jiffies - start_time)); 1303 1304 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx); 1305 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx); 1306 1307 ret = bcm_vk_free_ctx(vk, ctx); 1308 if (ret == 0) 1309 ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num); 1310 else 1311 ret = 0; 1312 1313 kref_put(&vk->kref, bcm_vk_release_data); 1314 1315 return ret; 1316} 1317 1318int bcm_vk_msg_init(struct bcm_vk *vk) 1319{ 1320 struct device *dev = &vk->pdev->dev; 1321 int ret; 1322 1323 if (bcm_vk_data_init(vk)) { 1324 dev_err(dev, "Error initializing internal data structures\n"); 1325 return -EINVAL; 1326 } 1327 1328 if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) || 1329 bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) { 1330 dev_err(dev, "Error initializing communication channel\n"); 1331 return -EIO; 1332 } 1333 1334 /* read msgq info if ready */ 1335 ret = bcm_vk_sync_msgq(vk, false); 1336 if (ret && (ret != -EAGAIN)) { 1337 dev_err(dev, "Error reading comm msg Q info\n"); 1338 return -EIO; 1339 } 1340 1341 return 0; 1342} 1343 1344void bcm_vk_msg_remove(struct bcm_vk *vk) 1345{ 1346 bcm_vk_blk_drv_access(vk); 1347 1348 /* drain all pending items */ 1349 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL); 1350 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL); 1351} 1352 1353