1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4#include <linux/dynamic_debug.h> 5 6#include "core.h" 7 8struct pdsc_wait_context { 9 struct pdsc_qcq *qcq; 10 struct completion wait_completion; 11}; 12 13static int pdsc_process_notifyq(struct pdsc_qcq *qcq) 14{ 15 union pds_core_notifyq_comp *comp; 16 struct pdsc *pdsc = qcq->pdsc; 17 struct pdsc_cq *cq = &qcq->cq; 18 struct pdsc_cq_info *cq_info; 19 int nq_work = 0; 20 u64 eid; 21 22 cq_info = &cq->info[cq->tail_idx]; 23 comp = cq_info->comp; 24 eid = le64_to_cpu(comp->event.eid); 25 while (eid > pdsc->last_eid) { 26 u16 ecode = le16_to_cpu(comp->event.ecode); 27 28 switch (ecode) { 29 case PDS_EVENT_LINK_CHANGE: 30 dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n", 31 ecode, eid); 32 pdsc_notify(PDS_EVENT_LINK_CHANGE, comp); 33 break; 34 35 case PDS_EVENT_RESET: 36 dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n", 37 ecode, eid); 38 pdsc_notify(PDS_EVENT_RESET, comp); 39 break; 40 41 case PDS_EVENT_XCVR: 42 dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n", 43 ecode, eid); 44 break; 45 46 default: 47 dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n", 48 ecode, eid); 49 break; 50 } 51 52 pdsc->last_eid = eid; 53 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 54 cq_info = &cq->info[cq->tail_idx]; 55 comp = cq_info->comp; 56 eid = le64_to_cpu(comp->event.eid); 57 58 nq_work++; 59 } 60 61 qcq->accum_work += nq_work; 62 63 return nq_work; 64} 65 66static bool pdsc_adminq_inc_if_up(struct pdsc *pdsc) 67{ 68 if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER) || 69 pdsc->state & BIT_ULL(PDSC_S_FW_DEAD)) 70 return false; 71 72 return refcount_inc_not_zero(&pdsc->adminq_refcnt); 73} 74 75void pdsc_process_adminq(struct pdsc_qcq *qcq) 76{ 77 union pds_core_adminq_comp *comp; 78 struct pdsc_queue *q = &qcq->q; 79 struct pdsc *pdsc = qcq->pdsc; 80 struct pdsc_cq *cq = &qcq->cq; 81 struct pdsc_q_info *q_info; 82 unsigned long irqflags; 83 int nq_work = 0; 84 int aq_work = 0; 85 86 /* Don't process AdminQ when it's not up */ 87 if (!pdsc_adminq_inc_if_up(pdsc)) { 88 dev_err(pdsc->dev, "%s: called while adminq is unavailable\n", 89 __func__); 90 return; 91 } 92 93 /* Check for NotifyQ event */ 94 nq_work = pdsc_process_notifyq(&pdsc->notifyqcq); 95 96 /* Check for empty queue, which can happen if the interrupt was 97 * for a NotifyQ event and there are no new AdminQ completions. 98 */ 99 if (q->tail_idx == q->head_idx) 100 goto credits; 101 102 /* Find the first completion to clean, 103 * run the callback in the related q_info, 104 * and continue while we still match done color 105 */ 106 spin_lock_irqsave(&pdsc->adminq_lock, irqflags); 107 comp = cq->info[cq->tail_idx].comp; 108 while (pdsc_color_match(comp->color, cq->done_color)) { 109 q_info = &q->info[q->tail_idx]; 110 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 111 112 /* Copy out the completion data */ 113 memcpy(q_info->dest, comp, sizeof(*comp)); 114 115 complete_all(&q_info->wc->wait_completion); 116 117 if (cq->tail_idx == cq->num_descs - 1) 118 cq->done_color = !cq->done_color; 119 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 120 comp = cq->info[cq->tail_idx].comp; 121 122 aq_work++; 123 } 124 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); 125 126 qcq->accum_work += aq_work; 127 128credits: 129 /* Return the interrupt credits, one for each completion */ 130 pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx], 131 nq_work + aq_work, 132 PDS_CORE_INTR_CRED_REARM); 133 refcount_dec(&pdsc->adminq_refcnt); 134} 135 136void pdsc_work_thread(struct work_struct *work) 137{ 138 struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work); 139 140 pdsc_process_adminq(qcq); 141} 142 143irqreturn_t pdsc_adminq_isr(int irq, void *data) 144{ 145 struct pdsc *pdsc = data; 146 struct pdsc_qcq *qcq; 147 148 /* Don't process AdminQ when it's not up */ 149 if (!pdsc_adminq_inc_if_up(pdsc)) { 150 dev_err(pdsc->dev, "%s: called while adminq is unavailable\n", 151 __func__); 152 return IRQ_HANDLED; 153 } 154 155 qcq = &pdsc->adminqcq; 156 queue_work(pdsc->wq, &qcq->work); 157 refcount_dec(&pdsc->adminq_refcnt); 158 159 return IRQ_HANDLED; 160} 161 162static int __pdsc_adminq_post(struct pdsc *pdsc, 163 struct pdsc_qcq *qcq, 164 union pds_core_adminq_cmd *cmd, 165 union pds_core_adminq_comp *comp, 166 struct pdsc_wait_context *wc) 167{ 168 struct pdsc_queue *q = &qcq->q; 169 struct pdsc_q_info *q_info; 170 unsigned long irqflags; 171 unsigned int avail; 172 int index; 173 int ret; 174 175 spin_lock_irqsave(&pdsc->adminq_lock, irqflags); 176 177 /* Check for space in the queue */ 178 avail = q->tail_idx; 179 if (q->head_idx >= avail) 180 avail += q->num_descs - q->head_idx - 1; 181 else 182 avail -= q->head_idx + 1; 183 if (!avail) { 184 ret = -ENOSPC; 185 goto err_out_unlock; 186 } 187 188 /* Check that the FW is running */ 189 if (!pdsc_is_fw_running(pdsc)) { 190 if (pdsc->info_regs) { 191 u8 fw_status = 192 ioread8(&pdsc->info_regs->fw_status); 193 194 dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n", 195 __func__, fw_status); 196 } else { 197 dev_info(pdsc->dev, "%s: post failed - BARs not setup\n", 198 __func__); 199 } 200 ret = -ENXIO; 201 202 goto err_out_unlock; 203 } 204 205 /* Post the request */ 206 index = q->head_idx; 207 q_info = &q->info[index]; 208 q_info->wc = wc; 209 q_info->dest = comp; 210 memcpy(q_info->desc, cmd, sizeof(*cmd)); 211 212 dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n", 213 q->head_idx, q->tail_idx); 214 dev_dbg(pdsc->dev, "post admin queue command:\n"); 215 dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1, 216 cmd, sizeof(*cmd), true); 217 218 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); 219 220 pds_core_dbell_ring(pdsc->kern_dbpage, 221 q->hw_type, q->dbval | q->head_idx); 222 ret = index; 223 224err_out_unlock: 225 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); 226 return ret; 227} 228 229int pdsc_adminq_post(struct pdsc *pdsc, 230 union pds_core_adminq_cmd *cmd, 231 union pds_core_adminq_comp *comp, 232 bool fast_poll) 233{ 234 struct pdsc_wait_context wc = { 235 .wait_completion = 236 COMPLETION_INITIALIZER_ONSTACK(wc.wait_completion), 237 }; 238 unsigned long poll_interval = 1; 239 unsigned long poll_jiffies; 240 unsigned long time_limit; 241 unsigned long time_start; 242 unsigned long time_done; 243 unsigned long remaining; 244 int err = 0; 245 int index; 246 247 if (!pdsc_adminq_inc_if_up(pdsc)) { 248 dev_dbg(pdsc->dev, "%s: preventing adminq cmd %u\n", 249 __func__, cmd->opcode); 250 return -ENXIO; 251 } 252 253 wc.qcq = &pdsc->adminqcq; 254 index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp, &wc); 255 if (index < 0) { 256 err = index; 257 goto err_out; 258 } 259 260 time_start = jiffies; 261 time_limit = time_start + HZ * pdsc->devcmd_timeout; 262 do { 263 /* Timeslice the actual wait to catch IO errors etc early */ 264 poll_jiffies = msecs_to_jiffies(poll_interval); 265 remaining = wait_for_completion_timeout(&wc.wait_completion, 266 poll_jiffies); 267 if (remaining) 268 break; 269 270 if (!pdsc_is_fw_running(pdsc)) { 271 if (pdsc->info_regs) { 272 u8 fw_status = 273 ioread8(&pdsc->info_regs->fw_status); 274 275 dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n", 276 __func__, fw_status); 277 } else { 278 dev_dbg(pdsc->dev, "%s: post wait failed - BARs not setup\n", 279 __func__); 280 } 281 err = -ENXIO; 282 break; 283 } 284 285 /* When fast_poll is not requested, prevent aggressive polling 286 * on failures due to timeouts by doing exponential back off. 287 */ 288 if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL) 289 poll_interval <<= 1; 290 } while (time_before(jiffies, time_limit)); 291 time_done = jiffies; 292 dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n", 293 __func__, jiffies_to_msecs(time_done - time_start)); 294 295 /* Check the results */ 296 if (time_after_eq(time_done, time_limit)) 297 err = -ETIMEDOUT; 298 299 dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index); 300 dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1, 301 comp, sizeof(*comp), true); 302 303 if (remaining && comp->status) 304 err = pdsc_err_to_errno(comp->status); 305 306err_out: 307 if (err) { 308 dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n", 309 __func__, cmd->opcode, comp->status, ERR_PTR(err)); 310 if (err == -ENXIO || err == -ETIMEDOUT) 311 queue_work(pdsc->wq, &pdsc->health_work); 312 } 313 314 refcount_dec(&pdsc->adminq_refcnt); 315 316 return err; 317} 318EXPORT_SYMBOL_GPL(pdsc_adminq_post); 319