1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
|
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_qpair.c 248768 2013-03-26 22:06:05Z jimharris $");
|
| 28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_qpair.c 248770 2013-03-26 22:11:34Z jimharris $"); |
29
30#include <sys/param.h>
31#include <sys/bus.h>
32
33#include <dev/pci/pcivar.h>
34
35#include "nvme_private.h"
36
37static void _nvme_qpair_submit_request(struct nvme_qpair *qpair,
38 struct nvme_request *req);
39
40static boolean_t
41nvme_completion_is_retry(const struct nvme_completion *cpl)
42{
43 /*
44 * TODO: spec is not clear how commands that are aborted due
45 * to TLER will be marked. So for now, it seems
46 * NAMESPACE_NOT_READY is the only case where we should
47 * look at the DNR bit.
48 */
49 switch (cpl->status.sct) {
50 case NVME_SCT_GENERIC:
51 switch (cpl->status.sc) {
52 case NVME_SC_ABORTED_BY_REQUEST:
53 case NVME_SC_NAMESPACE_NOT_READY:
54 if (cpl->status.dnr)
55 return (0);
56 else
57 return (1);
58 case NVME_SC_INVALID_OPCODE:
59 case NVME_SC_INVALID_FIELD:
60 case NVME_SC_COMMAND_ID_CONFLICT:
61 case NVME_SC_DATA_TRANSFER_ERROR:
62 case NVME_SC_ABORTED_POWER_LOSS:
63 case NVME_SC_INTERNAL_DEVICE_ERROR:
64 case NVME_SC_ABORTED_SQ_DELETION:
65 case NVME_SC_ABORTED_FAILED_FUSED:
66 case NVME_SC_ABORTED_MISSING_FUSED:
67 case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
68 case NVME_SC_COMMAND_SEQUENCE_ERROR:
69 case NVME_SC_LBA_OUT_OF_RANGE:
70 case NVME_SC_CAPACITY_EXCEEDED:
71 default:
72 return (0);
73 }
74 case NVME_SCT_COMMAND_SPECIFIC:
75 case NVME_SCT_MEDIA_ERROR:
76 case NVME_SCT_VENDOR_SPECIFIC:
77 default:
78 return (0);
79 }
80}
81
82static void
83nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
84 uint16_t cid)
85{
86
87 bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map);
88 bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map);
89
90 bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp,
91 sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0);
92
93 callout_init(&tr->timer, 1);
94 tr->cid = cid;
95 tr->qpair = qpair;
96}
97
98static void
99nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
100 struct nvme_completion *cpl, boolean_t print_on_error)
101{
102 struct nvme_request *req;
103 boolean_t retry, error;
104
105 req = tr->req;
106 error = nvme_completion_is_error(cpl);
107 retry = error && nvme_completion_is_retry(cpl) &&
108 req->retries < nvme_retry_count;
109
110 if (error && print_on_error) {
111 nvme_dump_completion(cpl);
112 nvme_dump_command(&req->cmd);
113 }
114
115 qpair->act_tr[cpl->cid] = NULL;
116
117 KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n"));
118
119 if (req->cb_fn && !retry)
120 req->cb_fn(req->cb_arg, cpl);
121
122 mtx_lock(&qpair->lock);
123 callout_stop(&tr->timer);
124
125 if (retry) {
126 req->retries++;
127 nvme_qpair_submit_tracker(qpair, tr);
128 } else {
129 if (req->payload_size > 0 || req->uio != NULL)
130 bus_dmamap_unload(qpair->dma_tag,
131 tr->payload_dma_map);
132
133 nvme_free_request(req);
134 tr->req = NULL;
135
136 TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq);
137 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
138
139 /*
140 * If the controller is in the middle of resetting, don't
141 * try to submit queued requests here - let the reset logic
142 * handle that instead.
143 */
144 if (!STAILQ_EMPTY(&qpair->queued_req) &&
145 !qpair->ctrlr->is_resetting) {
146 req = STAILQ_FIRST(&qpair->queued_req);
147 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
148 _nvme_qpair_submit_request(qpair, req);
149 }
150 }
151
152 mtx_unlock(&qpair->lock);
153}
154
155static void
156nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
157 struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
158 boolean_t print_on_error)
159{
160 struct nvme_completion cpl;
161
162 memset(&cpl, 0, sizeof(cpl));
163 cpl.sqid = qpair->id;
164 cpl.cid = tr->cid;
165 cpl.status.sct = sct;
166 cpl.status.sc = sc;
167 cpl.status.dnr = dnr;
168 nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
169}
170
171void
172nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
173 struct nvme_request *req, uint32_t sct, uint32_t sc,
174 boolean_t print_on_error)
175{
176 struct nvme_completion cpl;
177 boolean_t error;
178
179 memset(&cpl, 0, sizeof(cpl));
180 cpl.sqid = qpair->id;
181 cpl.status.sct = sct;
182 cpl.status.sc = sc;
183
184 error = nvme_completion_is_error(&cpl);
185
186 if (error && print_on_error) {
187 nvme_dump_completion(&cpl);
188 nvme_dump_command(&req->cmd);
189 }
190
191 if (req->cb_fn)
192 req->cb_fn(req->cb_arg, &cpl);
193
194 nvme_free_request(req);
195}
196
197void
198nvme_qpair_process_completions(struct nvme_qpair *qpair)
199{
200 struct nvme_tracker *tr;
201 struct nvme_completion *cpl;
202
203 qpair->num_intr_handler_calls++;
204
205 if (!qpair->is_enabled)
206 /*
207 * qpair is not enabled, likely because a controller reset is
208 * is in progress. Ignore the interrupt - any I/O that was
209 * associated with this interrupt will get retried when the
210 * reset is complete.
211 */
212 return;
213
214 while (1) {
215 cpl = &qpair->cpl[qpair->cq_head];
216
217 if (cpl->status.p != qpair->phase)
218 break;
219
220 tr = qpair->act_tr[cpl->cid];
221
222 if (tr != NULL) {
223 nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE);
224 qpair->sq_head = cpl->sqhd;
225 } else {
226 printf("cpl does not map to outstanding cmd\n");
227 nvme_dump_completion(cpl);
228 KASSERT(0, ("received completion for unknown cmd\n"));
229 }
230
231 if (++qpair->cq_head == qpair->num_entries) {
232 qpair->cq_head = 0;
233 qpair->phase = !qpair->phase;
234 }
235
236 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
237 qpair->cq_head);
238 }
239}
240
241static void
242nvme_qpair_msix_handler(void *arg)
243{
244 struct nvme_qpair *qpair = arg;
245
246 nvme_qpair_process_completions(qpair);
247}
248
249void
250nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
251 uint16_t vector, uint32_t num_entries, uint32_t num_trackers,
252 uint32_t max_xfer_size, struct nvme_controller *ctrlr)
253{
254 struct nvme_tracker *tr;
255 uint32_t i;
256
257 qpair->id = id;
258 qpair->vector = vector;
259 qpair->num_entries = num_entries;
260#ifdef CHATHAM2
261 /*
262 * Chatham prototype board starts having issues at higher queue
263 * depths. So use a conservative estimate here of no more than 64
264 * outstanding I/O per queue at any one point.
265 */
266 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
267 num_trackers = min(num_trackers, 64);
268#endif
269 qpair->num_trackers = num_trackers;
270 qpair->max_xfer_size = max_xfer_size;
271 qpair->ctrlr = ctrlr;
272
273 if (ctrlr->msix_enabled) {
274
275 /*
276 * MSI-X vector resource IDs start at 1, so we add one to
277 * the queue's vector to get the corresponding rid to use.
278 */
279 qpair->rid = vector + 1;
280
281 qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
282 &qpair->rid, RF_ACTIVE);
283
284 bus_setup_intr(ctrlr->dev, qpair->res,
285 INTR_TYPE_MISC | INTR_MPSAFE, NULL,
286 nvme_qpair_msix_handler, qpair, &qpair->tag);
287 }
288
289 mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
290
291 bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
292 sizeof(uint64_t), PAGE_SIZE, BUS_SPACE_MAXADDR,
293 BUS_SPACE_MAXADDR, NULL, NULL, qpair->max_xfer_size,
294 (qpair->max_xfer_size/PAGE_SIZE)+1, PAGE_SIZE, 0,
295 NULL, NULL, &qpair->dma_tag);
296
297 qpair->num_cmds = 0;
298 qpair->num_intr_handler_calls = 0;
299
|
300 /* TODO: error checking on contigmalloc, bus_dmamap_load calls */
|
300 qpair->cmd = contigmalloc(qpair->num_entries *
|
302 sizeof(struct nvme_command), M_NVME, M_ZERO | M_NOWAIT,
|
| 301 sizeof(struct nvme_command), M_NVME, M_ZERO, |
302 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
303 qpair->cpl = contigmalloc(qpair->num_entries *
|
305 sizeof(struct nvme_completion), M_NVME, M_ZERO | M_NOWAIT,
|
| 304 sizeof(struct nvme_completion), M_NVME, M_ZERO, |
305 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
306
307 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map);
308 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map);
309
310 bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map,
311 qpair->cmd, qpair->num_entries * sizeof(struct nvme_command),
312 nvme_single_map, &qpair->cmd_bus_addr, 0);
313 bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map,
314 qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion),
315 nvme_single_map, &qpair->cpl_bus_addr, 0);
316
317 qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
318 qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
319
320 TAILQ_INIT(&qpair->free_tr);
321 TAILQ_INIT(&qpair->outstanding_tr);
322 STAILQ_INIT(&qpair->queued_req);
323
324 for (i = 0; i < qpair->num_trackers; i++) {
|
326 tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT);
327
328 if (tr == NULL) {
329 printf("warning: nvme tracker malloc failed\n");
330 break;
331 }
332
|
| 325 tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_WAITOK); |
326 nvme_qpair_construct_tracker(qpair, tr, i);
327 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
328 }
329
330 qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries,
|
338 M_NVME, M_ZERO | M_NOWAIT);
|
| 331 M_NVME, M_ZERO | M_WAITOK); |
332}
333
334static void
335nvme_qpair_destroy(struct nvme_qpair *qpair)
336{
337 struct nvme_tracker *tr;
338
339 if (qpair->tag)
340 bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag);
341
342 if (qpair->res)
343 bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ,
344 rman_get_rid(qpair->res), qpair->res);
345
346 if (qpair->cmd) {
347 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map);
348 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map);
349 contigfree(qpair->cmd,
350 qpair->num_entries * sizeof(struct nvme_command), M_NVME);
351 }
352
353 if (qpair->cpl) {
354 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map);
355 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map);
356 contigfree(qpair->cpl,
357 qpair->num_entries * sizeof(struct nvme_completion),
358 M_NVME);
359 }
360
361 if (qpair->dma_tag)
362 bus_dma_tag_destroy(qpair->dma_tag);
363
364 if (qpair->act_tr)
365 free(qpair->act_tr, M_NVME);
366
367 while (!TAILQ_EMPTY(&qpair->free_tr)) {
368 tr = TAILQ_FIRST(&qpair->free_tr);
369 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
370 bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map);
371 bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map);
372 free(tr, M_NVME);
373 }
374}
375
376static void
377nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
378{
379 struct nvme_tracker *tr;
380
381 tr = TAILQ_FIRST(&qpair->outstanding_tr);
382 while (tr != NULL) {
383 if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
384 nvme_qpair_manual_complete_tracker(qpair, tr,
385 NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0,
386 FALSE);
387 tr = TAILQ_FIRST(&qpair->outstanding_tr);
388 } else {
389 tr = TAILQ_NEXT(tr, tailq);
390 }
391 }
392}
393
394void
395nvme_admin_qpair_destroy(struct nvme_qpair *qpair)
396{
397
398 nvme_admin_qpair_abort_aers(qpair);
399 nvme_qpair_destroy(qpair);
400}
401
402void
403nvme_io_qpair_destroy(struct nvme_qpair *qpair)
404{
405
406 nvme_qpair_destroy(qpair);
407}
408
409static void
410nvme_abort_complete(void *arg, const struct nvme_completion *status)
411{
412 struct nvme_tracker *tr = arg;
413
414 /*
415 * If cdw0 == 1, the controller was not able to abort the command
416 * we requested. We still need to check the active tracker array,
417 * to cover race where I/O timed out at same time controller was
418 * completing the I/O.
419 */
420 if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) {
421 /*
422 * An I/O has timed out, and the controller was unable to
423 * abort it for some reason. Construct a fake completion
424 * status, and then complete the I/O's tracker manually.
425 */
426 printf("abort command failed, aborting command manually\n");
427 nvme_qpair_manual_complete_tracker(tr->qpair, tr,
428 NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
429 }
430}
431
432static void
433nvme_timeout(void *arg)
434{
435 struct nvme_tracker *tr = arg;
436 struct nvme_qpair *qpair = tr->qpair;
437 struct nvme_controller *ctrlr = qpair->ctrlr;
438 union csts_register csts;
439
440 /* Read csts to get value of cfs - controller fatal status. */
441 csts.raw = nvme_mmio_read_4(ctrlr, csts);
442
443 if (ctrlr->enable_aborts && csts.bits.cfs == 0) {
444 /*
445 * If aborts are enabled, only use them if the controller is
446 * not reporting fatal status.
447 */
448 nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,
449 nvme_abort_complete, tr);
450 } else
451 nvme_ctrlr_reset(ctrlr);
452}
453
454void
455nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
456{
457 struct nvme_request *req;
458 struct nvme_controller *ctrlr;
459
460 mtx_assert(&qpair->lock, MA_OWNED);
461
462 req = tr->req;
463 req->cmd.cid = tr->cid;
464 qpair->act_tr[tr->cid] = tr;
465 ctrlr = qpair->ctrlr;
466
467 if (req->timeout)
468#if __FreeBSD_version >= 800030
469 callout_reset_curcpu(&tr->timer, ctrlr->timeout_period * hz,
470 nvme_timeout, tr);
471#else
472 callout_reset(&tr->timer, ctrlr->timeout_period * hz,
473 nvme_timeout, tr);
474#endif
475
476 /* Copy the command from the tracker to the submission queue. */
477 memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
478
479 if (++qpair->sq_tail == qpair->num_entries)
480 qpair->sq_tail = 0;
481
482 wmb();
483 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
484 qpair->sq_tail);
485
486 qpair->num_cmds++;
487}
488
489static void
490_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
491{
492 struct nvme_tracker *tr;
493 int err;
494
495 mtx_assert(&qpair->lock, MA_OWNED);
496
497 tr = TAILQ_FIRST(&qpair->free_tr);
498 req->qpair = qpair;
499
500 if (tr == NULL || !qpair->is_enabled) {
501 /*
502 * No tracker is available, or the qpair is disabled due to
503 * an in-progress controller-level reset or controller
504 * failure.
505 */
506
507 if (qpair->ctrlr->is_failed) {
508 /*
509 * The controller has failed. Post the request to a
510 * task where it will be aborted, so that we do not
511 * invoke the request's callback in the context
512 * of the submission.
513 */
514 nvme_ctrlr_post_failed_request(qpair->ctrlr, req);
515 } else {
516 /*
517 * Put the request on the qpair's request queue to be
518 * processed when a tracker frees up via a command
519 * completion or when the controller reset is
520 * completed.
521 */
522 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
523 }
524 return;
525 }
526
527 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
528 TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
529 tr->req = req;
530
531 if (req->uio == NULL) {
532 if (req->payload_size > 0) {
533 err = bus_dmamap_load(tr->qpair->dma_tag,
534 tr->payload_dma_map, req->payload,
535 req->payload_size,
536 nvme_payload_map, tr, 0);
537 if (err != 0)
538 panic("bus_dmamap_load returned non-zero!\n");
539 } else
540 nvme_qpair_submit_tracker(tr->qpair, tr);
541 } else {
542 err = bus_dmamap_load_uio(tr->qpair->dma_tag,
543 tr->payload_dma_map, req->uio,
544 nvme_payload_map_uio, tr, 0);
545 if (err != 0)
546 panic("bus_dmamap_load returned non-zero!\n");
547 }
548}
549
550void
551nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
552{
553
554 mtx_lock(&qpair->lock);
555 _nvme_qpair_submit_request(qpair, req);
556 mtx_unlock(&qpair->lock);
557}
558
559static void
560nvme_qpair_enable(struct nvme_qpair *qpair)
561{
562
563 qpair->is_enabled = TRUE;
564}
565
566void
567nvme_qpair_reset(struct nvme_qpair *qpair)
568{
569
570 qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;
571
572 /*
573 * First time through the completion queue, HW will set phase
574 * bit on completions to 1. So set this to 1 here, indicating
575 * we're looking for a 1 to know which entries have completed.
576 * we'll toggle the bit each time when the completion queue
577 * rolls over.
578 */
579 qpair->phase = 1;
580
581 memset(qpair->cmd, 0,
582 qpair->num_entries * sizeof(struct nvme_command));
583 memset(qpair->cpl, 0,
584 qpair->num_entries * sizeof(struct nvme_completion));
585}
586
587void
588nvme_admin_qpair_enable(struct nvme_qpair *qpair)
589{
590 struct nvme_tracker *tr;
591 struct nvme_tracker *tr_temp;
592
593 /*
594 * Manually abort each outstanding admin command. Do not retry
595 * admin commands found here, since they will be left over from
596 * a controller reset and its likely the context in which the
597 * command was issued no longer applies.
598 */
599 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
600 device_printf(qpair->ctrlr->dev,
601 "aborting outstanding admin command\n");
602 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
603 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE);
604 }
605
606 nvme_qpair_enable(qpair);
607}
608
609void
610nvme_io_qpair_enable(struct nvme_qpair *qpair)
611{
612 STAILQ_HEAD(, nvme_request) temp;
613 struct nvme_tracker *tr;
614 struct nvme_tracker *tr_temp;
615 struct nvme_request *req;
616
617 /*
618 * Manually abort each outstanding I/O. This normally results in a
619 * retry, unless the retry count on the associated request has
620 * reached its limit.
621 */
622 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
623 device_printf(qpair->ctrlr->dev,
624 "aborting outstanding i/o\n");
625 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
626 NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
627 }
628
629 mtx_lock(&qpair->lock);
630
631 nvme_qpair_enable(qpair);
632
633 STAILQ_INIT(&temp);
634 STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);
635
636 while (!STAILQ_EMPTY(&temp)) {
637 req = STAILQ_FIRST(&temp);
638 STAILQ_REMOVE_HEAD(&temp, stailq);
639 device_printf(qpair->ctrlr->dev,
640 "resubmitting queued i/o\n");
641 nvme_dump_command(&req->cmd);
642 _nvme_qpair_submit_request(qpair, req);
643 }
644
645 mtx_unlock(&qpair->lock);
646}
647
648static void
649nvme_qpair_disable(struct nvme_qpair *qpair)
650{
651 struct nvme_tracker *tr;
652
653 qpair->is_enabled = FALSE;
654 mtx_lock(&qpair->lock);
655 TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq)
656 callout_stop(&tr->timer);
657 mtx_unlock(&qpair->lock);
658}
659
660void
661nvme_admin_qpair_disable(struct nvme_qpair *qpair)
662{
663
664 nvme_qpair_disable(qpair);
665 nvme_admin_qpair_abort_aers(qpair);
666}
667
668void
669nvme_io_qpair_disable(struct nvme_qpair *qpair)
670{
671
672 nvme_qpair_disable(qpair);
673}
674
675void
676nvme_qpair_fail(struct nvme_qpair *qpair)
677{
678 struct nvme_tracker *tr;
679 struct nvme_request *req;
680
681 mtx_lock(&qpair->lock);
682
683 while (!STAILQ_EMPTY(&qpair->queued_req)) {
684 req = STAILQ_FIRST(&qpair->queued_req);
685 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
686 device_printf(qpair->ctrlr->dev,
687 "failing queued i/o\n");
688 mtx_unlock(&qpair->lock);
689 nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
690 NVME_SC_ABORTED_BY_REQUEST, TRUE);
691 mtx_lock(&qpair->lock);
692 }
693
694 /* Manually abort each outstanding I/O. */
695 while (!TAILQ_EMPTY(&qpair->outstanding_tr)) {
696 tr = TAILQ_FIRST(&qpair->outstanding_tr);
697 /*
698 * Do not remove the tracker. The abort_tracker path will
699 * do that for us.
700 */
701 device_printf(qpair->ctrlr->dev,
702 "failing outstanding i/o\n");
703 mtx_unlock(&qpair->lock);
704 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
705 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE);
706 mtx_lock(&qpair->lock);
707 }
708
709 mtx_unlock(&qpair->lock);
710}
711
|