Deleted Added
sdiff udiff text old ( 248766 ) new ( 248767 )
full compact
nvme_qpair.c (248766) nvme_qpair.c (248767)
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_qpair.c 248766 2013-03-26 21:48:41Z jimharris $");
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_qpair.c 248767 2013-03-26 21:58:38Z jimharris $");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32
33#include <dev/pci/pcivar.h>
34
35#include "nvme_private.h"
36

--- 8 unchanged lines hidden (view full) ---

45 * to TLER will be marked. So for now, it seems
46 * NAMESPACE_NOT_READY is the only case where we should
47 * look at the DNR bit.
48 */
49 switch (cpl->status.sct) {
50 case NVME_SCT_GENERIC:
51 switch (cpl->status.sc) {
52 case NVME_SC_ABORTED_BY_REQUEST:
29
30#include <sys/param.h>
31#include <sys/bus.h>
32
33#include <dev/pci/pcivar.h>
34
35#include "nvme_private.h"
36

--- 8 unchanged lines hidden (view full) ---

45 * to TLER will be marked. So for now, it seems
46 * NAMESPACE_NOT_READY is the only case where we should
47 * look at the DNR bit.
48 */
49 switch (cpl->status.sct) {
50 case NVME_SCT_GENERIC:
51 switch (cpl->status.sc) {
52 case NVME_SC_ABORTED_BY_REQUEST:
53 return (1);
54 case NVME_SC_NAMESPACE_NOT_READY:
55 if (cpl->status.dnr)
56 return (0);
57 else
58 return (1);
59 case NVME_SC_INVALID_OPCODE:
60 case NVME_SC_INVALID_FIELD:
61 case NVME_SC_COMMAND_ID_CONFLICT:

--- 88 unchanged lines hidden (view full) ---

150 }
151 }
152
153 mtx_unlock(&qpair->lock);
154}
155
156static void
157nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
53 case NVME_SC_NAMESPACE_NOT_READY:
54 if (cpl->status.dnr)
55 return (0);
56 else
57 return (1);
58 case NVME_SC_INVALID_OPCODE:
59 case NVME_SC_INVALID_FIELD:
60 case NVME_SC_COMMAND_ID_CONFLICT:

--- 88 unchanged lines hidden (view full) ---

149 }
150 }
151
152 mtx_unlock(&qpair->lock);
153}
154
155static void
156nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
158 struct nvme_tracker *tr, uint32_t sct, uint32_t sc,
157 struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr,
159 boolean_t print_on_error)
160{
161 struct nvme_completion cpl;
162
163 memset(&cpl, 0, sizeof(cpl));
164 cpl.sqid = qpair->id;
165 cpl.cid = tr->cid;
166 cpl.status.sct = sct;
167 cpl.status.sc = sc;
158 boolean_t print_on_error)
159{
160 struct nvme_completion cpl;
161
162 memset(&cpl, 0, sizeof(cpl));
163 cpl.sqid = qpair->id;
164 cpl.cid = tr->cid;
165 cpl.status.sct = sct;
166 cpl.status.sc = sc;
167 cpl.status.dnr = dnr;
168 nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
169}
170
171void
168 nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
169}
170
171void
172nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
173 struct nvme_request *req, uint32_t sct, uint32_t sc,
174 boolean_t print_on_error)
175{
176 struct nvme_completion cpl;
177 boolean_t error;
178
179 memset(&cpl, 0, sizeof(cpl));
180 cpl.sqid = qpair->id;
181 cpl.status.sct = sct;
182 cpl.status.sc = sc;
183
184 error = nvme_completion_is_error(&cpl);
185
186 if (error && print_on_error) {
187 nvme_dump_completion(&cpl);
188 nvme_dump_command(&req->cmd);
189 }
190
191 if (req->cb_fn)
192 req->cb_fn(req->cb_arg, &cpl);
193
194 nvme_free_request(req);
195}
196
197void
172nvme_qpair_process_completions(struct nvme_qpair *qpair)
173{
174 struct nvme_tracker *tr;
175 struct nvme_completion *cpl;
176
177 qpair->num_intr_handler_calls++;
178
179 if (!qpair->is_enabled)

--- 178 unchanged lines hidden (view full) ---

358nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
359{
360 struct nvme_tracker *tr;
361
362 tr = TAILQ_FIRST(&qpair->outstanding_tr);
363 while (tr != NULL) {
364 if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
365 nvme_qpair_manual_complete_tracker(qpair, tr,
198nvme_qpair_process_completions(struct nvme_qpair *qpair)
199{
200 struct nvme_tracker *tr;
201 struct nvme_completion *cpl;
202
203 qpair->num_intr_handler_calls++;
204
205 if (!qpair->is_enabled)

--- 178 unchanged lines hidden (view full) ---

384nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
385{
386 struct nvme_tracker *tr;
387
388 tr = TAILQ_FIRST(&qpair->outstanding_tr);
389 while (tr != NULL) {
390 if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
391 nvme_qpair_manual_complete_tracker(qpair, tr,
366 NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION,
392 NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0,
367 FALSE);
368 tr = TAILQ_FIRST(&qpair->outstanding_tr);
369 } else {
370 tr = TAILQ_NEXT(tr, tailq);
371 }
372 }
373}
374

--- 26 unchanged lines hidden (view full) ---

401 if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) {
402 /*
403 * An I/O has timed out, and the controller was unable to
404 * abort it for some reason. Construct a fake completion
405 * status, and then complete the I/O's tracker manually.
406 */
407 printf("abort command failed, aborting command manually\n");
408 nvme_qpair_manual_complete_tracker(tr->qpair, tr,
393 FALSE);
394 tr = TAILQ_FIRST(&qpair->outstanding_tr);
395 } else {
396 tr = TAILQ_NEXT(tr, tailq);
397 }
398 }
399}
400

--- 26 unchanged lines hidden (view full) ---

427 if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) {
428 /*
429 * An I/O has timed out, and the controller was unable to
430 * abort it for some reason. Construct a fake completion
431 * status, and then complete the I/O's tracker manually.
432 */
433 printf("abort command failed, aborting command manually\n");
434 nvme_qpair_manual_complete_tracker(tr->qpair, tr,
409 NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE);
435 NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
410 }
411}
412
413static void
414nvme_timeout(void *arg)
415{
416 struct nvme_tracker *tr = arg;
417 struct nvme_qpair *qpair = tr->qpair;

--- 53 unchanged lines hidden (view full) ---

471_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
472{
473 struct nvme_tracker *tr;
474 int err;
475
476 mtx_assert(&qpair->lock, MA_OWNED);
477
478 tr = TAILQ_FIRST(&qpair->free_tr);
436 }
437}
438
439static void
440nvme_timeout(void *arg)
441{
442 struct nvme_tracker *tr = arg;
443 struct nvme_qpair *qpair = tr->qpair;

--- 53 unchanged lines hidden (view full) ---

497_nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
498{
499 struct nvme_tracker *tr;
500 int err;
501
502 mtx_assert(&qpair->lock, MA_OWNED);
503
504 tr = TAILQ_FIRST(&qpair->free_tr);
505 req->qpair = qpair;
479
480 if (tr == NULL || !qpair->is_enabled) {
481 /*
482 * No tracker is available, or the qpair is disabled due to
506
507 if (tr == NULL || !qpair->is_enabled) {
508 /*
509 * No tracker is available, or the qpair is disabled due to
483 * an in-progress controller-level reset.
484 *
485 * Put the request on the qpair's request queue to be processed
486 * when a tracker frees up via a command completion or when
487 * the controller reset is completed.
510 * an in-progress controller-level reset or controller
511 * failure.
488 */
512 */
489 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
513
514 if (qpair->ctrlr->is_failed) {
515 /*
516 * The controller has failed. Post the request to a
517 * task where it will be aborted, so that we do not
518 * invoke the request's callback in the context
519 * of the submission.
520 */
521 nvme_ctrlr_post_failed_request(qpair->ctrlr, req);
522 } else {
523 /*
524 * Put the request on the qpair's request queue to be
525 * processed when a tracker frees up via a command
526 * completion or when the controller reset is
527 * completed.
528 */
529 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
530 }
490 return;
491 }
492
493 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
494 TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
495 tr->req = req;
496
497 if (req->uio == NULL) {

--- 71 unchanged lines hidden (view full) ---

569 * Manually abort each outstanding I/O. This normally results in a
570 * retry, unless the retry count on the associated request has
571 * reached its limit.
572 */
573 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
574 device_printf(qpair->ctrlr->dev,
575 "aborting outstanding i/o\n");
576 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
531 return;
532 }
533
534 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
535 TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
536 tr->req = req;
537
538 if (req->uio == NULL) {

--- 71 unchanged lines hidden (view full) ---

610 * Manually abort each outstanding I/O. This normally results in a
611 * retry, unless the retry count on the associated request has
612 * reached its limit.
613 */
614 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
615 device_printf(qpair->ctrlr->dev,
616 "aborting outstanding i/o\n");
617 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
577 NVME_SC_ABORTED_BY_REQUEST, TRUE);
618 NVME_SC_ABORTED_BY_REQUEST, 0, TRUE);
578 }
579
580 mtx_lock(&qpair->lock);
581
582 nvme_qpair_enable(qpair);
583
584 STAILQ_INIT(&temp);
585 STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);

--- 31 unchanged lines hidden (view full) ---

617}
618
619void
620nvme_io_qpair_disable(struct nvme_qpair *qpair)
621{
622
623 nvme_qpair_disable(qpair);
624}
619 }
620
621 mtx_lock(&qpair->lock);
622
623 nvme_qpair_enable(qpair);
624
625 STAILQ_INIT(&temp);
626 STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);

--- 31 unchanged lines hidden (view full) ---

658}
659
660void
661nvme_io_qpair_disable(struct nvme_qpair *qpair)
662{
663
664 nvme_qpair_disable(qpair);
665}
666
667void
668nvme_qpair_fail(struct nvme_qpair *qpair)
669{
670 struct nvme_tracker *tr;
671 struct nvme_request *req;
672
673 mtx_lock(&qpair->lock);
674
675 while (!STAILQ_EMPTY(&qpair->queued_req)) {
676 req = STAILQ_FIRST(&qpair->queued_req);
677 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
678 device_printf(qpair->ctrlr->dev,
679 "failing queued i/o\n");
680 mtx_unlock(&qpair->lock);
681 nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
682 NVME_SC_ABORTED_BY_REQUEST, TRUE);
683 mtx_lock(&qpair->lock);
684 }
685
686 /* Manually abort each outstanding I/O. */
687 while (!TAILQ_EMPTY(&qpair->outstanding_tr)) {
688 tr = TAILQ_FIRST(&qpair->outstanding_tr);
689 /*
690 * Do not remove the tracker. The abort_tracker path will
691 * do that for us.
692 */
693 device_printf(qpair->ctrlr->dev,
694 "failing outstanding i/o\n");
695 mtx_unlock(&qpair->lock);
696 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
697 NVME_SC_ABORTED_BY_REQUEST, 1 /* do not retry */, TRUE);
698 mtx_lock(&qpair->lock);
699 }
700
701 mtx_unlock(&qpair->lock);
702}
703