Lines Matching defs:qpair
37 static void _nvme_qpair_submit_request(struct nvme_qpair *qpair,
107 nvme_admin_qpair_print_command(struct nvme_qpair *qpair,
111 nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x "
113 get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid,
118 nvme_io_qpair_print_command(struct nvme_qpair *qpair,
127 nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d "
129 get_io_opcode_string(cmd->opc), qpair->id, cmd->cid,
136 nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n",
137 get_io_opcode_string(cmd->opc), qpair->id, cmd->cid,
141 nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n",
142 get_io_opcode_string(cmd->opc), cmd->opc, qpair->id,
149 nvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd)
151 if (qpair->id == 0)
152 nvme_admin_qpair_print_command(qpair, cmd);
154 nvme_io_qpair_print_command(qpair, cmd);
242 nvme_qpair_print_completion(struct nvme_qpair *qpair,
245 nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n",
293 nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
297 bus_dmamap_create(qpair->dma_tag_payload, 0, &tr->payload_dma_map);
298 bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map);
300 bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp,
305 tr->qpair = qpair;
309 nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
321 nvme_qpair_print_command(qpair, &req->cmd);
322 nvme_qpair_print_completion(qpair, cpl);
325 qpair->act_tr[cpl->cid] = NULL;
332 mtx_lock(&qpair->lock);
337 nvme_qpair_submit_tracker(qpair, tr);
340 bus_dmamap_unload(qpair->dma_tag_payload,
346 TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq);
347 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
354 if (!STAILQ_EMPTY(&qpair->queued_req) &&
355 !qpair->ctrlr->is_resetting) {
356 req = STAILQ_FIRST(&qpair->queued_req);
357 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
358 _nvme_qpair_submit_request(qpair, req);
362 mtx_unlock(&qpair->lock);
366 nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
373 cpl.sqid = qpair->id;
378 nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
382 nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
390 cpl.sqid = qpair->id;
397 nvme_qpair_print_command(qpair, &req->cmd);
398 nvme_qpair_print_completion(qpair, &cpl);
408 nvme_qpair_process_completions(struct nvme_qpair *qpair)
413 qpair->num_intr_handler_calls++;
415 if (!qpair->is_enabled)
417 * qpair is not enabled, likely because a controller reset is
425 cpl = &qpair->cpl[qpair->cq_head];
427 if (cpl->status.p != qpair->phase)
430 tr = qpair->act_tr[cpl->cid];
433 nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE);
434 qpair->sq_head = cpl->sqhd;
436 nvme_printf(qpair->ctrlr,
442 if (++qpair->cq_head == qpair->num_entries) {
443 qpair->cq_head = 0;
444 qpair->phase = !qpair->phase;
447 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
448 qpair->cq_head);
455 struct nvme_qpair *qpair = arg;
457 nvme_qpair_process_completions(qpair);
461 nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
469 qpair->id = id;
470 qpair->vector = vector;
471 qpair->num_entries = num_entries;
472 qpair->num_trackers = num_trackers;
473 qpair->ctrlr = ctrlr;
481 qpair->rid = vector + 1;
482 qpair->res = ctrlr->msi_res[vector];
484 bus_setup_intr(ctrlr->dev, qpair->res,
486 nvme_qpair_msix_handler, qpair, &qpair->tag);
489 mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
496 NULL, NULL, &qpair->dma_tag_payload);
503 NULL, NULL, &qpair->dma_tag);
507 qpair->num_cmds = 0;
508 qpair->num_intr_handler_calls = 0;
510 qpair->cmd = contigmalloc(qpair->num_entries *
513 qpair->cpl = contigmalloc(qpair->num_entries *
517 err = bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map);
521 err = bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map);
525 bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map,
526 qpair->cmd, qpair->num_entries * sizeof(struct nvme_command),
527 nvme_single_map, &qpair->cmd_bus_addr, 0);
528 bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map,
529 qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion),
530 nvme_single_map, &qpair->cpl_bus_addr, 0);
532 qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
533 qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
535 TAILQ_INIT(&qpair->free_tr);
536 TAILQ_INIT(&qpair->outstanding_tr);
537 STAILQ_INIT(&qpair->queued_req);
539 for (i = 0; i < qpair->num_trackers; i++) {
541 nvme_qpair_construct_tracker(qpair, tr, i);
542 TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq);
545 qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries,
550 nvme_qpair_destroy(struct nvme_qpair *qpair)
554 if (qpair->tag)
555 bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag);
557 if (qpair->res)
558 bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ,
559 rman_get_rid(qpair->res), qpair->res);
561 if (qpair->cmd) {
562 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map);
563 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map);
564 contigfree(qpair->cmd,
565 qpair->num_entries * sizeof(struct nvme_command), M_NVME);
568 if (qpair->cpl) {
569 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map);
570 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map);
571 contigfree(qpair->cpl,
572 qpair->num_entries * sizeof(struct nvme_completion),
576 if (qpair->dma_tag)
577 bus_dma_tag_destroy(qpair->dma_tag);
579 if (qpair->dma_tag_payload)
580 bus_dma_tag_destroy(qpair->dma_tag_payload);
582 if (qpair->act_tr)
583 free(qpair->act_tr, M_NVME);
585 while (!TAILQ_EMPTY(&qpair->free_tr)) {
586 tr = TAILQ_FIRST(&qpair->free_tr);
587 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
588 bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map);
589 bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map);
595 nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair)
599 tr = TAILQ_FIRST(&qpair->outstanding_tr);
602 nvme_qpair_manual_complete_tracker(qpair, tr,
605 tr = TAILQ_FIRST(&qpair->outstanding_tr);
613 nvme_admin_qpair_destroy(struct nvme_qpair *qpair)
616 nvme_admin_qpair_abort_aers(qpair);
617 nvme_qpair_destroy(qpair);
621 nvme_io_qpair_destroy(struct nvme_qpair *qpair)
624 nvme_qpair_destroy(qpair);
638 if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) {
644 nvme_printf(tr->qpair->ctrlr,
646 nvme_qpair_manual_complete_tracker(tr->qpair, tr,
655 struct nvme_qpair *qpair = tr->qpair;
656 struct nvme_controller *ctrlr = qpair->ctrlr;
667 nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id,
674 nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
679 mtx_assert(&qpair->lock, MA_OWNED);
683 qpair->act_tr[tr->cid] = tr;
684 ctrlr = qpair->ctrlr;
696 memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
698 if (++qpair->sq_tail == qpair->num_entries)
699 qpair->sq_tail = 0;
702 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
703 qpair->sq_tail);
705 qpair->num_cmds++;
720 nvme_printf(tr->qpair->ctrlr,
752 nvme_qpair_submit_tracker(tr->qpair, tr);
756 _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
761 mtx_assert(&qpair->lock, MA_OWNED);
763 tr = TAILQ_FIRST(&qpair->free_tr);
764 req->qpair = qpair;
766 if (tr == NULL || !qpair->is_enabled) {
768 * No tracker is available, or the qpair is disabled due to
773 if (qpair->ctrlr->is_failed) {
780 nvme_ctrlr_post_failed_request(qpair->ctrlr, req);
783 * Put the request on the qpair's request queue to be
788 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
793 TAILQ_REMOVE(&qpair->free_tr, tr, tailq);
794 TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
799 KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size,
801 req->payload_size, qpair->ctrlr->max_xfer_size));
802 err = bus_dmamap_load(tr->qpair->dma_tag_payload,
806 nvme_printf(qpair->ctrlr,
810 nvme_qpair_submit_tracker(tr->qpair, tr);
814 KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size,
817 qpair->ctrlr->max_xfer_size));
818 err = bus_dmamap_load_bio(tr->qpair->dma_tag_payload,
821 nvme_printf(qpair->ctrlr,
836 * with the qpair lock held.
838 mtx_unlock(&qpair->lock);
839 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
841 mtx_lock(&qpair->lock);
846 nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
849 mtx_lock(&qpair->lock);
850 _nvme_qpair_submit_request(qpair, req);
851 mtx_unlock(&qpair->lock);
855 nvme_qpair_enable(struct nvme_qpair *qpair)
858 qpair->is_enabled = TRUE;
862 nvme_qpair_reset(struct nvme_qpair *qpair)
865 qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;
874 qpair->phase = 1;
876 memset(qpair->cmd, 0,
877 qpair->num_entries * sizeof(struct nvme_command));
878 memset(qpair->cpl, 0,
879 qpair->num_entries * sizeof(struct nvme_completion));
883 nvme_admin_qpair_enable(struct nvme_qpair *qpair)
894 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
895 nvme_printf(qpair->ctrlr,
897 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
901 nvme_qpair_enable(qpair);
905 nvme_io_qpair_enable(struct nvme_qpair *qpair)
917 TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) {
918 nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n");
919 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
923 mtx_lock(&qpair->lock);
925 nvme_qpair_enable(qpair);
928 STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request);
933 nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n");
934 nvme_qpair_print_command(qpair, &req->cmd);
935 _nvme_qpair_submit_request(qpair, req);
938 mtx_unlock(&qpair->lock);
942 nvme_qpair_disable(struct nvme_qpair *qpair)
946 qpair->is_enabled = FALSE;
947 mtx_lock(&qpair->lock);
948 TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq)
950 mtx_unlock(&qpair->lock);
954 nvme_admin_qpair_disable(struct nvme_qpair *qpair)
957 nvme_qpair_disable(qpair);
958 nvme_admin_qpair_abort_aers(qpair);
962 nvme_io_qpair_disable(struct nvme_qpair *qpair)
965 nvme_qpair_disable(qpair);
969 nvme_qpair_fail(struct nvme_qpair *qpair)
974 mtx_lock(&qpair->lock);
976 while (!STAILQ_EMPTY(&qpair->queued_req)) {
977 req = STAILQ_FIRST(&qpair->queued_req);
978 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
979 nvme_printf(qpair->ctrlr, "failing queued i/o\n");
980 mtx_unlock(&qpair->lock);
981 nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
983 mtx_lock(&qpair->lock);
987 while (!TAILQ_EMPTY(&qpair->outstanding_tr)) {
988 tr = TAILQ_FIRST(&qpair->outstanding_tr);
993 nvme_printf(qpair->ctrlr, "failing outstanding i/o\n");
994 mtx_unlock(&qpair->lock);
995 nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
997 mtx_lock(&qpair->lock);
1000 mtx_unlock(&qpair->lock);