Deleted Added
sdiff udiff text old ( 248766 ) new ( 248767 )
full compact
nvme_ctrlr.c (248766) nvme_ctrlr.c (248767)
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (C) 2012 Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_ctrlr.c 248766 2013-03-26 21:48:41Z jimharris $");
28__FBSDID("$FreeBSD: head/sys/dev/nvme/nvme_ctrlr.c 248767 2013-03-26 21:58:38Z jimharris $");
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/conf.h>
33#include <sys/ioccom.h>
34#include <sys/smp.h>
35
36#include <dev/pci/pcireg.h>

--- 269 unchanged lines hidden (view full) ---

306
307 if (ctrlr->per_cpu_io_queues)
308 bus_bind_intr(ctrlr->dev, qpair->res, i);
309 }
310
311 return (0);
312}
313
29
30#include <sys/param.h>
31#include <sys/bus.h>
32#include <sys/conf.h>
33#include <sys/ioccom.h>
34#include <sys/smp.h>
35
36#include <dev/pci/pcireg.h>

--- 269 unchanged lines hidden (view full) ---

306
307 if (ctrlr->per_cpu_io_queues)
308 bus_bind_intr(ctrlr->dev, qpair->res, i);
309 }
310
311 return (0);
312}
313
314static void
315nvme_ctrlr_fail(struct nvme_controller *ctrlr)
316{
317 int i;
318
319 ctrlr->is_failed = TRUE;
320 nvme_qpair_fail(&ctrlr->adminq);
321 for (i = 0; i < ctrlr->num_io_queues; i++)
322 nvme_qpair_fail(&ctrlr->ioq[i]);
323 nvme_notify_fail_consumers(ctrlr);
324}
325
326void
327nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
328 struct nvme_request *req)
329{
330
331 mtx_lock(&ctrlr->fail_req_lock);
332 STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
333 mtx_unlock(&ctrlr->fail_req_lock);
334 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
335}
336
337static void
338nvme_ctrlr_fail_req_task(void *arg, int pending)
339{
340 struct nvme_controller *ctrlr = arg;
341 struct nvme_request *req;
342
343 mtx_lock(&ctrlr->fail_req_lock);
344 while (!STAILQ_EMPTY(&ctrlr->fail_req)) {
345 req = STAILQ_FIRST(&ctrlr->fail_req);
346 STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
347 nvme_qpair_manual_complete_request(req->qpair, req,
348 NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE);
349 }
350 mtx_unlock(&ctrlr->fail_req_lock);
351}
352
314static int
315nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr)
316{
317 int ms_waited;
318 union cc_register cc;
319 union csts_register csts;
320
321 cc.raw = nvme_mmio_read_4(ctrlr, cc);

--- 99 unchanged lines hidden (view full) ---

421
422void
423nvme_ctrlr_reset(struct nvme_controller *ctrlr)
424{
425 int cmpset;
426
427 cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
428
353static int
354nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr)
355{
356 int ms_waited;
357 union cc_register cc;
358 union csts_register csts;
359
360 cc.raw = nvme_mmio_read_4(ctrlr, cc);

--- 99 unchanged lines hidden (view full) ---

460
461void
462nvme_ctrlr_reset(struct nvme_controller *ctrlr)
463{
464 int cmpset;
465
466 cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
467
429 if (cmpset == 0)
430 /* Controller is already resetting. */
468 if (cmpset == 0 || ctrlr->is_failed)
469 /*
470 * Controller is already resetting or has failed. Return
471 * immediately since there is no need to kick off another
472 * reset in these cases.
473 */
431 return;
432
433 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
434}
435
436static int
437nvme_ctrlr_identify(struct nvme_controller *ctrlr)
438{

--- 301 unchanged lines hidden (view full) ---

740 int i;
741
742 nvme_qpair_reset(&ctrlr->adminq);
743 for (i = 0; i < ctrlr->num_io_queues; i++)
744 nvme_qpair_reset(&ctrlr->ioq[i]);
745
746 nvme_admin_qpair_enable(&ctrlr->adminq);
747
474 return;
475
476 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
477}
478
479static int
480nvme_ctrlr_identify(struct nvme_controller *ctrlr)
481{

--- 301 unchanged lines hidden (view full) ---

783 int i;
784
785 nvme_qpair_reset(&ctrlr->adminq);
786 for (i = 0; i < ctrlr->num_io_queues; i++)
787 nvme_qpair_reset(&ctrlr->ioq[i]);
788
789 nvme_admin_qpair_enable(&ctrlr->adminq);
790
748 if (nvme_ctrlr_identify(ctrlr) != 0)
791 if (nvme_ctrlr_identify(ctrlr) != 0) {
792 nvme_ctrlr_fail(ctrlr);
749 return;
793 return;
794 }
750
795
751 if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0)
796 if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
797 nvme_ctrlr_fail(ctrlr);
752 return;
798 return;
799 }
753
800
754 if (nvme_ctrlr_create_qpairs(ctrlr) != 0)
801 if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
802 nvme_ctrlr_fail(ctrlr);
755 return;
803 return;
804 }
756
805
757 if (nvme_ctrlr_construct_namespaces(ctrlr) != 0)
806 if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
807 nvme_ctrlr_fail(ctrlr);
758 return;
808 return;
809 }
759
760 nvme_ctrlr_configure_aer(ctrlr);
761 nvme_ctrlr_configure_int_coalescing(ctrlr);
762
763 for (i = 0; i < ctrlr->num_io_queues; i++)
764 nvme_io_qpair_enable(&ctrlr->ioq[i]);
765
766 /*

--- 30 unchanged lines hidden (view full) ---

797 * all nvme interrupts completed before proceeding with restarting the
798 * controller.
799 *
800 * XXX - any way to guarantee the interrupt handlers have quiesced?
801 */
802 pause("nvmereset", hz / 10);
803 if (status == 0)
804 nvme_ctrlr_start(ctrlr);
810
811 nvme_ctrlr_configure_aer(ctrlr);
812 nvme_ctrlr_configure_int_coalescing(ctrlr);
813
814 for (i = 0; i < ctrlr->num_io_queues; i++)
815 nvme_io_qpair_enable(&ctrlr->ioq[i]);
816
817 /*

--- 30 unchanged lines hidden (view full) ---

848 * all nvme interrupts completed before proceeding with restarting the
849 * controller.
850 *
851 * XXX - any way to guarantee the interrupt handlers have quiesced?
852 */
853 pause("nvmereset", hz / 10);
854 if (status == 0)
855 nvme_ctrlr_start(ctrlr);
856 else
857 nvme_ctrlr_fail(ctrlr);
805
806 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
807}
808
809static void
810nvme_ctrlr_intx_handler(void *arg)
811{
812 struct nvme_controller *ctrlr = arg;

--- 180 unchanged lines hidden (view full) ---

993 ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
994 "nvme%d", device_get_unit(dev));
995
996 if (ctrlr->cdev == NULL)
997 return (ENXIO);
998
999 ctrlr->cdev->si_drv1 = (void *)ctrlr;
1000
858
859 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
860}
861
862static void
863nvme_ctrlr_intx_handler(void *arg)
864{
865 struct nvme_controller *ctrlr = arg;

--- 180 unchanged lines hidden (view full) ---

1046 ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
1047 "nvme%d", device_get_unit(dev));
1048
1049 if (ctrlr->cdev == NULL)
1050 return (ENXIO);
1051
1052 ctrlr->cdev->si_drv1 = (void *)ctrlr;
1053
1001 TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1002 ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1003 taskqueue_thread_enqueue, &ctrlr->taskqueue);
1004 taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq");
1005
1006 ctrlr->is_resetting = 0;
1054 ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1055 taskqueue_thread_enqueue, &ctrlr->taskqueue);
1056 taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq");
1057
1058 ctrlr->is_resetting = 0;
1059 TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1007
1060
1061 TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
1062 mtx_init(&ctrlr->fail_req_lock, "nvme ctrlr fail req lock", NULL,
1063 MTX_DEF);
1064 STAILQ_INIT(&ctrlr->fail_req);
1065 ctrlr->is_failed = FALSE;
1066
1008 return (0);
1009}
1010
1011void
1012nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1013{
1014 int i;
1015

--- 80 unchanged lines hidden --- 		1067 return (0);
1068}
1069
1070void
1071nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1072{
1073 int i;
1074

--- 80 unchanged lines hidden ---