1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/errno.h>
28#include <sys/types.h>
29#include <sys/conf.h>
30#include <sys/kmem.h>
31#include <sys/ddi.h>
32#include <sys/stat.h>
33#include <sys/sunddi.h>
34#include <sys/file.h>
35#include <sys/open.h>
36#include <sys/modctl.h>
37#include <sys/ddi_impldefs.h>
38#include <sys/sysmacros.h>
39
40#include <sys/ioat.h>
41
42static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
43static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
44static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
45static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
46static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
47    void **result);
48static int ioat_quiesce(dev_info_t *dip);
49
50static 	struct cb_ops ioat_cb_ops = {
51	ioat_open,		/* cb_open */
52	ioat_close,		/* cb_close */
53	nodev,			/* cb_strategy */
54	nodev,			/* cb_print */
55	nodev,			/* cb_dump */
56	nodev,			/* cb_read */
57	nodev,			/* cb_write */
58	ioat_ioctl,		/* cb_ioctl */
59	nodev,			/* cb_devmap */
60	nodev,			/* cb_mmap */
61	nodev,			/* cb_segmap */
62	nochpoll,		/* cb_chpoll */
63	ddi_prop_op,		/* cb_prop_op */
64	NULL,			/* cb_stream */
65	D_NEW | D_MP | D_64BIT | D_DEVMAP,	/* cb_flag */
66	CB_REV
67};
68
69static struct dev_ops ioat_dev_ops = {
70	DEVO_REV,		/* devo_rev */
71	0,			/* devo_refcnt */
72	ioat_getinfo,		/* devo_getinfo */
73	nulldev,		/* devo_identify */
74	nulldev,		/* devo_probe */
75	ioat_attach,		/* devo_attach */
76	ioat_detach,		/* devo_detach */
77	nodev,			/* devo_reset */
78	&ioat_cb_ops,		/* devo_cb_ops */
79	NULL,			/* devo_bus_ops */
80	NULL,			/* devo_power */
81	ioat_quiesce,		/* devo_quiesce */
82};
83
84static struct modldrv ioat_modldrv = {
85	&mod_driverops,		/* Type of module.  This one is a driver */
86	"ioat driver",		/* Name of the module. */
87	&ioat_dev_ops,		/* driver ops */
88};
89
90static struct modlinkage ioat_modlinkage = {
91	MODREV_1,
92	(void *) &ioat_modldrv,
93	NULL
94};
95
96
97void *ioat_statep;
98
99static int ioat_chip_init(ioat_state_t *state);
100static void ioat_chip_fini(ioat_state_t *state);
101static int ioat_drv_init(ioat_state_t *state);
102static void ioat_drv_fini(ioat_state_t *state);
103static uint_t ioat_isr(caddr_t parm);
104static void ioat_intr_enable(ioat_state_t *state);
105static void ioat_intr_disable(ioat_state_t *state);
106void ioat_detach_finish(ioat_state_t *state);
107
108
109ddi_device_acc_attr_t ioat_acc_attr = {
110	DDI_DEVICE_ATTR_V0,		/* devacc_attr_version */
111	DDI_NEVERSWAP_ACC,		/* devacc_attr_endian_flags */
112	DDI_STORECACHING_OK_ACC,	/* devacc_attr_dataorder */
113	DDI_DEFAULT_ACC			/* devacc_attr_access */
114};
115
116/* dcopy callback interface */
117dcopy_device_cb_t ioat_cb = {
118	DCOPY_DEVICECB_V0,
119	0,		/* reserved */
120	ioat_channel_alloc,
121	ioat_channel_free,
122	ioat_cmd_alloc,
123	ioat_cmd_free,
124	ioat_cmd_post,
125	ioat_cmd_poll,
126	ioat_unregister_complete
127};
128
129/*
130 * _init()
131 */
132int
133_init(void)
134{
135	int e;
136
137	e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
138	if (e != 0) {
139		return (e);
140	}
141
142	e = mod_install(&ioat_modlinkage);
143	if (e != 0) {
144		ddi_soft_state_fini(&ioat_statep);
145		return (e);
146	}
147
148	return (0);
149}
150
151/*
152 * _info()
153 */
154int
155_info(struct modinfo *modinfop)
156{
157	return (mod_info(&ioat_modlinkage, modinfop));
158}
159
160/*
161 * _fini()
162 */
163int
164_fini(void)
165{
166	int e;
167
168	e = mod_remove(&ioat_modlinkage);
169	if (e != 0) {
170		return (e);
171	}
172
173	ddi_soft_state_fini(&ioat_statep);
174
175	return (0);
176}
177
178/*
179 * ioat_attach()
180 */
181static int
182ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
183{
184	ioat_state_t *state;
185	int instance;
186	int e;
187
188
189	switch (cmd) {
190	case DDI_ATTACH:
191		break;
192
193	case DDI_RESUME:
194		instance = ddi_get_instance(dip);
195		state = ddi_get_soft_state(ioat_statep, instance);
196		if (state == NULL) {
197			return (DDI_FAILURE);
198		}
199		e = ioat_channel_resume(state);
200		if (e != DDI_SUCCESS) {
201			return (DDI_FAILURE);
202		}
203		ioat_intr_enable(state);
204		return (DDI_SUCCESS);
205
206	default:
207		return (DDI_FAILURE);
208	}
209
210	instance = ddi_get_instance(dip);
211	e = ddi_soft_state_zalloc(ioat_statep, instance);
212	if (e != DDI_SUCCESS) {
213		return (DDI_FAILURE);
214	}
215	state = ddi_get_soft_state(ioat_statep, instance);
216	if (state == NULL) {
217		goto attachfail_get_soft_state;
218	}
219
220	state->is_dip = dip;
221	state->is_instance = instance;
222
223	/* setup the registers, save away some device info */
224	e = ioat_chip_init(state);
225	if (e != DDI_SUCCESS) {
226		goto attachfail_chip_init;
227	}
228
229	/* initialize driver state, must be after chip init */
230	e = ioat_drv_init(state);
231	if (e != DDI_SUCCESS) {
232		goto attachfail_drv_init;
233	}
234
235	/* create the minor node (for the ioctl) */
236	e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
237	    0);
238	if (e != DDI_SUCCESS) {
239		goto attachfail_minor_node;
240	}
241
242	/* Enable device interrupts */
243	ioat_intr_enable(state);
244
245	/* Report that driver was loaded */
246	ddi_report_dev(dip);
247
248	/* register with dcopy */
249	e = dcopy_device_register(state, &state->is_deviceinfo,
250	    &state->is_device_handle);
251	if (e != DCOPY_SUCCESS) {
252		goto attachfail_register;
253	}
254
255	return (DDI_SUCCESS);
256
257attachfail_register:
258	ioat_intr_disable(state);
259	ddi_remove_minor_node(dip, NULL);
260attachfail_minor_node:
261	ioat_drv_fini(state);
262attachfail_drv_init:
263	ioat_chip_fini(state);
264attachfail_chip_init:
265attachfail_get_soft_state:
266	(void) ddi_soft_state_free(ioat_statep, instance);
267
268	return (DDI_FAILURE);
269}
270
271/*
272 * ioat_detach()
273 */
274static int
275ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
276{
277	ioat_state_t *state;
278	int instance;
279	int e;
280
281
282	instance = ddi_get_instance(dip);
283	state = ddi_get_soft_state(ioat_statep, instance);
284	if (state == NULL) {
285		return (DDI_FAILURE);
286	}
287
288	switch (cmd) {
289	case DDI_DETACH:
290		break;
291
292	case DDI_SUSPEND:
293		ioat_channel_suspend(state);
294		return (DDI_SUCCESS);
295
296	default:
297		return (DDI_FAILURE);
298	}
299
300	/*
301	 * try to unregister from dcopy.  Since this driver doesn't follow the
302	 * traditional parent/child model, we may still be in use so we can't
303	 * detach yet.
304	 */
305	e = dcopy_device_unregister(&state->is_device_handle);
306	if (e != DCOPY_SUCCESS) {
307		if (e == DCOPY_PENDING) {
308			cmn_err(CE_NOTE, "device busy, performing asynchronous"
309			    " detach\n");
310		}
311		return (DDI_FAILURE);
312	}
313
314	ioat_detach_finish(state);
315
316	return (DDI_SUCCESS);
317}
318
319/*
320 * ioat_getinfo()
321 */
322/*ARGSUSED*/
323static int
324ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
325{
326	ioat_state_t *state;
327	int instance;
328	dev_t dev;
329	int e;
330
331
332	dev = (dev_t)arg;
333	instance = getminor(dev);
334
335	switch (cmd) {
336	case DDI_INFO_DEVT2DEVINFO:
337		state = ddi_get_soft_state(ioat_statep, instance);
338		if (state == NULL) {
339			return (DDI_FAILURE);
340		}
341		*result = (void *)state->is_dip;
342		e = DDI_SUCCESS;
343		break;
344
345	case DDI_INFO_DEVT2INSTANCE:
346		*result = (void *)(uintptr_t)instance;
347		e = DDI_SUCCESS;
348		break;
349
350	default:
351		e = DDI_FAILURE;
352		break;
353	}
354
355	return (e);
356}
357
358
359/*
360 * ioat_open()
361 */
362/*ARGSUSED*/
363static int
364ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
365{
366	ioat_state_t *state;
367	int instance;
368
369	instance = getminor(*devp);
370	state = ddi_get_soft_state(ioat_statep, instance);
371	if (state == NULL) {
372		return (ENXIO);
373	}
374
375	return (0);
376}
377
378
379/*
380 * ioat_close()
381 */
382/*ARGSUSED*/
383static int
384ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
385{
386	return (0);
387}
388
389
390/*
391 * ioat_chip_init()
392 */
393static int
394ioat_chip_init(ioat_state_t *state)
395{
396	ddi_device_acc_attr_t attr;
397	int e;
398
399
400	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
401	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
402	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
403
404	e =  ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
405	    0, 0, &attr, &state->is_reg_handle);
406	if (e != DDI_SUCCESS) {
407		goto chipinitfail_regsmap;
408	}
409
410	/* save away ioat chip info */
411	state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
412	    &state->is_genregs[IOAT_CHANCNT]);
413
414	/*
415	 * If we get a bogus value, something is wrong with the H/W, fail to
416	 * attach.
417	 */
418	if (state->is_num_channels == 0) {
419		goto chipinitfail_numchan;
420	}
421
422	state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
423	    &state->is_genregs[IOAT_XFERCAP]);
424	state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
425	    (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
426	state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
427	    &state->is_genregs[IOAT_CBVER]);
428	state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
429	    (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
430	state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
431	    (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
432	state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
433	    (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);
434
435	if (state->is_cbver & 0x10) {
436		state->is_ver = IOAT_CBv1;
437	} else if (state->is_cbver & 0x20) {
438		state->is_ver = IOAT_CBv2;
439	} else {
440		goto chipinitfail_version;
441	}
442
443	return (DDI_SUCCESS);
444
445chipinitfail_version:
446chipinitfail_numchan:
447	ddi_regs_map_free(&state->is_reg_handle);
448chipinitfail_regsmap:
449	return (DDI_FAILURE);
450}
451
452
453/*
454 * ioat_chip_fini()
455 */
456static void
457ioat_chip_fini(ioat_state_t *state)
458{
459	ddi_regs_map_free(&state->is_reg_handle);
460}
461
462
463/*
464 * ioat_drv_init()
465 */
466static int
467ioat_drv_init(ioat_state_t *state)
468{
469	ddi_acc_handle_t handle;
470	int e;
471
472
473	mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);
474
475	state->is_deviceinfo.di_dip = state->is_dip;
476	state->is_deviceinfo.di_num_dma = state->is_num_channels;
477	state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
478	state->is_deviceinfo.di_capabilities = state->is_capabilities;
479	state->is_deviceinfo.di_cb = &ioat_cb;
480
481	e = pci_config_setup(state->is_dip, &handle);
482	if (e != DDI_SUCCESS) {
483		goto drvinitfail_config_setup;
484	}
485
486	/* read in Vendor ID */
487	state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
488	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;
489
490	/* read in Device ID */
491	state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
492	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;
493
494	/* Add in chipset version */
495	state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
496	pci_config_teardown(&handle);
497
498	e = ddi_intr_hilevel(state->is_dip, 0);
499	if (e != 0) {
500		cmn_err(CE_WARN, "hilevel interrupt not supported\n");
501		goto drvinitfail_hilevel;
502	}
503
504	/* we don't support MSIs for v2 yet */
505	e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
506	    (caddr_t)state);
507	if (e != DDI_SUCCESS) {
508		goto drvinitfail_add_intr;
509	}
510
511	e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
512	if (e != DDI_SUCCESS) {
513		goto drvinitfail_iblock_cookie;
514	}
515
516	e = ioat_channel_init(state);
517	if (e != DDI_SUCCESS) {
518		goto drvinitfail_channel_init;
519	}
520
521	return (DDI_SUCCESS);
522
523drvinitfail_channel_init:
524drvinitfail_iblock_cookie:
525	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
526drvinitfail_add_intr:
527drvinitfail_hilevel:
528drvinitfail_config_setup:
529	mutex_destroy(&state->is_mutex);
530
531	return (DDI_FAILURE);
532}
533
534
535/*
536 * ioat_drv_fini()
537 */
538static void
539ioat_drv_fini(ioat_state_t *state)
540{
541	ioat_channel_fini(state);
542	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
543	mutex_destroy(&state->is_mutex);
544}
545
546
547/*
548 * ioat_unregister_complete()
549 */
550void
551ioat_unregister_complete(void *device_private, int status)
552{
553	ioat_state_t *state;
554
555
556	state = device_private;
557
558	if (status != DCOPY_SUCCESS) {
559		cmn_err(CE_WARN, "asynchronous detach aborted\n");
560		return;
561	}
562
563	cmn_err(CE_CONT, "detach completing\n");
564	ioat_detach_finish(state);
565}
566
567
568/*
569 * ioat_detach_finish()
570 */
571void
572ioat_detach_finish(ioat_state_t *state)
573{
574	ioat_intr_disable(state);
575	ddi_remove_minor_node(state->is_dip, NULL);
576	ioat_drv_fini(state);
577	ioat_chip_fini(state);
578	(void) ddi_soft_state_free(ioat_statep, state->is_instance);
579}
580
581
582/*
583 * ioat_intr_enable()
584 */
585static void
586ioat_intr_enable(ioat_state_t *state)
587{
588	uint32_t intr_status;
589
590
591	/* Clear any pending interrupts */
592	intr_status = ddi_get32(state->is_reg_handle,
593	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
594	if (intr_status != 0) {
595		ddi_put32(state->is_reg_handle,
596		    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
597		    intr_status);
598	}
599
600	/* Enable interrupts on the device */
601	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
602	    IOAT_INTRCTL_MASTER_EN);
603}
604
605
606/*
607 * ioat_intr_disable()
608 */
609static void
610ioat_intr_disable(ioat_state_t *state)
611{
612	/*
613	 * disable interrupts on the device. A read of the interrupt control
614	 * register clears the enable bit.
615	 */
616	(void) ddi_get8(state->is_reg_handle,
617	    &state->is_genregs[IOAT_INTRCTL]);
618}
619
620
621/*
622 * ioat_isr()
623 */
624static uint_t
625ioat_isr(caddr_t parm)
626{
627	uint32_t intr_status;
628	ioat_state_t *state;
629	uint8_t intrctrl;
630	uint32_t chan;
631	uint_t r;
632	int i;
633
634	state = (ioat_state_t *)parm;
635
636	intrctrl = ddi_get8(state->is_reg_handle,
637	    &state->is_genregs[IOAT_INTRCTL]);
638	/* master interrupt enable should always be set */
639	ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);
640
641	/* If the interrupt status bit isn't set, it's not ours */
642	if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
643		/* re-set master interrupt enable (since it clears on read) */
644		ddi_put8(state->is_reg_handle,
645		    &state->is_genregs[IOAT_INTRCTL], intrctrl);
646		return (DDI_INTR_UNCLAIMED);
647	}
648
649	/* see which channels generated the interrupt */
650	intr_status = ddi_get32(state->is_reg_handle,
651	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
652
653	/* call the intr handler for the channels */
654	r = DDI_INTR_UNCLAIMED;
655	chan = 1;
656	for (i = 0; i < state->is_num_channels; i++) {
657		if (intr_status & chan) {
658			ioat_channel_intr(&state->is_channel[i]);
659			r = DDI_INTR_CLAIMED;
660		}
661		chan = chan << 1;
662	}
663
664	/*
665	 * if interrupt status bit was set, there should have been an
666	 * attention status bit set too.
667	 */
668	ASSERT(r == DDI_INTR_CLAIMED);
669
670	/* re-set master interrupt enable (since it clears on read) */
671	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
672	    intrctrl);
673
674	return (r);
675}
676
677static int
678ioat_quiesce(dev_info_t *dip)
679{
680	ioat_state_t *state;
681	int instance;
682
683	instance = ddi_get_instance(dip);
684	state = ddi_get_soft_state(ioat_statep, instance);
685	if (state == NULL) {
686		return (DDI_FAILURE);
687	}
688
689	ioat_intr_disable(state);
690	ioat_channel_quiesce(state);
691
692	return (DDI_SUCCESS);
693}
694