1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD$");
28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35	__checkReturn	int
36efx_family(
37	__in		uint16_t venid,
38	__in		uint16_t devid,
39	__out		efx_family_t *efp)
40{
41#if EFSYS_OPT_FALCON
42	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_FALCON) {
43		*efp = EFX_FAMILY_FALCON;
44		return (0);
45	}
46#endif
47#if EFSYS_OPT_SIENA
48	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_BETHPAGE) {
49		*efp = EFX_FAMILY_SIENA;
50		return (0);
51	}
52	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_SIENA) {
53		*efp = EFX_FAMILY_SIENA;
54		return (0);
55	}
56	if (venid == EFX_PCI_VENID_SFC &&
57	    devid == EFX_PCI_DEVID_SIENA_F1_UNINIT) {
58		*efp = EFX_FAMILY_SIENA;
59		return (0);
60	}
61#endif
62	return (ENOTSUP);
63}
64
65/*
66 * To support clients which aren't provided with any PCI context infer
67 * the hardware family by inspecting the hardware. Obviously the caller
68 * must be damn sure they're really talking to a supported device.
69 */
70	__checkReturn	int
71efx_infer_family(
72	__in		efsys_bar_t *esbp,
73	__out		efx_family_t *efp)
74{
75	efx_family_t family;
76	efx_oword_t oword;
77	unsigned int portnum;
78	int rc;
79
80	EFSYS_BAR_READO(esbp, FR_AZ_CS_DEBUG_REG_OFST, &oword, B_TRUE);
81	portnum = EFX_OWORD_FIELD(oword, FRF_CZ_CS_PORT_NUM);
82	switch (portnum) {
83#if EFSYS_OPT_FALCON
84	case 0:
85		family = EFX_FAMILY_FALCON;
86		break;
87#endif
88#if EFSYS_OPT_SIENA
89	case 1:
90	case 2:
91		family = EFX_FAMILY_SIENA;
92		break;
93#endif
94	default:
95		rc = ENOTSUP;
96		goto fail1;
97	}
98
99	if (efp != NULL)
100		*efp = family;
101	return (0);
102
103fail1:
104	EFSYS_PROBE1(fail1, int, rc);
105
106	return (rc);
107}
108
109/*
110 * The built-in default value device id for port 1 of Siena is 0x0810.
111 * manftest needs to be able to cope with that.
112 */
113
114#define	EFX_BIU_MAGIC0	0x01234567
115#define	EFX_BIU_MAGIC1	0xfedcba98
116
117static	__checkReturn	int
118efx_nic_biu_test(
119	__in		efx_nic_t *enp)
120{
121	efx_oword_t oword;
122	int rc;
123
124	/*
125	 * Write magic values to scratch registers 0 and 1, then
126	 * verify that the values were written correctly.  Interleave
127	 * the accesses to ensure that the BIU is not just reading
128	 * back the cached value that was last written.
129	 */
130	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
131	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
132
133	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
134	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
135
136	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
137	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
138		rc = EIO;
139		goto fail1;
140	}
141
142	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
143	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
144		rc = EIO;
145		goto fail2;
146	}
147
148	/*
149	 * Perform the same test, with the values swapped.  This
150	 * ensures that subsequent tests don't start with the correct
151	 * values already written into the scratch registers.
152	 */
153	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
154	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
155
156	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
157	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
158
159	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
160	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
161		rc = EIO;
162		goto fail3;
163	}
164
165	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
166	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
167		rc = EIO;
168		goto fail4;
169	}
170
171	return (0);
172
173fail4:
174	EFSYS_PROBE(fail4);
175fail3:
176	EFSYS_PROBE(fail3);
177fail2:
178	EFSYS_PROBE(fail2);
179fail1:
180	EFSYS_PROBE1(fail1, int, rc);
181
182	return (rc);
183}
184
185#if EFSYS_OPT_FALCON
186
187static efx_nic_ops_t	__cs __efx_nic_falcon_ops = {
188	falcon_nic_probe,		/* eno_probe */
189	falcon_nic_reset,		/* eno_reset */
190	falcon_nic_init,		/* eno_init */
191#if EFSYS_OPT_DIAG
192	falcon_sram_test,		/* eno_sram_test */
193	falcon_nic_register_test,	/* eno_register_test */
194#endif	/* EFSYS_OPT_DIAG */
195	falcon_nic_fini,		/* eno_fini */
196	falcon_nic_unprobe,		/* eno_unprobe */
197};
198
199#endif	/* EFSYS_OPT_FALCON */
200
201#if EFSYS_OPT_SIENA
202
203static efx_nic_ops_t	__cs __efx_nic_siena_ops = {
204	siena_nic_probe,		/* eno_probe */
205	siena_nic_reset,		/* eno_reset */
206	siena_nic_init,			/* eno_init */
207#if EFSYS_OPT_DIAG
208	siena_sram_test,		/* eno_sram_test */
209	siena_nic_register_test,	/* eno_register_test */
210#endif	/* EFSYS_OPT_DIAG */
211	siena_nic_fini,			/* eno_fini */
212	siena_nic_unprobe,		/* eno_unprobe */
213};
214
215#endif	/* EFSYS_OPT_SIENA */
216
217	__checkReturn	int
218efx_nic_create(
219	__in		efx_family_t family,
220	__in		efsys_identifier_t *esip,
221	__in		efsys_bar_t *esbp,
222	__in		efsys_lock_t *eslp,
223	__deref_out	efx_nic_t **enpp)
224{
225	efx_nic_t *enp;
226	int rc;
227
228	EFSYS_ASSERT3U(family, >, EFX_FAMILY_INVALID);
229	EFSYS_ASSERT3U(family, <, EFX_FAMILY_NTYPES);
230
231	/* Allocate a NIC object */
232	EFSYS_KMEM_ALLOC(esip, sizeof (efx_nic_t), enp);
233
234	if (enp == NULL) {
235		rc = ENOMEM;
236		goto fail1;
237	}
238
239	enp->en_magic = EFX_NIC_MAGIC;
240
241	switch (family) {
242#if EFSYS_OPT_FALCON
243	case EFX_FAMILY_FALCON:
244		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_falcon_ops;
245		enp->en_features = 0;
246		break;
247#endif	/* EFSYS_OPT_FALCON */
248
249#if EFSYS_OPT_SIENA
250	case EFX_FAMILY_SIENA:
251		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_siena_ops;
252		enp->en_features = EFX_FEATURE_IPV6 |
253		    EFX_FEATURE_LFSR_HASH_INSERT |
254		    EFX_FEATURE_LINK_EVENTS | EFX_FEATURE_PERIODIC_MAC_STATS |
255		    EFX_FEATURE_WOL | EFX_FEATURE_MCDI |
256		    EFX_FEATURE_LOOKAHEAD_SPLIT | EFX_FEATURE_MAC_HEADER_FILTERS;
257		break;
258#endif	/* EFSYS_OPT_SIENA */
259
260	default:
261		rc = ENOTSUP;
262		goto fail2;
263	}
264
265	enp->en_family = family;
266	enp->en_esip = esip;
267	enp->en_esbp = esbp;
268	enp->en_eslp = eslp;
269
270	*enpp = enp;
271
272	return (0);
273
274fail2:
275	EFSYS_PROBE(fail3);
276
277	enp->en_magic = 0;
278
279	/* Free the NIC object */
280	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
281
282fail1:
283	EFSYS_PROBE1(fail1, int, rc);
284
285	return (rc);
286}
287
288	__checkReturn	int
289efx_nic_probe(
290	__in		efx_nic_t *enp)
291{
292	efx_nic_ops_t *enop;
293	efx_oword_t oword;
294	int rc;
295
296	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
297#if EFSYS_OPT_MCDI
298	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
299#endif	/* EFSYS_OPT_MCDI */
300	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_PROBE));
301
302	/* Test BIU */
303	if ((rc = efx_nic_biu_test(enp)) != 0)
304		goto fail1;
305
306	/* Clear the region register */
307	EFX_POPULATE_OWORD_4(oword,
308	    FRF_AZ_ADR_REGION0, 0,
309	    FRF_AZ_ADR_REGION1, (1 << 16),
310	    FRF_AZ_ADR_REGION2, (2 << 16),
311	    FRF_AZ_ADR_REGION3, (3 << 16));
312	EFX_BAR_WRITEO(enp, FR_AZ_ADR_REGION_REG, &oword);
313
314	enop = enp->en_enop;
315	if ((rc = enop->eno_probe(enp)) != 0)
316		goto fail2;
317
318	if ((rc = efx_phy_probe(enp)) != 0)
319		goto fail3;
320
321	enp->en_mod_flags |= EFX_MOD_PROBE;
322
323	return (0);
324
325fail3:
326	EFSYS_PROBE(fail3);
327
328	enop->eno_unprobe(enp);
329
330fail2:
331	EFSYS_PROBE(fail2);
332fail1:
333	EFSYS_PROBE1(fail1, int, rc);
334
335	return (rc);
336}
337
338#if EFSYS_OPT_PCIE_TUNE
339
340	__checkReturn	int
341efx_nic_pcie_tune(
342	__in		efx_nic_t *enp,
343	unsigned int	nlanes)
344{
345	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
346	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
347	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
348
349#if EFSYS_OPT_FALCON
350	if (enp->en_family == EFX_FAMILY_FALCON)
351		return (falcon_nic_pcie_tune(enp, nlanes));
352#endif
353	return (ENOTSUP);
354}
355
356	__checkReturn	int
357efx_nic_pcie_extended_sync(
358	__in		efx_nic_t *enp)
359{
360	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
361	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
362	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
363
364#if EFSYS_OPT_SIENA
365	if (enp->en_family == EFX_FAMILY_SIENA)
366		return (siena_nic_pcie_extended_sync(enp));
367#endif
368
369	return (ENOTSUP);
370}
371
372#endif	/* EFSYS_OPT_PCIE_TUNE */
373
374	__checkReturn	int
375efx_nic_init(
376	__in		efx_nic_t *enp)
377{
378	efx_nic_ops_t *enop = enp->en_enop;
379	int rc;
380
381	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
382	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
383
384	if (enp->en_mod_flags & EFX_MOD_NIC) {
385		rc = EINVAL;
386		goto fail1;
387	}
388
389	if ((rc = enop->eno_init(enp)) != 0)
390		goto fail2;
391
392	enp->en_mod_flags |= EFX_MOD_NIC;
393
394	return (0);
395
396fail2:
397	EFSYS_PROBE(fail2);
398fail1:
399	EFSYS_PROBE1(fail1, int, rc);
400
401	return (rc);
402}
403
404			void
405efx_nic_fini(
406	__in		efx_nic_t *enp)
407{
408	efx_nic_ops_t *enop = enp->en_enop;
409
410	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
411	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
412	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_NIC);
413	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
414	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
415	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
416	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
417
418	enop->eno_fini(enp);
419
420	enp->en_mod_flags &= ~EFX_MOD_NIC;
421}
422
423			void
424efx_nic_unprobe(
425	__in		efx_nic_t *enp)
426{
427	efx_nic_ops_t *enop = enp->en_enop;
428
429	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
430#if EFSYS_OPT_MCDI
431	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
432#endif	/* EFSYS_OPT_MCDI */
433	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
434	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
435	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
436	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
437	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
438	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
439
440	efx_phy_unprobe(enp);
441
442	enop->eno_unprobe(enp);
443
444	enp->en_mod_flags &= ~EFX_MOD_PROBE;
445}
446
447			void
448efx_nic_destroy(
449	__in	efx_nic_t *enp)
450{
451	efsys_identifier_t *esip = enp->en_esip;
452
453	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
454	EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0);
455
456	enp->en_family = 0;
457	enp->en_esip = NULL;
458	enp->en_esbp = NULL;
459	enp->en_eslp = NULL;
460
461	enp->en_enop = NULL;
462
463	enp->en_magic = 0;
464
465	/* Free the NIC object */
466	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
467}
468
469	__checkReturn	int
470efx_nic_reset(
471	__in		efx_nic_t *enp)
472{
473	efx_nic_ops_t *enop = enp->en_enop;
474	unsigned int mod_flags;
475	int rc;
476
477	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
478	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
479	/*
480	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON (which we
481	 * do not reset here) must have been shut down or never initialized.
482	 *
483	 * A rule of thumb here is: If the controller or MC reboots, is *any*
484	 * state lost. If it's lost and needs reapplying, then the module
485	 * *must* not be initialised during the reset.
486	 */
487	mod_flags = enp->en_mod_flags;
488	mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM |
489		    EFX_MOD_VPD | EFX_MOD_MON);
490	EFSYS_ASSERT3U(mod_flags, ==, 0);
491	if (mod_flags != 0) {
492		rc = EINVAL;
493		goto fail1;
494	}
495
496	if ((rc = enop->eno_reset(enp)) != 0)
497		goto fail2;
498
499	enp->en_reset_flags |= EFX_RESET_MAC;
500
501	return (0);
502
503fail2:
504	EFSYS_PROBE(fail2);
505fail1:
506	EFSYS_PROBE1(fail1, int, rc);
507
508	return (rc);
509}
510
511			const efx_nic_cfg_t *
512efx_nic_cfg_get(
513	__in		efx_nic_t *enp)
514{
515	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
516
517	return (&(enp->en_nic_cfg));
518}
519
520#if EFSYS_OPT_DIAG
521
522	__checkReturn	int
523efx_nic_register_test(
524	__in		efx_nic_t *enp)
525{
526	efx_nic_ops_t *enop = enp->en_enop;
527	int rc;
528
529	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
530	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
531	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
532
533	if ((rc = enop->eno_register_test(enp)) != 0)
534		goto fail1;
535
536	return (0);
537
538fail1:
539	EFSYS_PROBE1(fail1, int, rc);
540
541	return (rc);
542}
543
544	__checkReturn	int
545efx_nic_test_registers(
546	__in		efx_nic_t *enp,
547	__in		efx_register_set_t *rsp,
548	__in		size_t count)
549{
550	unsigned int bit;
551	efx_oword_t original;
552	efx_oword_t reg;
553	efx_oword_t buf;
554	int rc;
555
556	while (count > 0) {
557		/* This function is only suitable for registers */
558		EFSYS_ASSERT(rsp->rows == 1);
559
560		/* bit sweep on and off */
561		EFSYS_BAR_READO(enp->en_esbp, rsp->address, &original,
562			    B_TRUE);
563		for (bit = 0; bit < 128; bit++) {
564			/* Is this bit in the mask? */
565			if (~(rsp->mask.eo_u32[bit >> 5]) & (1 << bit))
566				continue;
567
568			/* Test this bit can be set in isolation */
569			reg = original;
570			EFX_AND_OWORD(reg, rsp->mask);
571			EFX_SET_OWORD_BIT(reg, bit);
572
573			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
574				    B_TRUE);
575			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
576				    B_TRUE);
577
578			EFX_AND_OWORD(buf, rsp->mask);
579			if (memcmp(&reg, &buf, sizeof (reg))) {
580				rc = EIO;
581				goto fail1;
582			}
583
584			/* Test this bit can be cleared in isolation */
585			EFX_OR_OWORD(reg, rsp->mask);
586			EFX_CLEAR_OWORD_BIT(reg, bit);
587
588			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
589				    B_TRUE);
590			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
591				    B_TRUE);
592
593			EFX_AND_OWORD(buf, rsp->mask);
594			if (memcmp(&reg, &buf, sizeof (reg))) {
595				rc = EIO;
596				goto fail2;
597			}
598		}
599
600		/* Restore the old value */
601		EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original,
602			    B_TRUE);
603
604		--count;
605		++rsp;
606	}
607
608	return (0);
609
610fail2:
611	EFSYS_PROBE(fail2);
612fail1:
613	EFSYS_PROBE1(fail1, int, rc);
614
615	/* Restore the old value */
616	EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, B_TRUE);
617
618	return (rc);
619}
620
621	__checkReturn	int
622efx_nic_test_tables(
623	__in		efx_nic_t *enp,
624	__in		efx_register_set_t *rsp,
625	__in		efx_pattern_type_t pattern,
626	__in		size_t count)
627{
628	efx_sram_pattern_fn_t func;
629	unsigned int index;
630	unsigned int address;
631	efx_oword_t reg;
632	efx_oword_t buf;
633	int rc;
634
635	EFSYS_ASSERT(pattern < EFX_PATTERN_NTYPES);
636	func = __efx_sram_pattern_fns[pattern];
637
638	while (count > 0) {
639		/* Write */
640		address = rsp->address;
641		for (index = 0; index < rsp->rows; ++index) {
642			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
643			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
644			EFX_AND_OWORD(reg, rsp->mask);
645			EFSYS_BAR_WRITEO(enp->en_esbp, address, &reg, B_TRUE);
646
647			address += rsp->step;
648		}
649
650		/* Read */
651		address = rsp->address;
652		for (index = 0; index < rsp->rows; ++index) {
653			func(2 * index + 0, B_FALSE, &reg.eo_qword[0]);
654			func(2 * index + 1, B_FALSE, &reg.eo_qword[1]);
655			EFX_AND_OWORD(reg, rsp->mask);
656			EFSYS_BAR_READO(enp->en_esbp, address, &buf, B_TRUE);
657			if (memcmp(&reg, &buf, sizeof (reg))) {
658				rc = EIO;
659				goto fail1;
660			}
661
662			address += rsp->step;
663		}
664
665		++rsp;
666		--count;
667	}
668
669	return (0);
670
671fail1:
672	EFSYS_PROBE1(fail1, int, rc);
673
674	return (rc);
675}
676
677#endif	/* EFSYS_OPT_DIAG */
678