1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25#include "efsys.h"
26#include "efx.h"
27#include "efx_types.h"
28#include "efx_regs.h"
29#include "efx_impl.h"
30
31	__checkReturn	int
32efx_family(
33	__in		uint16_t venid,
34	__in		uint16_t devid,
35	__out		efx_family_t *efp)
36{
37#if EFSYS_OPT_FALCON
38	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_FALCON) {
39		*efp = EFX_FAMILY_FALCON;
40		return (0);
41	}
42#endif
43#if EFSYS_OPT_SIENA
44	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_BETHPAGE) {
45		*efp = EFX_FAMILY_SIENA;
46		return (0);
47	}
48	if (venid == EFX_PCI_VENID_SFC && devid == EFX_PCI_DEVID_SIENA) {
49		*efp = EFX_FAMILY_SIENA;
50		return (0);
51	}
52	if (venid == EFX_PCI_VENID_SFC &&
53	    devid == EFX_PCI_DEVID_SIENA_F1_UNINIT) {
54		*efp = EFX_FAMILY_SIENA;
55		return (0);
56	}
57#endif
58	return (ENOTSUP);
59}
60
61/*
62 * To support clients which aren't provided with any PCI context infer
63 * the hardware family by inspecting the hardware. Obviously the caller
64 * must be damn sure they're really talking to a supported device.
65 */
66	__checkReturn	int
67efx_infer_family(
68	__in		efsys_bar_t *esbp,
69	__out		efx_family_t *efp)
70{
71	efx_family_t family;
72	efx_oword_t oword;
73	unsigned int portnum;
74	int rc;
75
76	EFSYS_BAR_READO(esbp, FR_AZ_CS_DEBUG_REG_OFST, &oword, B_TRUE);
77	portnum = EFX_OWORD_FIELD(oword, FRF_CZ_CS_PORT_NUM);
78	switch (portnum) {
79#if EFSYS_OPT_FALCON
80	case 0:
81		family = EFX_FAMILY_FALCON;
82		break;
83#endif
84#if EFSYS_OPT_SIENA
85	case 1:
86	case 2:
87		family = EFX_FAMILY_SIENA;
88		break;
89#endif
90	default:
91		rc = ENOTSUP;
92		goto fail1;
93	}
94
95	if (efp != NULL)
96		*efp = family;
97	return (0);
98
99fail1:
100	EFSYS_PROBE1(fail1, int, rc);
101
102	return (rc);
103}
104
105/*
106 * The built-in default value device id for port 1 of Siena is 0x0810.
107 * manftest needs to be able to cope with that.
108 */
109
110#define	EFX_BIU_MAGIC0	0x01234567
111#define	EFX_BIU_MAGIC1	0xfedcba98
112
113static	__checkReturn	int
114efx_nic_biu_test(
115	__in		efx_nic_t *enp)
116{
117	efx_oword_t oword;
118	int rc;
119
120	/*
121	 * Write magic values to scratch registers 0 and 1, then
122	 * verify that the values were written correctly.  Interleave
123	 * the accesses to ensure that the BIU is not just reading
124	 * back the cached value that was last written.
125	 */
126	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
127	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
128
129	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
130	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
131
132	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
133	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
134		rc = EIO;
135		goto fail1;
136	}
137
138	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
139	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
140		rc = EIO;
141		goto fail2;
142	}
143
144	/*
145	 * Perform the same test, with the values swapped.  This
146	 * ensures that subsequent tests don't start with the correct
147	 * values already written into the scratch registers.
148	 */
149	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC1);
150	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 0, &oword);
151
152	EFX_POPULATE_OWORD_1(oword, FRF_AZ_DRIVER_DW0, EFX_BIU_MAGIC0);
153	EFX_BAR_TBL_WRITEO(enp, FR_AZ_DRIVER_REG, 1, &oword);
154
155	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 0, &oword);
156	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC1) {
157		rc = EIO;
158		goto fail3;
159	}
160
161	EFX_BAR_TBL_READO(enp, FR_AZ_DRIVER_REG, 1, &oword);
162	if (EFX_OWORD_FIELD(oword, FRF_AZ_DRIVER_DW0) != EFX_BIU_MAGIC0) {
163		rc = EIO;
164		goto fail4;
165	}
166
167	return (0);
168
169fail4:
170	EFSYS_PROBE(fail4);
171fail3:
172	EFSYS_PROBE(fail3);
173fail2:
174	EFSYS_PROBE(fail2);
175fail1:
176	EFSYS_PROBE1(fail1, int, rc);
177
178	return (rc);
179}
180
181#if EFSYS_OPT_FALCON
182
183static efx_nic_ops_t	__cs __efx_nic_falcon_ops = {
184	falcon_nic_probe,		/* eno_probe */
185	falcon_nic_reset,		/* eno_reset */
186	falcon_nic_init,		/* eno_init */
187#if EFSYS_OPT_DIAG
188	falcon_sram_test,		/* eno_sram_test */
189	falcon_nic_register_test,	/* eno_register_test */
190#endif	/* EFSYS_OPT_DIAG */
191	falcon_nic_fini,		/* eno_fini */
192	falcon_nic_unprobe,		/* eno_unprobe */
193};
194
195#endif	/* EFSYS_OPT_FALCON */
196
197#if EFSYS_OPT_SIENA
198
199static efx_nic_ops_t	__cs __efx_nic_siena_ops = {
200	siena_nic_probe,		/* eno_probe */
201	siena_nic_reset,		/* eno_reset */
202	siena_nic_init,			/* eno_init */
203#if EFSYS_OPT_DIAG
204	siena_sram_test,		/* eno_sram_test */
205	siena_nic_register_test,	/* eno_register_test */
206#endif	/* EFSYS_OPT_DIAG */
207	siena_nic_fini,			/* eno_fini */
208	siena_nic_unprobe,		/* eno_unprobe */
209};
210
211#endif	/* EFSYS_OPT_SIENA */
212
213
214// efx_nic_create
215// It creates and initializes a efx_nic_t datastructure
216// based on the family of device, it sets up the function pointers
217// to call appropriate functions.  No actual functions are called here.
218	__checkReturn	int
219efx_nic_create(
220	__in		efx_family_t family,
221	__in		efsys_identifier_t *esip,
222	__in		efsys_bar_t *esbp,
223	__in		efsys_lock_t *eslp,
224	__deref_out	efx_nic_t **enpp)
225{
226	efx_nic_t *enp;
227	int rc;
228
229	EFSYS_ASSERT3U(family, >, EFX_FAMILY_INVALID);
230	EFSYS_ASSERT3U(family, <, EFX_FAMILY_NTYPES);
231
232	/* Allocate a NIC object */
233	EFSYS_KMEM_ALLOC(esip, sizeof (efx_nic_t), enp);
234
235	if (enp == NULL) {
236		rc = ENOMEM;
237		goto fail1;
238	}
239
240	enp->en_magic = EFX_NIC_MAGIC;
241
242	switch (family) {
243#if EFSYS_OPT_FALCON
244	case EFX_FAMILY_FALCON:
245		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_falcon_ops;
246		enp->en_features = 0;
247		break;
248#endif	/* EFSYS_OPT_FALCON */
249
250#if EFSYS_OPT_SIENA
251	case EFX_FAMILY_SIENA:
252		enp->en_enop = (efx_nic_ops_t *)&__efx_nic_siena_ops;
253		enp->en_features = EFX_FEATURE_IPV6 |
254		    EFX_FEATURE_LFSR_HASH_INSERT |
255		    EFX_FEATURE_LINK_EVENTS | EFX_FEATURE_PERIODIC_MAC_STATS |
256		    EFX_FEATURE_WOL | EFX_FEATURE_MCDI |
257		    EFX_FEATURE_LOOKAHEAD_SPLIT | EFX_FEATURE_MAC_HEADER_FILTERS;
258		break;
259#endif	/* EFSYS_OPT_SIENA */
260
261	default:
262		rc = ENOTSUP;
263		goto fail2;
264	}
265
266	enp->en_family = family;
267	enp->en_esip = esip;
268	enp->en_esbp = esbp;
269	enp->en_eslp = eslp;
270
271	*enpp = enp;
272
273	return (0);
274
275fail2:
276	EFSYS_PROBE(fail3);
277
278	enp->en_magic = 0;
279
280	/* Free the NIC object */
281	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
282
283fail1:
284	EFSYS_PROBE1(fail1, int, rc);
285
286	return (rc);
287}
288
289	__checkReturn	int
290efx_nic_probe(
291	__in		efx_nic_t *enp)
292{
293	efx_nic_ops_t *enop;
294	efx_oword_t oword;
295	int rc;
296
297	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
298#if EFSYS_OPT_MCDI
299	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
300#endif	/* EFSYS_OPT_MCDI */
301	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_PROBE));
302
303	/* Test BIU */
304	if ((rc = efx_nic_biu_test(enp)) != 0)
305		goto fail1;
306
307	/* Clear the region register */
308	EFX_POPULATE_OWORD_4(oword,
309	    FRF_AZ_ADR_REGION0, 0,
310	    FRF_AZ_ADR_REGION1, (1 << 16),
311	    FRF_AZ_ADR_REGION2, (2 << 16),
312	    FRF_AZ_ADR_REGION3, (3 << 16));
313	EFX_BAR_WRITEO(enp, FR_AZ_ADR_REGION_REG, &oword);
314
315	enop = enp->en_enop;
316	if ((rc = enop->eno_probe(enp)) != 0)
317		goto fail2;
318
319	if ((rc = efx_phy_probe(enp)) != 0)
320		goto fail3;
321
322	enp->en_mod_flags |= EFX_MOD_PROBE;
323
324	return (0);
325
326fail3:
327	EFSYS_PROBE(fail3);
328
329	enop->eno_unprobe(enp);
330
331fail2:
332	EFSYS_PROBE(fail2);
333fail1:
334	EFSYS_PROBE1(fail1, int, rc);
335
336	return (rc);
337}
338
339#if EFSYS_OPT_PCIE_TUNE
340
341	__checkReturn	int
342efx_nic_pcie_tune(
343	__in		efx_nic_t *enp,
344	unsigned int	nlanes)
345{
346	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
347	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
348	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
349
350#if EFSYS_OPT_FALCON
351	if (enp->en_family == EFX_FAMILY_FALCON)
352		return (falcon_nic_pcie_tune(enp, nlanes));
353#endif
354	return (ENOTSUP);
355}
356
357	__checkReturn	int
358efx_nic_pcie_extended_sync(
359	__in		efx_nic_t *enp)
360{
361	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
362	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
363	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
364
365#if EFSYS_OPT_SIENA
366	if (enp->en_family == EFX_FAMILY_SIENA)
367		return (siena_nic_pcie_extended_sync(enp));
368#endif
369
370	return (ENOTSUP);
371}
372
373#endif	/* EFSYS_OPT_PCIE_TUNE */
374
375	__checkReturn	int
376efx_nic_init(
377	__in		efx_nic_t *enp)
378{
379	efx_nic_ops_t *enop = enp->en_enop;
380	int rc;
381
382	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
383	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
384
385	if (enp->en_mod_flags & EFX_MOD_NIC) {
386		rc = EINVAL;
387		goto fail1;
388	}
389
390	if ((rc = enop->eno_init(enp)) != 0)
391		goto fail2;
392
393	enp->en_mod_flags |= EFX_MOD_NIC;
394
395	return (0);
396
397fail2:
398	EFSYS_PROBE(fail2);
399fail1:
400	EFSYS_PROBE1(fail1, int, rc);
401
402	return (rc);
403}
404
405			void
406efx_nic_fini(
407	__in		efx_nic_t *enp)
408{
409	efx_nic_ops_t *enop = enp->en_enop;
410
411	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
412	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
413	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_NIC);
414	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
415	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
416	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
417	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
418
419	enop->eno_fini(enp);
420
421	enp->en_mod_flags &= ~EFX_MOD_NIC;
422}
423
424			void
425efx_nic_unprobe(
426	__in		efx_nic_t *enp)
427{
428	efx_nic_ops_t *enop = enp->en_enop;
429
430	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
431#if EFSYS_OPT_MCDI
432	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
433#endif	/* EFSYS_OPT_MCDI */
434	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
435	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
436	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_INTR));
437	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV));
438	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
439	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
440
441	efx_phy_unprobe(enp);
442
443	enop->eno_unprobe(enp);
444
445	enp->en_mod_flags &= ~EFX_MOD_PROBE;
446}
447
448			void
449efx_nic_destroy(
450	__in	efx_nic_t *enp)
451{
452	efsys_identifier_t *esip = enp->en_esip;
453
454	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
455	EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0);
456
457	enp->en_family = 0;
458	enp->en_esip = NULL;
459	enp->en_esbp = NULL;
460	enp->en_eslp = NULL;
461
462	enp->en_enop = NULL;
463
464	enp->en_magic = 0;
465
466	/* Free the NIC object */
467	EFSYS_KMEM_FREE(esip, sizeof (efx_nic_t), enp);
468}
469
470	__checkReturn	int
471efx_nic_reset(
472	__in		efx_nic_t *enp)
473{
474	efx_nic_ops_t *enop = enp->en_enop;
475	unsigned int mod_flags;
476	int rc;
477
478	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
479	EFSYS_ASSERT(enp->en_mod_flags & EFX_MOD_PROBE);
480	/*
481	 * All modules except the MCDI, PROBE, NVRAM, VPD, MON (which we
482	 * do not reset here) must have been shut down or never initialized.
483	 *
484	 * A rule of thumb here is: If the controller or MC reboots, is *any*
485	 * state lost. If it's lost and needs reapplying, then the module
486	 * *must* not be initialised during the reset.
487	 */
488	mod_flags = enp->en_mod_flags;
489	mod_flags &= ~(EFX_MOD_MCDI | EFX_MOD_PROBE | EFX_MOD_NVRAM |
490		    EFX_MOD_VPD | EFX_MOD_MON);
491	EFSYS_ASSERT3U(mod_flags, ==, 0);
492	if (mod_flags != 0) {
493		rc = EINVAL;
494		goto fail1;
495	}
496
497	if ((rc = enop->eno_reset(enp)) != 0)
498		goto fail2;
499
500	enp->en_reset_flags |= EFX_RESET_MAC;
501
502	return (0);
503
504fail2:
505	EFSYS_PROBE(fail2);
506fail1:
507	EFSYS_PROBE1(fail1, int, rc);
508
509	return (rc);
510}
511
512			const efx_nic_cfg_t *
513efx_nic_cfg_get(
514	__in		efx_nic_t *enp)
515{
516	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
517
518	return (&(enp->en_nic_cfg));
519}
520
521#if EFSYS_OPT_DIAG
522
523	__checkReturn	int
524efx_nic_register_test(
525	__in		efx_nic_t *enp)
526{
527	efx_nic_ops_t *enop = enp->en_enop;
528	int rc;
529
530	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
531	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
532	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_NIC));
533
534	if ((rc = enop->eno_register_test(enp)) != 0)
535		goto fail1;
536
537	return (0);
538
539fail1:
540	EFSYS_PROBE1(fail1, int, rc);
541
542	return (rc);
543}
544
545	__checkReturn	int
546efx_nic_test_registers(
547	__in		efx_nic_t *enp,
548	__in		efx_register_set_t *rsp,
549	__in		size_t count)
550{
551	unsigned int bit;
552	efx_oword_t original;
553	efx_oword_t reg;
554	efx_oword_t buf;
555	int rc;
556
557	while (count > 0) {
558		/* This function is only suitable for registers */
559		EFSYS_ASSERT(rsp->rows == 1);
560
561		/* bit sweep on and off */
562		EFSYS_BAR_READO(enp->en_esbp, rsp->address, &original,
563			    B_TRUE);
564		for (bit = 0; bit < 128; bit++) {
565			/* Is this bit in the mask? */
566			if (~(rsp->mask.eo_u32[bit >> 5]) & (1 << bit))
567				continue;
568
569			/* Test this bit can be set in isolation */
570			reg = original;
571			EFX_AND_OWORD(reg, rsp->mask);
572			EFX_SET_OWORD_BIT(reg, bit);
573
574			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
575				    B_TRUE);
576			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
577				    B_TRUE);
578
579			EFX_AND_OWORD(buf, rsp->mask);
580			if (memcmp(&reg, &buf, sizeof (reg))) {
581				rc = EIO;
582				goto fail1;
583			}
584
585			/* Test this bit can be cleared in isolation */
586			EFX_OR_OWORD(reg, rsp->mask);
587			EFX_CLEAR_OWORD_BIT(reg, bit);
588
589			EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &reg,
590				    B_TRUE);
591			EFSYS_BAR_READO(enp->en_esbp, rsp->address, &buf,
592				    B_TRUE);
593
594			EFX_AND_OWORD(buf, rsp->mask);
595			if (memcmp(&reg, &buf, sizeof (reg))) {
596				rc = EIO;
597				goto fail2;
598			}
599		}
600
601		/* Restore the old value */
602		EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original,
603			    B_TRUE);
604
605		--count;
606		++rsp;
607	}
608
609	return (0);
610
611fail2:
612	EFSYS_PROBE(fail2);
613fail1:
614	EFSYS_PROBE1(fail1, int, rc);
615
616	/* Restore the old value */
617	EFSYS_BAR_WRITEO(enp->en_esbp, rsp->address, &original, B_TRUE);
618
619	return (rc);
620}
621
622	__checkReturn	int
623efx_nic_test_tables(
624	__in		efx_nic_t *enp,
625	__in		efx_register_set_t *rsp,
626	__in		efx_pattern_type_t pattern,
627	__in		size_t count)
628{
629	efx_sram_pattern_fn_t func;
630    unsigned int idx;
631	unsigned int address;
632	efx_oword_t reg;
633	efx_oword_t buf;
634	int rc;
635
636	EFSYS_ASSERT(pattern < EFX_PATTERN_NTYPES);
637	func = __efx_sram_pattern_fns[pattern];
638
639	while (count > 0) {
640		/* Write */
641		address = rsp->address;
642        for (idx = 0; idx < rsp->rows; ++idx) {
643            func(2 * idx + 0, B_FALSE, &reg.eo_qword[0]);
644            func(2 * idx + 1, B_FALSE, &reg.eo_qword[1]);
645			EFX_AND_OWORD(reg, rsp->mask);
646			EFSYS_BAR_WRITEO(enp->en_esbp, address, &reg, B_TRUE);
647
648			address += rsp->step;
649		}
650
651		/* Read */
652		address = rsp->address;
653        for (idx = 0; idx < rsp->rows; ++idx) {
654            func(2 * idx + 0, B_FALSE, &reg.eo_qword[0]);
655            func(2 * idx + 1, B_FALSE, &reg.eo_qword[1]);
656			EFX_AND_OWORD(reg, rsp->mask);
657			EFSYS_BAR_READO(enp->en_esbp, address, &buf, B_TRUE);
658			if (memcmp(&reg, &buf, sizeof (reg))) {
659				rc = EIO;
660				goto fail1;
661			}
662
663			address += rsp->step;
664		}
665
666		++rsp;
667		--count;
668	}
669
670	return (0);
671
672fail1:
673	EFSYS_PROBE1(fail1, int, rc);
674
675	return (rc);
676}
677
678#endif	/* EFSYS_OPT_DIAG */
679