1223313Snwhitehorn/*-
2223313Snwhitehorn * Copyright (C) 2011 glevand (geoffrey.levand@mail.ru)
3223313Snwhitehorn * All rights reserved.
4223313Snwhitehorn *
5223313Snwhitehorn * Redistribution and use in source and binary forms, with or without
6223313Snwhitehorn * modification, are permitted provided that the following conditions
7223313Snwhitehorn * are met:
8223313Snwhitehorn * 1. Redistributions of source code must retain the above copyright
9223313Snwhitehorn *    notice, this list of conditions and the following disclaimer.
10223313Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright
11223313Snwhitehorn *    notice, this list of conditions and the following disclaimer in the
12223313Snwhitehorn *    documentation and/or other materials provided with the distribution.
13223313Snwhitehorn *
14223313Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15223313Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16223313Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17223313Snwhitehorn * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18223313Snwhitehorn * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19223313Snwhitehorn * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20223313Snwhitehorn * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21223313Snwhitehorn * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22223313Snwhitehorn * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23223313Snwhitehorn * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24223313Snwhitehorn *
25223313Snwhitehorn */
26223313Snwhitehorn
27223313Snwhitehorn#include <sys/cdefs.h>
28223313Snwhitehorn__FBSDID("$FreeBSD: releng/11.0/sys/powerpc/ps3/ps3disk.c 295880 2016-02-22 09:02:20Z skra $");
29223313Snwhitehorn
30223313Snwhitehorn#include <sys/param.h>
31223313Snwhitehorn#include <sys/systm.h>
32223313Snwhitehorn#include <sys/sysctl.h>
33223313Snwhitehorn#include <sys/disk.h>
34223313Snwhitehorn#include <sys/bio.h>
35223313Snwhitehorn#include <sys/bus.h>
36223313Snwhitehorn#include <sys/conf.h>
37223313Snwhitehorn#include <sys/kernel.h>
38223313Snwhitehorn#include <sys/kthread.h>
39223313Snwhitehorn#include <sys/lock.h>
40223313Snwhitehorn#include <sys/malloc.h>
41223313Snwhitehorn#include <sys/module.h>
42223313Snwhitehorn#include <sys/mutex.h>
43223313Snwhitehorn
44223313Snwhitehorn#include <vm/vm.h>
45223313Snwhitehorn#include <vm/pmap.h>
46223313Snwhitehorn
47223313Snwhitehorn#include <machine/pio.h>
48223313Snwhitehorn#include <machine/bus.h>
49223313Snwhitehorn#include <machine/platform.h>
50223313Snwhitehorn#include <machine/resource.h>
51223313Snwhitehorn#include <sys/bus.h>
52223313Snwhitehorn#include <sys/rman.h>
53223313Snwhitehorn
54223313Snwhitehorn#include <geom/geom_disk.h>
55223313Snwhitehorn
56223313Snwhitehorn#include "ps3bus.h"
57223313Snwhitehorn#include "ps3-hvcall.h"
58223313Snwhitehorn
59223313Snwhitehorn#define PS3DISK_LOCK_INIT(_sc)		\
60223313Snwhitehorn	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), "ps3disk", MTX_DEF)
61223313Snwhitehorn#define PS3DISK_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
62223313Snwhitehorn#define PS3DISK_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
63223313Snwhitehorn#define	PS3DISK_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
64223313Snwhitehorn#define PS3DISK_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
65223313Snwhitehorn#define PS3DISK_ASSERT_UNLOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
66223313Snwhitehorn
67223313Snwhitehorn#define LV1_STORAGE_ATA_HDDOUT 		0x23
68223313Snwhitehorn
69227309Sedstatic SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD, 0,
70227309Sed    "PS3 Disk driver parameters");
71223313Snwhitehorn
72223313Snwhitehorn#ifdef PS3DISK_DEBUG
73223313Snwhitehornstatic int ps3disk_debug = 0;
74223313SnwhitehornSYSCTL_INT(_hw_ps3disk, OID_AUTO, debug, CTLFLAG_RW, &ps3disk_debug,
75223313Snwhitehorn	0, "control debugging printfs");
76223313SnwhitehornTUNABLE_INT("hw.ps3disk.debug", &ps3disk_debug);
77223313Snwhitehornenum {
78223313Snwhitehorn	PS3DISK_DEBUG_INTR	= 0x00000001,
79223313Snwhitehorn	PS3DISK_DEBUG_TASK	= 0x00000002,
80223313Snwhitehorn	PS3DISK_DEBUG_READ	= 0x00000004,
81223313Snwhitehorn	PS3DISK_DEBUG_WRITE	= 0x00000008,
82223313Snwhitehorn	PS3DISK_DEBUG_FLUSH	= 0x00000010,
83223313Snwhitehorn	PS3DISK_DEBUG_ANY	= 0xffffffff
84223313Snwhitehorn};
85223313Snwhitehorn#define	DPRINTF(sc, m, fmt, ...)				\
86223313Snwhitehorndo {								\
87223313Snwhitehorn	if (sc->sc_debug & (m))					\
88223313Snwhitehorn		printf(fmt, __VA_ARGS__);			\
89223313Snwhitehorn} while (0)
90223313Snwhitehorn#else
91223313Snwhitehorn#define	DPRINTF(sc, m, fmt, ...)
92223313Snwhitehorn#endif
93223313Snwhitehorn
94223313Snwhitehornstruct ps3disk_region {
95223313Snwhitehorn	uint64_t r_id;
96223313Snwhitehorn	uint64_t r_start;
97223313Snwhitehorn	uint64_t r_size;
98223313Snwhitehorn	uint64_t r_flags;
99223313Snwhitehorn};
100223313Snwhitehorn
101223313Snwhitehornstruct ps3disk_softc {
102223313Snwhitehorn	device_t sc_dev;
103223313Snwhitehorn
104223313Snwhitehorn	struct mtx sc_mtx;
105223313Snwhitehorn
106223313Snwhitehorn	uint64_t sc_blksize;
107223313Snwhitehorn	uint64_t sc_nblocks;
108223313Snwhitehorn
109223313Snwhitehorn	uint64_t sc_nregs;
110223313Snwhitehorn	struct ps3disk_region *sc_reg;
111223313Snwhitehorn
112223313Snwhitehorn	int sc_irqid;
113223313Snwhitehorn	struct resource	*sc_irq;
114223313Snwhitehorn	void *sc_irqctx;
115223313Snwhitehorn
116223313Snwhitehorn	struct disk **sc_disk;
117223313Snwhitehorn
118223313Snwhitehorn	struct bio_queue_head sc_bioq;
119223461Snwhitehorn	struct bio_queue_head sc_deferredq;
120223461Snwhitehorn	struct proc *sc_task;
121223313Snwhitehorn
122223461Snwhitehorn	bus_dma_tag_t sc_dmatag;
123223313Snwhitehorn
124223313Snwhitehorn	int sc_running;
125223313Snwhitehorn	int sc_debug;
126223313Snwhitehorn};
127223313Snwhitehorn
128223313Snwhitehornstatic int ps3disk_open(struct disk *dp);
129223313Snwhitehornstatic int ps3disk_close(struct disk *dp);
130223313Snwhitehornstatic void ps3disk_strategy(struct bio *bp);
131223461Snwhitehorn
132223313Snwhitehornstatic void ps3disk_task(void *arg);
133223313Snwhitehornstatic void ps3disk_intr(void *arg);
134223313Snwhitehornstatic int ps3disk_get_disk_geometry(struct ps3disk_softc *sc);
135223313Snwhitehornstatic int ps3disk_enum_regions(struct ps3disk_softc *sc);
136223461Snwhitehornstatic void ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs,
137223461Snwhitehorn    int error);
138223313Snwhitehorn
139223313Snwhitehornstatic void ps3disk_sysctlattach(struct ps3disk_softc *sc);
140223313Snwhitehorn
141223313Snwhitehornstatic MALLOC_DEFINE(M_PS3DISK, "ps3disk", "PS3 Disk");
142223313Snwhitehorn
143223313Snwhitehornstatic int
144223313Snwhitehornps3disk_probe(device_t dev)
145223313Snwhitehorn{
146223313Snwhitehorn	if (ps3bus_get_bustype(dev) != PS3_BUSTYPE_STORAGE ||
147223313Snwhitehorn	    ps3bus_get_devtype(dev) != PS3_DEVTYPE_DISK)
148223313Snwhitehorn		return (ENXIO);
149223313Snwhitehorn
150223313Snwhitehorn	device_set_desc(dev, "Playstation 3 Disk");
151223313Snwhitehorn
152223313Snwhitehorn	return (BUS_PROBE_SPECIFIC);
153223313Snwhitehorn}
154223313Snwhitehorn
155223313Snwhitehornstatic int
156223313Snwhitehornps3disk_attach(device_t dev)
157223313Snwhitehorn{
158223313Snwhitehorn	struct ps3disk_softc *sc;
159223313Snwhitehorn	struct disk *d;
160223313Snwhitehorn	intmax_t mb;
161223461Snwhitehorn	uint64_t junk;
162223313Snwhitehorn	char unit;
163223313Snwhitehorn	int i, err;
164223313Snwhitehorn
165223313Snwhitehorn	sc = device_get_softc(dev);
166223313Snwhitehorn	sc->sc_dev = dev;
167223313Snwhitehorn
168223313Snwhitehorn	PS3DISK_LOCK_INIT(sc);
169223313Snwhitehorn
170223313Snwhitehorn	err = ps3disk_get_disk_geometry(sc);
171223313Snwhitehorn	if (err) {
172223313Snwhitehorn		device_printf(dev, "Could not get disk geometry\n");
173223313Snwhitehorn		err = ENXIO;
174223313Snwhitehorn		goto fail_destroy_lock;
175223313Snwhitehorn	}
176223313Snwhitehorn
177223313Snwhitehorn	device_printf(dev, "block size %lu total blocks %lu\n",
178223313Snwhitehorn	    sc->sc_blksize, sc->sc_nblocks);
179223313Snwhitehorn
180223313Snwhitehorn	err = ps3disk_enum_regions(sc);
181223313Snwhitehorn	if (err) {
182223313Snwhitehorn		device_printf(dev, "Could not enumerate disk regions\n");
183223313Snwhitehorn		err = ENXIO;
184223313Snwhitehorn		goto fail_destroy_lock;
185223313Snwhitehorn	}
186223313Snwhitehorn
187223313Snwhitehorn	device_printf(dev, "Found %lu regions\n", sc->sc_nregs);
188223313Snwhitehorn
189223313Snwhitehorn	if (!sc->sc_nregs) {
190223313Snwhitehorn		err = ENXIO;
191223313Snwhitehorn		goto fail_destroy_lock;
192223313Snwhitehorn	}
193223313Snwhitehorn
194223313Snwhitehorn	/* Setup interrupt handler */
195223313Snwhitehorn	sc->sc_irqid = 0;
196223313Snwhitehorn	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqid,
197223313Snwhitehorn	    RF_ACTIVE);
198223313Snwhitehorn	if (!sc->sc_irq) {
199223313Snwhitehorn		device_printf(dev, "Could not allocate IRQ\n");
200223313Snwhitehorn		err = ENXIO;
201223313Snwhitehorn		goto fail_free_regions;
202223313Snwhitehorn	}
203223313Snwhitehorn
204223313Snwhitehorn	err = bus_setup_intr(dev, sc->sc_irq,
205223313Snwhitehorn	    INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY,
206223461Snwhitehorn	    NULL, ps3disk_intr, sc, &sc->sc_irqctx);
207223313Snwhitehorn	if (err) {
208223313Snwhitehorn		device_printf(dev, "Could not setup IRQ\n");
209223313Snwhitehorn		err = ENXIO;
210223313Snwhitehorn		goto fail_release_intr;
211223313Snwhitehorn	}
212223313Snwhitehorn
213223461Snwhitehorn	/* Setup DMA */
214223313Snwhitehorn	err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0,
215223313Snwhitehorn	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
216223461Snwhitehorn	    BUS_SPACE_UNRESTRICTED, 1, PAGE_SIZE, 0,
217223461Snwhitehorn	    busdma_lock_mutex, &sc->sc_mtx, &sc->sc_dmatag);
218223313Snwhitehorn	if (err) {
219223461Snwhitehorn		device_printf(dev, "Could not create DMA tag\n");
220223313Snwhitehorn		err = ENXIO;
221223313Snwhitehorn		goto fail_teardown_intr;
222223313Snwhitehorn	}
223223313Snwhitehorn
224223313Snwhitehorn	/* Setup disks */
225223313Snwhitehorn
226223313Snwhitehorn	sc->sc_disk = malloc(sc->sc_nregs * sizeof(struct disk *),
227223313Snwhitehorn	    M_PS3DISK, M_ZERO | M_WAITOK);
228223313Snwhitehorn	if (!sc->sc_disk) {
229223313Snwhitehorn		device_printf(dev, "Could not allocate disk(s)\n");
230223313Snwhitehorn		err = ENOMEM;
231223461Snwhitehorn		goto fail_teardown_intr;
232223313Snwhitehorn	}
233223313Snwhitehorn
234223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++) {
235223316Snwhitehorn		struct ps3disk_region *rp = &sc->sc_reg[i];
236223316Snwhitehorn
237223313Snwhitehorn		d = sc->sc_disk[i] = disk_alloc();
238223313Snwhitehorn		d->d_open = ps3disk_open;
239223313Snwhitehorn		d->d_close = ps3disk_close;
240223313Snwhitehorn		d->d_strategy = ps3disk_strategy;
241223313Snwhitehorn		d->d_name = "ps3disk";
242223313Snwhitehorn		d->d_drv1 = sc;
243223461Snwhitehorn		d->d_maxsize = PAGE_SIZE;
244223313Snwhitehorn		d->d_sectorsize = sc->sc_blksize;
245223313Snwhitehorn		d->d_unit = i;
246223313Snwhitehorn		d->d_mediasize = sc->sc_reg[i].r_size * sc->sc_blksize;
247223313Snwhitehorn		d->d_flags |= DISKFLAG_CANFLUSHCACHE;
248223313Snwhitehorn
249223313Snwhitehorn		mb = d->d_mediasize >> 20;
250223313Snwhitehorn		unit = 'M';
251223313Snwhitehorn		if (mb >= 10240) {
252223313Snwhitehorn			unit = 'G';
253223313Snwhitehorn			mb /= 1024;
254223313Snwhitehorn		}
255223313Snwhitehorn
256223316Snwhitehorn		/* Test to see if we can read this region */
257223316Snwhitehorn		err = lv1_storage_read(ps3bus_get_device(dev), d->d_unit,
258223461Snwhitehorn		    0, 0, rp->r_flags, 0, &junk);
259223316Snwhitehorn		device_printf(dev, "region %d %ju%cB%s\n", i, mb, unit,
260223461Snwhitehorn		    (err == LV1_DENIED_BY_POLICY) ?  " (hypervisor protected)"
261223461Snwhitehorn		    : "");
262223313Snwhitehorn
263223461Snwhitehorn		if (err != LV1_DENIED_BY_POLICY)
264223316Snwhitehorn			disk_create(d, DISK_VERSION);
265223313Snwhitehorn	}
266223316Snwhitehorn	err = 0;
267223313Snwhitehorn
268223313Snwhitehorn	bioq_init(&sc->sc_bioq);
269223461Snwhitehorn	bioq_init(&sc->sc_deferredq);
270223461Snwhitehorn	kproc_create(&ps3disk_task, sc, &sc->sc_task, 0, 0, "ps3disk");
271223313Snwhitehorn
272223313Snwhitehorn	ps3disk_sysctlattach(sc);
273223313Snwhitehorn	sc->sc_running = 1;
274223313Snwhitehorn	return (0);
275223313Snwhitehorn
276223313Snwhitehornfail_teardown_intr:
277223313Snwhitehorn	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
278223313Snwhitehornfail_release_intr:
279223313Snwhitehorn	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
280223313Snwhitehornfail_free_regions:
281223313Snwhitehorn	free(sc->sc_reg, M_PS3DISK);
282223313Snwhitehornfail_destroy_lock:
283223313Snwhitehorn	PS3DISK_LOCK_DESTROY(sc);
284223313Snwhitehorn	return (err);
285223313Snwhitehorn}
286223313Snwhitehorn
287223313Snwhitehornstatic int
288223313Snwhitehornps3disk_detach(device_t dev)
289223313Snwhitehorn{
290223313Snwhitehorn	struct ps3disk_softc *sc = device_get_softc(dev);
291223313Snwhitehorn	int i;
292223313Snwhitehorn
293223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++)
294223313Snwhitehorn		disk_destroy(sc->sc_disk[i]);
295223313Snwhitehorn
296223461Snwhitehorn	bus_dma_tag_destroy(sc->sc_dmatag);
297223313Snwhitehorn
298223313Snwhitehorn	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
299223313Snwhitehorn	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
300223313Snwhitehorn
301223313Snwhitehorn	free(sc->sc_disk, M_PS3DISK);
302223313Snwhitehorn	free(sc->sc_reg, M_PS3DISK);
303223313Snwhitehorn
304223313Snwhitehorn	PS3DISK_LOCK_DESTROY(sc);
305223313Snwhitehorn
306223313Snwhitehorn	return (0);
307223313Snwhitehorn}
308223313Snwhitehorn
309223313Snwhitehornstatic int
310223313Snwhitehornps3disk_open(struct disk *dp)
311223313Snwhitehorn{
312223313Snwhitehorn	return (0);
313223313Snwhitehorn}
314223313Snwhitehorn
315223313Snwhitehornstatic int
316223313Snwhitehornps3disk_close(struct disk *dp)
317223313Snwhitehorn{
318223313Snwhitehorn	return (0);
319223313Snwhitehorn}
320223313Snwhitehorn
321223461Snwhitehorn/* Process deferred blocks */
322223313Snwhitehornstatic void
323223313Snwhitehornps3disk_task(void *arg)
324223313Snwhitehorn{
325223313Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
326223313Snwhitehorn	struct bio *bp;
327223313Snwhitehorn
328223461Snwhitehorn
329223461Snwhitehorn	while (1) {
330223461Snwhitehorn		kproc_suspend_check(sc->sc_task);
331223461Snwhitehorn		tsleep(&sc->sc_deferredq, PRIBIO, "ps3disk", 10);
332223461Snwhitehorn
333223313Snwhitehorn		PS3DISK_LOCK(sc);
334223461Snwhitehorn		bp = bioq_takefirst(&sc->sc_deferredq);
335223313Snwhitehorn		PS3DISK_UNLOCK(sc);
336223313Snwhitehorn
337223461Snwhitehorn		if (bp == NULL)
338223461Snwhitehorn			continue;
339223313Snwhitehorn
340223461Snwhitehorn		if (bp->bio_driver1 != NULL) {
341223461Snwhitehorn			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
342223461Snwhitehorn			    bp->bio_driver1);
343223461Snwhitehorn			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
344223461Snwhitehorn			    bp->bio_driver1);
345223461Snwhitehorn		}
346223313Snwhitehorn
347223461Snwhitehorn		ps3disk_strategy(bp);
348223461Snwhitehorn	}
349223313Snwhitehorn
350223461Snwhitehorn	kproc_exit(0);
351223461Snwhitehorn}
352223313Snwhitehorn
353223461Snwhitehornstatic void
354223461Snwhitehornps3disk_strategy(struct bio *bp)
355223461Snwhitehorn{
356223461Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
357223461Snwhitehorn	int err;
358223313Snwhitehorn
359223461Snwhitehorn	if (sc == NULL) {
360223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
361223461Snwhitehorn		bp->bio_error = EINVAL;
362223461Snwhitehorn		biodone(bp);
363223461Snwhitehorn		return;
364223461Snwhitehorn	}
365223313Snwhitehorn
366223461Snwhitehorn	PS3DISK_LOCK(sc);
367223461Snwhitehorn	bp->bio_resid = bp->bio_bcount;
368223461Snwhitehorn	bioq_insert_tail(&sc->sc_bioq, bp);
369223313Snwhitehorn
370223461Snwhitehorn	DPRINTF(sc, PS3DISK_DEBUG_TASK, "%s: bio_cmd 0x%02x\n",
371223461Snwhitehorn	    __func__, bp->bio_cmd);
372223313Snwhitehorn
373223461Snwhitehorn	err = 0;
374223461Snwhitehorn	if (bp->bio_cmd == BIO_FLUSH) {
375223461Snwhitehorn		bp->bio_driver1 = 0;
376223461Snwhitehorn		err = lv1_storage_send_device_command(
377223461Snwhitehorn		    ps3bus_get_device(sc->sc_dev), LV1_STORAGE_ATA_HDDOUT,
378223461Snwhitehorn		    0, 0, 0, 0, (uint64_t *)&bp->bio_driver2);
379223461Snwhitehorn		if (err == LV1_BUSY)
380223461Snwhitehorn			err = EAGAIN;
381223461Snwhitehorn	} else if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
382223461Snwhitehorn		if (bp->bio_bcount % sc->sc_blksize != 0) {
383223461Snwhitehorn			err = EINVAL;
384223313Snwhitehorn		} else {
385223461Snwhitehorn			bus_dmamap_create(sc->sc_dmatag, BUS_DMA_COHERENT,
386223461Snwhitehorn			    (bus_dmamap_t *)(&bp->bio_driver1));
387223461Snwhitehorn			err = bus_dmamap_load(sc->sc_dmatag,
388223461Snwhitehorn			    (bus_dmamap_t)(bp->bio_driver1), bp->bio_data,
389223461Snwhitehorn			    bp->bio_bcount, ps3disk_transfer, bp, 0);
390223461Snwhitehorn			if (err == EINPROGRESS)
391223461Snwhitehorn				err = 0;
392223313Snwhitehorn		}
393223461Snwhitehorn	} else {
394223461Snwhitehorn		err = EINVAL;
395223461Snwhitehorn	}
396223313Snwhitehorn
397223461Snwhitehorn	if (err == EAGAIN) {
398223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
399223461Snwhitehorn		bioq_insert_tail(&sc->sc_deferredq, bp);
400223461Snwhitehorn	} else if (err != 0) {
401223461Snwhitehorn		bp->bio_error = err;
402223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
403223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
404223461Snwhitehorn		disk_err(bp, "hard error", -1, 1);
405223313Snwhitehorn		biodone(bp);
406223313Snwhitehorn	}
407223313Snwhitehorn
408223313Snwhitehorn	PS3DISK_UNLOCK(sc);
409223313Snwhitehorn}
410223313Snwhitehorn
411223313Snwhitehornstatic void
412223313Snwhitehornps3disk_intr(void *arg)
413223313Snwhitehorn{
414223313Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
415223313Snwhitehorn	device_t dev = sc->sc_dev;
416223313Snwhitehorn	uint64_t devid = ps3bus_get_device(dev);
417223461Snwhitehorn	struct bio *bp;
418223313Snwhitehorn	uint64_t tag, status;
419223313Snwhitehorn
420223461Snwhitehorn	if (lv1_storage_get_async_status(devid, &tag, &status) != 0)
421223461Snwhitehorn		return;
422223461Snwhitehorn
423223313Snwhitehorn	PS3DISK_LOCK(sc);
424223313Snwhitehorn
425223461Snwhitehorn	DPRINTF(sc, PS3DISK_DEBUG_INTR, "%s: tag 0x%016lx "
426223461Snwhitehorn	    "status 0x%016lx\n", __func__, tag, status);
427223313Snwhitehorn
428223461Snwhitehorn	/* Locate the matching request */
429223461Snwhitehorn	TAILQ_FOREACH(bp, &sc->sc_bioq.queue, bio_queue) {
430223461Snwhitehorn		if ((uint64_t)bp->bio_driver2 != tag)
431223461Snwhitehorn			continue;
432223313Snwhitehorn
433223461Snwhitehorn		if (status != 0) {
434223461Snwhitehorn			device_printf(sc->sc_dev, "%s error (%#lx)\n",
435223461Snwhitehorn			    (bp->bio_cmd == BIO_READ) ? "Read" : "Write",
436223461Snwhitehorn			    status);
437223461Snwhitehorn			bp->bio_error = EIO;
438223461Snwhitehorn			bp->bio_flags |= BIO_ERROR;
439223461Snwhitehorn		} else {
440223461Snwhitehorn			bp->bio_error = 0;
441223461Snwhitehorn			bp->bio_resid = 0;
442223461Snwhitehorn			bp->bio_flags |= BIO_DONE;
443223461Snwhitehorn		}
444223313Snwhitehorn
445223461Snwhitehorn		if (bp->bio_driver1 != NULL) {
446223461Snwhitehorn			if (bp->bio_cmd == BIO_READ)
447223461Snwhitehorn				bus_dmamap_sync(sc->sc_dmatag, (bus_dmamap_t)
448223461Snwhitehorn				    bp->bio_driver1, BUS_DMASYNC_POSTREAD);
449223461Snwhitehorn			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
450223461Snwhitehorn			    bp->bio_driver1);
451223461Snwhitehorn			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
452223461Snwhitehorn			    bp->bio_driver1);
453223461Snwhitehorn		}
454223461Snwhitehorn
455223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
456223461Snwhitehorn		biodone(bp);
457223461Snwhitehorn		break;
458223313Snwhitehorn	}
459223313Snwhitehorn
460223461Snwhitehorn	if (bioq_first(&sc->sc_deferredq) != NULL)
461223461Snwhitehorn		wakeup(&sc->sc_deferredq);
462223313Snwhitehorn
463223313Snwhitehorn	PS3DISK_UNLOCK(sc);
464223313Snwhitehorn}
465223313Snwhitehorn
466223313Snwhitehornstatic int
467223313Snwhitehornps3disk_get_disk_geometry(struct ps3disk_softc *sc)
468223313Snwhitehorn{
469223313Snwhitehorn	device_t dev = sc->sc_dev;
470223313Snwhitehorn	uint64_t bus_index = ps3bus_get_busidx(dev);
471223313Snwhitehorn	uint64_t dev_index = ps3bus_get_devidx(dev);
472223313Snwhitehorn	uint64_t junk;
473223313Snwhitehorn	int err;
474223313Snwhitehorn
475223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
476223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
477223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
478223313Snwhitehorn	    lv1_repository_string("blk_size"), 0, &sc->sc_blksize, &junk);
479223313Snwhitehorn	if (err) {
480223313Snwhitehorn		device_printf(dev, "Could not get block size (0x%08x)\n", err);
481223461Snwhitehorn		return (ENXIO);
482223313Snwhitehorn	}
483223313Snwhitehorn
484223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
485223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
486223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
487223313Snwhitehorn	    lv1_repository_string("n_blocks"), 0, &sc->sc_nblocks, &junk);
488223313Snwhitehorn	if (err) {
489223461Snwhitehorn		device_printf(dev, "Could not get total number of blocks "
490223461Snwhitehorn		    "(0x%08x)\n", err);
491223313Snwhitehorn		err = ENXIO;
492223313Snwhitehorn	}
493223313Snwhitehorn
494223313Snwhitehorn	return (err);
495223313Snwhitehorn}
496223313Snwhitehorn
497223313Snwhitehornstatic int
498223313Snwhitehornps3disk_enum_regions(struct ps3disk_softc *sc)
499223313Snwhitehorn{
500223313Snwhitehorn	device_t dev = sc->sc_dev;
501223313Snwhitehorn	uint64_t bus_index = ps3bus_get_busidx(dev);
502223313Snwhitehorn	uint64_t dev_index = ps3bus_get_devidx(dev);
503223313Snwhitehorn	uint64_t junk;
504223313Snwhitehorn	int i, err;
505223313Snwhitehorn
506223313Snwhitehorn	/* Read number of regions */
507223313Snwhitehorn
508223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
509223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
510223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
511223313Snwhitehorn	    lv1_repository_string("n_regs"), 0, &sc->sc_nregs, &junk);
512223313Snwhitehorn	if (err) {
513223313Snwhitehorn		device_printf(dev, "Could not get number of regions (0x%08x)\n",
514223313Snwhitehorn		    err);
515223313Snwhitehorn		err = ENXIO;
516223313Snwhitehorn		goto fail;
517223313Snwhitehorn	}
518223313Snwhitehorn
519223313Snwhitehorn	if (!sc->sc_nregs)
520223313Snwhitehorn		return 0;
521223313Snwhitehorn
522223313Snwhitehorn	sc->sc_reg = malloc(sc->sc_nregs * sizeof(struct ps3disk_region),
523223313Snwhitehorn	    M_PS3DISK, M_ZERO | M_WAITOK);
524223313Snwhitehorn	if (!sc->sc_reg) {
525223313Snwhitehorn		err = ENOMEM;
526223313Snwhitehorn		goto fail;
527223313Snwhitehorn	}
528223313Snwhitehorn
529223313Snwhitehorn	/* Setup regions */
530223313Snwhitehorn
531223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++) {
532223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
533223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
534223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
535223313Snwhitehorn		    lv1_repository_string("region") | i,
536223313Snwhitehorn		    lv1_repository_string("id"), &sc->sc_reg[i].r_id, &junk);
537223313Snwhitehorn		if (err) {
538223313Snwhitehorn			device_printf(dev, "Could not get region id (0x%08x)\n",
539223313Snwhitehorn			    err);
540223313Snwhitehorn			err = ENXIO;
541223313Snwhitehorn			goto fail;
542223313Snwhitehorn		}
543223313Snwhitehorn
544223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
545223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
546223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
547223313Snwhitehorn		    lv1_repository_string("region") | i,
548223461Snwhitehorn		    lv1_repository_string("start"), &sc->sc_reg[i].r_start,
549223461Snwhitehorn		    &junk);
550223313Snwhitehorn		if (err) {
551223461Snwhitehorn			device_printf(dev, "Could not get region start "
552223461Snwhitehorn			    "(0x%08x)\n", err);
553223313Snwhitehorn			err = ENXIO;
554223313Snwhitehorn			goto fail;
555223313Snwhitehorn		}
556223313Snwhitehorn
557223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
558223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
559223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
560223313Snwhitehorn		    lv1_repository_string("region") | i,
561223461Snwhitehorn		    lv1_repository_string("size"), &sc->sc_reg[i].r_size,
562223461Snwhitehorn		    &junk);
563223313Snwhitehorn		if (err) {
564223461Snwhitehorn			device_printf(dev, "Could not get region size "
565223461Snwhitehorn			    "(0x%08x)\n", err);
566223313Snwhitehorn			err = ENXIO;
567223313Snwhitehorn			goto fail;
568223313Snwhitehorn		}
569223313Snwhitehorn
570223313Snwhitehorn		if (i == 0)
571223313Snwhitehorn			sc->sc_reg[i].r_flags = 0x2;
572223313Snwhitehorn		else
573223313Snwhitehorn			sc->sc_reg[i].r_flags = 0;
574223313Snwhitehorn	}
575223313Snwhitehorn
576223313Snwhitehorn	return (0);
577223313Snwhitehorn
578223313Snwhitehornfail:
579223313Snwhitehorn
580223313Snwhitehorn	sc->sc_nregs = 0;
581223313Snwhitehorn	if (sc->sc_reg)
582223313Snwhitehorn		free(sc->sc_reg, M_PS3DISK);
583223313Snwhitehorn
584223313Snwhitehorn	return (err);
585223313Snwhitehorn}
586223313Snwhitehorn
587223461Snwhitehornstatic void
588223461Snwhitehornps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
589223313Snwhitehorn{
590223461Snwhitehorn	struct bio *bp = (struct bio *)(arg);
591223461Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
592223461Snwhitehorn	struct ps3disk_region *rp = &sc->sc_reg[bp->bio_disk->d_unit];
593223461Snwhitehorn	uint64_t devid = ps3bus_get_device(sc->sc_dev);
594223461Snwhitehorn	uint64_t block;
595223461Snwhitehorn	int i, err;
596223313Snwhitehorn
597223461Snwhitehorn	/* Locks already held by busdma */
598223461Snwhitehorn	PS3DISK_ASSERT_LOCKED(sc);
599223313Snwhitehorn
600223461Snwhitehorn	if (error) {
601223461Snwhitehorn		bp->bio_error = error;
602223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
603223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
604223461Snwhitehorn		biodone(bp);
605223461Snwhitehorn		return;
606223313Snwhitehorn	}
607223313Snwhitehorn
608223461Snwhitehorn	block = bp->bio_pblkno;
609223461Snwhitehorn	for (i = 0; i < nsegs; i++) {
610223461Snwhitehorn		KASSERT((segs[i].ds_len % sc->sc_blksize) == 0,
611223461Snwhitehorn		    ("DMA fragments not blocksize multiples"));
612223313Snwhitehorn
613223461Snwhitehorn		if (bp->bio_cmd == BIO_READ) {
614223461Snwhitehorn			err = lv1_storage_read(devid, rp->r_id,
615223461Snwhitehorn			    block, segs[i].ds_len/sc->sc_blksize,
616223461Snwhitehorn			    rp->r_flags, segs[i].ds_addr,
617223461Snwhitehorn			    (uint64_t *)&bp->bio_driver2);
618223461Snwhitehorn		} else {
619223461Snwhitehorn			bus_dmamap_sync(sc->sc_dmatag,
620223461Snwhitehorn			    (bus_dmamap_t)bp->bio_driver1,
621223461Snwhitehorn			    BUS_DMASYNC_PREWRITE);
622223461Snwhitehorn			err = lv1_storage_write(devid, rp->r_id,
623223461Snwhitehorn			    block, segs[i].ds_len/sc->sc_blksize,
624223461Snwhitehorn			    rp->r_flags, segs[i].ds_addr,
625223461Snwhitehorn			    (uint64_t *)&bp->bio_driver2);
626223461Snwhitehorn		}
627223313Snwhitehorn
628223461Snwhitehorn		if (err) {
629223461Snwhitehorn			if (err == LV1_BUSY) {
630223461Snwhitehorn				bioq_remove(&sc->sc_bioq, bp);
631223461Snwhitehorn				bioq_insert_tail(&sc->sc_deferredq, bp);
632223461Snwhitehorn			} else {
633223461Snwhitehorn				bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
634223461Snwhitehorn				    bp->bio_driver1);
635223461Snwhitehorn				bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
636223461Snwhitehorn				    bp->bio_driver1);
637223461Snwhitehorn				device_printf(sc->sc_dev, "Could not read "
638223461Snwhitehorn				    "sectors (0x%08x)\n", err);
639223461Snwhitehorn				bp->bio_error = EINVAL;
640223461Snwhitehorn				bp->bio_flags |= BIO_ERROR;
641223461Snwhitehorn				bioq_remove(&sc->sc_bioq, bp);
642223461Snwhitehorn				biodone(bp);
643223461Snwhitehorn			}
644223313Snwhitehorn
645223461Snwhitehorn			break;
646223461Snwhitehorn		}
647223313Snwhitehorn
648223461Snwhitehorn		DPRINTF(sc, PS3DISK_DEBUG_READ, "%s: tag 0x%016lx\n",
649223461Snwhitehorn		    __func__, sc->sc_bounce_tag);
650223313Snwhitehorn	}
651223313Snwhitehorn}
652223313Snwhitehorn
653223313Snwhitehorn#ifdef PS3DISK_DEBUG
654223313Snwhitehornstatic int
655223313Snwhitehornps3disk_sysctl_debug(SYSCTL_HANDLER_ARGS)
656223313Snwhitehorn{
657223313Snwhitehorn	struct ps3disk_softc *sc = arg1;
658223313Snwhitehorn	int debug, error;
659223313Snwhitehorn
660223313Snwhitehorn	debug = sc->sc_debug;
661223313Snwhitehorn
662223313Snwhitehorn	error = sysctl_handle_int(oidp, &debug, 0, req);
663223313Snwhitehorn	if (error || !req->newptr)
664223313Snwhitehorn		return error;
665223313Snwhitehorn
666223313Snwhitehorn	sc->sc_debug = debug;
667223313Snwhitehorn
668223313Snwhitehorn	return 0;
669223313Snwhitehorn}
670223313Snwhitehorn#endif
671223313Snwhitehorn
672223313Snwhitehornstatic void
673223313Snwhitehornps3disk_sysctlattach(struct ps3disk_softc *sc)
674223313Snwhitehorn{
675223313Snwhitehorn#ifdef PS3DISK_DEBUG
676223313Snwhitehorn	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
677223313Snwhitehorn	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
678223313Snwhitehorn
679223313Snwhitehorn	sc->sc_debug = ps3disk_debug;
680223313Snwhitehorn
681223313Snwhitehorn	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
682223313Snwhitehorn		"debug", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
683223313Snwhitehorn		ps3disk_sysctl_debug, "I", "control debugging printfs");
684223313Snwhitehorn#endif
685223313Snwhitehorn}
686223313Snwhitehorn
687223313Snwhitehornstatic device_method_t ps3disk_methods[] = {
688223313Snwhitehorn	DEVMETHOD(device_probe,		ps3disk_probe),
689223313Snwhitehorn	DEVMETHOD(device_attach,	ps3disk_attach),
690223313Snwhitehorn	DEVMETHOD(device_detach,	ps3disk_detach),
691223313Snwhitehorn	{0, 0},
692223313Snwhitehorn};
693223313Snwhitehorn
694223313Snwhitehornstatic driver_t ps3disk_driver = {
695223313Snwhitehorn	"ps3disk",
696223313Snwhitehorn	ps3disk_methods,
697223313Snwhitehorn	sizeof(struct ps3disk_softc),
698223313Snwhitehorn};
699223313Snwhitehorn
700223313Snwhitehornstatic devclass_t ps3disk_devclass;
701223313Snwhitehorn
702223313SnwhitehornDRIVER_MODULE(ps3disk, ps3bus, ps3disk_driver, ps3disk_devclass, 0, 0);
703