1223313Snwhitehorn/*-
2223313Snwhitehorn * Copyright (C) 2011 glevand (geoffrey.levand@mail.ru)
3223313Snwhitehorn * All rights reserved.
4223313Snwhitehorn *
5223313Snwhitehorn * Redistribution and use in source and binary forms, with or without
6223313Snwhitehorn * modification, are permitted provided that the following conditions
7223313Snwhitehorn * are met:
8223313Snwhitehorn * 1. Redistributions of source code must retain the above copyright
9223313Snwhitehorn *    notice, this list of conditions and the following disclaimer.
10223313Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright
11223313Snwhitehorn *    notice, this list of conditions and the following disclaimer in the
12223313Snwhitehorn *    documentation and/or other materials provided with the distribution.
13223313Snwhitehorn *
14223313Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15223313Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16223313Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17223313Snwhitehorn * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18223313Snwhitehorn * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19223313Snwhitehorn * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20223313Snwhitehorn * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21223313Snwhitehorn * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22223313Snwhitehorn * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23223313Snwhitehorn * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24223313Snwhitehorn *
25223313Snwhitehorn */
26223313Snwhitehorn
27223313Snwhitehorn#include <sys/cdefs.h>
28223313Snwhitehorn__FBSDID("$FreeBSD$");
29223313Snwhitehorn
30223313Snwhitehorn#include <sys/param.h>
31223313Snwhitehorn#include <sys/systm.h>
32223313Snwhitehorn#include <sys/sysctl.h>
33223313Snwhitehorn#include <sys/disk.h>
34223313Snwhitehorn#include <sys/bio.h>
35223313Snwhitehorn#include <sys/bus.h>
36223313Snwhitehorn#include <sys/conf.h>
37223313Snwhitehorn#include <sys/kernel.h>
38223313Snwhitehorn#include <sys/kthread.h>
39223313Snwhitehorn#include <sys/lock.h>
40223313Snwhitehorn#include <sys/malloc.h>
41223313Snwhitehorn#include <sys/module.h>
42223313Snwhitehorn#include <sys/mutex.h>
43223313Snwhitehorn
44223313Snwhitehorn#include <vm/vm.h>
45223313Snwhitehorn#include <vm/pmap.h>
46223313Snwhitehorn
47223313Snwhitehorn#include <machine/pio.h>
48223313Snwhitehorn#include <machine/bus.h>
49223313Snwhitehorn#include <machine/platform.h>
50223313Snwhitehorn#include <machine/pmap.h>
51223313Snwhitehorn#include <machine/resource.h>
52223313Snwhitehorn#include <sys/bus.h>
53223313Snwhitehorn#include <sys/rman.h>
54223313Snwhitehorn
55223313Snwhitehorn#include <geom/geom_disk.h>
56223313Snwhitehorn
57223313Snwhitehorn#include "ps3bus.h"
58223313Snwhitehorn#include "ps3-hvcall.h"
59223313Snwhitehorn
60223313Snwhitehorn#define PS3DISK_LOCK_INIT(_sc)		\
61223313Snwhitehorn	mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), "ps3disk", MTX_DEF)
62223313Snwhitehorn#define PS3DISK_LOCK_DESTROY(_sc)	mtx_destroy(&_sc->sc_mtx);
63223313Snwhitehorn#define PS3DISK_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
64223313Snwhitehorn#define	PS3DISK_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_mtx)
65223313Snwhitehorn#define PS3DISK_ASSERT_LOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_OWNED);
66223313Snwhitehorn#define PS3DISK_ASSERT_UNLOCKED(_sc)	mtx_assert(&_sc->sc_mtx, MA_NOTOWNED);
67223313Snwhitehorn
68223313Snwhitehorn#define LV1_STORAGE_ATA_HDDOUT 		0x23
69223313Snwhitehorn
70248085Smariusstatic SYSCTL_NODE(_hw, OID_AUTO, ps3disk, CTLFLAG_RD, 0,
71248085Smarius    "PS3 Disk driver parameters");
72223313Snwhitehorn
73223313Snwhitehorn#ifdef PS3DISK_DEBUG
74223313Snwhitehornstatic int ps3disk_debug = 0;
75223313SnwhitehornSYSCTL_INT(_hw_ps3disk, OID_AUTO, debug, CTLFLAG_RW, &ps3disk_debug,
76223313Snwhitehorn	0, "control debugging printfs");
77223313SnwhitehornTUNABLE_INT("hw.ps3disk.debug", &ps3disk_debug);
78223313Snwhitehornenum {
79223313Snwhitehorn	PS3DISK_DEBUG_INTR	= 0x00000001,
80223313Snwhitehorn	PS3DISK_DEBUG_TASK	= 0x00000002,
81223313Snwhitehorn	PS3DISK_DEBUG_READ	= 0x00000004,
82223313Snwhitehorn	PS3DISK_DEBUG_WRITE	= 0x00000008,
83223313Snwhitehorn	PS3DISK_DEBUG_FLUSH	= 0x00000010,
84223313Snwhitehorn	PS3DISK_DEBUG_ANY	= 0xffffffff
85223313Snwhitehorn};
86223313Snwhitehorn#define	DPRINTF(sc, m, fmt, ...)				\
87223313Snwhitehorndo {								\
88223313Snwhitehorn	if (sc->sc_debug & (m))					\
89223313Snwhitehorn		printf(fmt, __VA_ARGS__);			\
90223313Snwhitehorn} while (0)
91223313Snwhitehorn#else
92223313Snwhitehorn#define	DPRINTF(sc, m, fmt, ...)
93223313Snwhitehorn#endif
94223313Snwhitehorn
95223313Snwhitehornstruct ps3disk_region {
96223313Snwhitehorn	uint64_t r_id;
97223313Snwhitehorn	uint64_t r_start;
98223313Snwhitehorn	uint64_t r_size;
99223313Snwhitehorn	uint64_t r_flags;
100223313Snwhitehorn};
101223313Snwhitehorn
102223313Snwhitehornstruct ps3disk_softc {
103223313Snwhitehorn	device_t sc_dev;
104223313Snwhitehorn
105223313Snwhitehorn	struct mtx sc_mtx;
106223313Snwhitehorn
107223313Snwhitehorn	uint64_t sc_blksize;
108223313Snwhitehorn	uint64_t sc_nblocks;
109223313Snwhitehorn
110223313Snwhitehorn	uint64_t sc_nregs;
111223313Snwhitehorn	struct ps3disk_region *sc_reg;
112223313Snwhitehorn
113223313Snwhitehorn	int sc_irqid;
114223313Snwhitehorn	struct resource	*sc_irq;
115223313Snwhitehorn	void *sc_irqctx;
116223313Snwhitehorn
117223313Snwhitehorn	struct disk **sc_disk;
118223313Snwhitehorn
119223313Snwhitehorn	struct bio_queue_head sc_bioq;
120223461Snwhitehorn	struct bio_queue_head sc_deferredq;
121223461Snwhitehorn	struct proc *sc_task;
122223313Snwhitehorn
123223461Snwhitehorn	bus_dma_tag_t sc_dmatag;
124223313Snwhitehorn
125223313Snwhitehorn	int sc_running;
126223313Snwhitehorn	int sc_debug;
127223313Snwhitehorn};
128223313Snwhitehorn
129223313Snwhitehornstatic int ps3disk_open(struct disk *dp);
130223313Snwhitehornstatic int ps3disk_close(struct disk *dp);
131223313Snwhitehornstatic void ps3disk_strategy(struct bio *bp);
132223461Snwhitehorn
133223313Snwhitehornstatic void ps3disk_task(void *arg);
134223313Snwhitehornstatic void ps3disk_intr(void *arg);
135223313Snwhitehornstatic int ps3disk_get_disk_geometry(struct ps3disk_softc *sc);
136223313Snwhitehornstatic int ps3disk_enum_regions(struct ps3disk_softc *sc);
137223461Snwhitehornstatic void ps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs,
138223461Snwhitehorn    int error);
139223313Snwhitehorn
140223313Snwhitehornstatic void ps3disk_sysctlattach(struct ps3disk_softc *sc);
141223313Snwhitehorn
142223313Snwhitehornstatic MALLOC_DEFINE(M_PS3DISK, "ps3disk", "PS3 Disk");
143223313Snwhitehorn
144223313Snwhitehornstatic int
145223313Snwhitehornps3disk_probe(device_t dev)
146223313Snwhitehorn{
147223313Snwhitehorn	if (ps3bus_get_bustype(dev) != PS3_BUSTYPE_STORAGE ||
148223313Snwhitehorn	    ps3bus_get_devtype(dev) != PS3_DEVTYPE_DISK)
149223313Snwhitehorn		return (ENXIO);
150223313Snwhitehorn
151223313Snwhitehorn	device_set_desc(dev, "Playstation 3 Disk");
152223313Snwhitehorn
153223313Snwhitehorn	return (BUS_PROBE_SPECIFIC);
154223313Snwhitehorn}
155223313Snwhitehorn
156223313Snwhitehornstatic int
157223313Snwhitehornps3disk_attach(device_t dev)
158223313Snwhitehorn{
159223313Snwhitehorn	struct ps3disk_softc *sc;
160223313Snwhitehorn	struct disk *d;
161223313Snwhitehorn	intmax_t mb;
162223461Snwhitehorn	uint64_t junk;
163223313Snwhitehorn	char unit;
164223313Snwhitehorn	int i, err;
165223313Snwhitehorn
166223313Snwhitehorn	sc = device_get_softc(dev);
167223313Snwhitehorn	sc->sc_dev = dev;
168223313Snwhitehorn
169223313Snwhitehorn	PS3DISK_LOCK_INIT(sc);
170223313Snwhitehorn
171223313Snwhitehorn	err = ps3disk_get_disk_geometry(sc);
172223313Snwhitehorn	if (err) {
173223313Snwhitehorn		device_printf(dev, "Could not get disk geometry\n");
174223313Snwhitehorn		err = ENXIO;
175223313Snwhitehorn		goto fail_destroy_lock;
176223313Snwhitehorn	}
177223313Snwhitehorn
178223313Snwhitehorn	device_printf(dev, "block size %lu total blocks %lu\n",
179223313Snwhitehorn	    sc->sc_blksize, sc->sc_nblocks);
180223313Snwhitehorn
181223313Snwhitehorn	err = ps3disk_enum_regions(sc);
182223313Snwhitehorn	if (err) {
183223313Snwhitehorn		device_printf(dev, "Could not enumerate disk regions\n");
184223313Snwhitehorn		err = ENXIO;
185223313Snwhitehorn		goto fail_destroy_lock;
186223313Snwhitehorn	}
187223313Snwhitehorn
188223313Snwhitehorn	device_printf(dev, "Found %lu regions\n", sc->sc_nregs);
189223313Snwhitehorn
190223313Snwhitehorn	if (!sc->sc_nregs) {
191223313Snwhitehorn		err = ENXIO;
192223313Snwhitehorn		goto fail_destroy_lock;
193223313Snwhitehorn	}
194223313Snwhitehorn
195223313Snwhitehorn	/* Setup interrupt handler */
196223313Snwhitehorn	sc->sc_irqid = 0;
197223313Snwhitehorn	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqid,
198223313Snwhitehorn	    RF_ACTIVE);
199223313Snwhitehorn	if (!sc->sc_irq) {
200223313Snwhitehorn		device_printf(dev, "Could not allocate IRQ\n");
201223313Snwhitehorn		err = ENXIO;
202223313Snwhitehorn		goto fail_free_regions;
203223313Snwhitehorn	}
204223313Snwhitehorn
205223313Snwhitehorn	err = bus_setup_intr(dev, sc->sc_irq,
206223313Snwhitehorn	    INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY,
207223461Snwhitehorn	    NULL, ps3disk_intr, sc, &sc->sc_irqctx);
208223313Snwhitehorn	if (err) {
209223313Snwhitehorn		device_printf(dev, "Could not setup IRQ\n");
210223313Snwhitehorn		err = ENXIO;
211223313Snwhitehorn		goto fail_release_intr;
212223313Snwhitehorn	}
213223313Snwhitehorn
214223461Snwhitehorn	/* Setup DMA */
215223313Snwhitehorn	err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0,
216223313Snwhitehorn	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
217223461Snwhitehorn	    BUS_SPACE_UNRESTRICTED, 1, PAGE_SIZE, 0,
218223461Snwhitehorn	    busdma_lock_mutex, &sc->sc_mtx, &sc->sc_dmatag);
219223313Snwhitehorn	if (err) {
220223461Snwhitehorn		device_printf(dev, "Could not create DMA tag\n");
221223313Snwhitehorn		err = ENXIO;
222223313Snwhitehorn		goto fail_teardown_intr;
223223313Snwhitehorn	}
224223313Snwhitehorn
225223313Snwhitehorn	/* Setup disks */
226223313Snwhitehorn
227223313Snwhitehorn	sc->sc_disk = malloc(sc->sc_nregs * sizeof(struct disk *),
228223313Snwhitehorn	    M_PS3DISK, M_ZERO | M_WAITOK);
229223313Snwhitehorn	if (!sc->sc_disk) {
230223313Snwhitehorn		device_printf(dev, "Could not allocate disk(s)\n");
231223313Snwhitehorn		err = ENOMEM;
232223461Snwhitehorn		goto fail_teardown_intr;
233223313Snwhitehorn	}
234223313Snwhitehorn
235223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++) {
236223316Snwhitehorn		struct ps3disk_region *rp = &sc->sc_reg[i];
237223316Snwhitehorn
238223313Snwhitehorn		d = sc->sc_disk[i] = disk_alloc();
239223313Snwhitehorn		d->d_open = ps3disk_open;
240223313Snwhitehorn		d->d_close = ps3disk_close;
241223313Snwhitehorn		d->d_strategy = ps3disk_strategy;
242223313Snwhitehorn		d->d_name = "ps3disk";
243223313Snwhitehorn		d->d_drv1 = sc;
244223461Snwhitehorn		d->d_maxsize = PAGE_SIZE;
245223313Snwhitehorn		d->d_sectorsize = sc->sc_blksize;
246223313Snwhitehorn		d->d_unit = i;
247223313Snwhitehorn		d->d_mediasize = sc->sc_reg[i].r_size * sc->sc_blksize;
248223313Snwhitehorn		d->d_flags |= DISKFLAG_CANFLUSHCACHE;
249223313Snwhitehorn
250223313Snwhitehorn		mb = d->d_mediasize >> 20;
251223313Snwhitehorn		unit = 'M';
252223313Snwhitehorn		if (mb >= 10240) {
253223313Snwhitehorn			unit = 'G';
254223313Snwhitehorn			mb /= 1024;
255223313Snwhitehorn		}
256223313Snwhitehorn
257223316Snwhitehorn		/* Test to see if we can read this region */
258223316Snwhitehorn		err = lv1_storage_read(ps3bus_get_device(dev), d->d_unit,
259223461Snwhitehorn		    0, 0, rp->r_flags, 0, &junk);
260223316Snwhitehorn		device_printf(dev, "region %d %ju%cB%s\n", i, mb, unit,
261223461Snwhitehorn		    (err == LV1_DENIED_BY_POLICY) ?  " (hypervisor protected)"
262223461Snwhitehorn		    : "");
263223313Snwhitehorn
264223461Snwhitehorn		if (err != LV1_DENIED_BY_POLICY)
265223316Snwhitehorn			disk_create(d, DISK_VERSION);
266223313Snwhitehorn	}
267223316Snwhitehorn	err = 0;
268223313Snwhitehorn
269223313Snwhitehorn	bioq_init(&sc->sc_bioq);
270223461Snwhitehorn	bioq_init(&sc->sc_deferredq);
271223461Snwhitehorn	kproc_create(&ps3disk_task, sc, &sc->sc_task, 0, 0, "ps3disk");
272223313Snwhitehorn
273223313Snwhitehorn	ps3disk_sysctlattach(sc);
274223313Snwhitehorn	sc->sc_running = 1;
275223313Snwhitehorn	return (0);
276223313Snwhitehorn
277223313Snwhitehornfail_teardown_intr:
278223313Snwhitehorn	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
279223313Snwhitehornfail_release_intr:
280223313Snwhitehorn	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
281223313Snwhitehornfail_free_regions:
282223313Snwhitehorn	free(sc->sc_reg, M_PS3DISK);
283223313Snwhitehornfail_destroy_lock:
284223313Snwhitehorn	PS3DISK_LOCK_DESTROY(sc);
285223313Snwhitehorn	return (err);
286223313Snwhitehorn}
287223313Snwhitehorn
288223313Snwhitehornstatic int
289223313Snwhitehornps3disk_detach(device_t dev)
290223313Snwhitehorn{
291223313Snwhitehorn	struct ps3disk_softc *sc = device_get_softc(dev);
292223313Snwhitehorn	int i;
293223313Snwhitehorn
294223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++)
295223313Snwhitehorn		disk_destroy(sc->sc_disk[i]);
296223313Snwhitehorn
297223461Snwhitehorn	bus_dma_tag_destroy(sc->sc_dmatag);
298223313Snwhitehorn
299223313Snwhitehorn	bus_teardown_intr(dev, sc->sc_irq, sc->sc_irqctx);
300223313Snwhitehorn	bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irqid, sc->sc_irq);
301223313Snwhitehorn
302223313Snwhitehorn	free(sc->sc_disk, M_PS3DISK);
303223313Snwhitehorn	free(sc->sc_reg, M_PS3DISK);
304223313Snwhitehorn
305223313Snwhitehorn	PS3DISK_LOCK_DESTROY(sc);
306223313Snwhitehorn
307223313Snwhitehorn	return (0);
308223313Snwhitehorn}
309223313Snwhitehorn
310223313Snwhitehornstatic int
311223313Snwhitehornps3disk_open(struct disk *dp)
312223313Snwhitehorn{
313223313Snwhitehorn	return (0);
314223313Snwhitehorn}
315223313Snwhitehorn
316223313Snwhitehornstatic int
317223313Snwhitehornps3disk_close(struct disk *dp)
318223313Snwhitehorn{
319223313Snwhitehorn	return (0);
320223313Snwhitehorn}
321223313Snwhitehorn
322223461Snwhitehorn/* Process deferred blocks */
323223313Snwhitehornstatic void
324223313Snwhitehornps3disk_task(void *arg)
325223313Snwhitehorn{
326223313Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
327223313Snwhitehorn	struct bio *bp;
328223313Snwhitehorn
329223461Snwhitehorn
330223461Snwhitehorn	while (1) {
331223461Snwhitehorn		kproc_suspend_check(sc->sc_task);
332223461Snwhitehorn		tsleep(&sc->sc_deferredq, PRIBIO, "ps3disk", 10);
333223461Snwhitehorn
334223313Snwhitehorn		PS3DISK_LOCK(sc);
335223461Snwhitehorn		bp = bioq_takefirst(&sc->sc_deferredq);
336223313Snwhitehorn		PS3DISK_UNLOCK(sc);
337223313Snwhitehorn
338223461Snwhitehorn		if (bp == NULL)
339223461Snwhitehorn			continue;
340223313Snwhitehorn
341223461Snwhitehorn		if (bp->bio_driver1 != NULL) {
342223461Snwhitehorn			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
343223461Snwhitehorn			    bp->bio_driver1);
344223461Snwhitehorn			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
345223461Snwhitehorn			    bp->bio_driver1);
346223461Snwhitehorn		}
347223313Snwhitehorn
348223461Snwhitehorn		ps3disk_strategy(bp);
349223461Snwhitehorn	}
350223313Snwhitehorn
351223461Snwhitehorn	kproc_exit(0);
352223461Snwhitehorn}
353223313Snwhitehorn
354223461Snwhitehornstatic void
355223461Snwhitehornps3disk_strategy(struct bio *bp)
356223461Snwhitehorn{
357223461Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
358223461Snwhitehorn	int err;
359223313Snwhitehorn
360223461Snwhitehorn	if (sc == NULL) {
361223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
362223461Snwhitehorn		bp->bio_error = EINVAL;
363223461Snwhitehorn		biodone(bp);
364223461Snwhitehorn		return;
365223461Snwhitehorn	}
366223313Snwhitehorn
367223461Snwhitehorn	PS3DISK_LOCK(sc);
368223461Snwhitehorn	bp->bio_resid = bp->bio_bcount;
369223461Snwhitehorn	bioq_insert_tail(&sc->sc_bioq, bp);
370223313Snwhitehorn
371223461Snwhitehorn	DPRINTF(sc, PS3DISK_DEBUG_TASK, "%s: bio_cmd 0x%02x\n",
372223461Snwhitehorn	    __func__, bp->bio_cmd);
373223313Snwhitehorn
374223461Snwhitehorn	err = 0;
375223461Snwhitehorn	if (bp->bio_cmd == BIO_FLUSH) {
376223461Snwhitehorn		bp->bio_driver1 = 0;
377223461Snwhitehorn		err = lv1_storage_send_device_command(
378223461Snwhitehorn		    ps3bus_get_device(sc->sc_dev), LV1_STORAGE_ATA_HDDOUT,
379223461Snwhitehorn		    0, 0, 0, 0, (uint64_t *)&bp->bio_driver2);
380223461Snwhitehorn		if (err == LV1_BUSY)
381223461Snwhitehorn			err = EAGAIN;
382223461Snwhitehorn	} else if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
383223461Snwhitehorn		if (bp->bio_bcount % sc->sc_blksize != 0) {
384223461Snwhitehorn			err = EINVAL;
385223313Snwhitehorn		} else {
386223461Snwhitehorn			bus_dmamap_create(sc->sc_dmatag, BUS_DMA_COHERENT,
387223461Snwhitehorn			    (bus_dmamap_t *)(&bp->bio_driver1));
388223461Snwhitehorn			err = bus_dmamap_load(sc->sc_dmatag,
389223461Snwhitehorn			    (bus_dmamap_t)(bp->bio_driver1), bp->bio_data,
390223461Snwhitehorn			    bp->bio_bcount, ps3disk_transfer, bp, 0);
391223461Snwhitehorn			if (err == EINPROGRESS)
392223461Snwhitehorn				err = 0;
393223313Snwhitehorn		}
394223461Snwhitehorn	} else {
395223461Snwhitehorn		err = EINVAL;
396223461Snwhitehorn	}
397223313Snwhitehorn
398223461Snwhitehorn	if (err == EAGAIN) {
399223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
400223461Snwhitehorn		bioq_insert_tail(&sc->sc_deferredq, bp);
401223461Snwhitehorn	} else if (err != 0) {
402223461Snwhitehorn		bp->bio_error = err;
403223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
404223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
405223461Snwhitehorn		disk_err(bp, "hard error", -1, 1);
406223313Snwhitehorn		biodone(bp);
407223313Snwhitehorn	}
408223313Snwhitehorn
409223313Snwhitehorn	PS3DISK_UNLOCK(sc);
410223313Snwhitehorn}
411223313Snwhitehorn
412223313Snwhitehornstatic void
413223313Snwhitehornps3disk_intr(void *arg)
414223313Snwhitehorn{
415223313Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *) arg;
416223313Snwhitehorn	device_t dev = sc->sc_dev;
417223313Snwhitehorn	uint64_t devid = ps3bus_get_device(dev);
418223461Snwhitehorn	struct bio *bp;
419223313Snwhitehorn	uint64_t tag, status;
420223313Snwhitehorn
421223461Snwhitehorn	if (lv1_storage_get_async_status(devid, &tag, &status) != 0)
422223461Snwhitehorn		return;
423223461Snwhitehorn
424223313Snwhitehorn	PS3DISK_LOCK(sc);
425223313Snwhitehorn
426223461Snwhitehorn	DPRINTF(sc, PS3DISK_DEBUG_INTR, "%s: tag 0x%016lx "
427223461Snwhitehorn	    "status 0x%016lx\n", __func__, tag, status);
428223313Snwhitehorn
429223461Snwhitehorn	/* Locate the matching request */
430223461Snwhitehorn	TAILQ_FOREACH(bp, &sc->sc_bioq.queue, bio_queue) {
431223461Snwhitehorn		if ((uint64_t)bp->bio_driver2 != tag)
432223461Snwhitehorn			continue;
433223313Snwhitehorn
434223461Snwhitehorn		if (status != 0) {
435223461Snwhitehorn			device_printf(sc->sc_dev, "%s error (%#lx)\n",
436223461Snwhitehorn			    (bp->bio_cmd == BIO_READ) ? "Read" : "Write",
437223461Snwhitehorn			    status);
438223461Snwhitehorn			bp->bio_error = EIO;
439223461Snwhitehorn			bp->bio_flags |= BIO_ERROR;
440223461Snwhitehorn		} else {
441223461Snwhitehorn			bp->bio_error = 0;
442223461Snwhitehorn			bp->bio_resid = 0;
443223461Snwhitehorn			bp->bio_flags |= BIO_DONE;
444223461Snwhitehorn		}
445223313Snwhitehorn
446223461Snwhitehorn		if (bp->bio_driver1 != NULL) {
447223461Snwhitehorn			if (bp->bio_cmd == BIO_READ)
448223461Snwhitehorn				bus_dmamap_sync(sc->sc_dmatag, (bus_dmamap_t)
449223461Snwhitehorn				    bp->bio_driver1, BUS_DMASYNC_POSTREAD);
450223461Snwhitehorn			bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
451223461Snwhitehorn			    bp->bio_driver1);
452223461Snwhitehorn			bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
453223461Snwhitehorn			    bp->bio_driver1);
454223461Snwhitehorn		}
455223461Snwhitehorn
456223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
457223461Snwhitehorn		biodone(bp);
458223461Snwhitehorn		break;
459223313Snwhitehorn	}
460223313Snwhitehorn
461223461Snwhitehorn	if (bioq_first(&sc->sc_deferredq) != NULL)
462223461Snwhitehorn		wakeup(&sc->sc_deferredq);
463223313Snwhitehorn
464223313Snwhitehorn	PS3DISK_UNLOCK(sc);
465223313Snwhitehorn}
466223313Snwhitehorn
467223313Snwhitehornstatic int
468223313Snwhitehornps3disk_get_disk_geometry(struct ps3disk_softc *sc)
469223313Snwhitehorn{
470223313Snwhitehorn	device_t dev = sc->sc_dev;
471223313Snwhitehorn	uint64_t bus_index = ps3bus_get_busidx(dev);
472223313Snwhitehorn	uint64_t dev_index = ps3bus_get_devidx(dev);
473223313Snwhitehorn	uint64_t junk;
474223313Snwhitehorn	int err;
475223313Snwhitehorn
476223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
477223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
478223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
479223313Snwhitehorn	    lv1_repository_string("blk_size"), 0, &sc->sc_blksize, &junk);
480223313Snwhitehorn	if (err) {
481223313Snwhitehorn		device_printf(dev, "Could not get block size (0x%08x)\n", err);
482223461Snwhitehorn		return (ENXIO);
483223313Snwhitehorn	}
484223313Snwhitehorn
485223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
486223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
487223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
488223313Snwhitehorn	    lv1_repository_string("n_blocks"), 0, &sc->sc_nblocks, &junk);
489223313Snwhitehorn	if (err) {
490223461Snwhitehorn		device_printf(dev, "Could not get total number of blocks "
491223461Snwhitehorn		    "(0x%08x)\n", err);
492223313Snwhitehorn		err = ENXIO;
493223313Snwhitehorn	}
494223313Snwhitehorn
495223313Snwhitehorn	return (err);
496223313Snwhitehorn}
497223313Snwhitehorn
498223313Snwhitehornstatic int
499223313Snwhitehornps3disk_enum_regions(struct ps3disk_softc *sc)
500223313Snwhitehorn{
501223313Snwhitehorn	device_t dev = sc->sc_dev;
502223313Snwhitehorn	uint64_t bus_index = ps3bus_get_busidx(dev);
503223313Snwhitehorn	uint64_t dev_index = ps3bus_get_devidx(dev);
504223313Snwhitehorn	uint64_t junk;
505223313Snwhitehorn	int i, err;
506223313Snwhitehorn
507223313Snwhitehorn	/* Read number of regions */
508223313Snwhitehorn
509223313Snwhitehorn	err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
510223313Snwhitehorn	    (lv1_repository_string("bus") >> 32) | bus_index,
511223313Snwhitehorn	    lv1_repository_string("dev") | dev_index,
512223313Snwhitehorn	    lv1_repository_string("n_regs"), 0, &sc->sc_nregs, &junk);
513223313Snwhitehorn	if (err) {
514223313Snwhitehorn		device_printf(dev, "Could not get number of regions (0x%08x)\n",
515223313Snwhitehorn		    err);
516223313Snwhitehorn		err = ENXIO;
517223313Snwhitehorn		goto fail;
518223313Snwhitehorn	}
519223313Snwhitehorn
520223313Snwhitehorn	if (!sc->sc_nregs)
521223313Snwhitehorn		return 0;
522223313Snwhitehorn
523223313Snwhitehorn	sc->sc_reg = malloc(sc->sc_nregs * sizeof(struct ps3disk_region),
524223313Snwhitehorn	    M_PS3DISK, M_ZERO | M_WAITOK);
525223313Snwhitehorn	if (!sc->sc_reg) {
526223313Snwhitehorn		err = ENOMEM;
527223313Snwhitehorn		goto fail;
528223313Snwhitehorn	}
529223313Snwhitehorn
530223313Snwhitehorn	/* Setup regions */
531223313Snwhitehorn
532223313Snwhitehorn	for (i = 0; i < sc->sc_nregs; i++) {
533223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
534223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
535223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
536223313Snwhitehorn		    lv1_repository_string("region") | i,
537223313Snwhitehorn		    lv1_repository_string("id"), &sc->sc_reg[i].r_id, &junk);
538223313Snwhitehorn		if (err) {
539223313Snwhitehorn			device_printf(dev, "Could not get region id (0x%08x)\n",
540223313Snwhitehorn			    err);
541223313Snwhitehorn			err = ENXIO;
542223313Snwhitehorn			goto fail;
543223313Snwhitehorn		}
544223313Snwhitehorn
545223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
546223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
547223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
548223313Snwhitehorn		    lv1_repository_string("region") | i,
549223461Snwhitehorn		    lv1_repository_string("start"), &sc->sc_reg[i].r_start,
550223461Snwhitehorn		    &junk);
551223313Snwhitehorn		if (err) {
552223461Snwhitehorn			device_printf(dev, "Could not get region start "
553223461Snwhitehorn			    "(0x%08x)\n", err);
554223313Snwhitehorn			err = ENXIO;
555223313Snwhitehorn			goto fail;
556223313Snwhitehorn		}
557223313Snwhitehorn
558223313Snwhitehorn		err = lv1_get_repository_node_value(PS3_LPAR_ID_PME,
559223313Snwhitehorn		    (lv1_repository_string("bus") >> 32) | bus_index,
560223313Snwhitehorn		    lv1_repository_string("dev") | dev_index,
561223313Snwhitehorn		    lv1_repository_string("region") | i,
562223461Snwhitehorn		    lv1_repository_string("size"), &sc->sc_reg[i].r_size,
563223461Snwhitehorn		    &junk);
564223313Snwhitehorn		if (err) {
565223461Snwhitehorn			device_printf(dev, "Could not get region size "
566223461Snwhitehorn			    "(0x%08x)\n", err);
567223313Snwhitehorn			err = ENXIO;
568223313Snwhitehorn			goto fail;
569223313Snwhitehorn		}
570223313Snwhitehorn
571223313Snwhitehorn		if (i == 0)
572223313Snwhitehorn			sc->sc_reg[i].r_flags = 0x2;
573223313Snwhitehorn		else
574223313Snwhitehorn			sc->sc_reg[i].r_flags = 0;
575223313Snwhitehorn	}
576223313Snwhitehorn
577223313Snwhitehorn	return (0);
578223313Snwhitehorn
579223313Snwhitehornfail:
580223313Snwhitehorn
581223313Snwhitehorn	sc->sc_nregs = 0;
582223313Snwhitehorn	if (sc->sc_reg)
583223313Snwhitehorn		free(sc->sc_reg, M_PS3DISK);
584223313Snwhitehorn
585223313Snwhitehorn	return (err);
586223313Snwhitehorn}
587223313Snwhitehorn
588223461Snwhitehornstatic void
589223461Snwhitehornps3disk_transfer(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
590223313Snwhitehorn{
591223461Snwhitehorn	struct bio *bp = (struct bio *)(arg);
592223461Snwhitehorn	struct ps3disk_softc *sc = (struct ps3disk_softc *)bp->bio_disk->d_drv1;
593223461Snwhitehorn	struct ps3disk_region *rp = &sc->sc_reg[bp->bio_disk->d_unit];
594223461Snwhitehorn	uint64_t devid = ps3bus_get_device(sc->sc_dev);
595223461Snwhitehorn	uint64_t block;
596223461Snwhitehorn	int i, err;
597223313Snwhitehorn
598223461Snwhitehorn	/* Locks already held by busdma */
599223461Snwhitehorn	PS3DISK_ASSERT_LOCKED(sc);
600223313Snwhitehorn
601223461Snwhitehorn	if (error) {
602223461Snwhitehorn		bp->bio_error = error;
603223461Snwhitehorn		bp->bio_flags |= BIO_ERROR;
604223461Snwhitehorn		bioq_remove(&sc->sc_bioq, bp);
605223461Snwhitehorn		biodone(bp);
606223461Snwhitehorn		return;
607223313Snwhitehorn	}
608223313Snwhitehorn
609223461Snwhitehorn	block = bp->bio_pblkno;
610223461Snwhitehorn	for (i = 0; i < nsegs; i++) {
611223461Snwhitehorn		KASSERT((segs[i].ds_len % sc->sc_blksize) == 0,
612223461Snwhitehorn		    ("DMA fragments not blocksize multiples"));
613223313Snwhitehorn
614223461Snwhitehorn		if (bp->bio_cmd == BIO_READ) {
615223461Snwhitehorn			err = lv1_storage_read(devid, rp->r_id,
616223461Snwhitehorn			    block, segs[i].ds_len/sc->sc_blksize,
617223461Snwhitehorn			    rp->r_flags, segs[i].ds_addr,
618223461Snwhitehorn			    (uint64_t *)&bp->bio_driver2);
619223461Snwhitehorn		} else {
620223461Snwhitehorn			bus_dmamap_sync(sc->sc_dmatag,
621223461Snwhitehorn			    (bus_dmamap_t)bp->bio_driver1,
622223461Snwhitehorn			    BUS_DMASYNC_PREWRITE);
623223461Snwhitehorn			err = lv1_storage_write(devid, rp->r_id,
624223461Snwhitehorn			    block, segs[i].ds_len/sc->sc_blksize,
625223461Snwhitehorn			    rp->r_flags, segs[i].ds_addr,
626223461Snwhitehorn			    (uint64_t *)&bp->bio_driver2);
627223461Snwhitehorn		}
628223313Snwhitehorn
629223461Snwhitehorn		if (err) {
630223461Snwhitehorn			if (err == LV1_BUSY) {
631223461Snwhitehorn				bioq_remove(&sc->sc_bioq, bp);
632223461Snwhitehorn				bioq_insert_tail(&sc->sc_deferredq, bp);
633223461Snwhitehorn			} else {
634223461Snwhitehorn				bus_dmamap_unload(sc->sc_dmatag, (bus_dmamap_t)
635223461Snwhitehorn				    bp->bio_driver1);
636223461Snwhitehorn				bus_dmamap_destroy(sc->sc_dmatag, (bus_dmamap_t)
637223461Snwhitehorn				    bp->bio_driver1);
638223461Snwhitehorn				device_printf(sc->sc_dev, "Could not read "
639223461Snwhitehorn				    "sectors (0x%08x)\n", err);
640223461Snwhitehorn				bp->bio_error = EINVAL;
641223461Snwhitehorn				bp->bio_flags |= BIO_ERROR;
642223461Snwhitehorn				bioq_remove(&sc->sc_bioq, bp);
643223461Snwhitehorn				biodone(bp);
644223461Snwhitehorn			}
645223313Snwhitehorn
646223461Snwhitehorn			break;
647223461Snwhitehorn		}
648223313Snwhitehorn
649223461Snwhitehorn		DPRINTF(sc, PS3DISK_DEBUG_READ, "%s: tag 0x%016lx\n",
650223461Snwhitehorn		    __func__, sc->sc_bounce_tag);
651223313Snwhitehorn	}
652223313Snwhitehorn}
653223313Snwhitehorn
654223313Snwhitehorn#ifdef PS3DISK_DEBUG
655223313Snwhitehornstatic int
656223313Snwhitehornps3disk_sysctl_debug(SYSCTL_HANDLER_ARGS)
657223313Snwhitehorn{
658223313Snwhitehorn	struct ps3disk_softc *sc = arg1;
659223313Snwhitehorn	int debug, error;
660223313Snwhitehorn
661223313Snwhitehorn	debug = sc->sc_debug;
662223313Snwhitehorn
663223313Snwhitehorn	error = sysctl_handle_int(oidp, &debug, 0, req);
664223313Snwhitehorn	if (error || !req->newptr)
665223313Snwhitehorn		return error;
666223313Snwhitehorn
667223313Snwhitehorn	sc->sc_debug = debug;
668223313Snwhitehorn
669223313Snwhitehorn	return 0;
670223313Snwhitehorn}
671223313Snwhitehorn#endif
672223313Snwhitehorn
673223313Snwhitehornstatic void
674223313Snwhitehornps3disk_sysctlattach(struct ps3disk_softc *sc)
675223313Snwhitehorn{
676223313Snwhitehorn#ifdef PS3DISK_DEBUG
677223313Snwhitehorn	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
678223313Snwhitehorn	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
679223313Snwhitehorn
680223313Snwhitehorn	sc->sc_debug = ps3disk_debug;
681223313Snwhitehorn
682223313Snwhitehorn	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
683223313Snwhitehorn		"debug", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
684223313Snwhitehorn		ps3disk_sysctl_debug, "I", "control debugging printfs");
685223313Snwhitehorn#endif
686223313Snwhitehorn}
687223313Snwhitehorn
688223313Snwhitehornstatic device_method_t ps3disk_methods[] = {
689223313Snwhitehorn	DEVMETHOD(device_probe,		ps3disk_probe),
690223313Snwhitehorn	DEVMETHOD(device_attach,	ps3disk_attach),
691223313Snwhitehorn	DEVMETHOD(device_detach,	ps3disk_detach),
692223313Snwhitehorn	{0, 0},
693223313Snwhitehorn};
694223313Snwhitehorn
695223313Snwhitehornstatic driver_t ps3disk_driver = {
696223313Snwhitehorn	"ps3disk",
697223313Snwhitehorn	ps3disk_methods,
698223313Snwhitehorn	sizeof(struct ps3disk_softc),
699223313Snwhitehorn};
700223313Snwhitehorn
701223313Snwhitehornstatic devclass_t ps3disk_devclass;
702223313Snwhitehorn
703223313SnwhitehornDRIVER_MODULE(ps3disk, ps3bus, ps3disk_driver, ps3disk_devclass, 0, 0);
704