1/*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28// 45678901234567890123456789012345678901234567890123456789012345678901234567890
29
30#include "IOCopyMapper.h"
31#include <sys/sysctl.h>
32
33#if 0
34#define DEBG(fmt, args...)  	{ kprintf(fmt, ## args); }
35#else
36#define DEBG(fmt, args...)  	{}
37#endif
38
39extern "C" {
40extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
41extern void ml_get_bouncepool_info(
42			       vm_offset_t *phys_addr,
43			       vm_size_t   *size);
44extern unsigned int vm_lopage_max_count;
45extern unsigned int vm_himemory_mode;
46}
47
48#define super IOMapper
49
50OSDefineMetaClassAndStructors(IOCopyMapper, IOMapper);
51
52// Remember no value can be bigger than 31 bits as the sign bit indicates
53// that this entry is valid to the hardware and that would be bad if it wasn't
54typedef struct FreeDARTEntry {
55#if __BIG_ENDIAN__
56    unsigned int
57    /* bool */	    fValid : 1,
58    /* bool */	    fInUse : 1,	// Allocated but not inserted yet
59    /* bool */		   : 5,	// Align size on nibble boundary for debugging
60    /* uint */	    fSize  : 5,
61    /* uint */	           : 2,
62    /* uint */	    fNext  :18;	// offset of FreeDARTEntry's
63
64#elif __LITTLE_ENDIAN__
65    unsigned int
66    /* uint */	    fNext  :18,	// offset of FreeDARTEntry's
67    /* uint */	           : 2,
68    /* uint */	    fSize  : 5,
69    /* bool */		   : 5,	// Align size on nibble boundary for debugging
70    /* bool */	    fInUse : 1,	// Allocated but not inserted yet
71    /* bool */	    fValid : 1;
72#endif
73#if __BIG_ENDIAN__
74    unsigned int
75    /* uint */	           :14,
76    /* uint */	    fPrev  :18;	// offset of FreeDARTEntry's
77
78#elif __LITTLE_ENDIAN__
79    unsigned int
80    /* uint */	    fPrev  :18,	// offset of FreeDARTEntry's
81    /* uint */	           :14;
82#endif
83} FreeDARTEntry;
84
85typedef struct ActiveDARTEntry {
86#if __BIG_ENDIAN__
87    unsigned int
88    /* bool */	    fValid : 1,	// Must be set to one if valid
89    /* uint */	    fPPNum :31;	// ppnum_t page of translation
90#define ACTIVEDARTENTRY(page)	{ true, page }
91
92#elif __LITTLE_ENDIAN__
93    unsigned int
94    /* uint */	    fPPNum :31,	// ppnum_t page of translation
95    /* bool */	    fValid : 1;	// Must be set to one if valid
96#define ACTIVEDARTENTRY(page)	{ page, true }
97
98#endif
99};
100
101#define kActivePerFree (sizeof(freeDART[0]) / sizeof(ActiveDARTEntry))
102
103static SYSCTL_UINT(_kern, OID_AUTO, copyregionmax,
104				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
105				 NULL, 0, "");
106
107static SYSCTL_UINT(_kern, OID_AUTO, lowpagemax,
108				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
109				&vm_lopage_max_count, 0, "");
110
111static SYSCTL_UINT(_kern, OID_AUTO, himemorymode,
112				CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN,
113				&vm_himemory_mode, 0, "");
114
115bool IOCopyMapper::initHardware(IOService * provider)
116{
117    UInt32 dartSizePages = 0;
118
119    vm_offset_t phys_addr;
120    vm_size_t   size;
121    ml_get_bouncepool_info(&phys_addr, &size);
122
123    if (!size)
124	return (false);
125
126    fBufferPage = atop_32(phys_addr);
127    dartSizePages = (atop_32(size) + kTransPerPage - 1) / kTransPerPage;
128
129    fTableLock = IOLockAlloc();
130
131    if (!fTableLock)
132	return false;
133
134    if (!allocTable(dartSizePages * kMapperPage))
135	return false;
136
137    UInt32 canMapPages = dartSizePages * kTransPerPage;
138    fMapperRegionSize = canMapPages;
139    for (fNumZones = 0; canMapPages; fNumZones++)
140	canMapPages >>= 1;
141    fNumZones -= 3; // correct for overshoot and minumum 16K pages allocation
142
143    invalidateDART(0, fMapperRegionSize);
144
145    breakUp(0, fNumZones, 0);
146    ((FreeDARTEntry *) fTable)->fInUse = true;
147
148    fMapperRegionUsed    = kMinZoneSize;
149    fMapperRegionMaxUsed = fMapperRegionUsed;
150
151    sysctl__kern_copyregionmax.oid_arg1 = &fMapperRegionMaxUsed;
152
153    sysctl_register_oid(&sysctl__kern_copyregionmax);
154    sysctl_register_oid(&sysctl__kern_lowpagemax);
155    sysctl_register_oid(&sysctl__kern_himemorymode);
156
157    fDummyPage = IOMallocAligned(0x1000, 0x1000);
158    fDummyPageNumber =
159	pmap_find_phys(kernel_pmap, (addr64_t) (uintptr_t) fDummyPage);
160
161    return true;
162}
163
164void IOCopyMapper::free()
165{
166    if (fDummyPage) {
167	IOFreeAligned(fDummyPage, 0x1000);
168	fDummyPage = 0;
169	fDummyPageNumber = 0;
170    }
171
172    if (fTableLock) {
173	IOLockFree(fTableLock);
174	fTableLock = 0;
175    }
176
177    super::free();
178}
179
180// Must be called while locked
181void IOCopyMapper::breakUp(unsigned startIndex, unsigned endIndex, unsigned freeInd)
182{
183    unsigned int zoneSize;
184    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
185
186    do {
187	// Need to break up bigger blocks of memory till we get one in our
188	// desired zone.
189	endIndex--;
190	zoneSize = (kMinZoneSize/2 << endIndex);
191	ppnum_t tail = freeInd + zoneSize;
192
193	DEBG("breakup z %d start %x tail %x\n", endIndex, freeInd, tail);
194
195	// By definition free lists must be empty
196	fFreeLists[endIndex] = tail;
197	freeDART[tail].fSize = endIndex;
198	freeDART[tail].fNext = freeDART[tail].fPrev = 0;
199    } while (endIndex != startIndex);
200    freeDART[freeInd].fSize = endIndex;
201}
202
203// Zero is never a valid page to return
204ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages)
205{
206    unsigned int zone, zoneSize, z, cnt;
207    ppnum_t next, ret = 0;
208    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
209
210    // Can't alloc anything of less than minumum
211    if (pages < kMinZoneSize)
212	pages = kMinZoneSize;
213
214    // Can't alloc anything bigger than 1/2 table
215    if (pages >= fMapperRegionSize/2)
216    {
217	panic("iovmAlloc 0x%lx", pages);
218	return 0;
219    }
220
221    // Find the appropriate zone for this allocation
222    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
223	zoneSize <<= 1;
224
225    {
226	IOLockLock(fTableLock);
227
228	for (;;) {
229	    for (z = zone; z < fNumZones; z++) {
230		if ( (ret = fFreeLists[z]) )
231		    break;
232	    }
233	    if (ret)
234		break;
235
236	    fFreeSleepers++;
237	    IOLockSleep(fTableLock, fFreeLists, THREAD_UNINT);
238	    fFreeSleepers--;
239	}
240
241	// If we didn't find a entry in our size then break up the free block
242	// that we did find.
243	if (zone != z)
244	{
245	    DEBG("breakup %d, %d, 0x%x\n", zone, z, ret);
246	    breakUp(zone, z, ret);
247	}
248
249	freeDART[ret].fInUse = true;	// Mark entry as In Use
250	next = freeDART[ret].fNext;
251	DEBG("va:  0x%lx, %ld, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next);
252
253	fFreeLists[z] = next;
254	if (next)
255	    freeDART[next].fPrev = 0;
256
257	// ret is free list offset not page offset;
258	ret *= kActivePerFree;
259
260	ActiveDARTEntry pageEntry = ACTIVEDARTENTRY(fDummyPageNumber);
261	for (cnt = 0; cnt < pages; cnt++) {
262	    ActiveDARTEntry *activeDART = &fMappings[ret + cnt];
263	    *activeDART = pageEntry;
264	}
265
266	fMapperRegionUsed += pages;
267	if (fMapperRegionUsed > fMapperRegionMaxUsed)
268	    fMapperRegionMaxUsed = fMapperRegionUsed;
269
270	IOLockUnlock(fTableLock);
271    }
272
273    if (ret)
274	ret += fBufferPage;
275
276    return ret;
277}
278
279
280void IOCopyMapper::invalidateDART(ppnum_t pnum, IOItemCount size)
281{
282    bzero((void *) &fMappings[pnum], size * sizeof(fMappings[0]));
283}
284
285void IOCopyMapper::iovmFree(ppnum_t addr, IOItemCount pages)
286{
287    unsigned int zone, zoneSize, z;
288    FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable;
289
290    if (addr < fBufferPage)
291	IOPanic("addr < fBufferPage");
292    addr -= fBufferPage;
293
294    // Can't free anything of less than minumum
295    if (pages < kMinZoneSize)
296	pages = kMinZoneSize;
297
298    // Can't free anything bigger than 1/2 table
299    if (pages >= fMapperRegionSize/2)
300	return;
301
302    // Find the appropriate zone for this allocation
303    for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++)
304	zoneSize <<= 1;
305
306    // Grab lock that protects the dart
307    IOLockLock(fTableLock);
308
309    invalidateDART(addr, pages);
310
311    addr /= kActivePerFree;
312
313    // We are freeing a block, check to see if pairs are available for
314    // coalescing.  We will walk up the entire chain if we can.
315    for (z = zone; z < fNumZones; z++) {
316	ppnum_t pair = addr ^ (kMinZoneSize/2 << z);	// Find pair address
317	if (freeDART[pair].fValid || freeDART[pair].fInUse || (freeDART[pair].fSize != z))
318	    break;
319
320	// The paired alloc entry is free if we are here
321	ppnum_t next = freeDART[pair].fNext;
322	ppnum_t prev = freeDART[pair].fPrev;
323
324	// Remove the pair from its freeList
325	if (prev)
326	    freeDART[prev].fNext = next;
327	else
328	    fFreeLists[z] = next;
329
330	if (next)
331	    freeDART[next].fPrev = prev;
332
333	// Sort the addr and the pair
334	if (addr > pair)
335	    addr = pair;
336    }
337
338    DEBG("vf:  0x%lx, %ld, z %d, head %lx, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr);
339
340    // Add the allocation entry into it's free list and re-init it
341    freeDART[addr].fSize = z;
342    freeDART[addr].fNext = fFreeLists[z];
343    if (fFreeLists[z])
344	freeDART[fFreeLists[z]].fPrev = addr;
345    freeDART[addr].fPrev = 0;
346    fFreeLists[z] = addr;
347
348    fMapperRegionUsed -= pages;
349
350    if (fFreeSleepers)
351	IOLockWakeup(fTableLock, fFreeLists, /* oneThread */ false);
352
353    IOLockUnlock(fTableLock);
354}
355
356addr64_t IOCopyMapper::mapAddr(IOPhysicalAddress addr)
357{
358    if (addr < ptoa_32(fBufferPage))
359    {
360	return (addr64_t) addr;	// Not mapped by us anyway
361    }
362
363    addr -= ptoa_32(fBufferPage);
364    if (addr >= ptoa_32(fMapperRegionSize))
365    {
366	return (addr64_t) addr;	// Not mapped by us anyway
367    }
368    else
369    {
370	ActiveDARTEntry *activeDART = (ActiveDARTEntry *) fTable;
371	UInt offset = addr & PAGE_MASK;
372
373	ActiveDARTEntry mappedPage = activeDART[atop_32(addr)];
374	if (mappedPage.fValid)
375    {
376	    return (ptoa_64(mappedPage.fPPNum) | offset);
377	}
378
379	panic("%s::mapAddr(0x%08lx) not mapped for I/O\n", getName(), addr);
380	return 0;
381    }
382}
383
384void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page)
385{
386    addr -= fBufferPage;
387    addr += offset;	// Add the offset page to the base address
388
389    ActiveDARTEntry *activeDART = &fMappings[addr];
390    ActiveDARTEntry entry = ACTIVEDARTENTRY(page);
391    *activeDART = entry;
392}
393
394void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
395	ppnum_t *pageList, IOItemCount pageCount)
396{
397    addr -= fBufferPage;
398    addr += offset;	// Add the offset page to the base address
399
400    IOItemCount i;
401    ActiveDARTEntry *activeDART = &fMappings[addr];
402
403    for (i = 0; i < pageCount; i++)
404    {
405	ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i]);
406	activeDART[i] = entry;
407    }
408}
409
410void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
411	upl_page_info_t *pageList, IOItemCount pageCount)
412{
413    addr -= fBufferPage;
414    addr += offset;	// Add the offset page to the base address
415
416    IOItemCount i;
417    ActiveDARTEntry *activeDART = &fMappings[addr];
418
419    for (i = 0; i < pageCount; i++)
420    {
421	ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i].phys_addr);
422	activeDART[i] = entry;
423    }
424}
425
426
427