1/*
2 * Copyright 2008-2010, Axel D��rfler, axeld@pinc-software.de.
3 * Copyright 2004-2006, Rudolf Cornelissen. All rights reserved.
4 *
5 * Distributed under the terms of the MIT License.
6 */
7
8// TODO: rethink the AGP interface for more than one bridge/device!
9//	(should be done with the new driver API then)
10
11/*
12	Notes:
13	- currently we just setup all found devices with AGP interface to the same
14	highest common mode, we don't distinquish different AGP busses.
15	TODO: it might be a better idea to just setup one instead.
16
17	- AGP3 defines 'asynchronous request size' and 'calibration cycle' fields
18	in the status and command registers. Currently programming zero's which will
19	make it work, although further optimisation is possible.
20
21	- AGP3.5 also defines isochronous transfers which are not implemented here:
22	the hardware keeps them disabled by default.
23*/
24
25
26#include <AGP.h>
27
28#include <stdlib.h>
29
30#include <KernelExport.h>
31#include <PCI.h>
32
33#include <util/OpenHashTable.h>
34#include <kernel/lock.h>
35#include <vm/vm_page.h>
36#include <vm/vm_types.h>
37
38#include <lock.h>
39
40
41#define TRACE_AGP
42#ifdef TRACE_AGP
43#	define TRACE(x...) dprintf("\33[36mAGP:\33[0m " x)
44#else
45#	define TRACE(x...) ;
46#endif
47#define ERROR(x...) dprintf("\33[36mAGP:\33[0m " x)
48
49
50#define MAX_DEVICES	  8
51
52#define AGP_ID(address) (address)
53#define AGP_STATUS(address) (address + 4)
54#define AGP_COMMAND(address) (address + 8)
55
56/* read and write to PCI config space */
57#define get_pci_config(info, offset, size) \
58	(sPCI->read_pci_config((info).bus, (info).device, (info).function, \
59		(offset), (size)))
60#define set_pci_config(info, offset, size, value) \
61	(sPCI->write_pci_config((info).bus, (info).device, (info).function, \
62		(offset), (size), (value)))
63
64#define RESERVED_APERTURE			0x80000000
65#define ALLOCATED_APERTURE			0x40000000
66#define BIND_APERTURE				0x20000000
67#define APERTURE_PUBLIC_FLAGS_MASK	0x0000ffff
68
69struct aperture_memory {
70	aperture_memory *next;
71	aperture_memory *hash_link;
72	addr_t		base;
73	size_t		size;
74	uint32		flags;
75#if !defined(GART_TEST)
76	union {
77		vm_page	**pages;
78		vm_page *page;
79	};
80#ifdef DEBUG_PAGE_ACCESS
81	thread_id	allocating_thread;
82#endif
83#else
84	area_id		area;
85#endif
86};
87
88class Aperture;
89
90class MemoryHashDefinition {
91public:
92	typedef addr_t KeyType;
93	typedef aperture_memory ValueType;
94
95	MemoryHashDefinition(aperture_info &info) : fInfo(info) {}
96
97	size_t HashKey(const KeyType &base) const
98		{ return (base - fInfo.base) / B_PAGE_SIZE; }
99	size_t Hash(aperture_memory *memory) const
100		{ return (memory->base - fInfo.base) / B_PAGE_SIZE; }
101	bool Compare(const KeyType &base, aperture_memory *memory) const
102		{ return base == memory->base; }
103	aperture_memory *&GetLink(aperture_memory *memory) const
104		{ return memory->hash_link; }
105
106private:
107	aperture_info	&fInfo;
108};
109
110typedef BOpenHashTable<MemoryHashDefinition> MemoryHashTable;
111
112struct agp_device_info {
113	uint8		address;	/* location of AGP interface in PCI capabilities */
114	agp_info	info;
115};
116
117class Aperture {
118public:
119	Aperture(agp_gart_bus_module_info *module, void *aperture);
120	~Aperture();
121
122	status_t InitCheck() const { return fLock.sem >= B_OK ? B_OK : fLock.sem; }
123
124	void DeleteMemory(aperture_memory *memory);
125	aperture_memory *CreateMemory(size_t size, size_t alignment, uint32 flags);
126
127	status_t AllocateMemory(aperture_memory *memory, uint32 flags);
128
129	status_t UnbindMemory(aperture_memory *memory);
130	status_t BindMemory(aperture_memory *memory, addr_t base, size_t size);
131
132	status_t GetInfo(aperture_info *info);
133
134	aperture_memory *GetMemory(addr_t base) { return fHashTable.Lookup(base); }
135
136	addr_t Base() const { return fInfo.base; }
137	addr_t Size() const { return fInfo.size; }
138	int32 ID() const { return fID; }
139	struct lock &Lock() { return fLock; }
140
141private:
142	bool _AdaptToReserved(addr_t &base, size_t &size, int32 *_offset = NULL);
143	void _Free(aperture_memory *memory);
144	void _Remove(aperture_memory *memory);
145	status_t _Insert(aperture_memory *memory, size_t size, size_t alignment,
146		uint32 flags);
147
148	struct lock					fLock;
149	agp_gart_bus_module_info	*fModule;
150	int32						fID;
151	aperture_info				fInfo;
152	MemoryHashTable				fHashTable;
153	aperture_memory				*fFirstMemory;
154	void						*fPrivateAperture;
155
156public:
157	Aperture					*fNext;
158};
159
160class ApertureHashDefinition {
161public:
162	typedef int32 KeyType;
163	typedef Aperture ValueType;
164
165	size_t HashKey(const KeyType &id) const
166		{ return id; }
167	size_t Hash(Aperture *aperture) const
168		{ return aperture->ID(); }
169	bool Compare(const KeyType &id, Aperture *aperture) const
170		{ return id == aperture->ID(); }
171	Aperture *&GetLink(Aperture *aperture) const
172		{ return aperture->fNext; }
173};
174
175typedef BOpenHashTable<ApertureHashDefinition> ApertureHashTable;
176
177
178static agp_device_info sDeviceInfos[MAX_DEVICES];
179static uint32 sDeviceCount;
180static pci_module_info *sPCI;
181static int32 sAcquired;
182static ApertureHashTable sApertureHashTable;
183static int32 sNextApertureID;
184static struct lock sLock;
185
186
187//	#pragma mark - private support functions
188
189
190/*!	Makes sure that all bits lower than the maximum supported rate is set. */
191static uint32
192fix_rate_support(uint32 command)
193{
194	if ((command & AGP_3_MODE) != 0) {
195		if ((command & AGP_3_8x) != 0)
196			command |= AGP_3_4x;
197
198		command &= ~AGP_RATE_MASK | AGP_3_8x | AGP_3_4x;
199		command |= AGP_SBA;
200			// SBA is required for AGP3
201	} else {
202		/* AGP 2.0 scheme applies */
203		if ((command & AGP_2_4x) != 0)
204			command |= AGP_2_2x;
205		if ((command & AGP_2_2x) != 0)
206			command |= AGP_2_1x;
207	}
208
209	return command;
210}
211
212
213/*!	Makes sure that only the highest rate bit is set. */
214static uint32
215fix_rate_command(uint32 command)
216{
217	if ((command & AGP_3_MODE) != 0) {
218		if ((command & AGP_3_8x) != 0)
219			command &= ~AGP_3_4x;
220	} else {
221		/* AGP 2.0 scheme applies */
222		if ((command & AGP_2_4x) != 0)
223			command &= ~(AGP_2_2x | AGP_2_1x);
224		if ((command & AGP_2_2x) != 0)
225			command &= ~AGP_2_1x;
226	}
227
228	return command;
229}
230
231
232/*!	Checks the capabilities of the device, and removes everything from
233	\a command that the device does not support.
234*/
235static void
236check_capabilities(agp_device_info &deviceInfo, uint32 &command)
237{
238	uint32 agpStatus = deviceInfo.info.interface.status;
239	if (deviceInfo.info.class_base == PCI_bridge) {
240		// make sure the AGP rate support mask is correct
241		// (ie. has the lower bits set)
242		agpStatus = fix_rate_support(agpStatus);
243	}
244
245	TRACE("device %u.%u.%u has AGP capabilities %" B_PRIx32 "\n", deviceInfo.info.bus,
246		deviceInfo.info.device, deviceInfo.info.function, agpStatus);
247
248	// block non-supported AGP modes
249	command &= (agpStatus & (AGP_3_MODE | AGP_RATE_MASK))
250		| ~(AGP_3_MODE | AGP_RATE_MASK);
251
252	// If no AGP mode is supported at all, nothing remains:
253	// devices exist that have the AGP style connector with AGP style registers,
254	// but not the features!
255	// (confirmed Matrox Millenium II AGP for instance)
256	if ((agpStatus & AGP_RATE_MASK) == 0)
257		command = 0;
258
259	// block side band adressing if not supported
260	if ((agpStatus & AGP_SBA) == 0)
261		command &= ~AGP_SBA;
262
263	// block fast writes if not supported
264	if ((agpStatus & AGP_FAST_WRITE) == 0)
265		command &= ~AGP_FAST_WRITE;
266
267	// adjust maximum request depth to least depth supported
268	// note: this is writable only in the graphics card
269	uint8 requestDepth = ((agpStatus & AGP_REQUEST) >> AGP_REQUEST_SHIFT);
270	if (requestDepth < ((command & AGP_REQUEST) >> AGP_REQUEST_SHIFT)) {
271		command &= ~AGP_REQUEST;
272		command |= (requestDepth << AGP_REQUEST_SHIFT);
273	}
274}
275
276
277/*!	Checks the PCI capabilities if the device is an AGP device
278*/
279static bool
280is_agp_device(pci_info &info, uint8 *_address)
281{
282	// Check if device implements a list of capabilities
283	if ((get_pci_config(info, PCI_status, 2) & PCI_status_capabilities) == 0)
284		return false;
285
286	// Get pointer to PCI capabilities list
287	// (AGP devices only, no need to take cardbus into account)
288	uint8 address = get_pci_config(info, PCI_capabilities_ptr, 1);
289
290	while (true) {
291		uint8 id = get_pci_config(info, address, 1);
292		uint8 next = get_pci_config(info, address + 1, 1) & ~0x3;
293
294		if (id == PCI_cap_id_agp) {
295			// is an AGP device
296			if (_address != NULL)
297				*_address = address;
298			return true;
299		}
300		if (next == 0) {
301			// end of list
302			break;
303		}
304
305		address = next;
306	}
307
308	return false;
309}
310
311
312static status_t
313get_next_agp_device(uint32 *_cookie, pci_info &info, agp_device_info &device)
314{
315	uint32 index = *_cookie;
316
317	// find devices
318
319	for (; sPCI->get_nth_pci_info(index, &info) == B_OK; index++) {
320		// is it a bridge or a graphics card?
321		if ((info.class_base != PCI_bridge || info.class_sub != PCI_host)
322			&& info.class_base != PCI_display)
323			continue;
324
325		if (is_agp_device(info, &device.address)) {
326			device.info.vendor_id = info.vendor_id;
327			device.info.device_id = info.device_id;
328			device.info.bus = info.bus;
329			device.info.device = info.device;
330			device.info.function = info.function;
331			device.info.class_sub = info.class_sub;
332			device.info.class_base = info.class_base;
333
334			/* get the contents of the AGP registers from this device */
335			device.info.interface.capability_id = get_pci_config(info,
336				AGP_ID(device.address), 4);
337			device.info.interface.status = get_pci_config(info,
338				AGP_STATUS(device.address), 4);
339			device.info.interface.command = get_pci_config(info,
340				AGP_COMMAND(device.address), 4);
341
342			*_cookie = index + 1;
343			return B_OK;
344		}
345	}
346
347	return B_ENTRY_NOT_FOUND;
348}
349
350
351static void
352set_agp_command(agp_device_info &deviceInfo, uint32 command)
353{
354	set_pci_config(deviceInfo.info, AGP_COMMAND(deviceInfo.address), 4, command);
355	deviceInfo.info.interface.command = get_pci_config(deviceInfo.info,
356		AGP_COMMAND(deviceInfo.address), 4);
357}
358
359
360static void
361set_pci_mode()
362{
363	TRACE("set PCI mode on all AGP capable devices.\n");
364
365	// First program all graphics cards
366
367	for (uint32 index = 0; index < sDeviceCount; index++) {
368		agp_device_info &deviceInfo = sDeviceInfos[index];
369		if (deviceInfo.info.class_base != PCI_display)
370			continue;
371
372		set_agp_command(deviceInfo, 0);
373	}
374
375	// Then program all bridges - it's the other around for AGP mode
376
377	for (uint32 index = 0; index < sDeviceCount; index++) {
378		agp_device_info &deviceInfo = sDeviceInfos[index];
379		if (deviceInfo.info.class_base != PCI_bridge)
380			continue;
381
382		set_agp_command(deviceInfo, 0);
383	}
384
385	// Wait 10mS for the bridges to recover (failsafe!)
386	// Note: some SiS bridge chipsets apparantly require 5mS to recover
387	// or the master (graphics card) cannot be initialized correctly!
388	snooze(10000);
389}
390
391
392status_t
393get_area_base_and_size(area_id area, addr_t &base, size_t &size)
394{
395	area_info info;
396	status_t status = get_area_info(area, &info);
397	if (status < B_OK)
398		return status;
399
400	base = (addr_t)info.address;
401	size = info.size;
402	return B_OK;
403}
404
405
406Aperture *
407get_aperture(aperture_id id)
408{
409	Autolock _(sLock);
410	return sApertureHashTable.Lookup(id);
411}
412
413
414//	#pragma mark - Aperture
415
416
417Aperture::Aperture(agp_gart_bus_module_info *module, void *aperture)
418	:
419	fModule(module),
420	fInfo(),
421	fHashTable(fInfo),
422	fFirstMemory(NULL),
423	fPrivateAperture(aperture)
424{
425	fModule->get_aperture_info(fPrivateAperture, &fInfo);
426	fID = atomic_add(&sNextApertureID, 1);
427	init_lock(&fLock, "aperture");
428}
429
430
431Aperture::~Aperture()
432{
433	while (fFirstMemory != NULL) {
434		DeleteMemory(fFirstMemory);
435	}
436
437	fModule->delete_aperture(fPrivateAperture);
438	put_module(fModule->info.name);
439}
440
441
442status_t
443Aperture::GetInfo(aperture_info *info)
444{
445	if (info == NULL)
446		return B_BAD_VALUE;
447
448	*info = fInfo;
449	return B_OK;
450}
451
452
453void
454Aperture::DeleteMemory(aperture_memory *memory)
455{
456	TRACE("delete memory %p\n", memory);
457
458	UnbindMemory(memory);
459	_Free(memory);
460	_Remove(memory);
461	fHashTable.Remove(memory);
462	delete memory;
463}
464
465
466aperture_memory *
467Aperture::CreateMemory(size_t size, size_t alignment, uint32 flags)
468{
469	aperture_memory *memory = new(std::nothrow) aperture_memory;
470	if (memory == NULL)
471		return NULL;
472
473	status_t status = _Insert(memory, size, alignment, flags);
474	if (status < B_OK) {
475		ERROR("Aperture::CreateMemory(): did not find a free space large for "
476			"this memory object\n");
477		delete memory;
478		return NULL;
479	}
480
481	TRACE("create memory %p, base %" B_PRIxADDR ", size %" B_PRIxSIZE
482		", flags %" B_PRIx32 "\n", memory, memory->base, memory->size, flags);
483
484	memory->flags = flags;
485#if !defined(GART_TEST)
486	memory->pages = NULL;
487#else
488	memory->area = -1;
489#endif
490
491	fHashTable.Insert(memory);
492	return memory;
493}
494
495
496bool
497Aperture::_AdaptToReserved(addr_t &base, size_t &size, int32 *_offset)
498{
499	addr_t reservedEnd = fInfo.base + fInfo.reserved_size;
500	if (reservedEnd <= base)
501		return false;
502
503	if (reservedEnd >= base + size) {
504		size = 0;
505		return true;
506	}
507
508	if (_offset != NULL)
509		*_offset = reservedEnd - base;
510
511	size -= reservedEnd - base;
512	base = reservedEnd;
513	return true;
514}
515
516
517status_t
518Aperture::AllocateMemory(aperture_memory *memory, uint32 flags)
519{
520	// We don't need to allocate reserved memory - it's
521	// already there for us to use
522	addr_t base = memory->base;
523	size_t size = memory->size;
524	if (_AdaptToReserved(base, size)) {
525		if (size == 0) {
526			TRACE("allocation is made of reserved memory\n");
527			return B_OK;
528		}
529
530		memset((void *)memory->base, 0, memory->size - size);
531	}
532	TRACE("allocate %ld bytes out of %ld\n", size, memory->size);
533
534#if !defined(GART_TEST)
535	uint32 count = size / B_PAGE_SIZE;
536
537	if ((flags & B_APERTURE_NEED_PHYSICAL) != 0) {
538		physical_address_restrictions restrictions = {};
539#if B_HAIKU_PHYSICAL_BITS > 32
540		restrictions.high_address = (phys_addr_t)1 << 32;
541			// TODO: Work-around until intel_gart can deal with physical
542			// addresses > 4 GB.
543#endif
544		memory->page = vm_page_allocate_page_run(
545			PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR, count, &restrictions,
546			VM_PRIORITY_SYSTEM);
547		if (memory->page == NULL) {
548			ERROR("Aperture::AllocateMemory(): vm_page_allocate_page_run() "
549				"failed (with B_APERTURE_NEED_PHYSICAL)\n");
550			return B_NO_MEMORY;
551		}
552	} else {
553		// Allocate table to hold the pages
554		memory->pages = (vm_page **)malloc(count * sizeof(vm_page *));
555		if (memory->pages == NULL)
556			return B_NO_MEMORY;
557
558#if B_HAIKU_PHYSICAL_BITS > 32
559		// TODO: Work-around until intel_gart can deal with physical
560		// addresses > 4 GB.
561		physical_address_restrictions restrictions = {};
562		restrictions.high_address = (phys_addr_t)1 << 32;
563		vm_page* page = vm_page_allocate_page_run(
564			PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR, count, &restrictions,
565			VM_PRIORITY_SYSTEM);
566		if (page == NULL) {
567			ERROR("Aperture::AllocateMemory(): vm_page_allocate_page_run() "
568				"failed (without B_APERTURE_NEED_PHYSICAL)\n");
569			return B_NO_MEMORY;
570		}
571
572		for (uint32 i = 0; i < count; i++)
573			memory->pages[i] = page + i;
574#else
575		vm_page_reservation reservation;
576		vm_page_reserve_pages(&reservation, count, VM_PRIORITY_SYSTEM);
577		for (uint32 i = 0; i < count; i++) {
578			memory->pages[i] = vm_page_allocate_page(&reservation,
579				PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
580		}
581		vm_page_unreserve_pages(&reservation);
582#endif
583	}
584
585#ifdef DEBUG_PAGE_ACCESS
586	memory->allocating_thread = find_thread(NULL);
587#endif
588
589#else	// GART_TEST
590	void *address;
591	memory->area = create_area("GART memory", &address, B_ANY_KERNEL_ADDRESS,
592		size, B_FULL_LOCK | ((flags & B_APERTURE_NEED_PHYSICAL) != 0
593			? B_CONTIGUOUS : 0), B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
594	if (memory->area < B_OK) {
595		ERROR("Aperture::AllocateMemory(): create_area() failed\n");
596		return B_NO_MEMORY;
597	}
598#endif
599
600	memory->flags |= ALLOCATED_APERTURE;
601	return B_OK;
602}
603
604
605status_t
606Aperture::UnbindMemory(aperture_memory *memory)
607{
608	if ((memory->flags & BIND_APERTURE) == 0)
609		return B_BAD_VALUE;
610
611	// We must not unbind reserved memory
612	addr_t base = memory->base;
613	size_t size = memory->size;
614	if (_AdaptToReserved(base, size) && size == 0) {
615		memory->flags &= ~BIND_APERTURE;
616		return B_OK;
617	}
618
619	addr_t start = base - Base();
620	TRACE("unbind %ld bytes at %lx\n", size, start);
621
622	for (addr_t offset = 0; offset < memory->size; offset += B_PAGE_SIZE) {
623		status_t status = fModule->unbind_page(fPrivateAperture, start + offset);
624		if (status < B_OK)
625			return status;
626	}
627
628	memory->flags &= ~BIND_APERTURE;
629	fModule->flush_tlbs(fPrivateAperture);
630	return B_OK;
631}
632
633
634status_t
635Aperture::BindMemory(aperture_memory *memory, addr_t address, size_t size)
636{
637	bool physical = false;
638
639	if ((memory->flags & ALLOCATED_APERTURE) != 0) {
640		// We allocated this memory, get the base and size from there
641		size = memory->size;
642		physical = true;
643	}
644
645	// We don't need to bind reserved memory
646	addr_t base = memory->base;
647	int32 offset;
648	if (_AdaptToReserved(base, size, &offset)) {
649		if (size == 0) {
650			TRACE("reserved memory already bound\n");
651			memory->flags |= BIND_APERTURE;
652			return B_OK;
653		}
654
655		address += offset;
656	}
657
658	addr_t start = base - Base();
659	TRACE("bind %ld bytes at %lx\n", size, base);
660
661	for (addr_t offset = 0; offset < size; offset += B_PAGE_SIZE) {
662		phys_addr_t physicalAddress = 0;
663		status_t status;
664
665		if (!physical) {
666			physical_entry entry;
667			status = get_memory_map((void *)(address + offset), B_PAGE_SIZE,
668				&entry, 1);
669			if (status < B_OK) {
670				ERROR("Aperture::BindMemory(): get_memory_map() failed\n");
671				return status;
672			}
673
674			physicalAddress = entry.address;
675		} else {
676			uint32 index = offset >> PAGE_SHIFT;
677			vm_page *page;
678			if ((memory->flags & B_APERTURE_NEED_PHYSICAL) != 0)
679				page = memory->page + index;
680			else
681				page = memory->pages[index];
682
683			physicalAddress
684				= (phys_addr_t)page->physical_page_number << PAGE_SHIFT;
685		}
686
687		status = fModule->bind_page(fPrivateAperture, start + offset,
688			physicalAddress);
689		if (status < B_OK) {
690			ERROR("Aperture::BindMemory(): bind_page() failed\n");
691			return status;
692		}
693	}
694
695	memory->flags |= BIND_APERTURE;
696	fModule->flush_tlbs(fPrivateAperture);
697	return B_OK;
698}
699
700
701void
702Aperture::_Free(aperture_memory *memory)
703{
704	if ((memory->flags & ALLOCATED_APERTURE) == 0)
705		return;
706
707#if !defined(GART_TEST)
708	// Remove the stolen area from the allocation
709	size_t size = memory->size;
710	addr_t reservedEnd = fInfo.base + fInfo.reserved_size;
711	if (memory->base < reservedEnd)
712		size -= reservedEnd - memory->base;
713
714	// Free previously allocated pages and page table
715	uint32 count = size / B_PAGE_SIZE;
716
717	if ((memory->flags & B_APERTURE_NEED_PHYSICAL) != 0) {
718		vm_page *page = memory->page;
719		for (uint32 i = 0; i < count; i++, page++) {
720			DEBUG_PAGE_ACCESS_TRANSFER(page, memory->allocating_thread);
721			vm_page_set_state(page, PAGE_STATE_FREE);
722		}
723
724		memory->page = NULL;
725	} else {
726		for (uint32 i = 0; i < count; i++) {
727			DEBUG_PAGE_ACCESS_TRANSFER(memory->pages[i],
728				memory->allocating_thread);
729			vm_page_set_state(memory->pages[i], PAGE_STATE_FREE);
730		}
731
732		free(memory->pages);
733		memory->pages = NULL;
734	}
735#else
736	delete_area(memory->area);
737	memory->area = -1;
738#endif
739
740	memory->flags &= ~ALLOCATED_APERTURE;
741}
742
743
744void
745Aperture::_Remove(aperture_memory *memory)
746{
747	aperture_memory *current = fFirstMemory, *last = NULL;
748
749	while (current != NULL) {
750		if (memory == current) {
751			if (last != NULL) {
752				last->next = current->next;
753			} else {
754				fFirstMemory = current->next;
755			}
756			break;
757		}
758
759		last = current;
760		current = current->next;
761	}
762}
763
764
765status_t
766Aperture::_Insert(aperture_memory *memory, size_t size, size_t alignment,
767	uint32 flags)
768{
769	aperture_memory *last = NULL;
770	aperture_memory *next;
771	bool foundSpot = false;
772
773	// do some sanity checking
774	if (size == 0 || size > fInfo.size)
775		return B_BAD_VALUE;
776
777	if (alignment < B_PAGE_SIZE)
778		alignment = B_PAGE_SIZE;
779
780	addr_t start = fInfo.base;
781	if ((flags & (B_APERTURE_NON_RESERVED | B_APERTURE_NEED_PHYSICAL)) != 0)
782		start += fInfo.reserved_size;
783
784	start = ROUNDUP(start, alignment);
785	if (start > fInfo.base - 1 + fInfo.size || start < fInfo.base)
786		return B_NO_MEMORY;
787
788	// walk up to the spot where we should start searching
789
790	next = fFirstMemory;
791	while (next) {
792		if (next->base >= start + size) {
793			// we have a winner
794			break;
795		}
796		last = next;
797		next = next->next;
798	}
799
800	// find a big enough hole
801	if (last == NULL) {
802		// see if we can build it at the beginning of the virtual map
803		if (next == NULL || (next->base >= ROUNDUP(start, alignment) + size)) {
804			memory->base = ROUNDUP(start, alignment);
805			foundSpot = true;
806		} else {
807			last = next;
808			next = next->next;
809		}
810	}
811
812	if (!foundSpot) {
813		// keep walking
814		while (next != NULL) {
815			if (next->base >= ROUNDUP(last->base + last->size, alignment) + size) {
816				// we found a spot (it'll be filled up below)
817				break;
818			}
819			last = next;
820			next = next->next;
821		}
822
823		if ((fInfo.base + (fInfo.size - 1)) >= (ROUNDUP(last->base + last->size,
824				alignment) + (size - 1))) {
825			// got a spot
826			foundSpot = true;
827			memory->base = ROUNDUP(last->base + last->size, alignment);
828			if (memory->base < start)
829				memory->base = start;
830		}
831
832		if (!foundSpot)
833			return B_NO_MEMORY;
834	}
835
836	memory->size = size;
837	if (last) {
838		memory->next = last->next;
839		last->next = memory;
840	} else {
841		memory->next = fFirstMemory;
842		fFirstMemory = memory;
843	}
844
845	return B_OK;
846}
847
848
849//	#pragma mark - AGP module interface
850
851
852status_t
853get_nth_agp_info(uint32 index, agp_info *info)
854{
855	TRACE("get_nth_agp_info(index %" B_PRIu32 ")\n", index);
856
857	if (index >= sDeviceCount)
858		return B_BAD_VALUE;
859
860	// refresh from the contents of the AGP registers from this device
861	sDeviceInfos[index].info.interface.status = get_pci_config(
862		sDeviceInfos[index].info, AGP_STATUS(sDeviceInfos[index].address), 4);
863	sDeviceInfos[index].info.interface.command = get_pci_config(
864		sDeviceInfos[index].info, AGP_COMMAND(sDeviceInfos[index].address), 4);
865
866	*info = sDeviceInfos[index].info;
867	return B_OK;
868}
869
870
871status_t
872acquire_agp(void)
873{
874	if (atomic_or(&sAcquired, 1) == 1)
875		return B_BUSY;
876
877	return B_OK;
878}
879
880
881void
882release_agp(void)
883{
884	atomic_and(&sAcquired, 0);
885}
886
887
888uint32
889set_agp_mode(uint32 command)
890{
891	TRACE("set_agp_mode(command %" B_PRIx32 ")\n", command);
892
893	if ((command & AGP_ENABLE) == 0) {
894		set_pci_mode();
895		return 0;
896	}
897
898	// Make sure we accept all modes lower than requested one and we
899	// reset reserved bits
900	command = fix_rate_support(command);
901
902	// iterate through our device list to find the common capabilities supported
903	for (uint32 index = 0; index < sDeviceCount; index++) {
904		agp_device_info &deviceInfo = sDeviceInfos[index];
905
906		// Refresh from the contents of the AGP capability registers
907		// (note: some graphics driver may have been tweaking, like nvidia)
908		deviceInfo.info.interface.status = get_pci_config(deviceInfo.info,
909			AGP_STATUS(deviceInfo.address), 4);
910
911		check_capabilities(deviceInfo, command);
912	}
913
914	command = fix_rate_command(command);
915	TRACE("set AGP command %" B_PRIx32 " on all capable devices.\n", command);
916
917	// The order of programming differs for enabling/disabling AGP mode
918	// (see AGP specification)
919
920	// First program all bridges (master)
921
922	for (uint32 index = 0; index < sDeviceCount; index++) {
923		agp_device_info &deviceInfo = sDeviceInfos[index];
924		if (deviceInfo.info.class_base != PCI_bridge)
925			continue;
926
927		set_agp_command(deviceInfo, command);
928	}
929
930	// Wait 10mS for the bridges to recover (failsafe, see set_pci_mode()!)
931	snooze(10000);
932
933	// Then all graphics cards (target)
934
935	for (uint32 index = 0; index < sDeviceCount; index++) {
936		agp_device_info &deviceInfo = sDeviceInfos[index];
937		if (deviceInfo.info.class_base != PCI_display)
938			continue;
939
940		set_agp_command(deviceInfo, command);
941	}
942
943	return command;
944}
945
946
947//	#pragma mark - GART module interface
948
949
950static aperture_id
951map_aperture(uint8 bus, uint8 device, uint8 function, size_t size,
952	addr_t *_apertureBase)
953{
954	void *iterator = open_module_list("busses/agp_gart");
955	status_t status = B_ENTRY_NOT_FOUND;
956	Aperture *aperture = NULL;
957
958	Autolock _(sLock);
959
960	while (true) {
961		char name[256];
962		size_t nameLength = sizeof(name);
963		if (read_next_module_name(iterator, name, &nameLength) != B_OK)
964			break;
965
966		agp_gart_bus_module_info *module;
967		if (get_module(name, (module_info **)&module) == B_OK) {
968			void *privateAperture;
969			status = module->create_aperture(bus, device, function, size,
970				&privateAperture);
971			if (status < B_OK) {
972				put_module(name);
973				continue;
974			}
975
976			aperture = new(std::nothrow) Aperture(module, privateAperture);
977			status = aperture->InitCheck();
978			if (status == B_OK) {
979				if (_apertureBase != NULL)
980					*_apertureBase = aperture->Base();
981
982				sApertureHashTable.Insert(aperture);
983			} else {
984				delete aperture;
985				aperture = NULL;
986			}
987			break;
988		}
989	}
990
991	close_module_list(iterator);
992	return aperture != NULL ? aperture->ID() : status;
993}
994
995
996static aperture_id
997map_custom_aperture(gart_bus_module_info *module, addr_t *_apertureBase)
998{
999	return B_ERROR;
1000}
1001
1002
1003static status_t
1004unmap_aperture(aperture_id id)
1005{
1006	Autolock _(sLock);
1007	Aperture *aperture = sApertureHashTable.Lookup(id);
1008	if (aperture == NULL)
1009		return B_ENTRY_NOT_FOUND;
1010
1011	sApertureHashTable.Remove(aperture);
1012	delete aperture;
1013	return B_OK;
1014}
1015
1016
1017static status_t
1018get_aperture_info(aperture_id id, aperture_info *info)
1019{
1020	Aperture *aperture = get_aperture(id);
1021	if (aperture == NULL)
1022		return B_ENTRY_NOT_FOUND;
1023
1024	Autolock _(aperture->Lock());
1025	return aperture->GetInfo(info);
1026}
1027
1028
1029static status_t
1030allocate_memory(aperture_id id, size_t size, size_t alignment, uint32 flags,
1031	addr_t *_apertureBase, phys_addr_t *_physicalBase)
1032{
1033	if ((flags & ~APERTURE_PUBLIC_FLAGS_MASK) != 0 || _apertureBase == NULL)
1034		return B_BAD_VALUE;
1035
1036	Aperture *aperture = get_aperture(id);
1037	if (aperture == NULL)
1038		return B_ENTRY_NOT_FOUND;
1039
1040	size = ROUNDUP(size, B_PAGE_SIZE);
1041
1042	Autolock _(aperture->Lock());
1043
1044	aperture_memory *memory = aperture->CreateMemory(size, alignment, flags);
1045	if (memory == NULL)
1046		return B_NO_MEMORY;
1047
1048	status_t status = aperture->AllocateMemory(memory, flags);
1049	if (status == B_OK)
1050		status = aperture->BindMemory(memory, 0, 0);
1051	if (status < B_OK) {
1052		aperture->DeleteMemory(memory);
1053		return status;
1054	}
1055
1056	if (_physicalBase != NULL && (flags & B_APERTURE_NEED_PHYSICAL) != 0) {
1057#if !defined(GART_TEST)
1058		*_physicalBase
1059			= (phys_addr_t)memory->page->physical_page_number * B_PAGE_SIZE;
1060#else
1061		physical_entry entry;
1062		status = get_memory_map((void *)memory->base, B_PAGE_SIZE, &entry, 1);
1063		if (status < B_OK) {
1064			aperture->DeleteMemory(memory);
1065			return status;
1066		}
1067
1068		*_physicalBase = entry.address;
1069#endif
1070	}
1071
1072	*_apertureBase = memory->base;
1073	return B_OK;
1074}
1075
1076
1077static status_t
1078free_memory(aperture_id id, addr_t base)
1079{
1080	Aperture *aperture = get_aperture(id);
1081	if (aperture == NULL)
1082		return B_ENTRY_NOT_FOUND;
1083
1084	Autolock _(aperture->Lock());
1085	aperture_memory *memory = aperture->GetMemory(base);
1086	if (memory == NULL)
1087		return B_BAD_VALUE;
1088
1089	aperture->DeleteMemory(memory);
1090	return B_OK;
1091}
1092
1093
1094static status_t
1095reserve_aperture(aperture_id id, size_t size, addr_t *_apertureBase)
1096{
1097	Aperture *aperture = get_aperture(id);
1098	if (aperture == NULL)
1099		return B_ENTRY_NOT_FOUND;
1100
1101	return B_ERROR;
1102}
1103
1104
1105static status_t
1106unreserve_aperture(aperture_id id, addr_t apertureBase)
1107{
1108	Aperture *aperture = get_aperture(id);
1109	if (aperture == NULL)
1110		return B_ENTRY_NOT_FOUND;
1111
1112	return B_ERROR;
1113}
1114
1115
1116static status_t
1117bind_aperture(aperture_id id, area_id area, addr_t base, size_t size,
1118	size_t alignment, addr_t reservedBase, addr_t *_apertureBase)
1119{
1120	Aperture *aperture = get_aperture(id);
1121	if (aperture == NULL)
1122		return B_ENTRY_NOT_FOUND;
1123
1124	if (area < 0) {
1125		if (size == 0 || size > aperture->Size()
1126			|| (base & (B_PAGE_SIZE - 1)) != 0
1127			|| base == 0)
1128			return B_BAD_VALUE;
1129
1130		size = ROUNDUP(size, B_PAGE_SIZE);
1131	}
1132
1133	if (area >= 0) {
1134		status_t status = get_area_base_and_size(area, base, size);
1135		if (status < B_OK)
1136			return status;
1137	}
1138
1139	Autolock _(aperture->Lock());
1140	aperture_memory *memory = NULL;
1141	if (reservedBase != 0) {
1142		// use reserved aperture to bind the pages
1143		memory = aperture->GetMemory(reservedBase);
1144		if (memory == NULL)
1145			return B_BAD_VALUE;
1146	} else {
1147		// create new memory object
1148		memory = aperture->CreateMemory(size, alignment,
1149			B_APERTURE_NON_RESERVED);
1150		if (memory == NULL)
1151			return B_NO_MEMORY;
1152	}
1153
1154	// just bind the physical pages backing the memory into the GART
1155
1156	status_t status = aperture->BindMemory(memory, base, size);
1157	if (status < B_OK) {
1158		if (reservedBase != 0)
1159			aperture->DeleteMemory(memory);
1160
1161		return status;
1162	}
1163
1164	if (_apertureBase != NULL)
1165		*_apertureBase = memory->base;
1166
1167	return B_OK;
1168}
1169
1170
1171static status_t
1172unbind_aperture(aperture_id id, addr_t base)
1173{
1174	Aperture *aperture = get_aperture(id);
1175	if (aperture == NULL)
1176		return B_ENTRY_NOT_FOUND;
1177
1178	Autolock _(aperture->Lock());
1179	aperture_memory *memory = aperture->GetMemory(base);
1180	if (memory == NULL || (memory->flags & BIND_APERTURE) == 0)
1181		return B_BAD_VALUE;
1182
1183	if ((memory->flags & ALLOCATED_APERTURE) != 0)
1184		panic("unbind memory %lx (%p) allocated by agp_gart.", base, memory);
1185
1186	status_t status = aperture->UnbindMemory(memory);
1187	if (status < B_OK)
1188		return status;
1189
1190	if ((memory->flags & RESERVED_APERTURE) == 0)
1191		aperture->DeleteMemory(memory);
1192
1193	return B_OK;
1194}
1195
1196
1197//	#pragma mark -
1198
1199
1200static status_t
1201agp_init(void)
1202{
1203	TRACE("bus manager init\n");
1204
1205	if (get_module(B_PCI_MODULE_NAME, (module_info **)&sPCI) != B_OK)
1206		return B_ERROR;
1207
1208	uint32 cookie = 0;
1209	sDeviceCount = 0;
1210	pci_info info;
1211	while (get_next_agp_device(&cookie, info, sDeviceInfos[sDeviceCount])
1212			== B_OK) {
1213		sDeviceCount++;
1214	}
1215
1216	TRACE("found %" B_PRId32 " AGP devices\n", sDeviceCount);
1217
1218	// Since there can be custom aperture modules (for memory management only),
1219	// we always succeed if we could get the resources we need.
1220
1221	new(&sApertureHashTable) ApertureHashTable();
1222	return init_lock(&sLock, "agp_gart");
1223}
1224
1225
1226void
1227agp_uninit(void)
1228{
1229	TRACE("bus manager uninit\n");
1230
1231	ApertureHashTable::Iterator iterator = sApertureHashTable.GetIterator();
1232	while (iterator.HasNext()) {
1233		Aperture *aperture = iterator.Next();
1234		sApertureHashTable.Remove(aperture);
1235		delete aperture;
1236	}
1237
1238	put_module(B_PCI_MODULE_NAME);
1239}
1240
1241
1242static int32
1243agp_std_ops(int32 op, ...)
1244{
1245	switch (op) {
1246		case B_MODULE_INIT:
1247			return agp_init();
1248		case B_MODULE_UNINIT:
1249			agp_uninit();
1250			return B_OK;
1251	}
1252
1253	return B_BAD_VALUE;
1254}
1255
1256
1257static struct agp_gart_module_info sAGPModuleInfo = {
1258	{
1259		{
1260			B_AGP_GART_MODULE_NAME,
1261			B_KEEP_LOADED,		// Keep loaded, even if no driver requires it
1262			agp_std_ops
1263		},
1264		NULL 					// the rescan function
1265	},
1266	get_nth_agp_info,
1267	acquire_agp,
1268	release_agp,
1269	set_agp_mode,
1270
1271	map_aperture,
1272	map_custom_aperture,
1273	unmap_aperture,
1274	get_aperture_info,
1275	allocate_memory,
1276	free_memory,
1277	reserve_aperture,
1278	unreserve_aperture,
1279	bind_aperture,
1280	unbind_aperture,
1281};
1282
1283module_info *modules[] = {
1284	(module_info *)&sAGPModuleInfo,
1285	NULL
1286};
1287