1/*
2 * Copyright 2013, 2018, J��r��me Duval, jerome.duval@gmail.com.
3 * Distributed under the terms of the MIT License.
4 */
5
6
7#include <new>
8#include <stdio.h>
9#include <string.h>
10
11#include <ByteOrder.h>
12#include <KernelExport.h>
13#include <device_manager.h>
14
15#include <drivers/ACPI.h>
16#include <drivers/bus/FDT.h>
17
18#include <debug.h>
19#include <AutoDeleterDrivers.h>
20
21#include <acpi.h>
22
23#include <virtio.h>
24#include <virtio_defs.h>
25#include "VirtioDevice.h"
26
27
28#define VIRTIO_MMIO_DEVICE_MODULE_NAME "busses/virtio/virtio_mmio/driver_v1"
29#define VIRTIO_MMIO_CONTROLLER_TYPE_NAME "virtio MMIO controller"
30
31
32device_manager_info* gDeviceManager;
33
34
35static const char *
36virtio_get_feature_name(uint64 feature)
37{
38	switch (feature) {
39		case VIRTIO_FEATURE_NOTIFY_ON_EMPTY:
40			return "notify on empty";
41		case VIRTIO_FEATURE_ANY_LAYOUT:
42			return "any layout";
43		case VIRTIO_FEATURE_RING_INDIRECT_DESC:
44			return "ring indirect";
45		case VIRTIO_FEATURE_RING_EVENT_IDX:
46			return "ring event index";
47		case VIRTIO_FEATURE_BAD_FEATURE:
48			return "bad feature";
49	}
50	return NULL;
51}
52
53
54static void
55virtio_dump_features(const char* title, uint64 features,
56	const char* (*get_feature_name)(uint64))
57{
58	char features_string[512] = "";
59	for (uint64 i = 0; i < 32; i++) {
60		uint64 feature = features & (1 << i);
61		if (feature == 0)
62			continue;
63		const char* name = virtio_get_feature_name(feature);
64		if (name == NULL)
65			name = get_feature_name(feature);
66		if (name != NULL) {
67			strlcat(features_string, "[", sizeof(features_string));
68			strlcat(features_string, name, sizeof(features_string));
69			strlcat(features_string, "] ", sizeof(features_string));
70		}
71	}
72	TRACE("%s: %s\n", title, features_string);
73}
74
75
76//#pragma mark Device
77
78
79static float
80virtio_device_supports_device(device_node* parent)
81{
82	TRACE("supports_device(%p)\n", parent);
83
84	const char* name;
85	const char* bus;
86
87	status_t status = gDeviceManager->get_attr_string(parent,
88		B_DEVICE_PRETTY_NAME, &name, false);
89
90	if (status >= B_OK)
91		TRACE("  name: %s\n", name);
92
93	status = gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false);
94
95	if (status < B_OK)
96		return -1.0f;
97
98	// detect virtio device from FDT
99	if (strcmp(bus, "fdt") == 0) {
100		const char* compatible;
101		status = gDeviceManager->get_attr_string(parent, "fdt/compatible",
102			&compatible, false);
103
104		if (status < B_OK)
105			return -1.0f;
106
107		if (strcmp(compatible, "virtio,mmio") != 0)
108			return 0.0f;
109
110		return 1.0f;
111	}
112
113	// detect virtio device from ACPI
114	if (strcmp(bus, "acpi") == 0) {
115		const char* hid;
116		status = gDeviceManager->get_attr_string(parent, "acpi/hid",
117			&hid, false);
118
119		if (status < B_OK)
120			return -1.0f;
121
122		if (strcmp(hid, "LNRO0005") != 0)
123			return 0.0f;
124
125		return 1.0f;
126	}
127
128	return 0.0f;
129}
130
131
132struct virtio_memory_range {
133	uint64 base;
134	uint64 length;
135};
136
137static acpi_status
138virtio_crs_find_address(acpi_resource *res, void *context)
139{
140	virtio_memory_range &range = *((virtio_memory_range *)context);
141
142	if (res->type == ACPI_RESOURCE_TYPE_FIXED_MEMORY32) {
143		range.base = res->data.fixed_memory32.address;
144		range.length = res->data.fixed_memory32.address_length;
145	}
146
147	return B_OK;
148}
149
150
151static acpi_status
152virtio_crs_find_interrupt(acpi_resource *res, void *context)
153{
154	uint64 &interrupt = *((uint64 *)context);
155
156	if (res->type == ACPI_RESOURCE_TYPE_EXTENDED_IRQ)
157		interrupt = res->data.extended_irq.interrupts[0];
158
159	return B_OK;
160}
161
162
163static status_t
164virtio_device_register_device(device_node* parent)
165{
166	TRACE("register_device(%p)\n", parent);
167
168	const char* bus;
169
170	status_t status = gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false);
171
172	if (status < B_OK)
173		return -1.0f;
174
175	uint64 regs = 0;
176	uint64 regsLen = 0;
177
178	// initialize virtio device from FDT
179	if (strcmp(bus, "fdt") == 0) {
180		fdt_device_module_info *parentModule;
181		fdt_device* parentDev;
182		if (gDeviceManager->get_driver(parent, (driver_module_info**)&parentModule,
183				(void**)&parentDev)) {
184			ERROR("can't get parent node driver");
185			return B_ERROR;
186		}
187
188		if (!parentModule->get_reg(parentDev, 0, &regs, &regsLen)) {
189			ERROR("no regs");
190			return B_ERROR;
191		}
192	}
193
194	// initialize virtio device from ACPI
195	if (strcmp(bus, "acpi") == 0) {
196		acpi_device_module_info *parentModule;
197		acpi_device parentDev;
198		if (gDeviceManager->get_driver(parent, (driver_module_info**)&parentModule,
199				(void**)&parentDev)) {
200			ERROR("can't get parent node driver");
201			return B_ERROR;
202		}
203
204		virtio_memory_range range = { 0, 0 };
205		parentModule->walk_resources(parentDev, (char *)"_CRS",
206			virtio_crs_find_address, &range);
207		regs = range.base;
208		regsLen = range.length;
209	}
210
211	VirtioRegs *volatile mappedRegs;
212	AreaDeleter fRegsArea(map_physical_memory("Virtio MMIO", regs, regsLen,
213		B_ANY_KERNEL_ADDRESS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
214		(void **)&mappedRegs));
215
216	if (!fRegsArea.IsSet()) {
217		ERROR("cant't map regs");
218		return B_ERROR;
219	}
220
221	if (mappedRegs->signature != kVirtioSignature) {
222		ERROR("bad signature: 0x%08" B_PRIx32 ", should be 0x%08" B_PRIx32 "\n",
223			mappedRegs->signature, (uint32)kVirtioSignature);
224		return B_ERROR;
225	}
226
227	TRACE("  version: 0x%08" B_PRIx32 "\n",   mappedRegs->version);
228	TRACE("  deviceId: 0x%08" B_PRIx32 "\n",  mappedRegs->deviceId);
229	TRACE("  vendorId: 0x%08" B_PRIx32 "\n",  mappedRegs->vendorId);
230
231	device_attr attrs[] = {
232		{ B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "Virtio MMIO"} },
233		{ B_DEVICE_BUS,         B_STRING_TYPE, {.string = "virtio"} },
234		{ "virtio/version",     B_UINT32_TYPE, {.ui32 = mappedRegs->version} },
235		{ "virtio/device_id",   B_UINT32_TYPE, {.ui32 = mappedRegs->deviceId} },
236		{ "virtio/type",        B_UINT16_TYPE, {.ui16 = (uint16)mappedRegs->deviceId} },
237		{ "virtio/vendor_id",   B_UINT32_TYPE, {.ui32 = mappedRegs->vendorId} },
238		{ NULL }
239	};
240
241	return gDeviceManager->register_node(parent, VIRTIO_MMIO_DEVICE_MODULE_NAME,
242		attrs, NULL, NULL);
243}
244
245
246static status_t
247virtio_device_init_device(device_node* node, void** cookie)
248{
249	TRACE("init_device(%p)\n", node);
250
251	DeviceNodePutter<&gDeviceManager>
252		parent(gDeviceManager->get_parent_node(node));
253
254	const char* bus;
255
256	status_t status = gDeviceManager->get_attr_string(parent.Get(), B_DEVICE_BUS, &bus, false);
257
258	if (status < B_OK)
259		return -1.0f;
260
261	uint64 regs = 0;
262	uint64 regsLen = 0;
263	uint64 interrupt = 0;
264
265	// initialize virtio device from FDT
266	if (strcmp(bus, "fdt") == 0) {
267		fdt_device_module_info *parentModule;
268		fdt_device* parentDev;
269		if (gDeviceManager->get_driver(parent.Get(),
270				(driver_module_info**)&parentModule, (void**)&parentDev))
271			panic("can't get parent node driver");
272
273		TRACE("  bus: %p\n", parentModule->get_bus(parentDev));
274		TRACE("  compatible: %s\n", (const char*)parentModule->get_prop(parentDev,
275			"compatible", NULL));
276
277		for (uint32 i = 0; parentModule->get_reg(parentDev, i, &regs, &regsLen);
278				i++) {
279			TRACE("  reg[%" B_PRIu32 "]: (0x%" B_PRIx64 ", 0x%" B_PRIx64 ")\n",
280				i, regs, regsLen);
281		}
282
283		device_node* interruptController;
284		for (uint32 i = 0; parentModule->get_interrupt(parentDev,
285				i, &interruptController, &interrupt); i++) {
286
287			const char* name;
288			if (interruptController == NULL
289				|| gDeviceManager->get_attr_string(interruptController, "fdt/name",
290					&name, false) < B_OK) {
291				name = NULL;
292			}
293
294			TRACE("  interrupt[%" B_PRIu32 "]: ('%s', 0x%" B_PRIx64 ")\n", i,
295				name, interrupt);
296		}
297
298		if (!parentModule->get_reg(parentDev, 0, &regs, &regsLen)) {
299			TRACE("  no regs\n");
300			return B_ERROR;
301		}
302
303		if (!parentModule->get_interrupt(parentDev, 0, &interruptController,
304				&interrupt)) {
305			TRACE("  no interrupts\n");
306			return B_ERROR;
307		}
308	}
309
310	// initialize virtio device from ACPI
311	if (strcmp(bus, "acpi") == 0) {
312		acpi_device_module_info *parentModule;
313		acpi_device parentDev;
314		if (gDeviceManager->get_driver(parent.Get(), (driver_module_info**)&parentModule,
315				(void**)&parentDev)) {
316			ERROR("can't get parent node driver");
317			return B_ERROR;
318		}
319
320		virtio_memory_range range = { 0, 0 };
321		parentModule->walk_resources(parentDev, (char *)"_CRS",
322			virtio_crs_find_address, &range);
323		regs = range.base;
324		regsLen = range.length;
325
326		parentModule->walk_resources(parentDev, (char *)"_CRS",
327			virtio_crs_find_interrupt, &interrupt);
328
329		TRACE("  regs: (0x%" B_PRIx64 ", 0x%" B_PRIx64 ")\n",
330			regs, regsLen);
331		TRACE("  interrupt: 0x%" B_PRIx64 "\n",
332			interrupt);
333	}
334
335	ObjectDeleter<VirtioDevice> dev(new(std::nothrow) VirtioDevice());
336	if (!dev.IsSet())
337		return B_NO_MEMORY;
338
339	status_t res = dev->Init(regs, regsLen, interrupt, 1);
340	if (res < B_OK)
341		return res;
342
343	*cookie = dev.Detach();
344	return B_OK;
345}
346
347
348static void
349virtio_device_uninit_device(void* cookie)
350{
351	TRACE("uninit_device(%p)\n", cookie);
352	ObjectDeleter<VirtioDevice> dev((VirtioDevice*)cookie);
353}
354
355
356static status_t
357virtio_device_register_child_devices(void* cookie)
358{
359	TRACE("register_child_devices(%p)\n", cookie);
360	return B_OK;
361}
362
363
364//#pragma mark driver API
365
366
367static status_t
368virtio_device_negotiate_features(virtio_device cookie, uint64 supported,
369	uint64* negotiated, const char* (*get_feature_name)(uint64))
370{
371	TRACE("virtio_device_negotiate_features(%p)\n", cookie);
372	VirtioDevice* dev = (VirtioDevice*)cookie;
373
374	dev->fRegs->status |= kVirtioConfigSAcknowledge;
375	dev->fRegs->status |= kVirtioConfigSDriver;
376
377	uint64 features = dev->fRegs->deviceFeatures;
378	virtio_dump_features("read features", features, get_feature_name);
379	features &= supported;
380
381	// filter our own features
382	features &= (VIRTIO_FEATURE_TRANSPORT_MASK
383		| VIRTIO_FEATURE_RING_INDIRECT_DESC | VIRTIO_FEATURE_RING_EVENT_IDX);
384	*negotiated = features;
385
386	virtio_dump_features("negotiated features", features, get_feature_name);
387
388	dev->fRegs->driverFeatures = features;
389	dev->fRegs->status |= kVirtioConfigSFeaturesOk;
390	dev->fRegs->status |= kVirtioConfigSDriverOk;
391	dev->fRegs->guestPageSize = B_PAGE_SIZE;
392
393	return B_OK;
394}
395
396
397static status_t
398virtio_device_clear_feature(virtio_device cookie, uint64 feature)
399{
400	panic("not implemented");
401	return B_ERROR;
402}
403
404
405static status_t
406virtio_device_read_device_config(virtio_device cookie, uint8 offset,
407	void* buffer, size_t bufferSize)
408{
409	TRACE("virtio_device_read_device_config(%p, %d, %" B_PRIuSIZE ")\n", cookie,
410		offset, bufferSize);
411	VirtioDevice* dev = (VirtioDevice*)cookie;
412
413	// TODO: check ARM support, ARM seems support only 32 bit aligned MMIO access.
414	vuint8* src = &dev->fRegs->config[offset];
415	uint8* dst = (uint8*)buffer;
416	while (bufferSize > 0) {
417		uint8 size = 4;
418		if (bufferSize == 1) {
419			size = 1;
420			*dst = *src;
421		} else if (bufferSize <= 3) {
422			size = 2;
423			*(uint16*)dst = *(vuint16*)src;
424		} else
425			*(uint32*)dst = *(vuint32*)src;
426
427		dst += size;
428		bufferSize -= size;
429		src += size;
430	}
431
432	return B_OK;
433}
434
435
436static status_t
437virtio_device_write_device_config(virtio_device cookie, uint8 offset,
438	const void* buffer, size_t bufferSize)
439{
440	TRACE("virtio_device_write_device_config(%p, %d, %" B_PRIuSIZE ")\n",
441		cookie, offset, bufferSize);
442	VirtioDevice* dev = (VirtioDevice*)cookie;
443
444	// See virtio_device_read_device_config
445	uint8* src = (uint8*)buffer;
446	vuint8* dst = &dev->fRegs->config[offset];
447	while (bufferSize > 0) {
448		uint8 size = 4;
449		if (bufferSize == 1) {
450			size = 1;
451			*dst = *src;
452		} else if (bufferSize <= 3) {
453			size = 2;
454			*(vuint16*)dst = *(uint16*)src;
455		} else
456			*(vuint32*)dst = *(uint32*)src;
457
458		dst += size;
459		bufferSize -= size;
460		src += size;
461	}
462
463	return B_OK;
464}
465
466
467static status_t
468virtio_device_alloc_queues(virtio_device cookie, size_t count,
469	virtio_queue* queues)
470{
471	TRACE("virtio_device_alloc_queues(%p, %" B_PRIuSIZE ")\n", cookie, count);
472	VirtioDevice* dev = (VirtioDevice*)cookie;
473
474	ArrayDeleter<ObjectDeleter<VirtioQueue> > newQueues(new(std::nothrow)
475		ObjectDeleter<VirtioQueue>[count]);
476
477	if (!newQueues.IsSet())
478		return B_NO_MEMORY;
479
480	for (size_t i = 0; i < count; i++) {
481		newQueues[i].SetTo(new(std::nothrow) VirtioQueue(dev, i));
482
483		if (!newQueues[i].IsSet())
484			return B_NO_MEMORY;
485
486		status_t res = newQueues[i]->Init();
487		if (res < B_OK)
488			return res;
489	}
490
491	dev->fQueueCnt = count;
492	dev->fQueues.SetTo(newQueues.Detach());
493
494	for (size_t i = 0; i < count; i++)
495		queues[i] = dev->fQueues[i].Get();
496
497	return B_OK;
498}
499
500
501static void
502virtio_device_free_queues(virtio_device cookie)
503{
504	TRACE("virtio_device_free_queues(%p)\n", cookie);
505	VirtioDevice* dev = (VirtioDevice*)cookie;
506
507	dev->fQueues.Unset();
508	dev->fQueueCnt = 0;
509}
510
511
512static status_t
513virtio_device_setup_interrupt(virtio_device cookie,
514	virtio_intr_func config_handler, void* driverCookie)
515{
516	VirtioDevice* dev = (VirtioDevice*)cookie;
517	TRACE("virtio_device_setup_interrupt(%p, %#" B_PRIxADDR ")\n", dev,
518		(addr_t)config_handler);
519
520	dev->fConfigHandler = config_handler;
521	dev->fConfigHandlerCookie = driverCookie;
522	dev->fConfigHandlerRef.SetTo((config_handler == NULL)
523		? NULL : &dev->fIrqHandler);
524
525	return B_OK;
526}
527
528
529static status_t
530virtio_device_free_interrupts(virtio_device cookie)
531{
532	VirtioDevice* dev = (VirtioDevice*)cookie;
533	TRACE("virtio_device_free_interrupts(%p)\n", dev);
534
535	for (int32 i = 0; i < dev->fQueueCnt; i++) {
536		VirtioQueue* queue = dev->fQueues[i].Get();
537		queue->fQueueHandler = NULL;
538		queue->fQueueHandlerCookie = NULL;
539		queue->fQueueHandlerRef.Unset();
540	}
541
542	dev->fConfigHandler = NULL;
543	dev->fConfigHandlerCookie = NULL;
544	dev->fConfigHandlerRef.Unset();
545
546	return B_OK;
547}
548
549
550static status_t
551virtio_device_queue_setup_interrupt(virtio_queue aQueue,
552	virtio_callback_func handler, void* cookie)
553{
554	TRACE("virtio_device_queue_setup_interrupt(%p, %p)\n", aQueue, handler);
555
556	VirtioQueue* queue = (VirtioQueue*)aQueue;
557	VirtioDevice* dev = queue->fDev;
558
559	queue->fQueueHandler = handler;
560	queue->fQueueHandlerCookie = cookie;
561	queue->fQueueHandlerRef.SetTo((handler == NULL) ? NULL : &dev->fIrqHandler);
562
563	return B_OK;
564}
565
566
567static status_t
568virtio_device_queue_request_v(virtio_queue aQueue,
569	const physical_entry* vector,
570	size_t readVectorCount, size_t writtenVectorCount,
571	void* cookie)
572{
573	// TRACE("virtio_device_queue_request_v(%p, %" B_PRIuSIZE ", %" B_PRIuSIZE
574	//	", %p)\n", aQueue, readVectorCount, writtenVectorCount, cookie);
575	VirtioQueue* queue = (VirtioQueue*)aQueue;
576
577	return queue->Enqueue(vector, readVectorCount, writtenVectorCount, cookie);
578}
579
580
581static status_t
582virtio_device_queue_request(virtio_queue aQueue,
583	const physical_entry* readEntry,
584	const physical_entry* writtenEntry, void* cookie)
585{
586	VirtioQueue* queue = (VirtioQueue*)aQueue;
587
588	physical_entry vector[2];
589	physical_entry* vectorEnd = vector;
590
591	if (readEntry != NULL)
592		*vectorEnd++ = *readEntry;
593
594	if (writtenEntry != NULL)
595		*vectorEnd++ = *writtenEntry;
596
597	return queue->Enqueue(vector, (readEntry != NULL) ? 1 : 0,
598		(writtenEntry != NULL) ? 1 : 0, cookie);
599}
600
601
602static bool
603virtio_device_queue_is_full(virtio_queue queue)
604{
605	panic("not implemented");
606	return false;
607}
608
609
610static bool
611virtio_device_queue_is_empty(virtio_queue aQueue)
612{
613	VirtioQueue *queue = (VirtioQueue *)aQueue;
614	return queue->fUsed->idx == queue->fLastUsed;
615}
616
617
618static uint16
619virtio_device_queue_size(virtio_queue aQueue)
620{
621	VirtioQueue *queue = (VirtioQueue *)aQueue;
622	return (uint16)queue->fQueueLen;
623}
624
625
626static bool
627virtio_device_queue_dequeue(virtio_queue aQueue, void** _cookie,
628	uint32* _usedLength)
629{
630	// TRACE("virtio_device_queue_dequeue(%p)\n", aQueue);
631	VirtioQueue* queue = (VirtioQueue*)aQueue;
632	return queue->Dequeue(_cookie, _usedLength);
633}
634
635
636//#pragma mark -
637
638
639module_dependency module_dependencies[] = {
640	{ B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager },
641	{ NULL }
642};
643
644
645static virtio_device_interface sVirtioDevice = {
646	{
647		{
648			VIRTIO_MMIO_DEVICE_MODULE_NAME,
649			0,
650			NULL
651		},
652
653		virtio_device_supports_device,
654		virtio_device_register_device,
655		virtio_device_init_device,
656		virtio_device_uninit_device,
657		virtio_device_register_child_devices,
658		NULL,	// rescan
659		NULL,	// device removed
660	},
661	virtio_device_negotiate_features,
662	virtio_device_clear_feature,
663	virtio_device_read_device_config,
664	virtio_device_write_device_config,
665	virtio_device_alloc_queues,
666	virtio_device_free_queues,
667	virtio_device_setup_interrupt,
668	virtio_device_free_interrupts,
669	virtio_device_queue_setup_interrupt,
670	virtio_device_queue_request,
671	virtio_device_queue_request_v,
672	virtio_device_queue_is_full,
673	virtio_device_queue_is_empty,
674	virtio_device_queue_size,
675	virtio_device_queue_dequeue,
676};
677
678
679module_info* modules[] = {
680	(module_info* )&sVirtioDevice,
681	NULL
682};
683