1/******************************************************************************
2 * gnttab.c
3 *
4 * Two sets of functionality:
5 * 1. Granting foreign access to our memory reservation.
6 * 2. Accessing others' memory reservations via grant references.
7 * (i.e., mechanisms for both sender and recipient of grant references)
8 *
9 * Copyright (c) 2005, Christopher Clark
10 * Copyright (c) 2004, K A Fraser
11 */
12
13#include <sys/cdefs.h>
14__FBSDID("$FreeBSD$");
15
16#include <sys/param.h>
17#include <sys/systm.h>
18#include <sys/bus.h>
19#include <sys/conf.h>
20#include <sys/module.h>
21#include <sys/kernel.h>
22#include <sys/lock.h>
23#include <sys/malloc.h>
24#include <sys/mman.h>
25#include <sys/limits.h>
26#include <sys/rman.h>
27#include <machine/resource.h>
28#include <machine/cpu.h>
29
30#include <xen/xen-os.h>
31#include <xen/hypervisor.h>
32#include <machine/xen/synch_bitops.h>
33
34#include <xen/hypervisor.h>
35#include <xen/gnttab.h>
36
37#include <vm/vm.h>
38#include <vm/vm_kern.h>
39#include <vm/vm_extern.h>
40#include <vm/pmap.h>
41
42/* External tools reserve first few grant table entries. */
43#define NR_RESERVED_ENTRIES 8
44#define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
45
46static grant_ref_t **gnttab_list;
47static unsigned int nr_grant_frames;
48static unsigned int boot_max_nr_grant_frames;
49static int gnttab_free_count;
50static grant_ref_t gnttab_free_head;
51static struct mtx gnttab_list_lock;
52
53/*
54 * Resource representing allocated physical address space
55 * for the grant table metainfo
56 */
57static struct resource *gnttab_pseudo_phys_res;
58
59/* Resource id for allocated physical address space. */
60static int gnttab_pseudo_phys_res_id;
61
62static grant_entry_t *shared;
63
64static struct gnttab_free_callback *gnttab_free_callback_list = NULL;
65
66static int gnttab_expand(unsigned int req_entries);
67
68#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
69#define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
70
71static int
72get_free_entries(int count, int *entries)
73{
74	int ref, error;
75	grant_ref_t head;
76
77	mtx_lock(&gnttab_list_lock);
78	if ((gnttab_free_count < count) &&
79	    ((error = gnttab_expand(count - gnttab_free_count)) != 0)) {
80		mtx_unlock(&gnttab_list_lock);
81		return (error);
82	}
83	ref = head = gnttab_free_head;
84	gnttab_free_count -= count;
85	while (count-- > 1)
86		head = gnttab_entry(head);
87	gnttab_free_head = gnttab_entry(head);
88	gnttab_entry(head) = GNTTAB_LIST_END;
89	mtx_unlock(&gnttab_list_lock);
90
91	*entries = ref;
92	return (0);
93}
94
95static void
96do_free_callbacks(void)
97{
98	struct gnttab_free_callback *callback, *next;
99
100	callback = gnttab_free_callback_list;
101	gnttab_free_callback_list = NULL;
102
103	while (callback != NULL) {
104		next = callback->next;
105		if (gnttab_free_count >= callback->count) {
106			callback->next = NULL;
107			callback->fn(callback->arg);
108		} else {
109			callback->next = gnttab_free_callback_list;
110			gnttab_free_callback_list = callback;
111		}
112		callback = next;
113	}
114}
115
116static inline void
117check_free_callbacks(void)
118{
119	if (__predict_false(gnttab_free_callback_list != NULL))
120		do_free_callbacks();
121}
122
123static void
124put_free_entry(grant_ref_t ref)
125{
126
127	mtx_lock(&gnttab_list_lock);
128	gnttab_entry(ref) = gnttab_free_head;
129	gnttab_free_head = ref;
130	gnttab_free_count++;
131	check_free_callbacks();
132	mtx_unlock(&gnttab_list_lock);
133}
134
135/*
136 * Public grant-issuing interface functions
137 */
138
139int
140gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly,
141	grant_ref_t *result)
142{
143	int error, ref;
144
145	error = get_free_entries(1, &ref);
146
147	if (__predict_false(error))
148		return (error);
149
150	shared[ref].frame = frame;
151	shared[ref].domid = domid;
152	wmb();
153	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
154
155	if (result)
156		*result = ref;
157
158	return (0);
159}
160
161void
162gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
163				unsigned long frame, int readonly)
164{
165
166	shared[ref].frame = frame;
167	shared[ref].domid = domid;
168	wmb();
169	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
170}
171
172int
173gnttab_query_foreign_access(grant_ref_t ref)
174{
175	uint16_t nflags;
176
177	nflags = shared[ref].flags;
178
179	return (nflags & (GTF_reading|GTF_writing));
180}
181
182int
183gnttab_end_foreign_access_ref(grant_ref_t ref)
184{
185	uint16_t flags, nflags;
186
187	nflags = shared[ref].flags;
188	do {
189		if ( (flags = nflags) & (GTF_reading|GTF_writing) ) {
190			printf("%s: WARNING: g.e. still in use!\n", __func__);
191			return (0);
192		}
193	} while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) !=
194	       flags);
195
196	return (1);
197}
198
199void
200gnttab_end_foreign_access(grant_ref_t ref, void *page)
201{
202	if (gnttab_end_foreign_access_ref(ref)) {
203		put_free_entry(ref);
204		if (page != NULL) {
205			free(page, M_DEVBUF);
206		}
207	}
208	else {
209		/* XXX This needs to be fixed so that the ref and page are
210		   placed on a list to be freed up later. */
211		printf("%s: WARNING: leaking g.e. and page still in use!\n",
212		       __func__);
213	}
214}
215
216void
217gnttab_end_foreign_access_references(u_int count, grant_ref_t *refs)
218{
219	grant_ref_t *last_ref;
220	grant_ref_t  head;
221	grant_ref_t  tail;
222
223	head = GNTTAB_LIST_END;
224	tail = *refs;
225	last_ref = refs + count;
226	while (refs != last_ref) {
227
228		if (gnttab_end_foreign_access_ref(*refs)) {
229			gnttab_entry(*refs) = head;
230			head = *refs;
231		} else {
232			/*
233			 * XXX This needs to be fixed so that the ref
234			 * is placed on a list to be freed up later.
235			 */
236			printf("%s: WARNING: leaking g.e. still in use!\n",
237			       __func__);
238			count--;
239		}
240		refs++;
241	}
242
243	if (count != 0) {
244		mtx_lock(&gnttab_list_lock);
245		gnttab_free_count += count;
246		gnttab_entry(tail) = gnttab_free_head;
247		gnttab_free_head = head;
248		mtx_unlock(&gnttab_list_lock);
249	}
250}
251
252int
253gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn,
254    grant_ref_t *result)
255{
256	int error, ref;
257
258	error = get_free_entries(1, &ref);
259	if (__predict_false(error))
260		return (error);
261
262	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
263
264	*result = ref;
265	return (0);
266}
267
268void
269gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
270	unsigned long pfn)
271{
272	shared[ref].frame = pfn;
273	shared[ref].domid = domid;
274	wmb();
275	shared[ref].flags = GTF_accept_transfer;
276}
277
278unsigned long
279gnttab_end_foreign_transfer_ref(grant_ref_t ref)
280{
281	unsigned long frame;
282	uint16_t      flags;
283
284	/*
285         * If a transfer is not even yet started, try to reclaim the grant
286         * reference and return failure (== 0).
287         */
288	while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
289		if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags )
290			return (0);
291		cpu_spinwait();
292	}
293
294	/* If a transfer is in progress then wait until it is completed. */
295	while (!(flags & GTF_transfer_completed)) {
296		flags = shared[ref].flags;
297		cpu_spinwait();
298	}
299
300	/* Read the frame number /after/ reading completion status. */
301	rmb();
302	frame = shared[ref].frame;
303	KASSERT(frame != 0, ("grant table inconsistent"));
304
305	return (frame);
306}
307
308unsigned long
309gnttab_end_foreign_transfer(grant_ref_t ref)
310{
311	unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
312
313	put_free_entry(ref);
314	return (frame);
315}
316
317void
318gnttab_free_grant_reference(grant_ref_t ref)
319{
320
321	put_free_entry(ref);
322}
323
324void
325gnttab_free_grant_references(grant_ref_t head)
326{
327	grant_ref_t ref;
328	int count = 1;
329
330	if (head == GNTTAB_LIST_END)
331		return;
332
333	ref = head;
334	while (gnttab_entry(ref) != GNTTAB_LIST_END) {
335		ref = gnttab_entry(ref);
336		count++;
337	}
338	mtx_lock(&gnttab_list_lock);
339	gnttab_entry(ref) = gnttab_free_head;
340	gnttab_free_head = head;
341	gnttab_free_count += count;
342	check_free_callbacks();
343	mtx_unlock(&gnttab_list_lock);
344}
345
346int
347gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
348{
349	int ref, error;
350
351	error = get_free_entries(count, &ref);
352	if (__predict_false(error))
353		return (error);
354
355	*head = ref;
356	return (0);
357}
358
359int
360gnttab_empty_grant_references(const grant_ref_t *private_head)
361{
362
363	return (*private_head == GNTTAB_LIST_END);
364}
365
366int
367gnttab_claim_grant_reference(grant_ref_t *private_head)
368{
369	grant_ref_t g = *private_head;
370
371	if (__predict_false(g == GNTTAB_LIST_END))
372		return (g);
373	*private_head = gnttab_entry(g);
374	return (g);
375}
376
377void
378gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t  release)
379{
380
381	gnttab_entry(release) = *private_head;
382	*private_head = release;
383}
384
385void
386gnttab_request_free_callback(struct gnttab_free_callback *callback,
387    void (*fn)(void *), void *arg, uint16_t count)
388{
389
390	mtx_lock(&gnttab_list_lock);
391	if (callback->next)
392		goto out;
393	callback->fn = fn;
394	callback->arg = arg;
395	callback->count = count;
396	callback->next = gnttab_free_callback_list;
397	gnttab_free_callback_list = callback;
398	check_free_callbacks();
399 out:
400	mtx_unlock(&gnttab_list_lock);
401
402}
403
404void
405gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
406{
407	struct gnttab_free_callback **pcb;
408
409	mtx_lock(&gnttab_list_lock);
410	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
411		if (*pcb == callback) {
412			*pcb = callback->next;
413			break;
414		}
415	}
416	mtx_unlock(&gnttab_list_lock);
417}
418
419
420static int
421grow_gnttab_list(unsigned int more_frames)
422{
423	unsigned int new_nr_grant_frames, extra_entries, i;
424
425	new_nr_grant_frames = nr_grant_frames + more_frames;
426	extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
427
428	for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
429	{
430		gnttab_list[i] = (grant_ref_t *)
431			malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
432
433		if (!gnttab_list[i])
434			goto grow_nomem;
435	}
436
437	for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
438	     i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
439		gnttab_entry(i) = i + 1;
440
441	gnttab_entry(i) = gnttab_free_head;
442	gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
443	gnttab_free_count += extra_entries;
444
445	nr_grant_frames = new_nr_grant_frames;
446
447	check_free_callbacks();
448
449	return (0);
450
451grow_nomem:
452	for ( ; i >= nr_grant_frames; i--)
453		free(gnttab_list[i], M_DEVBUF);
454	return (ENOMEM);
455}
456
457static unsigned int
458__max_nr_grant_frames(void)
459{
460	struct gnttab_query_size query;
461	int rc;
462
463	query.dom = DOMID_SELF;
464
465	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
466	if ((rc < 0) || (query.status != GNTST_okay))
467		return (4); /* Legacy max supported number of frames */
468
469	return (query.max_nr_frames);
470}
471
472static inline
473unsigned int max_nr_grant_frames(void)
474{
475	unsigned int xen_max = __max_nr_grant_frames();
476
477	if (xen_max > boot_max_nr_grant_frames)
478		return (boot_max_nr_grant_frames);
479	return (xen_max);
480}
481
482#ifdef notyet
483/*
484 * XXX needed for backend support
485 *
486 */
487static int
488map_pte_fn(pte_t *pte, struct page *pmd_page,
489		      unsigned long addr, void *data)
490{
491	unsigned long **frames = (unsigned long **)data;
492
493	set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL));
494	(*frames)++;
495	return 0;
496}
497
498static int
499unmap_pte_fn(pte_t *pte, struct page *pmd_page,
500			unsigned long addr, void *data)
501{
502
503	set_pte_at(&init_mm, addr, pte, __pte(0));
504	return 0;
505}
506#endif
507
508static vm_paddr_t resume_frames;
509
510static int
511gnttab_map(unsigned int start_idx, unsigned int end_idx)
512{
513	struct xen_add_to_physmap xatp;
514	unsigned int i = end_idx;
515
516	/*
517	 * Loop backwards, so that the first hypercall has the largest index,
518	 * ensuring that the table will grow only once.
519	 */
520	do {
521		xatp.domid = DOMID_SELF;
522		xatp.idx = i;
523		xatp.space = XENMAPSPACE_grant_table;
524		xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
525		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
526			panic("HYPERVISOR_memory_op failed to map gnttab");
527	} while (i-- > start_idx);
528
529	if (shared == NULL) {
530		vm_offset_t area;
531
532		area = kva_alloc(PAGE_SIZE * max_nr_grant_frames());
533		KASSERT(area, ("can't allocate VM space for grant table"));
534		shared = (grant_entry_t *)area;
535	}
536
537	for (i = start_idx; i <= end_idx; i++) {
538		pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE,
539		    resume_frames + i * PAGE_SIZE);
540	}
541
542	return (0);
543}
544
545int
546gnttab_resume(device_t dev)
547{
548	unsigned int max_nr_gframes, nr_gframes;
549
550	nr_gframes = nr_grant_frames;
551	max_nr_gframes = max_nr_grant_frames();
552	if (max_nr_gframes < nr_gframes)
553		return (ENOSYS);
554
555	if (!resume_frames) {
556		KASSERT(dev != NULL,
557		    ("No resume frames and no device provided"));
558
559		gnttab_pseudo_phys_res = xenmem_alloc(dev,
560		    &gnttab_pseudo_phys_res_id, PAGE_SIZE * max_nr_gframes);
561		if (gnttab_pseudo_phys_res == NULL)
562			panic("Unable to reserve physical memory for gnttab");
563		resume_frames = rman_get_start(gnttab_pseudo_phys_res);
564	}
565
566	return (gnttab_map(0, nr_gframes - 1));
567}
568
569static int
570gnttab_expand(unsigned int req_entries)
571{
572	int error;
573	unsigned int cur, extra;
574
575	cur = nr_grant_frames;
576	extra = howmany(req_entries, GREFS_PER_GRANT_FRAME);
577	if (cur + extra > max_nr_grant_frames())
578		return (ENOSPC);
579
580	error = gnttab_map(cur, cur + extra - 1);
581	if (!error)
582		error = grow_gnttab_list(extra);
583
584	return (error);
585}
586
587MTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF);
588
589/*------------------ Private Device Attachment Functions  --------------------*/
590/**
591 * \brief Identify instances of this device type in the system.
592 *
593 * \param driver  The driver performing this identify action.
594 * \param parent  The NewBus parent device for any devices this method adds.
595 */
596static void
597granttable_identify(driver_t *driver __unused, device_t parent)
598{
599
600	KASSERT(xen_domain(),
601	    ("Trying to attach grant-table device on non Xen domain"));
602	/*
603	 * A single device instance for our driver is always present
604	 * in a system operating under Xen.
605	 */
606	if (BUS_ADD_CHILD(parent, 0, driver->name, 0) == NULL)
607		panic("unable to attach Xen Grant-table device");
608}
609
610/**
611 * \brief Probe for the existence of the Xen Grant-table device
612 *
613 * \param dev  NewBus device_t for this instance.
614 *
615 * \return  Always returns 0 indicating success.
616 */
617static int
618granttable_probe(device_t dev)
619{
620
621	device_set_desc(dev, "Xen Grant-table Device");
622	return (BUS_PROBE_NOWILDCARD);
623}
624
625/**
626 * \brief Attach the Xen Grant-table device.
627 *
628 * \param dev  NewBus device_t for this instance.
629 *
630 * \return  On success, 0. Otherwise an errno value indicating the
631 *          type of failure.
632 */
633static int
634granttable_attach(device_t dev)
635{
636	int i;
637	unsigned int max_nr_glist_frames;
638	unsigned int nr_init_grefs;
639
640	nr_grant_frames = 1;
641	boot_max_nr_grant_frames = __max_nr_grant_frames();
642
643	/* Determine the maximum number of frames required for the
644	 * grant reference free list on the current hypervisor.
645	 */
646	max_nr_glist_frames = (boot_max_nr_grant_frames *
647			       GREFS_PER_GRANT_FRAME /
648			       (PAGE_SIZE / sizeof(grant_ref_t)));
649
650	gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *),
651	    M_DEVBUF, M_NOWAIT);
652
653	if (gnttab_list == NULL)
654		return (ENOMEM);
655
656	for (i = 0; i < nr_grant_frames; i++) {
657		gnttab_list[i] = (grant_ref_t *)
658			malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
659		if (gnttab_list[i] == NULL)
660			goto ini_nomem;
661	}
662
663	if (gnttab_resume(dev))
664		return (ENODEV);
665
666	nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
667
668	for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
669		gnttab_entry(i) = i + 1;
670
671	gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
672	gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
673	gnttab_free_head  = NR_RESERVED_ENTRIES;
674
675	if (bootverbose)
676		printf("Grant table initialized\n");
677
678	return (0);
679
680ini_nomem:
681	for (i--; i >= 0; i--)
682		free(gnttab_list[i], M_DEVBUF);
683	free(gnttab_list, M_DEVBUF);
684	return (ENOMEM);
685}
686
687/*-------------------- Private Device Attachment Data  -----------------------*/
688static device_method_t granttable_methods[] = {
689	/* Device interface */
690	DEVMETHOD(device_identify,	granttable_identify),
691	DEVMETHOD(device_probe,         granttable_probe),
692	DEVMETHOD(device_attach,        granttable_attach),
693
694	DEVMETHOD_END
695};
696
697DEFINE_CLASS_0(granttable, granttable_driver, granttable_methods, 0);
698devclass_t granttable_devclass;
699
700DRIVER_MODULE_ORDERED(granttable, xenpv, granttable_driver, granttable_devclass,
701    NULL, NULL, SI_ORDER_FIRST);
702