1/*-
2 * Copyright (c) 2011 NetApp, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#include <sys/kernel.h>
34#include <sys/systm.h>
35#include <sys/malloc.h>
36
37#include <vm/vm.h>
38#include <vm/pmap.h>
39
40#include <dev/pci/pcireg.h>
41
42#include <machine/vmparam.h>
43#include <contrib/dev/acpica/include/acpi.h>
44
45#include "io/iommu.h"
46
47/*
48 * Documented in the "Intel Virtualization Technology for Directed I/O",
49 * Architecture Spec, September 2008.
50 */
51
52/* Section 10.4 "Register Descriptions" */
53struct vtdmap {
54	volatile uint32_t	version;
55	volatile uint32_t	res0;
56	volatile uint64_t	cap;
57	volatile uint64_t	ext_cap;
58	volatile uint32_t	gcr;
59	volatile uint32_t	gsr;
60	volatile uint64_t	rta;
61	volatile uint64_t	ccr;
62};
63
64#define	VTD_CAP_SAGAW(cap)	(((cap) >> 8) & 0x1F)
65#define	VTD_CAP_ND(cap)		((cap) & 0x7)
66#define	VTD_CAP_CM(cap)		(((cap) >> 7) & 0x1)
67#define	VTD_CAP_SPS(cap)	(((cap) >> 34) & 0xF)
68#define	VTD_CAP_RWBF(cap)	(((cap) >> 4) & 0x1)
69
70#define	VTD_ECAP_DI(ecap)	(((ecap) >> 2) & 0x1)
71#define	VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1)
72#define	VTD_ECAP_IRO(ecap)	(((ecap) >> 8) & 0x3FF)
73
74#define	VTD_GCR_WBF		(1 << 27)
75#define	VTD_GCR_SRTP		(1 << 30)
76#define	VTD_GCR_TE		(1U << 31)
77
78#define	VTD_GSR_WBFS		(1 << 27)
79#define	VTD_GSR_RTPS		(1 << 30)
80#define	VTD_GSR_TES		(1U << 31)
81
82#define	VTD_CCR_ICC		(1UL << 63)	/* invalidate context cache */
83#define	VTD_CCR_CIRG_GLOBAL	(1UL << 61)	/* global invalidation */
84
85#define	VTD_IIR_IVT		(1UL << 63)	/* invalidation IOTLB */
86#define	VTD_IIR_IIRG_GLOBAL	(1ULL << 60)	/* global IOTLB invalidation */
87#define	VTD_IIR_IIRG_DOMAIN	(2ULL << 60)	/* domain IOTLB invalidation */
88#define	VTD_IIR_IIRG_PAGE	(3ULL << 60)	/* page IOTLB invalidation */
89#define	VTD_IIR_DRAIN_READS	(1ULL << 49)	/* drain pending DMA reads */
90#define	VTD_IIR_DRAIN_WRITES	(1ULL << 48)	/* drain pending DMA writes */
91#define	VTD_IIR_DOMAIN_P	32
92
93#define	VTD_ROOT_PRESENT	0x1
94#define	VTD_CTX_PRESENT		0x1
95#define	VTD_CTX_TT_ALL		(1UL << 2)
96
97#define	VTD_PTE_RD		(1UL << 0)
98#define	VTD_PTE_WR		(1UL << 1)
99#define	VTD_PTE_SUPERPAGE	(1UL << 7)
100#define	VTD_PTE_ADDR_M		(0x000FFFFFFFFFF000UL)
101
102#define VTD_RID2IDX(rid)	(((rid) & 0xff) * 2)
103
104struct domain {
105	uint64_t	*ptp;		/* first level page table page */
106	int		pt_levels;	/* number of page table levels */
107	int		addrwidth;	/* 'AW' field in context entry */
108	int		spsmask;	/* supported super page sizes */
109	u_int		id;		/* domain id */
110	vm_paddr_t	maxaddr;	/* highest address to be mapped */
111	SLIST_ENTRY(domain) next;
112};
113
114static SLIST_HEAD(, domain) domhead;
115
116#define	DRHD_MAX_UNITS	8
117static int		drhd_num;
118static struct vtdmap	*vtdmaps[DRHD_MAX_UNITS];
119static int		max_domains;
120typedef int		(*drhd_ident_func_t)(void);
121
122static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
123static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
124
125static MALLOC_DEFINE(M_VTD, "vtd", "vtd");
126
127static int
128vtd_max_domains(struct vtdmap *vtdmap)
129{
130	int nd;
131
132	nd = VTD_CAP_ND(vtdmap->cap);
133
134	switch (nd) {
135	case 0:
136		return (16);
137	case 1:
138		return (64);
139	case 2:
140		return (256);
141	case 3:
142		return (1024);
143	case 4:
144		return (4 * 1024);
145	case 5:
146		return (16 * 1024);
147	case 6:
148		return (64 * 1024);
149	default:
150		panic("vtd_max_domains: invalid value of nd (0x%0x)", nd);
151	}
152}
153
154static u_int
155domain_id(void)
156{
157	u_int id;
158	struct domain *dom;
159
160	/* Skip domain id 0 - it is reserved when Caching Mode field is set */
161	for (id = 1; id < max_domains; id++) {
162		SLIST_FOREACH(dom, &domhead, next) {
163			if (dom->id == id)
164				break;
165		}
166		if (dom == NULL)
167			break;		/* found it */
168	}
169
170	if (id >= max_domains)
171		panic("domain ids exhausted");
172
173	return (id);
174}
175
176static void
177vtd_wbflush(struct vtdmap *vtdmap)
178{
179
180	if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0)
181		pmap_invalidate_cache();
182
183	if (VTD_CAP_RWBF(vtdmap->cap)) {
184		vtdmap->gcr = VTD_GCR_WBF;
185		while ((vtdmap->gsr & VTD_GSR_WBFS) != 0)
186			;
187	}
188}
189
190static void
191vtd_ctx_global_invalidate(struct vtdmap *vtdmap)
192{
193
194	vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL;
195	while ((vtdmap->ccr & VTD_CCR_ICC) != 0)
196		;
197}
198
199static void
200vtd_iotlb_global_invalidate(struct vtdmap *vtdmap)
201{
202	int offset;
203	volatile uint64_t *iotlb_reg, val;
204
205	vtd_wbflush(vtdmap);
206
207	offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16;
208	iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8);
209
210	*iotlb_reg =  VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL |
211		      VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES;
212
213	while (1) {
214		val = *iotlb_reg;
215		if ((val & VTD_IIR_IVT) == 0)
216			break;
217	}
218}
219
220static void
221vtd_translation_enable(struct vtdmap *vtdmap)
222{
223
224	vtdmap->gcr = VTD_GCR_TE;
225	while ((vtdmap->gsr & VTD_GSR_TES) == 0)
226		;
227}
228
229static void
230vtd_translation_disable(struct vtdmap *vtdmap)
231{
232
233	vtdmap->gcr = 0;
234	while ((vtdmap->gsr & VTD_GSR_TES) != 0)
235		;
236}
237
238static int
239vtd_init(void)
240{
241	int i, units, remaining;
242	struct vtdmap *vtdmap;
243	vm_paddr_t ctx_paddr;
244	char *end, envname[32];
245	unsigned long mapaddr;
246	ACPI_STATUS status;
247	ACPI_TABLE_DMAR *dmar;
248	ACPI_DMAR_HEADER *hdr;
249	ACPI_DMAR_HARDWARE_UNIT *drhd;
250
251	/*
252	 * Allow the user to override the ACPI DMAR table by specifying the
253	 * physical address of each remapping unit.
254	 *
255	 * The following example specifies two remapping units at
256	 * physical addresses 0xfed90000 and 0xfeda0000 respectively.
257	 * set vtd.regmap.0.addr=0xfed90000
258	 * set vtd.regmap.1.addr=0xfeda0000
259	 */
260	for (units = 0; units < DRHD_MAX_UNITS; units++) {
261		snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units);
262		if (getenv_ulong(envname, &mapaddr) == 0)
263			break;
264		vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr);
265	}
266
267	if (units > 0)
268		goto skip_dmar;
269
270	/* Search for DMAR table. */
271	status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar);
272	if (ACPI_FAILURE(status))
273		return (ENXIO);
274
275	end = (char *)dmar + dmar->Header.Length;
276	remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR);
277	while (remaining > sizeof(ACPI_DMAR_HEADER)) {
278		hdr = (ACPI_DMAR_HEADER *)(end - remaining);
279		if (hdr->Length > remaining)
280			break;
281		/*
282		 * From Intel VT-d arch spec, version 1.3:
283		 * BIOS implementations must report mapping structures
284		 * in numerical order, i.e. All remapping structures of
285		 * type 0 (DRHD) enumerated before remapping structures of
286		 * type 1 (RMRR) and so forth.
287		 */
288		if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT)
289			break;
290
291		drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr;
292		vtdmaps[units++] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address);
293		if (units >= DRHD_MAX_UNITS)
294			break;
295		remaining -= hdr->Length;
296	}
297
298	if (units <= 0)
299		return (ENXIO);
300
301skip_dmar:
302	drhd_num = units;
303	vtdmap = vtdmaps[0];
304
305	if (VTD_CAP_CM(vtdmap->cap) != 0)
306		panic("vtd_init: invalid caching mode");
307
308	max_domains = vtd_max_domains(vtdmap);
309
310	/*
311	 * Set up the root-table to point to the context-entry tables
312	 */
313	for (i = 0; i < 256; i++) {
314		ctx_paddr = vtophys(ctx_tables[i]);
315		if (ctx_paddr & PAGE_MASK)
316			panic("ctx table (0x%0lx) not page aligned", ctx_paddr);
317
318		root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT;
319	}
320
321	return (0);
322}
323
324static void
325vtd_cleanup(void)
326{
327}
328
329static void
330vtd_enable(void)
331{
332	int i;
333	struct vtdmap *vtdmap;
334
335	for (i = 0; i < drhd_num; i++) {
336		vtdmap = vtdmaps[i];
337		vtd_wbflush(vtdmap);
338
339		/* Update the root table address */
340		vtdmap->rta = vtophys(root_table);
341		vtdmap->gcr = VTD_GCR_SRTP;
342		while ((vtdmap->gsr & VTD_GSR_RTPS) == 0)
343			;
344
345		vtd_ctx_global_invalidate(vtdmap);
346		vtd_iotlb_global_invalidate(vtdmap);
347
348		vtd_translation_enable(vtdmap);
349	}
350}
351
352static void
353vtd_disable(void)
354{
355	int i;
356	struct vtdmap *vtdmap;
357
358	for (i = 0; i < drhd_num; i++) {
359		vtdmap = vtdmaps[i];
360		vtd_translation_disable(vtdmap);
361	}
362}
363
364static void
365vtd_add_device(void *arg, uint16_t rid)
366{
367	int idx;
368	uint64_t *ctxp;
369	struct domain *dom = arg;
370	vm_paddr_t pt_paddr;
371	struct vtdmap *vtdmap;
372	uint8_t bus;
373
374	vtdmap = vtdmaps[0];
375	bus = PCI_RID2BUS(rid);
376	ctxp = ctx_tables[bus];
377	pt_paddr = vtophys(dom->ptp);
378	idx = VTD_RID2IDX(rid);
379
380	if (ctxp[idx] & VTD_CTX_PRESENT) {
381		panic("vtd_add_device: device %x is already owned by "
382		      "domain %d", rid,
383		      (uint16_t)(ctxp[idx + 1] >> 8));
384	}
385
386	/*
387	 * Order is important. The 'present' bit is set only after all fields
388	 * of the context pointer are initialized.
389	 */
390	ctxp[idx + 1] = dom->addrwidth | (dom->id << 8);
391
392	if (VTD_ECAP_DI(vtdmap->ext_cap))
393		ctxp[idx] = VTD_CTX_TT_ALL;
394	else
395		ctxp[idx] = 0;
396
397	ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT;
398
399	/*
400	 * 'Not Present' entries are not cached in either the Context Cache
401	 * or in the IOTLB, so there is no need to invalidate either of them.
402	 */
403}
404
405static void
406vtd_remove_device(void *arg, uint16_t rid)
407{
408	int i, idx;
409	uint64_t *ctxp;
410	struct vtdmap *vtdmap;
411	uint8_t bus;
412
413	bus = PCI_RID2BUS(rid);
414	ctxp = ctx_tables[bus];
415	idx = VTD_RID2IDX(rid);
416
417	/*
418	 * Order is important. The 'present' bit is must be cleared first.
419	 */
420	ctxp[idx] = 0;
421	ctxp[idx + 1] = 0;
422
423	/*
424	 * Invalidate the Context Cache and the IOTLB.
425	 *
426	 * XXX use device-selective invalidation for Context Cache
427	 * XXX use domain-selective invalidation for IOTLB
428	 */
429	for (i = 0; i < drhd_num; i++) {
430		vtdmap = vtdmaps[i];
431		vtd_ctx_global_invalidate(vtdmap);
432		vtd_iotlb_global_invalidate(vtdmap);
433	}
434}
435
436#define	CREATE_MAPPING	0
437#define	REMOVE_MAPPING	1
438
439static uint64_t
440vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,
441		   int remove)
442{
443	struct domain *dom;
444	int i, spshift, ptpshift, ptpindex, nlevels;
445	uint64_t spsize, *ptp;
446
447	dom = arg;
448	ptpindex = 0;
449	ptpshift = 0;
450
451	KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__,
452	    gpa, len));
453	KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond "
454	    "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr));
455
456	if (gpa & PAGE_MASK)
457		panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa);
458
459	if (hpa & PAGE_MASK)
460		panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa);
461
462	if (len & PAGE_MASK)
463		panic("vtd_create_mapping: unaligned len 0x%0lx", len);
464
465	/*
466	 * Compute the size of the mapping that we can accommodate.
467	 *
468	 * This is based on three factors:
469	 * - supported super page size
470	 * - alignment of the region starting at 'gpa' and 'hpa'
471	 * - length of the region 'len'
472	 */
473	spshift = 48;
474	for (i = 3; i >= 0; i--) {
475		spsize = 1UL << spshift;
476		if ((dom->spsmask & (1 << i)) != 0 &&
477		    (gpa & (spsize - 1)) == 0 &&
478		    (hpa & (spsize - 1)) == 0 &&
479		    (len >= spsize)) {
480			break;
481		}
482		spshift -= 9;
483	}
484
485	ptp = dom->ptp;
486	nlevels = dom->pt_levels;
487	while (--nlevels >= 0) {
488		ptpshift = 12 + nlevels * 9;
489		ptpindex = (gpa >> ptpshift) & 0x1FF;
490
491		/* We have reached the leaf mapping */
492		if (spshift >= ptpshift) {
493			break;
494		}
495
496		/*
497		 * We are working on a non-leaf page table page.
498		 *
499		 * Create a downstream page table page if necessary and point
500		 * to it from the current page table.
501		 */
502		if (ptp[ptpindex] == 0) {
503			void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO);
504			ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR;
505		}
506
507		ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M);
508	}
509
510	if ((gpa & ((1UL << ptpshift) - 1)) != 0)
511		panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift);
512
513	/*
514	 * Update the 'gpa' -> 'hpa' mapping
515	 */
516	if (remove) {
517		ptp[ptpindex] = 0;
518	} else {
519		ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR;
520
521		if (nlevels > 0)
522			ptp[ptpindex] |= VTD_PTE_SUPERPAGE;
523	}
524
525	return (1UL << ptpshift);
526}
527
528static uint64_t
529vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
530{
531
532	return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING));
533}
534
535static uint64_t
536vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
537{
538
539	return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING));
540}
541
542static void
543vtd_invalidate_tlb(void *dom)
544{
545	int i;
546	struct vtdmap *vtdmap;
547
548	/*
549	 * Invalidate the IOTLB.
550	 * XXX use domain-selective invalidation for IOTLB
551	 */
552	for (i = 0; i < drhd_num; i++) {
553		vtdmap = vtdmaps[i];
554		vtd_iotlb_global_invalidate(vtdmap);
555	}
556}
557
558static void *
559vtd_create_domain(vm_paddr_t maxaddr)
560{
561	struct domain *dom;
562	vm_paddr_t addr;
563	int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth;
564	struct vtdmap *vtdmap;
565
566	if (drhd_num <= 0)
567		panic("vtd_create_domain: no dma remapping hardware available");
568
569	vtdmap = vtdmaps[0];
570
571	/*
572	 * Calculate AGAW.
573	 * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec.
574	 */
575	addr = 0;
576	for (gaw = 0; addr < maxaddr; gaw++)
577		addr = 1ULL << gaw;
578
579	res = (gaw - 12) % 9;
580	if (res == 0)
581		agaw = gaw;
582	else
583		agaw = gaw + 9 - res;
584
585	if (agaw > 64)
586		agaw = 64;
587
588	/*
589	 * Select the smallest Supported AGAW and the corresponding number
590	 * of page table levels.
591	 */
592	pt_levels = 2;
593	sagaw = 30;
594	addrwidth = 0;
595	tmp = VTD_CAP_SAGAW(vtdmap->cap);
596	for (i = 0; i < 5; i++) {
597		if ((tmp & (1 << i)) != 0 && sagaw >= agaw)
598			break;
599		pt_levels++;
600		addrwidth++;
601		sagaw += 9;
602		if (sagaw > 64)
603			sagaw = 64;
604	}
605
606	if (i >= 5) {
607		panic("vtd_create_domain: SAGAW 0x%lx does not support AGAW %d",
608		      VTD_CAP_SAGAW(vtdmap->cap), agaw);
609	}
610
611	dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK);
612	dom->pt_levels = pt_levels;
613	dom->addrwidth = addrwidth;
614	dom->id = domain_id();
615	dom->maxaddr = maxaddr;
616	dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK);
617	if ((uintptr_t)dom->ptp & PAGE_MASK)
618		panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp);
619
620#ifdef notyet
621	/*
622	 * XXX superpage mappings for the iommu do not work correctly.
623	 *
624	 * By default all physical memory is mapped into the host_domain.
625	 * When a VM is allocated wired memory the pages belonging to it
626	 * are removed from the host_domain and added to the vm's domain.
627	 *
628	 * If the page being removed was mapped using a superpage mapping
629	 * in the host_domain then we need to demote the mapping before
630	 * removing the page.
631	 *
632	 * There is not any code to deal with the demotion at the moment
633	 * so we disable superpage mappings altogether.
634	 */
635	dom->spsmask = VTD_CAP_SPS(vtdmap->cap);
636#endif
637
638	SLIST_INSERT_HEAD(&domhead, dom, next);
639
640	return (dom);
641}
642
643static void
644vtd_free_ptp(uint64_t *ptp, int level)
645{
646	int i;
647	uint64_t *nlp;
648
649	if (level > 1) {
650		for (i = 0; i < 512; i++) {
651			if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0)
652				continue;
653			if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0)
654				continue;
655			nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M);
656			vtd_free_ptp(nlp, level - 1);
657		}
658	}
659
660	bzero(ptp, PAGE_SIZE);
661	free(ptp, M_VTD);
662}
663
664static void
665vtd_destroy_domain(void *arg)
666{
667	struct domain *dom;
668
669	dom = arg;
670
671	SLIST_REMOVE(&domhead, dom, domain, next);
672	vtd_free_ptp(dom->ptp, dom->pt_levels);
673	free(dom, M_VTD);
674}
675
676struct iommu_ops iommu_ops_intel = {
677	vtd_init,
678	vtd_cleanup,
679	vtd_enable,
680	vtd_disable,
681	vtd_create_domain,
682	vtd_destroy_domain,
683	vtd_create_mapping,
684	vtd_remove_mapping,
685	vtd_add_device,
686	vtd_remove_device,
687	vtd_invalidate_tlb,
688};
689