1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Copyright IBM Corporation, 2008
19 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
20 *
21 * Author: Allen M. Kay <allen.m.kay@intel.com>
22 * Author: Weidong Han <weidong.han@intel.com>
23 * Author: Ben-Ami Yassour <benami@il.ibm.com>
24 */
25
26#include <linux/list.h>
27#include <linux/kvm_host.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/iommu.h>
31#include <linux/intel-iommu.h>
32
33static int kvm_iommu_unmap_memslots(struct kvm *kvm);
34static void kvm_iommu_put_pages(struct kvm *kvm,
35				gfn_t base_gfn, unsigned long npages);
36
37static pfn_t kvm_pin_pages(struct kvm *kvm, struct kvm_memory_slot *slot,
38			   gfn_t gfn, unsigned long size)
39{
40	gfn_t end_gfn;
41	pfn_t pfn;
42
43	pfn     = gfn_to_pfn_memslot(kvm, slot, gfn);
44	end_gfn = gfn + (size >> PAGE_SHIFT);
45	gfn    += 1;
46
47	if (is_error_pfn(pfn))
48		return pfn;
49
50	while (gfn < end_gfn)
51		gfn_to_pfn_memslot(kvm, slot, gfn++);
52
53	return pfn;
54}
55
56int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
57{
58	gfn_t gfn, end_gfn;
59	pfn_t pfn;
60	int r = 0;
61	struct iommu_domain *domain = kvm->arch.iommu_domain;
62	int flags;
63
64	/* check if iommu exists and in use */
65	if (!domain)
66		return 0;
67
68	gfn     = slot->base_gfn;
69	end_gfn = gfn + slot->npages;
70
71	flags = IOMMU_READ | IOMMU_WRITE;
72	if (kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY)
73		flags |= IOMMU_CACHE;
74
75
76	while (gfn < end_gfn) {
77		unsigned long page_size;
78
79		/* Check if already mapped */
80		if (iommu_iova_to_phys(domain, gfn_to_gpa(gfn))) {
81			gfn += 1;
82			continue;
83		}
84
85		/* Get the page size we could use to map */
86		page_size = kvm_host_page_size(kvm, gfn);
87
88		/* Make sure the page_size does not exceed the memslot */
89		while ((gfn + (page_size >> PAGE_SHIFT)) > end_gfn)
90			page_size >>= 1;
91
92		/* Make sure gfn is aligned to the page size we want to map */
93		while ((gfn << PAGE_SHIFT) & (page_size - 1))
94			page_size >>= 1;
95
96		/*
97		 * Pin all pages we are about to map in memory. This is
98		 * important because we unmap and unpin in 4kb steps later.
99		 */
100		pfn = kvm_pin_pages(kvm, slot, gfn, page_size);
101		if (is_error_pfn(pfn)) {
102			gfn += 1;
103			continue;
104		}
105
106		/* Map into IO address space */
107		r = iommu_map(domain, gfn_to_gpa(gfn), pfn_to_hpa(pfn),
108			      get_order(page_size), flags);
109		if (r) {
110			printk(KERN_ERR "kvm_iommu_map_address:"
111			       "iommu failed to map pfn=%llx\n", pfn);
112			goto unmap_pages;
113		}
114
115		gfn += page_size >> PAGE_SHIFT;
116
117
118	}
119
120	return 0;
121
122unmap_pages:
123	kvm_iommu_put_pages(kvm, slot->base_gfn, gfn);
124	return r;
125}
126
127static int kvm_iommu_map_memslots(struct kvm *kvm)
128{
129	int i, idx, r = 0;
130	struct kvm_memslots *slots;
131
132	idx = srcu_read_lock(&kvm->srcu);
133	slots = kvm_memslots(kvm);
134
135	for (i = 0; i < slots->nmemslots; i++) {
136		r = kvm_iommu_map_pages(kvm, &slots->memslots[i]);
137		if (r)
138			break;
139	}
140	srcu_read_unlock(&kvm->srcu, idx);
141
142	return r;
143}
144
145int kvm_assign_device(struct kvm *kvm,
146		      struct kvm_assigned_dev_kernel *assigned_dev)
147{
148	struct pci_dev *pdev = NULL;
149	struct iommu_domain *domain = kvm->arch.iommu_domain;
150	int r, last_flags;
151
152	/* check if iommu exists and in use */
153	if (!domain)
154		return 0;
155
156	pdev = assigned_dev->dev;
157	if (pdev == NULL)
158		return -ENODEV;
159
160	r = iommu_attach_device(domain, &pdev->dev);
161	if (r) {
162		printk(KERN_ERR "assign device %x:%x:%x.%x failed",
163			pci_domain_nr(pdev->bus),
164			pdev->bus->number,
165			PCI_SLOT(pdev->devfn),
166			PCI_FUNC(pdev->devfn));
167		return r;
168	}
169
170	last_flags = kvm->arch.iommu_flags;
171	if (iommu_domain_has_cap(kvm->arch.iommu_domain,
172				 IOMMU_CAP_CACHE_COHERENCY))
173		kvm->arch.iommu_flags |= KVM_IOMMU_CACHE_COHERENCY;
174
175	/* Check if need to update IOMMU page table for guest memory */
176	if ((last_flags ^ kvm->arch.iommu_flags) ==
177			KVM_IOMMU_CACHE_COHERENCY) {
178		kvm_iommu_unmap_memslots(kvm);
179		r = kvm_iommu_map_memslots(kvm);
180		if (r)
181			goto out_unmap;
182	}
183
184	printk(KERN_DEBUG "assign device %x:%x:%x.%x\n",
185		assigned_dev->host_segnr,
186		assigned_dev->host_busnr,
187		PCI_SLOT(assigned_dev->host_devfn),
188		PCI_FUNC(assigned_dev->host_devfn));
189
190	return 0;
191out_unmap:
192	kvm_iommu_unmap_memslots(kvm);
193	return r;
194}
195
196int kvm_deassign_device(struct kvm *kvm,
197			struct kvm_assigned_dev_kernel *assigned_dev)
198{
199	struct iommu_domain *domain = kvm->arch.iommu_domain;
200	struct pci_dev *pdev = NULL;
201
202	/* check if iommu exists and in use */
203	if (!domain)
204		return 0;
205
206	pdev = assigned_dev->dev;
207	if (pdev == NULL)
208		return -ENODEV;
209
210	iommu_detach_device(domain, &pdev->dev);
211
212	printk(KERN_DEBUG "deassign device %x:%x:%x.%x\n",
213		assigned_dev->host_segnr,
214		assigned_dev->host_busnr,
215		PCI_SLOT(assigned_dev->host_devfn),
216		PCI_FUNC(assigned_dev->host_devfn));
217
218	return 0;
219}
220
221int kvm_iommu_map_guest(struct kvm *kvm)
222{
223	int r;
224
225	if (!iommu_found()) {
226		printk(KERN_ERR "%s: iommu not found\n", __func__);
227		return -ENODEV;
228	}
229
230	kvm->arch.iommu_domain = iommu_domain_alloc();
231	if (!kvm->arch.iommu_domain)
232		return -ENOMEM;
233
234	r = kvm_iommu_map_memslots(kvm);
235	if (r)
236		goto out_unmap;
237
238	return 0;
239
240out_unmap:
241	kvm_iommu_unmap_memslots(kvm);
242	return r;
243}
244
245static void kvm_unpin_pages(struct kvm *kvm, pfn_t pfn, unsigned long npages)
246{
247	unsigned long i;
248
249	for (i = 0; i < npages; ++i)
250		kvm_release_pfn_clean(pfn + i);
251}
252
253static void kvm_iommu_put_pages(struct kvm *kvm,
254				gfn_t base_gfn, unsigned long npages)
255{
256	struct iommu_domain *domain;
257	gfn_t end_gfn, gfn;
258	pfn_t pfn;
259	u64 phys;
260
261	domain  = kvm->arch.iommu_domain;
262	end_gfn = base_gfn + npages;
263	gfn     = base_gfn;
264
265	/* check if iommu exists and in use */
266	if (!domain)
267		return;
268
269	while (gfn < end_gfn) {
270		unsigned long unmap_pages;
271		int order;
272
273		/* Get physical address */
274		phys = iommu_iova_to_phys(domain, gfn_to_gpa(gfn));
275		pfn  = phys >> PAGE_SHIFT;
276
277		/* Unmap address from IO address space */
278		order       = iommu_unmap(domain, gfn_to_gpa(gfn), 0);
279		unmap_pages = 1ULL << order;
280
281		/* Unpin all pages we just unmapped to not leak any memory */
282		kvm_unpin_pages(kvm, pfn, unmap_pages);
283
284		gfn += unmap_pages;
285	}
286}
287
288static int kvm_iommu_unmap_memslots(struct kvm *kvm)
289{
290	int i, idx;
291	struct kvm_memslots *slots;
292
293	idx = srcu_read_lock(&kvm->srcu);
294	slots = kvm_memslots(kvm);
295
296	for (i = 0; i < slots->nmemslots; i++) {
297		kvm_iommu_put_pages(kvm, slots->memslots[i].base_gfn,
298				    slots->memslots[i].npages);
299	}
300	srcu_read_unlock(&kvm->srcu, idx);
301
302	return 0;
303}
304
305int kvm_iommu_unmap_guest(struct kvm *kvm)
306{
307	struct iommu_domain *domain = kvm->arch.iommu_domain;
308
309	/* check if iommu exists and in use */
310	if (!domain)
311		return 0;
312
313	kvm_iommu_unmap_memslots(kvm);
314	iommu_domain_free(domain);
315	return 0;
316}
317