Deleted Added
sdiff udiff text old ( 257512 ) new ( 280260 )
full compact
intel_qi.c (257512) intel_qi.c (280260)
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 14 unchanged lines hidden (view full) ---

23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_qi.c 257512 2013-11-01 17:38:52Z kib $");
31__FBSDID("$FreeBSD: head/sys/x86/iommu/intel_qi.c 280260 2015-03-19 13:57:47Z kib $");
32
33#include "opt_acpi.h"
34
35#include <sys/param.h>
36#include <sys/bus.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/memdesc.h>
40#include <sys/module.h>
41#include <sys/rman.h>
42#include <sys/taskqueue.h>
43#include <sys/tree.h>
32
33#include "opt_acpi.h"
34
35#include <sys/param.h>
36#include <sys/bus.h>
37#include <sys/kernel.h>
38#include <sys/malloc.h>
39#include <sys/memdesc.h>
40#include <sys/module.h>
41#include <sys/rman.h>
42#include <sys/taskqueue.h>
43#include <sys/tree.h>
44#include <sys/vmem.h>
44#include <machine/bus.h>
45#include <contrib/dev/acpica/include/acpi.h>
46#include <contrib/dev/acpica/include/accommon.h>
47#include <dev/acpica/acpivar.h>
48#include <vm/vm.h>
49#include <vm/vm_extern.h>
50#include <vm/vm_kern.h>
51#include <vm/vm_page.h>

--- 137 unchanged lines hidden (view full) ---

189 }
190 seq = unit->inv_waitd_seq++;
191 pseq->gen = unit->inv_waitd_gen;
192 pseq->seq = seq;
193 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
194}
195
196static void
45#include <machine/bus.h>
46#include <contrib/dev/acpica/include/acpi.h>
47#include <contrib/dev/acpica/include/accommon.h>
48#include <dev/acpica/acpivar.h>
49#include <vm/vm.h>
50#include <vm/vm_extern.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_page.h>

--- 137 unchanged lines hidden (view full) ---

190 }
191 seq = unit->inv_waitd_seq++;
192 pseq->gen = unit->inv_waitd_gen;
193 pseq->seq = seq;
194 dmar_qi_emit_wait_descr(unit, seq, true, true, false);
195}
196
197static void
197dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq)
198dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
199 bool nowait)
198{
199
200 DMAR_ASSERT_LOCKED(unit);
201 unit->inv_seq_waiters++;
202 while (!dmar_qi_seq_processed(unit, gseq)) {
200{
201
202 DMAR_ASSERT_LOCKED(unit);
203 unit->inv_seq_waiters++;
204 while (!dmar_qi_seq_processed(unit, gseq)) {
203 if (cold) {
205 if (cold || nowait) {
204 cpu_spinwait();
205 } else {
206 msleep(&unit->inv_seq_waiters, &unit->lock, 0,
207 "dmarse", hz);
208 }
209 }
210 unit->inv_seq_waiters--;
211}

--- 29 unchanged lines hidden (view full) ---

241{
242 struct dmar_qi_genseq gseq;
243
244 DMAR_ASSERT_LOCKED(unit);
245 dmar_qi_ensure(unit, 2);
246 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
247 dmar_qi_emit_wait_seq(unit, &gseq);
248 dmar_qi_advance_tail(unit);
206 cpu_spinwait();
207 } else {
208 msleep(&unit->inv_seq_waiters, &unit->lock, 0,
209 "dmarse", hz);
210 }
211 }
212 unit->inv_seq_waiters--;
213}

--- 29 unchanged lines hidden (view full) ---

243{
244 struct dmar_qi_genseq gseq;
245
246 DMAR_ASSERT_LOCKED(unit);
247 dmar_qi_ensure(unit, 2);
248 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
249 dmar_qi_emit_wait_seq(unit, &gseq);
250 dmar_qi_advance_tail(unit);
249 dmar_qi_wait_for_seq(unit, &gseq);
251 dmar_qi_wait_for_seq(unit, &gseq, false);
250}
251
252void
253dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
254{
255 struct dmar_qi_genseq gseq;
256
257 DMAR_ASSERT_LOCKED(unit);
258 dmar_qi_ensure(unit, 2);
259 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
260 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
261 dmar_qi_emit_wait_seq(unit, &gseq);
262 dmar_qi_advance_tail(unit);
252}
253
254void
255dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
256{
257 struct dmar_qi_genseq gseq;
258
259 DMAR_ASSERT_LOCKED(unit);
260 dmar_qi_ensure(unit, 2);
261 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
262 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
263 dmar_qi_emit_wait_seq(unit, &gseq);
264 dmar_qi_advance_tail(unit);
263 dmar_qi_wait_for_seq(unit, &gseq);
265 dmar_qi_wait_for_seq(unit, &gseq, false);
264}
265
266}
267
268void
269dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
270{
271 struct dmar_qi_genseq gseq;
272
273 DMAR_ASSERT_LOCKED(unit);
274 dmar_qi_ensure(unit, 2);
275 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
276 dmar_qi_emit_wait_seq(unit, &gseq);
277 dmar_qi_advance_tail(unit);
278 dmar_qi_wait_for_seq(unit, &gseq, false);
279}
280
281void
282dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
283{
284 struct dmar_qi_genseq gseq;
285 u_int c, l;
286
287 DMAR_ASSERT_LOCKED(unit);
288 KASSERT(start < unit->irte_cnt && start < start + cnt &&
289 start + cnt <= unit->irte_cnt,
290 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
291 for (; cnt > 0; cnt -= c, start += c) {
292 l = ffs(start | cnt) - 1;
293 c = 1 << l;
294 dmar_qi_ensure(unit, 1);
295 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
296 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
297 DMAR_IQ_DESCR_IEC_IM(l), 0);
298 }
299 dmar_qi_ensure(unit, 1);
300 dmar_qi_emit_wait_seq(unit, &gseq);
301 dmar_qi_advance_tail(unit);
302
303 /*
304 * The caller of the function, in particular,
305 * dmar_ir_program_irte(), may be called from the context
306 * where the sleeping is forbidden (in fact, the
307 * intr_table_lock mutex may be held, locked from
308 * intr_shuffle_irqs()). Wait for the invalidation completion
309 * using the busy wait.
310 *
311 * The impact on the interrupt input setup code is small, the
312 * expected overhead is comparable with the chipset register
313 * read. It is more harmful for the parallel DMA operations,
314 * since we own the dmar unit lock until whole invalidation
315 * queue is processed, which includes requests possibly issued
316 * before our request.
317 */
318 dmar_qi_wait_for_seq(unit, &gseq, true);
319}
320
266int
267dmar_qi_intr(void *arg)
268{
269 struct dmar_unit *unit;
270
271 unit = arg;
272 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
273 taskqueue_enqueue_fast(unit->qi_taskqueue, &unit->qi_task);

--- 98 unchanged lines hidden (view full) ---

372 taskqueue_free(unit->qi_taskqueue);
373 unit->qi_taskqueue = NULL;
374
375 DMAR_LOCK(unit);
376 /* quisce */
377 dmar_qi_ensure(unit, 1);
378 dmar_qi_emit_wait_seq(unit, &gseq);
379 dmar_qi_advance_tail(unit);
321int
322dmar_qi_intr(void *arg)
323{
324 struct dmar_unit *unit;
325
326 unit = arg;
327 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
328 taskqueue_enqueue_fast(unit->qi_taskqueue, &unit->qi_task);

--- 98 unchanged lines hidden (view full) ---

427 taskqueue_free(unit->qi_taskqueue);
428 unit->qi_taskqueue = NULL;
429
430 DMAR_LOCK(unit);
431 /* quisce */
432 dmar_qi_ensure(unit, 1);
433 dmar_qi_emit_wait_seq(unit, &gseq);
434 dmar_qi_advance_tail(unit);
380 dmar_qi_wait_for_seq(unit, &gseq);
435 dmar_qi_wait_for_seq(unit, &gseq, false);
381 /* only after the quisce, disable queue */
382 dmar_disable_qi(unit);
383 KASSERT(unit->inv_seq_waiters == 0,
384 ("dmar%d: waiters on disabled queue", unit->unit));
385 DMAR_UNLOCK(unit);
386
387 kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
388 unit->inv_queue = 0;

--- 26 unchanged lines hidden --- 		436 /* only after the quisce, disable queue */
437 dmar_disable_qi(unit);
438 KASSERT(unit->inv_seq_waiters == 0,
439 ("dmar%d: waiters on disabled queue", unit->unit));
440 DMAR_UNLOCK(unit);
441
442 kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
443 unit->inv_queue = 0;

--- 26 unchanged lines hidden ---