x86/iommu/intel_qi.c

62 dmar_qi_seq_processed(const struct dmar_unit *unit,
66 	return (pseq->gen < unit->inv_waitd_gen ||
67 	    (pseq->gen == unit->inv_waitd_gen &&
68 	     pseq->seq <= unit->inv_waitd_seq_hw));
72 dmar_enable_qi(struct dmar_unit *unit)
76 	DMAR_ASSERT_LOCKED(unit);
77 	unit->hw_gcmd |= DMAR_GCMD_QIE;
78 	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
79 	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
85 dmar_disable_qi(struct dmar_unit *unit)
89 	DMAR_ASSERT_LOCKED(unit);
90 	unit->hw_gcmd &= ~DMAR_GCMD_QIE;
91 	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
92 	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
98 dmar_qi_advance_tail(struct dmar_unit *unit)
101 	DMAR_ASSERT_LOCKED(unit);
102 	dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
106 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
111 	DMAR_ASSERT_LOCKED(unit);
114 		if (bytes <= unit->inv_queue_avail)
117 		head = dmar_read4(unit, DMAR_IQH_REG);
119 		unit->inv_queue_avail = head - unit->inv_queue_tail -
121 		if (head <= unit->inv_queue_tail)
122 			unit->inv_queue_avail += unit->inv_queue_size;
123 		if (bytes <= unit->inv_queue_avail)
133 		dmar_qi_advance_tail(unit);
134 		unit->inv_queue_full++;
137 	unit->inv_queue_avail -= bytes;
141 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
144 	DMAR_ASSERT_LOCKED(unit);
145 	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
146 	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
147 	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
148 	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
149 	    (uintmax_t)unit->inv_queue_size));
150 	unit->inv_queue_tail &= unit->inv_queue_size - 1;
151 	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
152 	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
153 	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
154 	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
155 	    (uintmax_t)unit->inv_queue_size));
156 	unit->inv_queue_tail &= unit->inv_queue_size - 1;
160 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
164 	DMAR_ASSERT_LOCKED(unit);
165 	dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
170 	    memw ? unit->inv_waitd_seq_hw_phys : 0);
174 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
181 	DMAR_ASSERT_LOCKED(unit);
182 	if (unit->inv_waitd_seq == 0xffffffff) {
183 		gsec.gen = unit->inv_waitd_gen;
184 		gsec.seq = unit->inv_waitd_seq;
185 		dmar_qi_ensure(unit, 1);
186 		dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
187 		dmar_qi_advance_tail(unit);
188 		while (!dmar_qi_seq_processed(unit, &gsec))
190 		unit->inv_waitd_gen++;
191 		unit->inv_waitd_seq = 1;
193 	seq = unit->inv_waitd_seq++;
194 	pseq->gen = unit->inv_waitd_gen;
197 		dmar_qi_ensure(unit, 1);
198 		dmar_qi_emit_wait_descr(unit, seq, true, true, false);
203 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
207 	DMAR_ASSERT_LOCKED(unit);
208 	unit->inv_seq_waiters++;
209 	while (!dmar_qi_seq_processed(unit, gseq)) {
213 			msleep(&unit->inv_seq_waiters, &unit->lock, 0,
217 	unit->inv_seq_waiters--;
224 	struct dmar_unit *unit;
228 	unit = domain->dmar;
229 	DMAR_ASSERT_LOCKED(unit);
231 		am = calc_am(unit, base, size, &isize);
232 		dmar_qi_ensure(unit, 1);
233 		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
239 	dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
240 	dmar_qi_advance_tail(unit);
244 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
248 	DMAR_ASSERT_LOCKED(unit);
249 	dmar_qi_ensure(unit, 2);
250 	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
251 	dmar_qi_emit_wait_seq(unit, &gseq, true);
252 	dmar_qi_advance_tail(unit);
253 	dmar_qi_wait_for_seq(unit, &gseq, false);
257 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
261 	DMAR_ASSERT_LOCKED(unit);
262 	dmar_qi_ensure(unit, 2);
263 	dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
265 	dmar_qi_emit_wait_seq(unit, &gseq, true);
266 	dmar_qi_advance_tail(unit);
267 	dmar_qi_wait_for_seq(unit, &gseq, false);
271 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
275 	DMAR_ASSERT_LOCKED(unit);
276 	dmar_qi_ensure(unit, 2);
277 	dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
278 	dmar_qi_emit_wait_seq(unit, &gseq, true);
279 	dmar_qi_advance_tail(unit);
280 	dmar_qi_wait_for_seq(unit, &gseq, false);
284 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
289 	DMAR_ASSERT_LOCKED(unit);
290 	KASSERT(start < unit->irte_cnt && start < start + cnt &&
291 	    start + cnt <= unit->irte_cnt,
292 	    ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
296 		dmar_qi_ensure(unit, 1);
297 		dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
301 	dmar_qi_ensure(unit, 1);
302 	dmar_qi_emit_wait_seq(unit, &gseq, true);
303 	dmar_qi_advance_tail(unit);
316 	 * since we own the dmar unit lock until whole invalidation
320 	dmar_qi_wait_for_seq(unit, &gseq, true);
326 	struct dmar_unit *unit;
328 	unit = arg;
329 	KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
330 	taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
337 	struct dmar_unit *unit;
341 	unit = arg;
343 	DMAR_LOCK(unit);
345 		entry = TAILQ_FIRST(&unit->tlb_flush_entries);
348 		if (!dmar_qi_seq_processed(unit, &entry->gseq))
350 		TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
351 		DMAR_UNLOCK(unit);
354 		DMAR_LOCK(unit);
356 	ics = dmar_read4(unit, DMAR_ICS_REG);
359 		dmar_write4(unit, DMAR_ICS_REG, ics);
361 	if (unit->inv_seq_waiters > 0)
362 		wakeup(&unit->inv_seq_waiters);
363 	DMAR_UNLOCK(unit);
367 dmar_init_qi(struct dmar_unit *unit)
373 	if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
375 	unit->qi_enabled = 1;
376 	TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
377 	if (!unit->qi_enabled)
380 	TAILQ_INIT(&unit->tlb_flush_entries);
381 	TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
382 	unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
383 	    taskqueue_thread_enqueue, &unit->qi_taskqueue);
384 	taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
385 	    "dmar%d qi taskq", unit->unit);
387 	unit->inv_waitd_gen = 0;
388 	unit->inv_waitd_seq = 1;
394 	unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
396 	unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
399 	unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
401 	unit->inv_waitd_seq_hw_phys = pmap_kextract(
402 	    (vm_offset_t)&unit->inv_waitd_seq_hw);
404 	DMAR_LOCK(unit);
405 	dmar_write8(unit, DMAR_IQT_REG, 0);
406 	iqa = pmap_kextract(unit->inv_queue);
408 	dmar_write8(unit, DMAR_IQA_REG, iqa);
409 	dmar_enable_qi(unit);
410 	ics = dmar_read4(unit, DMAR_ICS_REG);
413 		dmar_write4(unit, DMAR_ICS_REG, ics);
415 	dmar_enable_qi_intr(unit);
416 	DMAR_UNLOCK(unit);
422 dmar_fini_qi(struct dmar_unit *unit)
426 	if (unit->qi_enabled)
428 	taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
429 	taskqueue_free(unit->qi_taskqueue);
430 	unit->qi_taskqueue = NULL;
432 	DMAR_LOCK(unit);
434 	dmar_qi_ensure(unit, 1);
435 	dmar_qi_emit_wait_seq(unit, &gseq, true);
436 	dmar_qi_advance_tail(unit);
437 	dmar_qi_wait_for_seq(unit, &gseq, false);
439 	dmar_disable_qi_intr(unit);
440 	dmar_disable_qi(unit);
441 	KASSERT(unit->inv_seq_waiters == 0,
442 	    ("dmar%d: waiters on disabled queue", unit->unit));
443 	DMAR_UNLOCK(unit);
445 	kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
446 	unit->inv_queue = 0;
447 	unit->inv_queue_size = 0;
448 	unit->qi_enabled = 0;
452 dmar_enable_qi_intr(struct dmar_unit *unit)
456 	DMAR_ASSERT_LOCKED(unit);
457 	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
458 	iectl = dmar_read4(unit, DMAR_IECTL_REG);
460 	dmar_write4(unit, DMAR_IECTL_REG, iectl);
464 dmar_disable_qi_intr(struct dmar_unit *unit)
468 	DMAR_ASSERT_LOCKED(unit);
469 	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
470 	iectl = dmar_read4(unit, DMAR_IECTL_REG);
471 	dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);