mm/book3s64/radix_tlb.c

29 					unsigned int pid,
36 	rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
100 static __always_inline void __tlbiel_pid(unsigned long pid, int set,
107 	rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
116 static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric)
121 	rs = pid << PPC_BITLSHIFT(31);
158 static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid,
165 	rs = pid << PPC_BITLSHIFT(31);
174 static __always_inline void __tlbie_va(unsigned long va, unsigned long pid,
181 	rs = pid << PPC_BITLSHIFT(31);
207 static inline void fixup_tlbie_va(unsigned long va, unsigned long pid,
217 		__tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
221 static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid,
231 		__tlbie_va(va, pid, ap, RIC_FLUSH_TLB);
235 static inline void fixup_tlbie_pid(unsigned long pid)
250 		__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
290 static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
300 		__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
304 		__tlbiel_pid(pid, 0, RIC_FLUSH_TLB);
313 		__tlbiel_pid(pid, 0, RIC_FLUSH_ALL);
319 			__tlbiel_pid(pid, set, RIC_FLUSH_TLB);
326 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
337 		__tlbie_pid(pid, RIC_FLUSH_TLB);
338 		fixup_tlbie_pid(pid);
341 		__tlbie_pid(pid, RIC_FLUSH_PWC);
345 		__tlbie_pid(pid, RIC_FLUSH_ALL);
346 		fixup_tlbie_pid(pid);
352 	unsigned long pid;
361 		_tlbiel_pid(t->pid, RIC_FLUSH_TLB);
363 		_tlbiel_pid(t->pid, RIC_FLUSH_PWC);
365 		_tlbiel_pid(t->pid, RIC_FLUSH_ALL);
369 				unsigned long pid, unsigned long ric)
372 	struct tlbiel_pid t = { .pid = pid, .ric = ric };
381 		_tlbie_pid(pid, RIC_FLUSH_ALL);
432 				    unsigned long pid, unsigned long page_size,
439 		__tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
442 static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid,
448 	__tlbiel_va(va, pid, ap, ric);
453 				    unsigned long pid, unsigned long page_size,
458 		__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
459 	__tlbiel_va_range(start, end, pid, page_size, psize);
464 				    unsigned long pid, unsigned long page_size,
471 		__tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
473 	fixup_tlbie_va_range(addr - page_size, pid, ap);
476 static __always_inline void _tlbie_va(unsigned long va, unsigned long pid,
482 	__tlbie_va(va, pid, ap, ric);
483 	fixup_tlbie_va(va, pid, ap);
488 	unsigned long pid;
499 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB);
501 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC);
503 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL);
507 				unsigned long va, unsigned long pid,
511 	struct tlbiel_va t = { .va = va, .pid = pid, .psize = psize, .ric = ric };
514 		_tlbie_va(va, pid, psize, RIC_FLUSH_TLB);
518 	unsigned long pid;
530 	_tlbiel_va_range(t->start, t->end, t->pid, t->page_size,
546 				    unsigned long pid, unsigned long page_size,
551 		__tlbie_pid(pid, RIC_FLUSH_PWC);
552 	__tlbie_va_range(start, end, pid, page_size, psize);
558 				unsigned long pid, unsigned long page_size,
563 				.pid = pid, .page_size = page_size,
568 		_tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
584 	unsigned long pid = mm->context.id;
586 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
590 	_tlbiel_pid(pid, RIC_FLUSH_TLB);
598 	unsigned long pid = mm->context.id;
600 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
604 	_tlbiel_pid(pid, RIC_FLUSH_ALL);
618 	unsigned long pid = mm->context.id;
620 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
624 	_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
665 	unsigned long pid = mm->context.id;
711 		_tlbiel_pid(pid, RIC_FLUSH_ALL);
836 	unsigned long pid;
839 	pid = mm->context.id;
840 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
852 		_tlbiel_pid(pid, RIC_FLUSH_TLB);
859 			pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
863 				_tlbie_pid(pid, RIC_FLUSH_ALL);
865 				_tlbie_pid(pid, RIC_FLUSH_TLB);
867 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB);
877 	unsigned long pid;
880 	pid = mm->context.id;
881 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
888 		_tlbiel_pid(pid, RIC_FLUSH_ALL);
897 			pseries_rpt_invalidate(pid, tgt, type,
900 			_tlbie_pid(pid, RIC_FLUSH_ALL);
902 			_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
917 	unsigned long pid;
920 	pid = mm->context.id;
921 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
928 		_tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
939 			pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB,
943 			_tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
945 			_tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB);
1013  * individual page flushes to full-pid flushes.
1021 	unsigned long pid;
1028 	pid = mm->context.id;
1029 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
1045 	 * full pid flush already does the PWC flush. if it is not full pid
1063 		pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
1069 			_tlbiel_pid(pid, RIC_FLUSH_ALL);
1072 				_tlbie_pid(pid, RIC_FLUSH_ALL);
1074 				_tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL);
1089 				__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
1090 			__tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize);
1092 				__tlbiel_va_range(hstart, hend, pid,
1098 				__tlbie_pid(pid, RIC_FLUSH_PWC);
1099 			__tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize);
1101 				__tlbie_va_range(hstart, hend, pid,
1106 					start, end, pid, page_size, mmu_virtual_psize, flush_pwc);
1109 					hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, flush_pwc);
1239 	unsigned long pid;
1246 	pid = mm->context.id;
1247 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
1272 		pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end);
1275 			_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
1281 				_tlbie_pid(pid,
1284 				_tlbiel_pid_multicast(mm, pid,
1291 			_tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
1293 			_tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
1296 					start, end, pid, page_size, psize, also_pwc);
1318 	unsigned long pid, end;
1321 	pid = mm->context.id;
1322 	if (WARN_ON_ONCE(pid == MMU_NO_CONTEXT))
1338 		_tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1350 			pseries_rpt_invalidate(pid, tgt, type, pg_sizes,
1353 			_tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1356 					addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
1402 static __always_inline void __tlbie_pid_lpid(unsigned long pid,
1409 	rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31)));
1418 static __always_inline void __tlbie_va_lpid(unsigned long va, unsigned long pid,
1426 	rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31)));
1435 static inline void fixup_tlbie_pid_lpid(unsigned long pid, unsigned long lpid)
1450 		__tlbie_va_lpid(va, pid, lpid, mmu_get_ap(MMU_PAGE_64K),
1455 static inline void _tlbie_pid_lpid(unsigned long pid, unsigned long lpid,
1467 		__tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
1468 		fixup_tlbie_pid_lpid(pid, lpid);
1471 		__tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
1475 		__tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL);
1476 		fixup_tlbie_pid_lpid(pid, lpid);
1482 					     unsigned long pid,
1493 		__tlbie_va_lpid(va, pid, lpid, ap, RIC_FLUSH_TLB);
1498 					 unsigned long pid, unsigned long lpid,
1506 		__tlbie_va_lpid(addr, pid, lpid, ap, RIC_FLUSH_TLB);
1508 	fixup_tlbie_va_range_lpid(addr - page_size, pid, lpid, ap);
1512 					unsigned long pid, unsigned long lpid,
1518 		__tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
1519 	__tlbie_va_range_lpid(start, end, pid, lpid, page_size, psize);
1527 void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid,
1540 		_tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL);
1545 		_tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC);
1549 		return _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
1567 			_tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB);
1570 		_tlbie_va_range_lpid(start, end, pid, lpid,