1// SPDX-License-Identifier: GPL-2.0
2#include <linux/atomic.h>
3#include <linux/mmu_context.h>
4#include <linux/percpu.h>
5#include <linux/spinlock.h>
6
7static DEFINE_RAW_SPINLOCK(cpu_mmid_lock);
8
9static atomic64_t mmid_version;
10static unsigned int num_mmids;
11static unsigned long *mmid_map;
12
13static DEFINE_PER_CPU(u64, reserved_mmids);
14static cpumask_t tlb_flush_pending;
15
16static bool asid_versions_eq(int cpu, u64 a, u64 b)
17{
18	return ((a ^ b) & asid_version_mask(cpu)) == 0;
19}
20
21void get_new_mmu_context(struct mm_struct *mm)
22{
23	unsigned int cpu;
24	u64 asid;
25
26	/*
27	 * This function is specific to ASIDs, and should not be called when
28	 * MMIDs are in use.
29	 */
30	if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
31		return;
32
33	cpu = smp_processor_id();
34	asid = asid_cache(cpu);
35
36	if (!((asid += cpu_asid_inc()) & cpu_asid_mask(&cpu_data[cpu]))) {
37		if (cpu_has_vtag_icache)
38			flush_icache_all();
39		local_flush_tlb_all();	/* start new asid cycle */
40	}
41
42	set_cpu_context(cpu, mm, asid);
43	asid_cache(cpu) = asid;
44}
45EXPORT_SYMBOL_GPL(get_new_mmu_context);
46
47void check_mmu_context(struct mm_struct *mm)
48{
49	unsigned int cpu = smp_processor_id();
50
51	/*
52	 * This function is specific to ASIDs, and should not be called when
53	 * MMIDs are in use.
54	 */
55	if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
56		return;
57
58	/* Check if our ASID is of an older version and thus invalid */
59	if (!asid_versions_eq(cpu, cpu_context(cpu, mm), asid_cache(cpu)))
60		get_new_mmu_context(mm);
61}
62EXPORT_SYMBOL_GPL(check_mmu_context);
63
64static void flush_context(void)
65{
66	u64 mmid;
67	int cpu;
68
69	/* Update the list of reserved MMIDs and the MMID bitmap */
70	bitmap_zero(mmid_map, num_mmids);
71
72	/* Reserve an MMID for kmap/wired entries */
73	__set_bit(MMID_KERNEL_WIRED, mmid_map);
74
75	for_each_possible_cpu(cpu) {
76		mmid = xchg_relaxed(&cpu_data[cpu].asid_cache, 0);
77
78		/*
79		 * If this CPU has already been through a
80		 * rollover, but hasn't run another task in
81		 * the meantime, we must preserve its reserved
82		 * MMID, as this is the only trace we have of
83		 * the process it is still running.
84		 */
85		if (mmid == 0)
86			mmid = per_cpu(reserved_mmids, cpu);
87
88		__set_bit(mmid & cpu_asid_mask(&cpu_data[cpu]), mmid_map);
89		per_cpu(reserved_mmids, cpu) = mmid;
90	}
91
92	/*
93	 * Queue a TLB invalidation for each CPU to perform on next
94	 * context-switch
95	 */
96	cpumask_setall(&tlb_flush_pending);
97}
98
99static bool check_update_reserved_mmid(u64 mmid, u64 newmmid)
100{
101	bool hit;
102	int cpu;
103
104	/*
105	 * Iterate over the set of reserved MMIDs looking for a match.
106	 * If we find one, then we can update our mm to use newmmid
107	 * (i.e. the same MMID in the current generation) but we can't
108	 * exit the loop early, since we need to ensure that all copies
109	 * of the old MMID are updated to reflect the mm. Failure to do
110	 * so could result in us missing the reserved MMID in a future
111	 * generation.
112	 */
113	hit = false;
114	for_each_possible_cpu(cpu) {
115		if (per_cpu(reserved_mmids, cpu) == mmid) {
116			hit = true;
117			per_cpu(reserved_mmids, cpu) = newmmid;
118		}
119	}
120
121	return hit;
122}
123
124static u64 get_new_mmid(struct mm_struct *mm)
125{
126	static u32 cur_idx = MMID_KERNEL_WIRED + 1;
127	u64 mmid, version, mmid_mask;
128
129	mmid = cpu_context(0, mm);
130	version = atomic64_read(&mmid_version);
131	mmid_mask = cpu_asid_mask(&boot_cpu_data);
132
133	if (!asid_versions_eq(0, mmid, 0)) {
134		u64 newmmid = version | (mmid & mmid_mask);
135
136		/*
137		 * If our current MMID was active during a rollover, we
138		 * can continue to use it and this was just a false alarm.
139		 */
140		if (check_update_reserved_mmid(mmid, newmmid)) {
141			mmid = newmmid;
142			goto set_context;
143		}
144
145		/*
146		 * We had a valid MMID in a previous life, so try to re-use
147		 * it if possible.
148		 */
149		if (!__test_and_set_bit(mmid & mmid_mask, mmid_map)) {
150			mmid = newmmid;
151			goto set_context;
152		}
153	}
154
155	/* Allocate a free MMID */
156	mmid = find_next_zero_bit(mmid_map, num_mmids, cur_idx);
157	if (mmid != num_mmids)
158		goto reserve_mmid;
159
160	/* We're out of MMIDs, so increment the global version */
161	version = atomic64_add_return_relaxed(asid_first_version(0),
162					      &mmid_version);
163
164	/* Note currently active MMIDs & mark TLBs as requiring flushes */
165	flush_context();
166
167	/* We have more MMIDs than CPUs, so this will always succeed */
168	mmid = find_first_zero_bit(mmid_map, num_mmids);
169
170reserve_mmid:
171	__set_bit(mmid, mmid_map);
172	cur_idx = mmid;
173	mmid |= version;
174set_context:
175	set_cpu_context(0, mm, mmid);
176	return mmid;
177}
178
179void check_switch_mmu_context(struct mm_struct *mm)
180{
181	unsigned int cpu = smp_processor_id();
182	u64 ctx, old_active_mmid;
183	unsigned long flags;
184
185	if (!cpu_has_mmid) {
186		check_mmu_context(mm);
187		write_c0_entryhi(cpu_asid(cpu, mm));
188		goto setup_pgd;
189	}
190
191	/*
192	 * MMID switch fast-path, to avoid acquiring cpu_mmid_lock when it's
193	 * unnecessary.
194	 *
195	 * The memory ordering here is subtle. If our active_mmids is non-zero
196	 * and the MMID matches the current version, then we update the CPU's
197	 * asid_cache with a relaxed cmpxchg. Racing with a concurrent rollover
198	 * means that either:
199	 *
200	 * - We get a zero back from the cmpxchg and end up waiting on
201	 *   cpu_mmid_lock in check_mmu_context(). Taking the lock synchronises
202	 *   with the rollover and so we are forced to see the updated
203	 *   generation.
204	 *
205	 * - We get a valid MMID back from the cmpxchg, which means the
206	 *   relaxed xchg in flush_context will treat us as reserved
207	 *   because atomic RmWs are totally ordered for a given location.
208	 */
209	ctx = cpu_context(cpu, mm);
210	old_active_mmid = READ_ONCE(cpu_data[cpu].asid_cache);
211	if (!old_active_mmid ||
212	    !asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)) ||
213	    !cmpxchg_relaxed(&cpu_data[cpu].asid_cache, old_active_mmid, ctx)) {
214		raw_spin_lock_irqsave(&cpu_mmid_lock, flags);
215
216		ctx = cpu_context(cpu, mm);
217		if (!asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)))
218			ctx = get_new_mmid(mm);
219
220		WRITE_ONCE(cpu_data[cpu].asid_cache, ctx);
221		raw_spin_unlock_irqrestore(&cpu_mmid_lock, flags);
222	}
223
224	/*
225	 * Invalidate the local TLB if needed. Note that we must only clear our
226	 * bit in tlb_flush_pending after this is complete, so that the
227	 * cpu_has_shared_ftlb_entries case below isn't misled.
228	 */
229	if (cpumask_test_cpu(cpu, &tlb_flush_pending)) {
230		if (cpu_has_vtag_icache)
231			flush_icache_all();
232		local_flush_tlb_all();
233		cpumask_clear_cpu(cpu, &tlb_flush_pending);
234	}
235
236	write_c0_memorymapid(ctx & cpu_asid_mask(&boot_cpu_data));
237
238	/*
239	 * If this CPU shares FTLB entries with its siblings and one or more of
240	 * those siblings hasn't yet invalidated its TLB following a version
241	 * increase then we need to invalidate any TLB entries for our MMID
242	 * that we might otherwise pick up from a sibling.
243	 *
244	 * We ifdef on CONFIG_SMP because cpu_sibling_map isn't defined in
245	 * CONFIG_SMP=n kernels.
246	 */
247#ifdef CONFIG_SMP
248	if (cpu_has_shared_ftlb_entries &&
249	    cpumask_intersects(&tlb_flush_pending, &cpu_sibling_map[cpu])) {
250		/* Ensure we operate on the new MMID */
251		mtc0_tlbw_hazard();
252
253		/*
254		 * Invalidate all TLB entries associated with the new
255		 * MMID, and wait for the invalidation to complete.
256		 */
257		ginvt_mmid();
258		sync_ginv();
259	}
260#endif
261
262setup_pgd:
263	TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
264}
265EXPORT_SYMBOL_GPL(check_switch_mmu_context);
266
267static int mmid_init(void)
268{
269	if (!cpu_has_mmid)
270		return 0;
271
272	/*
273	 * Expect allocation after rollover to fail if we don't have at least
274	 * one more MMID than CPUs.
275	 */
276	num_mmids = asid_first_version(0);
277	WARN_ON(num_mmids <= num_possible_cpus());
278
279	atomic64_set(&mmid_version, asid_first_version(0));
280	mmid_map = bitmap_zalloc(num_mmids, GFP_KERNEL);
281	if (!mmid_map)
282		panic("Failed to allocate bitmap for %u MMIDs\n", num_mmids);
283
284	/* Reserve an MMID for kmap/wired entries */
285	__set_bit(MMID_KERNEL_WIRED, mmid_map);
286
287	pr_info("MMID allocator initialised with %u entries\n", num_mmids);
288	return 0;
289}
290early_initcall(mmid_init);
291