osfmk/x86_64/pmap_pcid.c

63 	int ccpu = cpu_number();
67 	pmap_pcid_log("PCID configure invoked on CPU %d\n", ccpu);
98 		cpu_datap(ccpu)->cpu_pmap_pcid_enabled = FALSE;
107 		cpu_datap(ccpu)->cpu_pmap_pcid_enabled = FALSE;
108 		pmap_pcid_log("PMAP: PCID not detected CPU %d\n", ccpu);
112 		cpu_datap(ccpu)->cpu_pmap_pcid_enabled = TRUE;
113 		pmap_pcid_log("PMAP: PCID already enabled %d\n", ccpu);
117 		pmap_pcid_log("Pre-PCID:CR0: 0x%lx, CR3: 0x%lx, CR4(CPU %d): 0x%lx\n", get_cr0(), get_cr3_raw(), ccpu, cr4);
124 			pmap_pcid_log("Toggled PGE ON (CPU: %d\n", ccpu);
127 		pmap_pcid_log("Post PCID: CR0: 0x%lx, CR3: 0x%lx, CR4(CPU %d): 0x%lx\n", get_cr0(), get_cr3_raw(), ccpu, get_cr4());
129 		cpu_datap(ccpu)->cpu_pmap_pcid_enabled = TRUE;
134 		cpu_datap(ccpu)->cpu_pmap_pcid_coherentp =
135 		    cpu_datap(ccpu)->cpu_pmap_pcid_coherentp_kernel =
136 		    &(kernel_pmap->pmap_pcid_coherency_vector[ccpu]);
137 		cpu_datap(ccpu)->cpu_pcid_refcounts[0] = 1;
166 pcid_t	pmap_pcid_allocate_pcid(int ccpu) {
170 	pcid_ref_t	*cpu_pcid_refcounts = &cpu_datap(ccpu)->cpu_pcid_refcounts[0];
173 	if ((i = cpu_datap(ccpu)->cpu_pcid_free_hint) != 0) {
176 			cpu_datap(ccpu)->cpu_pcid_free_hint = 0;
214 void	pmap_pcid_deallocate_pcid(int ccpu, pmap_t tpmap) {
219 	pcid = tpmap->pmap_pcid_cpus[ccpu];
224 	lp = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid];
226 	pmap_assert(cpu_datap(ccpu)->cpu_pcid_refcounts[pcid] >= 1);
229 		(void)__sync_bool_compare_and_swap(&cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[pcid], tpmap, PMAP_INVALID);
231 	if ((prior_count = __sync_fetch_and_sub(&cpu_datap(ccpu)->cpu_pcid_refcounts[pcid], 1)) == 1) {
232 		    cpu_datap(ccpu)->cpu_pcid_free_hint = pcid;
245 pcid_t	pcid_for_pmap_cpu_tuple(pmap_t pmap, int ccpu) {
246 	return pmap->pmap_pcid_cpus[ccpu];
253 void	pmap_pcid_activate(pmap_t tpmap, int ccpu) {
254 	pcid_t		new_pcid = tpmap->pmap_pcid_cpus[ccpu];
258 	pmap_assert(cpu_datap(ccpu)->cpu_pmap_pcid_enabled);
260 		new_pcid = tpmap->pmap_pcid_cpus[ccpu] = pmap_pcid_allocate_pcid(ccpu);
264 	cpu_datap(ccpu)->cpu_last_pcid = cpu_datap(ccpu)->cpu_active_pcid;
266 	cpu_datap(ccpu)->cpu_active_pcid = new_pcid;
268 	pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
270 		last_pmap = cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid];
274 		pmap_pcid_validate_cpu(tpmap, ccpu);
277 	cpu_datap(ccpu)->cpu_pcid_last_pmap_dispatched[new_pcid] = tpmap;
282 	pcid_record_array[ccpu % PCID_RECORD_SIZE] = tpmap->pm_cr3 | new_pcid | (((uint64_t)(!(pending_flush || pcid_conflict))) <<63);
300 		pending_flush = (tpmap->pmap_pcid_coherency_vector[ccpu] != 0);
302 			pmap_pcid_validate_cpu(tpmap, ccpu);
306 	cpu_datap(ccpu)->cpu_pmap_pcid_coherentp = &(tpmap->pmap_pcid_coherency_vector[ccpu]);