1/*
2 * Copyright 2023, Puck Meerburg, puck@puckipedia.com.
3 * Copyright 2013, Pawe�� Dziepak, pdziepak@quarnos.org.
4 * Copyright 2002-2005, Axel D��rfler, axeld@pinc-software.de. All rights reserved.
5 * Distributed under the terms of the MIT License.
6 *
7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8 * Distributed under the terms of the NewOS License.
9 */
10
11
12#include <boot/kernel_args.h>
13#include <vm/vm.h>
14#include <cpu.h>
15#include <int.h>
16#include <smp.h>
17#include <smp_priv.h>
18
19#include <arch/atomic.h>
20#include <arch/cpu.h>
21#include <arch/vm.h>
22#include <arch/smp.h>
23
24#include <arch/x86/apic.h>
25#include <arch/x86/arch_smp.h>
26#include <arch/x86/smp_priv.h>
27#include <arch/x86/timer.h>
28
29#include <string.h>
30#include <stdio.h>
31
32#include <algorithm>
33
34
35//#define TRACE_ARCH_SMP
36#ifdef TRACE_ARCH_SMP
37#	define TRACE(x) dprintf x
38#else
39#	define TRACE(x) ;
40#endif
41
42
43#define	ICI_VECTOR		0xfd
44
45
46static uint32 sCPUAPICIds[SMP_MAX_CPUS];
47static uint32 sAPICVersions[SMP_MAX_CPUS];
48
49
50static int32
51x86_ici_interrupt(void *data)
52{
53	// genuine inter-cpu interrupt
54	int cpu = smp_get_current_cpu();
55	TRACE(("inter-cpu interrupt on cpu %d\n", cpu));
56	return smp_intercpu_int_handler(cpu);
57}
58
59
60static int32
61x86_spurious_interrupt(void *data)
62{
63	// spurious interrupt
64	TRACE(("spurious interrupt on cpu %" B_PRId32 "\n", smp_get_current_cpu()));
65
66	// spurious interrupts must not be acknowledged as it does not expect
67	// a end of interrupt - if we still do it we would loose the next best
68	// interrupt
69	return B_HANDLED_INTERRUPT;
70}
71
72
73static int32
74x86_smp_error_interrupt(void *data)
75{
76	// smp error interrupt
77	TRACE(("smp error interrupt on cpu %" B_PRId32 "\n", smp_get_current_cpu()));
78	return B_HANDLED_INTERRUPT;
79}
80
81
82uint32
83x86_get_cpu_apic_id(int32 cpu)
84{
85	ASSERT(cpu >= 0 && cpu < SMP_MAX_CPUS);
86	return sCPUAPICIds[cpu];
87}
88
89
90status_t
91arch_smp_init(kernel_args *args)
92{
93	TRACE(("%s: entry\n", __func__));
94
95	if (!apic_available()) {
96		// if we don't have an apic we can't do smp
97		TRACE(("%s: apic not available for smp\n", __func__));
98		return B_OK;
99	}
100
101	// setup some globals
102	memcpy(sCPUAPICIds, args->arch_args.cpu_apic_id, sizeof(args->arch_args.cpu_apic_id));
103	memcpy(sAPICVersions, args->arch_args.cpu_apic_version, sizeof(args->arch_args.cpu_apic_version));
104
105	// set up the local apic on the boot cpu
106	arch_smp_per_cpu_init(args, 0);
107
108	if (args->num_cpus > 1) {
109		// I/O interrupts start at ARCH_INTERRUPT_BASE, so all interrupts are shifted
110		reserve_io_interrupt_vectors(3, 0xfd - ARCH_INTERRUPT_BASE,
111			INTERRUPT_TYPE_ICI);
112		install_io_interrupt_handler(0xfd - ARCH_INTERRUPT_BASE, &x86_ici_interrupt, NULL, B_NO_LOCK_VECTOR);
113		install_io_interrupt_handler(0xfe - ARCH_INTERRUPT_BASE, &x86_smp_error_interrupt, NULL, B_NO_LOCK_VECTOR);
114		install_io_interrupt_handler(0xff - ARCH_INTERRUPT_BASE, &x86_spurious_interrupt, NULL, B_NO_LOCK_VECTOR);
115	}
116
117	return B_OK;
118}
119
120
121status_t
122arch_smp_per_cpu_init(kernel_args *args, int32 cpu)
123{
124	// set up the local apic on the current cpu
125	TRACE(("arch_smp_init_percpu: setting up the apic on cpu %" B_PRId32 "\n",
126		cpu));
127	apic_per_cpu_init(args, cpu);
128
129	// setup FPU and SSE if supported
130	x86_init_fpu();
131
132	return B_OK;
133}
134
135
136static void
137send_multicast_ici_physical(CPUSet& cpuSet)
138{
139	int32 cpuCount = smp_get_num_cpus();
140	int32 currentCpu = smp_get_current_cpu();
141
142	for (int32 i = 0; i < cpuCount; i++) {
143		if (cpuSet.GetBit(i) && i != currentCpu) {
144			uint32 destination = sCPUAPICIds[i];
145			uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
146					| APIC_INTR_COMMAND_1_ASSERT
147					| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
148					| APIC_INTR_COMMAND_1_DEST_FIELD;
149
150			while (!apic_interrupt_delivered())
151				cpu_pause();
152			apic_set_interrupt_command(destination, mode);
153		}
154	}
155}
156
157
158void
159arch_smp_send_multicast_ici(CPUSet& cpuSet)
160{
161#if KDEBUG
162	if (are_interrupts_enabled())
163		panic("arch_smp_send_multicast_ici: called with interrupts enabled");
164#endif
165
166	memory_write_barrier();
167
168	if (!x2apic_available()) {
169		send_multicast_ici_physical(cpuSet);
170		return;
171	}
172
173	// WRMSR on the x2APIC MSRs is neither serializing, nor a load-store
174	// operation, requiring both memory serialization *and* a load fence, which is
175	// the only way to ensure the MSR doesn't get executed before the write
176	// barrier.
177	memory_read_barrier();
178
179	int32 cpuCount = smp_get_num_cpus();
180	int32 currentCpu = smp_get_current_cpu();
181
182	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
183			| APIC_INTR_COMMAND_1_ASSERT
184			| APIC_INTR_COMMAND_1_DEST_MODE_LOGICAL
185			| APIC_INTR_COMMAND_1_DEST_FIELD;
186
187	for (int32 i = 0; i < cpuCount; i++) {
188		if (!cpuSet.GetBit(i) || i == currentCpu)
189			continue;
190
191		apic_set_interrupt_command(gCPU[i].arch.logical_apic_id, mode);
192	}
193}
194
195
196void
197arch_smp_send_broadcast_ici(void)
198{
199#if KDEBUG
200	if (are_interrupts_enabled())
201		panic("arch_smp_send_broadcast_ici: called with interrupts enabled");
202#endif
203
204	memory_write_barrier();
205
206	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
207			| APIC_INTR_COMMAND_1_ASSERT
208			| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
209			| APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF;
210
211	while (!apic_interrupt_delivered())
212		cpu_pause();
213	apic_set_interrupt_command(0, mode);
214}
215
216
217void
218arch_smp_send_ici(int32 target_cpu)
219{
220#if KDEBUG
221	if (are_interrupts_enabled())
222		panic("arch_smp_send_ici: called with interrupts enabled");
223#endif
224
225	memory_write_barrier();
226
227	uint32 destination = sCPUAPICIds[target_cpu];
228	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
229			| APIC_INTR_COMMAND_1_ASSERT
230			| APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL
231			| APIC_INTR_COMMAND_1_DEST_FIELD;
232
233	while (!apic_interrupt_delivered())
234		cpu_pause();
235	apic_set_interrupt_command(destination, mode);
236}
237
238