1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8#include <mode/smp/ipi.h> 9#include <smp/ipi.h> 10#include <smp/lock.h> 11 12#ifdef ENABLE_SMP_SUPPORT 13 14static IpiModeRemoteCall_t remoteCall; /* the remote call being requested */ 15 16static inline void init_ipi_args(IpiRemoteCall_t func, 17 word_t data1, word_t data2, word_t data3, 18 word_t mask) 19{ 20 remoteCall = (IpiModeRemoteCall_t)func; 21 ipi_args[0] = data1; 22 ipi_args[1] = data2; 23 ipi_args[2] = data3; 24 25 /* get number of cores involved in this IPI */ 26 totalCoreBarrier = popcountl(mask); 27} 28 29static void handleRemoteCall(IpiModeRemoteCall_t call, word_t arg0, 30 word_t arg1, word_t arg2, bool_t irqPath) 31{ 32 /* we gets spurious irq_remote_call_ipi calls, e.g. when handling IPI 33 * in lock while hardware IPI is pending. Guard against spurious IPIs! */ 34 if (clh_is_ipi_pending(getCurrentCPUIndex())) { 35 switch ((IpiRemoteCall_t)call) { 36 case IpiRemoteCall_Stall: 37 ipiStallCoreCallback(irqPath); 38 break; 39 40 case IpiRemoteCall_InvalidatePageStructureCacheASID: 41 invalidateLocalPageStructureCacheASID(arg0, arg1); 42 break; 43 44 case IpiRemoteCall_InvalidateTranslationSingle: 45 invalidateLocalTranslationSingle(arg0); 46 break; 47 48 case IpiRemoteCall_InvalidateTranslationSingleASID: 49 invalidateLocalTranslationSingleASID(arg0, arg1); 50 break; 51 52 case IpiRemoteCall_InvalidateTranslationAll: 53 invalidateLocalTranslationAll(); 54 break; 55 56 case IpiRemoteCall_switchFpuOwner: 57 switchLocalFpuOwner((user_fpu_state_t *)arg0); 58 break; 59 60#ifdef CONFIG_VTX 61 case IpiRemoteCall_ClearCurrentVCPU: 62 clearCurrentVCPU(); 63 break; 64 case IpiRemoteCall_VMCheckBoundNotification: 65 VMCheckBoundNotification((tcb_t *)arg0); 66 break; 67#endif 68 default: 69 Mode_handleRemoteCall(call, arg0, arg1, arg2); 70 break; 71 } 72 73 big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0; 74 ipi_wait(totalCoreBarrier); 75 } 76} 77 78/* make sure all cpu IDs for number of core fit in bitwise word */ 79compile_assert(invalid_number_of_supported_nodes, CONFIG_MAX_NUM_NODES <= wordBits); 80 81#ifdef CONFIG_USE_LOGICAL_IDS 82static void x86_ipi_send_mask(interrupt_t ipi, word_t mask, bool_t isBlocking) 83{ 84 word_t nr_target_clusters = 0; 85 word_t target_clusters[CONFIG_MAX_NUM_NODES]; 86 87 do { 88 int core = wordBits - 1 - clzl(mask); 89 target_clusters[nr_target_clusters] = 0; 90 91 /* get mask of all cores in bitmask which are in same cluster as 'core' */ 92 word_t sub_mask = mask & cpu_mapping.other_indexes_in_cluster[core]; 93 target_clusters[nr_target_clusters] |= cpu_mapping.index_to_logical_id[core]; 94 if (isBlocking) { 95 big_kernel_lock.node_owners[core].ipi = 1; 96 } 97 98 /* check if there is any other core in this cluster */ 99 while (sub_mask) { 100 int index = wordBits - 1 - clzl(sub_mask); 101 target_clusters[nr_target_clusters] |= cpu_mapping.index_to_logical_id[index]; 102 if (isBlocking) { 103 big_kernel_lock.node_owners[index].ipi = 1; 104 } 105 sub_mask &= ~BIT(index); 106 } 107 108 mask &= ~(cpu_mapping.other_indexes_in_cluster[core] | BIT(core)); 109 nr_target_clusters++; 110 } while (mask != 0); 111 112 /* broadcast IPIs to clusters... */ 113 IPI_ICR_BARRIER; 114 for (int i = 0; i < nr_target_clusters; i++) { 115 apic_send_ipi_cluster(ipi, target_clusters[i]); 116 } 117} 118#endif /* CONFIG_USE_LOGICAL_IDS */ 119 120void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking) 121{ 122 interrupt_t interrupt_ipi = ipi + IRQ_INT_OFFSET; 123 124#ifdef CONFIG_USE_LOGICAL_IDS 125 x86_ipi_send_mask(interrupt_ipi, mask, isBlocking); 126#else 127 generic_ipi_send_mask(interrupt_ipi, mask, isBlocking); 128#endif /* CONFIG_USE_LOGICAL_IDS */ 129} 130#endif /* ENABLE_SMP_SUPPORT */ 131