1//===-- xray_fdr_controller.h ---------------------------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file is a part of XRay, a function call tracing system. 11// 12//===----------------------------------------------------------------------===// 13#ifndef COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ 14#define COMPILER_RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ 15 16#include <limits> 17#include <time.h> 18 19#include "xray/xray_interface.h" 20#include "xray/xray_records.h" 21#include "xray_buffer_queue.h" 22#include "xray_fdr_log_writer.h" 23 24namespace __xray { 25 26template <size_t Version = 5> class FDRController { 27 BufferQueue *BQ; 28 BufferQueue::Buffer &B; 29 FDRLogWriter &W; 30 int (*WallClockReader)(clockid_t, struct timespec *) = 0; 31 uint64_t CycleThreshold = 0; 32 33 uint64_t LastFunctionEntryTSC = 0; 34 uint64_t LatestTSC = 0; 35 uint16_t LatestCPU = 0; 36 tid_t TId = 0; 37 pid_t PId = 0; 38 bool First = true; 39 40 uint32_t UndoableFunctionEnters = 0; 41 uint32_t UndoableTailExits = 0; 42 43 bool finalized() const XRAY_NEVER_INSTRUMENT { 44 return BQ == nullptr || BQ->finalizing(); 45 } 46 47 bool hasSpace(size_t S) XRAY_NEVER_INSTRUMENT { 48 return B.Data != nullptr && B.Generation == BQ->generation() && 49 W.getNextRecord() + S <= reinterpret_cast<char *>(B.Data) + B.Size; 50 } 51 52 constexpr int32_t mask(int32_t FuncId) const XRAY_NEVER_INSTRUMENT { 53 return FuncId & ((1 << 29) - 1); 54 } 55 56 bool getNewBuffer() XRAY_NEVER_INSTRUMENT { 57 if (BQ->getBuffer(B) != BufferQueue::ErrorCode::Ok) 58 return false; 59 60 W.resetRecord(); 61 DCHECK_EQ(W.getNextRecord(), B.Data); 62 LatestTSC = 0; 63 LatestCPU = 0; 64 First = true; 65 UndoableFunctionEnters = 0; 66 UndoableTailExits = 0; 67 atomic_store(B.Extents, 0, memory_order_release); 68 return true; 69 } 70 71 bool setupNewBuffer() XRAY_NEVER_INSTRUMENT { 72 if (finalized()) 73 return false; 74 75 DCHECK(hasSpace(sizeof(MetadataRecord) * 3)); 76 TId = GetTid(); 77 PId = internal_getpid(); 78 struct timespec TS { 79 0, 0 80 }; 81 WallClockReader(CLOCK_MONOTONIC, &TS); 82 83 MetadataRecord Metadata[] = { 84 // Write out a MetadataRecord to signify that this is the start of a new 85 // buffer, associated with a particular thread, with a new CPU. For the 86 // data, we have 15 bytes to squeeze as much information as we can. At 87 // this point we only write down the following bytes: 88 // - Thread ID (tid_t, cast to 4 bytes type due to Darwin being 8 89 // bytes) 90 createMetadataRecord<MetadataRecord::RecordKinds::NewBuffer>( 91 static_cast<int32_t>(TId)), 92 93 // Also write the WalltimeMarker record. We only really need microsecond 94 // precision here, and enforce across platforms that we need 64-bit 95 // seconds and 32-bit microseconds encoded in the Metadata record. 96 createMetadataRecord<MetadataRecord::RecordKinds::WalltimeMarker>( 97 static_cast<int64_t>(TS.tv_sec), 98 static_cast<int32_t>(TS.tv_nsec / 1000)), 99 100 // Also write the Pid record. 101 createMetadataRecord<MetadataRecord::RecordKinds::Pid>( 102 static_cast<int32_t>(PId)), 103 }; 104 105 if (finalized()) 106 return false; 107 return W.writeMetadataRecords(Metadata); 108 } 109 110 bool prepareBuffer(size_t S) XRAY_NEVER_INSTRUMENT { 111 if (finalized()) 112 return returnBuffer(); 113 114 if (UNLIKELY(!hasSpace(S))) { 115 if (!returnBuffer()) 116 return false; 117 if (!getNewBuffer()) 118 return false; 119 if (!setupNewBuffer()) 120 return false; 121 } 122 123 if (First) { 124 First = false; 125 W.resetRecord(); 126 atomic_store(B.Extents, 0, memory_order_release); 127 return setupNewBuffer(); 128 } 129 130 return true; 131 } 132 133 bool returnBuffer() XRAY_NEVER_INSTRUMENT { 134 if (BQ == nullptr) 135 return false; 136 137 First = true; 138 if (finalized()) { 139 BQ->releaseBuffer(B); // ignore result. 140 return false; 141 } 142 143 return BQ->releaseBuffer(B) == BufferQueue::ErrorCode::Ok; 144 } 145 146 enum class PreambleResult { NoChange, WroteMetadata, InvalidBuffer }; 147 PreambleResult recordPreamble(uint64_t TSC, 148 uint16_t CPU) XRAY_NEVER_INSTRUMENT { 149 if (UNLIKELY(LatestCPU != CPU || LatestTSC == 0)) { 150 // We update our internal tracking state for the Latest TSC and CPU we've 151 // seen, then write out the appropriate metadata and function records. 152 LatestTSC = TSC; 153 LatestCPU = CPU; 154 155 if (B.Generation != BQ->generation()) 156 return PreambleResult::InvalidBuffer; 157 158 W.writeMetadata<MetadataRecord::RecordKinds::NewCPUId>(CPU, TSC); 159 return PreambleResult::WroteMetadata; 160 } 161 162 DCHECK_EQ(LatestCPU, CPU); 163 164 if (UNLIKELY(LatestTSC > TSC || 165 TSC - LatestTSC > 166 uint64_t{std::numeric_limits<int32_t>::max()})) { 167 // Either the TSC has wrapped around from the last TSC we've seen or the 168 // delta is too large to fit in a 32-bit signed integer, so we write a 169 // wrap-around record. 170 LatestTSC = TSC; 171 172 if (B.Generation != BQ->generation()) 173 return PreambleResult::InvalidBuffer; 174 175 W.writeMetadata<MetadataRecord::RecordKinds::TSCWrap>(TSC); 176 return PreambleResult::WroteMetadata; 177 } 178 179 return PreambleResult::NoChange; 180 } 181 182 bool rewindRecords(int32_t FuncId, uint64_t TSC, 183 uint16_t CPU) XRAY_NEVER_INSTRUMENT { 184 // Undo one enter record, because at this point we are either at the state 185 // of: 186 // - We are exiting a function that we recently entered. 187 // - We are exiting a function that was the result of a sequence of tail 188 // exits, and we can check whether the tail exits can be re-wound. 189 // 190 FunctionRecord F; 191 W.undoWrites(sizeof(FunctionRecord)); 192 if (B.Generation != BQ->generation()) 193 return false; 194 internal_memcpy(&F, W.getNextRecord(), sizeof(FunctionRecord)); 195 196 DCHECK(F.RecordKind == 197 uint8_t(FunctionRecord::RecordKinds::FunctionEnter) && 198 "Expected to find function entry recording when rewinding."); 199 DCHECK_EQ(F.FuncId, FuncId & ~(0x0F << 28)); 200 201 LatestTSC -= F.TSCDelta; 202 if (--UndoableFunctionEnters != 0) { 203 LastFunctionEntryTSC -= F.TSCDelta; 204 return true; 205 } 206 207 LastFunctionEntryTSC = 0; 208 auto RewindingTSC = LatestTSC; 209 auto RewindingRecordPtr = W.getNextRecord() - sizeof(FunctionRecord); 210 while (UndoableTailExits) { 211 if (B.Generation != BQ->generation()) 212 return false; 213 internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); 214 DCHECK_EQ(F.RecordKind, 215 uint8_t(FunctionRecord::RecordKinds::FunctionTailExit)); 216 RewindingTSC -= F.TSCDelta; 217 RewindingRecordPtr -= sizeof(FunctionRecord); 218 if (B.Generation != BQ->generation()) 219 return false; 220 internal_memcpy(&F, RewindingRecordPtr, sizeof(FunctionRecord)); 221 222 // This tail call exceeded the threshold duration. It will not be erased. 223 if ((TSC - RewindingTSC) >= CycleThreshold) { 224 UndoableTailExits = 0; 225 return true; 226 } 227 228 --UndoableTailExits; 229 W.undoWrites(sizeof(FunctionRecord) * 2); 230 LatestTSC = RewindingTSC; 231 } 232 return true; 233 } 234 235public: 236 template <class WallClockFunc> 237 FDRController(BufferQueue *BQ, BufferQueue::Buffer &B, FDRLogWriter &W, 238 WallClockFunc R, uint64_t C) XRAY_NEVER_INSTRUMENT 239 : BQ(BQ), 240 B(B), 241 W(W), 242 WallClockReader(R), 243 CycleThreshold(C) {} 244 245 bool functionEnter(int32_t FuncId, uint64_t TSC, 246 uint16_t CPU) XRAY_NEVER_INSTRUMENT { 247 if (finalized() || 248 !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) 249 return returnBuffer(); 250 251 auto PreambleStatus = recordPreamble(TSC, CPU); 252 if (PreambleStatus == PreambleResult::InvalidBuffer) 253 return returnBuffer(); 254 255 if (PreambleStatus == PreambleResult::WroteMetadata) { 256 UndoableFunctionEnters = 1; 257 UndoableTailExits = 0; 258 } else { 259 ++UndoableFunctionEnters; 260 } 261 262 auto Delta = TSC - LatestTSC; 263 LastFunctionEntryTSC = TSC; 264 LatestTSC = TSC; 265 return W.writeFunction(FDRLogWriter::FunctionRecordKind::Enter, 266 mask(FuncId), Delta); 267 } 268 269 bool functionTailExit(int32_t FuncId, uint64_t TSC, 270 uint16_t CPU) XRAY_NEVER_INSTRUMENT { 271 if (finalized()) 272 return returnBuffer(); 273 274 if (!prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) 275 return returnBuffer(); 276 277 auto PreambleStatus = recordPreamble(TSC, CPU); 278 if (PreambleStatus == PreambleResult::InvalidBuffer) 279 return returnBuffer(); 280 281 if (PreambleStatus == PreambleResult::NoChange && 282 UndoableFunctionEnters != 0 && 283 TSC - LastFunctionEntryTSC < CycleThreshold) 284 return rewindRecords(FuncId, TSC, CPU); 285 286 UndoableTailExits = UndoableFunctionEnters ? UndoableTailExits + 1 : 0; 287 UndoableFunctionEnters = 0; 288 auto Delta = TSC - LatestTSC; 289 LatestTSC = TSC; 290 return W.writeFunction(FDRLogWriter::FunctionRecordKind::TailExit, 291 mask(FuncId), Delta); 292 } 293 294 bool functionEnterArg(int32_t FuncId, uint64_t TSC, uint16_t CPU, 295 uint64_t Arg) XRAY_NEVER_INSTRUMENT { 296 if (finalized() || 297 !prepareBuffer((2 * sizeof(MetadataRecord)) + sizeof(FunctionRecord)) || 298 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) 299 return returnBuffer(); 300 301 auto Delta = TSC - LatestTSC; 302 LatestTSC = TSC; 303 LastFunctionEntryTSC = 0; 304 UndoableFunctionEnters = 0; 305 UndoableTailExits = 0; 306 307 return W.writeFunctionWithArg(FDRLogWriter::FunctionRecordKind::EnterArg, 308 mask(FuncId), Delta, Arg); 309 } 310 311 bool functionExit(int32_t FuncId, uint64_t TSC, 312 uint16_t CPU) XRAY_NEVER_INSTRUMENT { 313 if (finalized() || 314 !prepareBuffer(sizeof(MetadataRecord) + sizeof(FunctionRecord))) 315 return returnBuffer(); 316 317 auto PreambleStatus = recordPreamble(TSC, CPU); 318 if (PreambleStatus == PreambleResult::InvalidBuffer) 319 return returnBuffer(); 320 321 if (PreambleStatus == PreambleResult::NoChange && 322 UndoableFunctionEnters != 0 && 323 TSC - LastFunctionEntryTSC < CycleThreshold) 324 return rewindRecords(FuncId, TSC, CPU); 325 326 auto Delta = TSC - LatestTSC; 327 LatestTSC = TSC; 328 UndoableFunctionEnters = 0; 329 UndoableTailExits = 0; 330 return W.writeFunction(FDRLogWriter::FunctionRecordKind::Exit, mask(FuncId), 331 Delta); 332 } 333 334 bool customEvent(uint64_t TSC, uint16_t CPU, const void *Event, 335 int32_t EventSize) XRAY_NEVER_INSTRUMENT { 336 if (finalized() || 337 !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || 338 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) 339 return returnBuffer(); 340 341 auto Delta = TSC - LatestTSC; 342 LatestTSC = TSC; 343 UndoableFunctionEnters = 0; 344 UndoableTailExits = 0; 345 return W.writeCustomEvent(Delta, Event, EventSize); 346 } 347 348 bool typedEvent(uint64_t TSC, uint16_t CPU, uint16_t EventType, 349 const void *Event, int32_t EventSize) XRAY_NEVER_INSTRUMENT { 350 if (finalized() || 351 !prepareBuffer((2 * sizeof(MetadataRecord)) + EventSize) || 352 recordPreamble(TSC, CPU) == PreambleResult::InvalidBuffer) 353 return returnBuffer(); 354 355 auto Delta = TSC - LatestTSC; 356 LatestTSC = TSC; 357 UndoableFunctionEnters = 0; 358 UndoableTailExits = 0; 359 return W.writeTypedEvent(Delta, EventType, Event, EventSize); 360 } 361 362 bool flush() XRAY_NEVER_INSTRUMENT { 363 if (finalized()) { 364 returnBuffer(); // ignore result. 365 return true; 366 } 367 return returnBuffer(); 368 } 369}; 370 371} // namespace __xray 372 373#endif // COMPILER-RT_LIB_XRAY_XRAY_FDR_CONTROLLER_H_ 374