AArch64ErrataFix.cpp revision 360784
1//===- AArch64ErrataFix.cpp -----------------------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// This file implements Section Patching for the purpose of working around 9// the AArch64 Cortex-53 errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 10// versions of the core. 11// 12// The general principle is that an erratum sequence of one or 13// more instructions is detected in the instruction stream, one of the 14// instructions in the sequence is replaced with a branch to a patch sequence 15// of replacement instructions. At the end of the replacement sequence the 16// patch branches back to the instruction stream. 17 18// This technique is only suitable for fixing an erratum when: 19// - There is a set of necessary conditions required to trigger the erratum that 20// can be detected at static link time. 21// - There is a set of replacement instructions that can be used to remove at 22// least one of the necessary conditions that trigger the erratum. 23// - We can overwrite an instruction in the erratum sequence with a branch to 24// the replacement sequence. 25// - We can place the replacement sequence within range of the branch. 26//===----------------------------------------------------------------------===// 27 28#include "AArch64ErrataFix.h" 29#include "Config.h" 30#include "LinkerScript.h" 31#include "OutputSections.h" 32#include "Relocations.h" 33#include "Symbols.h" 34#include "SyntheticSections.h" 35#include "Target.h" 36#include "lld/Common/Memory.h" 37#include "lld/Common/Strings.h" 38#include "llvm/Support/Endian.h" 39#include "llvm/Support/raw_ostream.h" 40#include <algorithm> 41 42using namespace llvm; 43using namespace llvm::ELF; 44using namespace llvm::object; 45using namespace llvm::support; 46using namespace llvm::support::endian; 47 48namespace lld { 49namespace elf { 50 51// Helper functions to identify instructions and conditions needed to trigger 52// the Cortex-A53-843419 erratum. 53 54// ADRP 55// | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) | 56static bool isADRP(uint32_t instr) { 57 return (instr & 0x9f000000) == 0x90000000; 58} 59 60// Load and store bit patterns from ARMv8-A ARM ARM. 61// Instructions appear in order of appearance starting from table in 62// C4.1.3 Loads and Stores. 63 64// All loads and stores have 1 (at bit position 27), (0 at bit position 25). 65// | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) | 66static bool isLoadStoreClass(uint32_t instr) { 67 return (instr & 0x0a000000) == 0x08000000; 68} 69 70// LDN/STN multiple no offset 71// | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) | 72// LDN/STN multiple post-indexed 73// | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) | 74// L == 0 for stores. 75 76// Utility routine to decode opcode field of LDN/STN multiple structure 77// instructions to find the ST1 instructions. 78// opcode == 0010 ST1 4 registers. 79// opcode == 0110 ST1 3 registers. 80// opcode == 0111 ST1 1 register. 81// opcode == 1010 ST1 2 registers. 82static bool isST1MultipleOpcode(uint32_t instr) { 83 return (instr & 0x0000f000) == 0x00002000 || 84 (instr & 0x0000f000) == 0x00006000 || 85 (instr & 0x0000f000) == 0x00007000 || 86 (instr & 0x0000f000) == 0x0000a000; 87} 88 89static bool isST1Multiple(uint32_t instr) { 90 return (instr & 0xbfff0000) == 0x0c000000 && isST1MultipleOpcode(instr); 91} 92 93// Writes to Rn (writeback). 94static bool isST1MultiplePost(uint32_t instr) { 95 return (instr & 0xbfe00000) == 0x0c800000 && isST1MultipleOpcode(instr); 96} 97 98// LDN/STN single no offset 99// | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)| 100// LDN/STN single post-indexed 101// | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)| 102// L == 0 for stores 103 104// Utility routine to decode opcode field of LDN/STN single structure 105// instructions to find the ST1 instructions. 106// R == 0 for ST1 and ST3, R == 1 for ST2 and ST4. 107// opcode == 000 ST1 8-bit. 108// opcode == 010 ST1 16-bit. 109// opcode == 100 ST1 32 or 64-bit (Size determines which). 110static bool isST1SingleOpcode(uint32_t instr) { 111 return (instr & 0x0040e000) == 0x00000000 || 112 (instr & 0x0040e000) == 0x00004000 || 113 (instr & 0x0040e000) == 0x00008000; 114} 115 116static bool isST1Single(uint32_t instr) { 117 return (instr & 0xbfff0000) == 0x0d000000 && isST1SingleOpcode(instr); 118} 119 120// Writes to Rn (writeback). 121static bool isST1SinglePost(uint32_t instr) { 122 return (instr & 0xbfe00000) == 0x0d800000 && isST1SingleOpcode(instr); 123} 124 125static bool isST1(uint32_t instr) { 126 return isST1Multiple(instr) || isST1MultiplePost(instr) || 127 isST1Single(instr) || isST1SinglePost(instr); 128} 129 130// Load/store exclusive 131// | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) | 132// L == 0 for Stores. 133static bool isLoadStoreExclusive(uint32_t instr) { 134 return (instr & 0x3f000000) == 0x08000000; 135} 136 137static bool isLoadExclusive(uint32_t instr) { 138 return (instr & 0x3f400000) == 0x08400000; 139} 140 141// Load register literal 142// | opc (2) 01 | 1 V 00 | imm19 | Rt (5) | 143static bool isLoadLiteral(uint32_t instr) { 144 return (instr & 0x3b000000) == 0x18000000; 145} 146 147// Load/store no-allocate pair 148// (offset) 149// | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 150// L == 0 for stores. 151// Never writes to register 152static bool isSTNP(uint32_t instr) { 153 return (instr & 0x3bc00000) == 0x28000000; 154} 155 156// Load/store register pair 157// (post-indexed) 158// | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 159// L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP 160// Writes to Rn. 161static bool isSTPPost(uint32_t instr) { 162 return (instr & 0x3bc00000) == 0x28800000; 163} 164 165// (offset) 166// | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 167static bool isSTPOffset(uint32_t instr) { 168 return (instr & 0x3bc00000) == 0x29000000; 169} 170 171// (pre-index) 172// | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) | 173// Writes to Rn. 174static bool isSTPPre(uint32_t instr) { 175 return (instr & 0x3bc00000) == 0x29800000; 176} 177 178static bool isSTP(uint32_t instr) { 179 return isSTPPost(instr) || isSTPOffset(instr) || isSTPPre(instr); 180} 181 182// Load/store register (unscaled immediate) 183// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) | 184// V == 0 for Scalar, V == 1 for Simd/FP. 185static bool isLoadStoreUnscaled(uint32_t instr) { 186 return (instr & 0x3b000c00) == 0x38000000; 187} 188 189// Load/store register (immediate post-indexed) 190// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) | 191static bool isLoadStoreImmediatePost(uint32_t instr) { 192 return (instr & 0x3b200c00) == 0x38000400; 193} 194 195// Load/store register (unprivileged) 196// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) | 197static bool isLoadStoreUnpriv(uint32_t instr) { 198 return (instr & 0x3b200c00) == 0x38000800; 199} 200 201// Load/store register (immediate pre-indexed) 202// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) | 203static bool isLoadStoreImmediatePre(uint32_t instr) { 204 return (instr & 0x3b200c00) == 0x38000c00; 205} 206 207// Load/store register (register offset) 208// | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt | 209static bool isLoadStoreRegisterOff(uint32_t instr) { 210 return (instr & 0x3b200c00) == 0x38200800; 211} 212 213// Load/store register (unsigned immediate) 214// | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) | 215static bool isLoadStoreRegisterUnsigned(uint32_t instr) { 216 return (instr & 0x3b000000) == 0x39000000; 217} 218 219// Rt is always in bit position 0 - 4. 220static uint32_t getRt(uint32_t instr) { return (instr & 0x1f); } 221 222// Rn is always in bit position 5 - 9. 223static uint32_t getRn(uint32_t instr) { return (instr >> 5) & 0x1f; } 224 225// C4.1.2 Branches, Exception Generating and System instructions 226// | op0 (3) 1 | 01 op1 (4) | x (22) | 227// op0 == 010 101 op1 == 0xxx Conditional Branch. 228// op0 == 110 101 op1 == 1xxx Unconditional Branch Register. 229// op0 == x00 101 op1 == xxxx Unconditional Branch immediate. 230// op0 == x01 101 op1 == 0xxx Compare and branch immediate. 231// op0 == x01 101 op1 == 1xxx Test and branch immediate. 232static bool isBranch(uint32_t instr) { 233 return ((instr & 0xfe000000) == 0xd6000000) || // Cond branch. 234 ((instr & 0xfe000000) == 0x54000000) || // Uncond branch reg. 235 ((instr & 0x7c000000) == 0x14000000) || // Uncond branch imm. 236 ((instr & 0x7c000000) == 0x34000000); // Compare and test branch. 237} 238 239static bool isV8SingleRegisterNonStructureLoadStore(uint32_t instr) { 240 return isLoadStoreUnscaled(instr) || isLoadStoreImmediatePost(instr) || 241 isLoadStoreUnpriv(instr) || isLoadStoreImmediatePre(instr) || 242 isLoadStoreRegisterOff(instr) || isLoadStoreRegisterUnsigned(instr); 243} 244 245// Note that this function refers to v8.0 only and does not include the 246// additional load and store instructions added for in later revisions of 247// the architecture such as the Atomic memory operations introduced 248// in v8.1. 249static bool isV8NonStructureLoad(uint32_t instr) { 250 if (isLoadExclusive(instr)) 251 return true; 252 if (isLoadLiteral(instr)) 253 return true; 254 else if (isV8SingleRegisterNonStructureLoadStore(instr)) { 255 // For Load and Store single register, Loads are derived from a 256 // combination of the Size, V and Opc fields. 257 uint32_t size = (instr >> 30) & 0xff; 258 uint32_t v = (instr >> 26) & 0x1; 259 uint32_t opc = (instr >> 22) & 0x3; 260 // For the load and store instructions that we are decoding. 261 // Opc == 0 are all stores. 262 // Opc == 1 with a couple of exceptions are loads. The exceptions are: 263 // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and 264 // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch. 265 return opc != 0 && !(size == 0 && v == 1 && opc == 2) && 266 !(size == 3 && v == 0 && opc == 2); 267 } 268 return false; 269} 270 271// The following decode instructions are only complete up to the instructions 272// needed for errata 843419. 273 274// Instruction with writeback updates the index register after the load/store. 275static bool hasWriteback(uint32_t instr) { 276 return isLoadStoreImmediatePre(instr) || isLoadStoreImmediatePost(instr) || 277 isSTPPre(instr) || isSTPPost(instr) || isST1SinglePost(instr) || 278 isST1MultiplePost(instr); 279} 280 281// For the load and store class of instructions, a load can write to the 282// destination register, a load and a store can write to the base register when 283// the instruction has writeback. 284static bool doesLoadStoreWriteToReg(uint32_t instr, uint32_t reg) { 285 return (isV8NonStructureLoad(instr) && getRt(instr) == reg) || 286 (hasWriteback(instr) && getRn(instr) == reg); 287} 288 289// Scanner for Cortex-A53 errata 843419 290// Full details are available in the Cortex A53 MPCore revision 0 Software 291// Developers Errata Notice (ARM-EPM-048406). 292// 293// The instruction sequence that triggers the erratum is common in compiled 294// AArch64 code, however it is sensitive to the offset of the sequence within 295// a 4k page. This means that by scanning and fixing the patch after we have 296// assigned addresses we only need to disassemble and fix instances of the 297// sequence in the range of affected offsets. 298// 299// In summary the erratum conditions are a series of 4 instructions: 300// 1.) An ADRP instruction that writes to register Rn with low 12 bits of 301// address of instruction either 0xff8 or 0xffc. 302// 2.) A load or store instruction that can be: 303// - A single register load or store, of either integer or vector registers. 304// - An STP or STNP, of either integer or vector registers. 305// - An Advanced SIMD ST1 store instruction. 306// - Must not write to Rn, but may optionally read from it. 307// 3.) An optional instruction that is not a branch and does not write to Rn. 308// 4.) A load or store from the Load/store register (unsigned immediate) class 309// that uses Rn as the base address register. 310// 311// Note that we do not attempt to scan for Sequence 2 as described in the 312// Software Developers Errata Notice as this has been assessed to be extremely 313// unlikely to occur in compiled code. This matches gold and ld.bfd behavior. 314 315// Return true if the Instruction sequence Adrp, Instr2, and Instr4 match 316// the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.), 317// and 4.) in the Scanner for Cortex-A53 errata comment above. 318static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2, 319 uint32_t instr4) { 320 if (!isADRP(instr1)) 321 return false; 322 323 uint32_t rn = getRt(instr1); 324 return isLoadStoreClass(instr2) && 325 (isLoadStoreExclusive(instr2) || isLoadLiteral(instr2) || 326 isV8SingleRegisterNonStructureLoadStore(instr2) || isSTP(instr2) || 327 isSTNP(instr2) || isST1(instr2)) && 328 !doesLoadStoreWriteToReg(instr2, rn) && 329 isLoadStoreRegisterUnsigned(instr4) && getRn(instr4) == rn; 330} 331 332// Scan the instruction sequence starting at Offset Off from the base of 333// InputSection isec. We update Off in this function rather than in the caller 334// as we can skip ahead much further into the section when we know how many 335// instructions we've scanned. 336// Return the offset of the load or store instruction in isec that we want to 337// patch or 0 if no patch required. 338static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off, 339 uint64_t limit) { 340 uint64_t isecAddr = isec->getVA(0); 341 342 // Advance Off so that (isecAddr + Off) modulo 0x1000 is at least 0xff8. 343 uint64_t initialPageOff = (isecAddr + off) & 0xfff; 344 if (initialPageOff < 0xff8) 345 off += 0xff8 - initialPageOff; 346 347 bool optionalAllowed = limit - off > 12; 348 if (off >= limit || limit - off < 12) { 349 // Need at least 3 4-byte sized instructions to trigger erratum. 350 off = limit; 351 return 0; 352 } 353 354 uint64_t patchOff = 0; 355 const uint8_t *buf = isec->data().begin(); 356 const ulittle32_t *instBuf = reinterpret_cast<const ulittle32_t *>(buf + off); 357 uint32_t instr1 = *instBuf++; 358 uint32_t instr2 = *instBuf++; 359 uint32_t instr3 = *instBuf++; 360 if (is843419ErratumSequence(instr1, instr2, instr3)) { 361 patchOff = off + 8; 362 } else if (optionalAllowed && !isBranch(instr3)) { 363 uint32_t instr4 = *instBuf++; 364 if (is843419ErratumSequence(instr1, instr2, instr4)) 365 patchOff = off + 12; 366 } 367 if (((isecAddr + off) & 0xfff) == 0xff8) 368 off += 4; 369 else 370 off += 0xffc; 371 return patchOff; 372} 373 374class Patch843419Section : public SyntheticSection { 375public: 376 Patch843419Section(InputSection *p, uint64_t off); 377 378 void writeTo(uint8_t *buf) override; 379 380 size_t getSize() const override { return 8; } 381 382 uint64_t getLDSTAddr() const; 383 384 static bool classof(const SectionBase *d) { 385 return d->kind() == InputSectionBase::Synthetic && d->name == ".text.patch"; 386 } 387 388 // The Section we are patching. 389 const InputSection *patchee; 390 // The offset of the instruction in the patchee section we are patching. 391 uint64_t patcheeOffset; 392 // A label for the start of the Patch that we can use as a relocation target. 393 Symbol *patchSym; 394}; 395 396Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) 397 : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4, 398 ".text.patch"), 399 patchee(p), patcheeOffset(off) { 400 this->parent = p->getParent(); 401 patchSym = addSyntheticLocal( 402 saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())), STT_FUNC, 0, 403 getSize(), *this); 404 addSyntheticLocal(saver.save("$x"), STT_NOTYPE, 0, 0, *this); 405} 406 407uint64_t Patch843419Section::getLDSTAddr() const { 408 return patchee->getVA(patcheeOffset); 409} 410 411void Patch843419Section::writeTo(uint8_t *buf) { 412 // Copy the instruction that we will be replacing with a branch in the 413 // patchee Section. 414 write32le(buf, read32le(patchee->data().begin() + patcheeOffset)); 415 416 // Apply any relocation transferred from the original patchee section. 417 // For a SyntheticSection Buf already has outSecOff added, but relocateAlloc 418 // also adds outSecOff so we need to subtract to avoid double counting. 419 this->relocateAlloc(buf - outSecOff, buf - outSecOff + getSize()); 420 421 // Return address is the next instruction after the one we have just copied. 422 uint64_t s = getLDSTAddr() + 4; 423 uint64_t p = patchSym->getVA() + 4; 424 target->relocateOne(buf + 4, R_AARCH64_JUMP26, s - p); 425} 426 427void AArch64Err843419Patcher::init() { 428 // The AArch64 ABI permits data in executable sections. We must avoid scanning 429 // this data as if it were instructions to avoid false matches. We use the 430 // mapping symbols in the InputObjects to identify this data, caching the 431 // results in sectionMap so we don't have to recalculate it each pass. 432 433 // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe 434 // half open intervals [Symbol Value, Next Symbol Value) of code and data 435 // within sections. If there is no next symbol then the half open interval is 436 // [Symbol Value, End of section). The type, code or data, is determined by 437 // the mapping symbol name, $x for code, $d for data. 438 auto isCodeMapSymbol = [](const Symbol *b) { 439 return b->getName() == "$x" || b->getName().startswith("$x."); 440 }; 441 auto isDataMapSymbol = [](const Symbol *b) { 442 return b->getName() == "$d" || b->getName().startswith("$d."); 443 }; 444 445 // Collect mapping symbols for every executable InputSection. 446 for (InputFile *file : objectFiles) { 447 auto *f = cast<ObjFile<ELF64LE>>(file); 448 for (Symbol *b : f->getLocalSymbols()) { 449 auto *def = dyn_cast<Defined>(b); 450 if (!def) 451 continue; 452 if (!isCodeMapSymbol(def) && !isDataMapSymbol(def)) 453 continue; 454 if (auto *sec = dyn_cast_or_null<InputSection>(def->section)) 455 if (sec->flags & SHF_EXECINSTR) 456 sectionMap[sec].push_back(def); 457 } 458 } 459 // For each InputSection make sure the mapping symbols are in sorted in 460 // ascending order and free from consecutive runs of mapping symbols with 461 // the same type. For example we must remove the redundant $d.1 from $x.0 462 // $d.0 $d.1 $x.1. 463 for (auto &kv : sectionMap) { 464 std::vector<const Defined *> &mapSyms = kv.second; 465 llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) { 466 return a->value < b->value; 467 }); 468 mapSyms.erase( 469 std::unique(mapSyms.begin(), mapSyms.end(), 470 [=](const Defined *a, const Defined *b) { 471 return isCodeMapSymbol(a) == isCodeMapSymbol(b); 472 }), 473 mapSyms.end()); 474 // Always start with a Code Mapping Symbol. 475 if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front())) 476 mapSyms.erase(mapSyms.begin()); 477 } 478 initialized = true; 479} 480 481// Insert the PatchSections we have created back into the 482// InputSectionDescription. As inserting patches alters the addresses of 483// InputSections that follow them, we try and place the patches after all the 484// executable sections, although we may need to insert them earlier if the 485// InputSectionDescription is larger than the maximum branch range. 486void AArch64Err843419Patcher::insertPatches( 487 InputSectionDescription &isd, std::vector<Patch843419Section *> &patches) { 488 uint64_t isecLimit; 489 uint64_t prevIsecLimit = isd.sections.front()->outSecOff; 490 uint64_t patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); 491 uint64_t outSecAddr = isd.sections.front()->getParent()->addr; 492 493 // Set the outSecOff of patches to the place where we want to insert them. 494 // We use a similar strategy to Thunk placement. Place patches roughly 495 // every multiple of maximum branch range. 496 auto patchIt = patches.begin(); 497 auto patchEnd = patches.end(); 498 for (const InputSection *isec : isd.sections) { 499 isecLimit = isec->outSecOff + isec->getSize(); 500 if (isecLimit > patchUpperBound) { 501 while (patchIt != patchEnd) { 502 if ((*patchIt)->getLDSTAddr() - outSecAddr >= prevIsecLimit) 503 break; 504 (*patchIt)->outSecOff = prevIsecLimit; 505 ++patchIt; 506 } 507 patchUpperBound = prevIsecLimit + target->getThunkSectionSpacing(); 508 } 509 prevIsecLimit = isecLimit; 510 } 511 for (; patchIt != patchEnd; ++patchIt) { 512 (*patchIt)->outSecOff = isecLimit; 513 } 514 515 // Merge all patch sections. We use the outSecOff assigned above to 516 // determine the insertion point. This is ok as we only merge into an 517 // InputSectionDescription once per pass, and at the end of the pass 518 // assignAddresses() will recalculate all the outSecOff values. 519 std::vector<InputSection *> tmp; 520 tmp.reserve(isd.sections.size() + patches.size()); 521 auto mergeCmp = [](const InputSection *a, const InputSection *b) { 522 if (a->outSecOff != b->outSecOff) 523 return a->outSecOff < b->outSecOff; 524 return isa<Patch843419Section>(a) && !isa<Patch843419Section>(b); 525 }; 526 std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(), 527 patches.end(), std::back_inserter(tmp), mergeCmp); 528 isd.sections = std::move(tmp); 529} 530 531// Given an erratum sequence that starts at address adrpAddr, with an 532// instruction that we need to patch at patcheeOffset from the start of 533// InputSection isec, create a Patch843419 Section and add it to the 534// Patches that we need to insert. 535static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset, 536 InputSection *isec, 537 std::vector<Patch843419Section *> &patches) { 538 // There may be a relocation at the same offset that we are patching. There 539 // are four cases that we need to consider. 540 // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this 541 // instance of the erratum on a previous patch and altered the relocation. We 542 // have nothing more to do. 543 // Case 2: A TLS Relaxation R_RELAX_TLS_IE_TO_LE. In this case the ADRP that 544 // we read will be transformed into a MOVZ later so we actually don't match 545 // the sequence and have nothing more to do. 546 // Case 3: A load/store register (unsigned immediate) class relocation. There 547 // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and 548 // they are both absolute. We need to add the same relocation to the patch, 549 // and replace the relocation with a R_AARCH_JUMP26 branch relocation. 550 // Case 4: No relocation. We must create a new R_AARCH64_JUMP26 branch 551 // relocation at the offset. 552 auto relIt = llvm::find_if(isec->relocations, [=](const Relocation &r) { 553 return r.offset == patcheeOffset; 554 }); 555 if (relIt != isec->relocations.end() && 556 (relIt->type == R_AARCH64_JUMP26 || relIt->expr == R_RELAX_TLS_IE_TO_LE)) 557 return; 558 559 log("detected cortex-a53-843419 erratum sequence starting at " + 560 utohexstr(adrpAddr) + " in unpatched output."); 561 562 auto *ps = make<Patch843419Section>(isec, patcheeOffset); 563 patches.push_back(ps); 564 565 auto makeRelToPatch = [](uint64_t offset, Symbol *patchSym) { 566 return Relocation{R_PC, R_AARCH64_JUMP26, offset, 0, patchSym}; 567 }; 568 569 if (relIt != isec->relocations.end()) { 570 ps->relocations.push_back( 571 {relIt->expr, relIt->type, 0, relIt->addend, relIt->sym}); 572 *relIt = makeRelToPatch(patcheeOffset, ps->patchSym); 573 } else 574 isec->relocations.push_back(makeRelToPatch(patcheeOffset, ps->patchSym)); 575} 576 577// Scan all the instructions in InputSectionDescription, for each instance of 578// the erratum sequence create a Patch843419Section. We return the list of 579// Patch843419Sections that need to be applied to the InputSectionDescription. 580std::vector<Patch843419Section *> 581AArch64Err843419Patcher::patchInputSectionDescription( 582 InputSectionDescription &isd) { 583 std::vector<Patch843419Section *> patches; 584 for (InputSection *isec : isd.sections) { 585 // LLD doesn't use the erratum sequence in SyntheticSections. 586 if (isa<SyntheticSection>(isec)) 587 continue; 588 // Use sectionMap to make sure we only scan code and not inline data. 589 // We have already sorted MapSyms in ascending order and removed consecutive 590 // mapping symbols of the same type. Our range of executable instructions to 591 // scan is therefore [codeSym->value, dataSym->value) or [codeSym->value, 592 // section size). 593 std::vector<const Defined *> &mapSyms = sectionMap[isec]; 594 595 auto codeSym = mapSyms.begin(); 596 while (codeSym != mapSyms.end()) { 597 auto dataSym = std::next(codeSym); 598 uint64_t off = (*codeSym)->value; 599 uint64_t limit = 600 (dataSym == mapSyms.end()) ? isec->data().size() : (*dataSym)->value; 601 602 while (off < limit) { 603 uint64_t startAddr = isec->getVA(off); 604 if (uint64_t patcheeOffset = 605 scanCortexA53Errata843419(isec, off, limit)) 606 implementPatch(startAddr, patcheeOffset, isec, patches); 607 } 608 if (dataSym == mapSyms.end()) 609 break; 610 codeSym = std::next(dataSym); 611 } 612 } 613 return patches; 614} 615 616// For each InputSectionDescription make one pass over the executable sections 617// looking for the erratum sequence; creating a synthetic Patch843419Section 618// for each instance found. We insert these synthetic patch sections after the 619// executable code in each InputSectionDescription. 620// 621// PreConditions: 622// The Output and Input Sections have had their final addresses assigned. 623// 624// PostConditions: 625// Returns true if at least one patch was added. The addresses of the 626// Output and Input Sections may have been changed. 627// Returns false if no patches were required and no changes were made. 628bool AArch64Err843419Patcher::createFixes() { 629 if (!initialized) 630 init(); 631 632 bool addressesChanged = false; 633 for (OutputSection *os : outputSections) { 634 if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR)) 635 continue; 636 for (BaseCommand *bc : os->sectionCommands) 637 if (auto *isd = dyn_cast<InputSectionDescription>(bc)) { 638 std::vector<Patch843419Section *> patches = 639 patchInputSectionDescription(*isd); 640 if (!patches.empty()) { 641 insertPatches(*isd, patches); 642 addressesChanged = true; 643 } 644 } 645 } 646 return addressesChanged; 647} 648} // namespace elf 649} // namespace lld 650