1/* 2 * Copyright (C) 2013, 2014 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#include "config.h" 27#include "FTLOSRExitCompiler.h" 28 29#if ENABLE(FTL_JIT) 30 31#include "DFGOSRExitCompilerCommon.h" 32#include "DFGOSRExitPreparation.h" 33#include "FTLExitArgumentForOperand.h" 34#include "FTLJITCode.h" 35#include "FTLOSRExit.h" 36#include "FTLState.h" 37#include "FTLSaveRestore.h" 38#include "LinkBuffer.h" 39#include "MaxFrameExtentForSlowPathCall.h" 40#include "OperandsInlines.h" 41#include "JSCInlines.h" 42#include "RegisterPreservationWrapperGenerator.h" 43#include "RepatchBuffer.h" 44 45namespace JSC { namespace FTL { 46 47using namespace DFG; 48 49static void compileStub( 50 unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock) 51{ 52 StackMaps::Record* record = nullptr; 53 54 for (unsigned i = jitCode->stackmaps.records.size(); i--;) { 55 record = &jitCode->stackmaps.records[i]; 56 if (record->patchpointID == exit.m_stackmapID) 57 break; 58 } 59 60 RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID); 61 62 // This code requires framePointerRegister is the same as callFrameRegister 63 static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same"); 64 65 CCallHelpers jit(vm, codeBlock); 66 67 // We need scratch space to save all registers and to build up the JSStack. 68 // Use a scratch buffer to transfer all values. 69 ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes() + jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t)); 70 EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0; 71 char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size()); 72 uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes()); 73 74 // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave(). 75 // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use 76 // that slot for saveAllRegisters(). 77 78 saveAllRegisters(jit, registerScratch); 79 80 // Bring the stack back into a sane form and assert that it's sane. 81 jit.popToRestore(GPRInfo::regT0); 82 jit.checkStackPointerAlignment(); 83 84 if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) { 85 Profiler::Database& database = *vm->m_perBytecodeProfiler; 86 Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get(); 87 88 Profiler::OSRExit* profilerExit = compilation->addOSRExit( 89 exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin), 90 exit.m_kind, isWatchpoint(exit.m_kind)); 91 jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress())); 92 } 93 94 // The remaining code assumes that SP/FP are in the same state that they were in the FTL's 95 // call frame. 96 97 // Get the call frame and tag thingies. 98 // Restore the exiting function's callFrame value into a regT4 99 jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister); 100 jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister); 101 102 // Do some value profiling. 103 if (exit.m_profileValueFormat != InvalidValueFormat) { 104 record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0); 105 reboxAccordingToFormat( 106 exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2); 107 108 if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) { 109 CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile; 110 if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) { 111 jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1); 112 jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID()); 113 jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1); 114 jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2); 115 jit.lshift32(GPRInfo::regT1, GPRInfo::regT2); 116 jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes())); 117 } 118 } 119 120 if (!!exit.m_valueProfile) 121 jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0)); 122 } 123 124 // Save all state from wherever the exit data tells us it was, into the appropriate place in 125 // the scratch buffer. This doesn't rebox any values yet. 126 127 for (unsigned index = exit.m_values.size(); index--;) { 128 ExitValue value = exit.m_values[index]; 129 130 switch (value.kind()) { 131 case ExitValueDead: 132 jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0); 133 break; 134 135 case ExitValueConstant: 136 jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0); 137 break; 138 139 case ExitValueArgument: 140 record->locations[value.exitArgument().argument()].restoreInto( 141 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0); 142 break; 143 144 case ExitValueInJSStack: 145 case ExitValueInJSStackAsInt32: 146 case ExitValueInJSStackAsInt52: 147 case ExitValueInJSStackAsDouble: 148 jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0); 149 break; 150 151 case ExitValueArgumentsObjectThatWasNotCreated: 152 // We can't actually recover this yet, but we can make the stack look sane. This is 153 // a prerequisite to running the actual arguments recovery. 154 jit.move(MacroAssembler::TrustedImm64(JSValue::encode(JSValue())), GPRInfo::regT0); 155 break; 156 157 case ExitValueRecovery: 158 record->locations[value.rightRecoveryArgument()].restoreInto( 159 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT1); 160 record->locations[value.leftRecoveryArgument()].restoreInto( 161 jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0); 162 switch (value.recoveryOpcode()) { 163 case AddRecovery: 164 switch (value.recoveryFormat()) { 165 case ValueFormatInt32: 166 jit.add32(GPRInfo::regT1, GPRInfo::regT0); 167 break; 168 case ValueFormatInt52: 169 jit.add64(GPRInfo::regT1, GPRInfo::regT0); 170 break; 171 default: 172 RELEASE_ASSERT_NOT_REACHED(); 173 break; 174 } 175 break; 176 case SubRecovery: 177 switch (value.recoveryFormat()) { 178 case ValueFormatInt32: 179 jit.sub32(GPRInfo::regT1, GPRInfo::regT0); 180 break; 181 case ValueFormatInt52: 182 jit.sub64(GPRInfo::regT1, GPRInfo::regT0); 183 break; 184 default: 185 RELEASE_ASSERT_NOT_REACHED(); 186 break; 187 } 188 break; 189 default: 190 RELEASE_ASSERT_NOT_REACHED(); 191 break; 192 } 193 break; 194 195 default: 196 RELEASE_ASSERT_NOT_REACHED(); 197 break; 198 } 199 200 jit.store64(GPRInfo::regT0, scratch + index); 201 } 202 203 // Henceforth we make it look like the exiting function was called through a register 204 // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then 205 // we restore the various things according to either exit.m_values or by copying from the 206 // old frame, and finally we save the various callee-save registers into where the 207 // restoration thunk would restore them from. 208 209 ptrdiff_t offset = registerPreservationOffset(); 210 RegisterSet toSave = registersToPreserve(); 211 212 // Before we start messing with the frame, we need to set aside any registers that the 213 // FTL code was preserving. 214 for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) { 215 RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i]; 216 jit.load64( 217 MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()), 218 GPRInfo::regT0); 219 jit.store64(GPRInfo::regT0, unwindScratch + i); 220 } 221 222 jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2); 223 224 // Let's say that the FTL function had failed its arity check. In that case, the stack will 225 // contain some extra stuff. 226 // 227 // First we compute the padded stack space: 228 // 229 // paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1) 230 // 231 // The stack will have regT2 + CallFrameHeaderSize stuff, but above it there will be 232 // paddedStackSpace gunk used by the arity check fail restoration thunk. When that happens 233 // we want to make the stack look like this, from higher addresses down: 234 // 235 // - register preservation return PC 236 // - preserved registers 237 // - arity check fail return PC 238 // - argument padding 239 // - actual arguments 240 // - call frame header 241 // 242 // So that the actual call frame header appears to return to the arity check fail return 243 // PC, and that then returns to the register preservation thunk. The arity check thunk that 244 // we return to will have the padding size encoded into it. It will then know to return 245 // into the register preservation thunk, which uses the argument count to figure out where 246 // registers are preserved. 247 248 // This code assumes that we're dealing with FunctionCode. 249 RELEASE_ASSERT(codeBlock->codeType() == FunctionCode); 250 251 jit.add32( 252 MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2, 253 GPRInfo::regT3); 254 MacroAssembler::Jump arityIntact = jit.branch32( 255 MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0)); 256 jit.neg32(GPRInfo::regT3); 257 jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3); 258 jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3); 259 jit.add32(GPRInfo::regT3, GPRInfo::regT2); 260 arityIntact.link(&jit); 261 262 // First set up SP so that our data doesn't get clobbered by signals. 263 unsigned conservativeStackDelta = 264 registerPreservationOffset() + 265 exit.m_values.numberOfLocals() * sizeof(Register) + 266 maxFrameExtentForSlowPathCall; 267 conservativeStackDelta = WTF::roundUpToMultipleOf( 268 stackAlignmentBytes(), conservativeStackDelta); 269 jit.addPtr( 270 MacroAssembler::TrustedImm32(-conservativeStackDelta), 271 MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister); 272 jit.checkStackPointerAlignment(); 273 274 jit.subPtr( 275 MacroAssembler::TrustedImm32(registerPreservationOffset()), 276 MacroAssembler::framePointerRegister); 277 278 // Copy the old frame data into its new location. 279 jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2); 280 jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1); 281 MacroAssembler::Label loop = jit.label(); 282 jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2); 283 jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0); 284 jit.store64(GPRInfo::regT0, GPRInfo::regT1); 285 jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1); 286 jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit); 287 288 // At this point regT1 points to where we would save our registers. Save them here. 289 ptrdiff_t currentOffset = 0; 290 for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { 291 if (!toSave.get(reg)) 292 continue; 293 currentOffset += sizeof(Register); 294 unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg); 295 if (unwindIndex == UINT_MAX) { 296 // The FTL compilation didn't preserve this register. This means that it also 297 // didn't use the register. So its value at the beginning of OSR exit should be 298 // preserved by the thunk. Luckily, we saved all registers into the register 299 // scratch buffer, so we can restore them from there. 300 jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0); 301 } else { 302 // The FTL compilation preserved the register. Its new value is therefore 303 // irrelevant, but we can get the value that was preserved by using the unwind 304 // data. We've already copied all unwind-able preserved registers into the unwind 305 // scratch buffer, so we can get it from there. 306 jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0); 307 } 308 jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset)); 309 } 310 311 // We need to make sure that we return into the register restoration thunk. This works 312 // differently depending on whether or not we had arity issues. 313 MacroAssembler::Jump arityIntactForReturnPC = jit.branch32( 314 MacroAssembler::GreaterThanOrEqual, 315 CCallHelpers::payloadFor(JSStack::ArgumentCount), 316 MacroAssembler::TrustedImm32(codeBlock->numParameters())); 317 318 // The return PC in the call frame header points at exactly the right arity restoration 319 // thunk. We don't want to change that. But the arity restoration thunk's frame has a 320 // return PC and we want to reroute that to our register restoration thunk. The arity 321 // restoration's return PC just just below regT1, and the register restoration's return PC 322 // is right at regT1. 323 jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0); 324 jit.storePtr(GPRInfo::regT0, GPRInfo::regT1); 325 jit.storePtr( 326 MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()), 327 MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register)))); 328 329 MacroAssembler::Jump arityReturnPCReady = jit.jump(); 330 331 arityIntactForReturnPC.link(&jit); 332 333 jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0); 334 jit.storePtr(GPRInfo::regT0, GPRInfo::regT1); 335 jit.storePtr( 336 MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()), 337 MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset())); 338 339 arityReturnPCReady.link(&jit); 340 341 // Now get state out of the scratch buffer and place it back into the stack. This part does 342 // all reboxing. 343 for (unsigned index = exit.m_values.size(); index--;) { 344 int operand = exit.m_values.operandForIndex(index); 345 ExitValue value = exit.m_values[index]; 346 347 jit.load64(scratch + index, GPRInfo::regT0); 348 reboxAccordingToFormat( 349 value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2); 350 jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand))); 351 } 352 353 handleExitCounts(jit, exit); 354 reifyInlinedCallFrames(jit, exit); 355 356 ArgumentsRecoveryGenerator argumentsRecovery; 357 for (unsigned index = exit.m_values.size(); index--;) { 358 if (!exit.m_values[index].isArgumentsObjectThatWasNotCreated()) 359 continue; 360 int operand = exit.m_values.operandForIndex(index); 361 argumentsRecovery.generateFor(operand, exit.m_codeOrigin, jit); 362 } 363 364 adjustAndJumpToTarget(jit, exit); 365 366 LinkBuffer patchBuffer(*vm, jit, codeBlock); 367 exit.m_code = FINALIZE_CODE_IF( 368 shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(), 369 patchBuffer, 370 ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s", 371 exitID, toCString(exit.m_codeOrigin).data(), 372 exitKindToString(exit.m_kind), toCString(*codeBlock).data(), 373 toCString(ignoringContext<DumpContext>(exit.m_values)).data(), 374 toCString(*record).data())); 375} 376 377extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID) 378{ 379 SamplingRegion samplingRegion("FTL OSR Exit Compilation"); 380 381 if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit()) 382 dataLog("Compiling OSR exit with exitID = ", exitID, "\n"); 383 384 CodeBlock* codeBlock = exec->codeBlock(); 385 386 ASSERT(codeBlock); 387 ASSERT(codeBlock->jitType() == JITCode::FTLJIT); 388 389 VM* vm = &exec->vm(); 390 391 // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't 392 // really be profitable. 393 DeferGCForAWhile deferGC(vm->heap); 394 395 JITCode* jitCode = codeBlock->jitCode()->ftl(); 396 OSRExit& exit = jitCode->osrExit[exitID]; 397 398 prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin); 399 400 compileStub(exitID, jitCode, exit, vm, codeBlock); 401 402 RepatchBuffer repatchBuffer(codeBlock); 403 repatchBuffer.relink( 404 exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code())); 405 406 return exit.m_code.code().executableAddress(); 407} 408 409} } // namespace JSC::FTL 410 411#endif // ENABLE(FTL_JIT) 412 413