1/* 2 * Copyright (C) 2011, 2013, 2014 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#include "config.h" 27#include "DFGOSRExitCompiler.h" 28 29#if ENABLE(DFG_JIT) && USE(JSVALUE64) 30 31#include "DFGOperations.h" 32#include "DFGOSRExitCompilerCommon.h" 33#include "DFGSpeculativeJIT.h" 34#include "JSCInlines.h" 35#include "VirtualRegister.h" 36 37#include <wtf/DataLog.h> 38 39namespace JSC { namespace DFG { 40 41void OSRExitCompiler::compileExit(const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery) 42{ 43 m_jit.jitAssertTagsInPlace(); 44 45 // 1) Pro-forma stuff. 46 if (Options::printEachOSRExit()) { 47 SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo; 48 debugInfo->codeBlock = m_jit.codeBlock(); 49 debugInfo->kind = exit.m_kind; 50 debugInfo->bytecodeOffset = exit.m_codeOrigin.bytecodeIndex; 51 52 m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo); 53 } 54 55 // Need to ensure that the stack pointer accounts for the worst-case stack usage at exit. 56 m_jit.addPtr( 57 CCallHelpers::TrustedImm32( 58 -m_jit.codeBlock()->jitCode()->dfgCommon()->requiredRegisterCountForExit * sizeof(Register)), 59 CCallHelpers::framePointerRegister, CCallHelpers::stackPointerRegister); 60 61 // 2) Perform speculation recovery. This only comes into play when an operation 62 // starts mutating state before verifying the speculation it has already made. 63 64 if (recovery) { 65 switch (recovery->type()) { 66 case SpeculativeAdd: 67 m_jit.sub32(recovery->src(), recovery->dest()); 68 m_jit.or64(GPRInfo::tagTypeNumberRegister, recovery->dest()); 69 break; 70 71 case BooleanSpeculationCheck: 72 m_jit.xor64(AssemblyHelpers::TrustedImm32(static_cast<int32_t>(ValueFalse)), recovery->dest()); 73 break; 74 75 default: 76 break; 77 } 78 } 79 80 // 3) Refine some array and/or value profile, if appropriate. 81 82 if (!!exit.m_jsValueSource) { 83 if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) { 84 // If the instruction that this originated from has an array profile, then 85 // refine it. If it doesn't, then do nothing. The latter could happen for 86 // hoisted checks, or checks emitted for operations that didn't have array 87 // profiling - either ops that aren't array accesses at all, or weren't 88 // known to be array acceses in the bytecode. The latter case is a FIXME 89 // while the former case is an outcome of a CheckStructure not knowing why 90 // it was emitted (could be either due to an inline cache of a property 91 // property access, or due to an array profile). 92 93 CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile; 94 if (ArrayProfile* arrayProfile = m_jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) { 95 GPRReg usedRegister; 96 if (exit.m_jsValueSource.isAddress()) 97 usedRegister = exit.m_jsValueSource.base(); 98 else 99 usedRegister = exit.m_jsValueSource.gpr(); 100 101 GPRReg scratch1; 102 GPRReg scratch2; 103 scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister); 104 scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister, scratch1); 105 106#if CPU(ARM64) 107 m_jit.pushToSave(scratch1); 108 m_jit.pushToSave(scratch2); 109#else 110 m_jit.push(scratch1); 111 m_jit.push(scratch2); 112#endif 113 114 GPRReg value; 115 if (exit.m_jsValueSource.isAddress()) { 116 value = scratch1; 117 m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value); 118 } else 119 value = exit.m_jsValueSource.gpr(); 120 121 m_jit.load32(AssemblyHelpers::Address(value, JSCell::structureIDOffset()), scratch1); 122 m_jit.store32(scratch1, arrayProfile->addressOfLastSeenStructureID()); 123 m_jit.load8(AssemblyHelpers::Address(value, JSCell::indexingTypeOffset()), scratch1); 124 m_jit.move(AssemblyHelpers::TrustedImm32(1), scratch2); 125 m_jit.lshift32(scratch1, scratch2); 126 m_jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes())); 127 128#if CPU(ARM64) 129 m_jit.popToRestore(scratch2); 130 m_jit.popToRestore(scratch1); 131#else 132 m_jit.pop(scratch2); 133 m_jit.pop(scratch1); 134#endif 135 } 136 } 137 138 if (!!exit.m_valueProfile) { 139 EncodedJSValue* bucket = exit.m_valueProfile.getSpecFailBucket(0); 140 141 if (exit.m_jsValueSource.isAddress()) { 142 // We can't be sure that we have a spare register. So use the tagTypeNumberRegister, 143 // since we know how to restore it. 144 m_jit.load64(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::tagTypeNumberRegister); 145 m_jit.store64(GPRInfo::tagTypeNumberRegister, bucket); 146 m_jit.move(AssemblyHelpers::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister); 147 } else 148 m_jit.store64(exit.m_jsValueSource.gpr(), bucket); 149 } 150 } 151 152 // What follows is an intentionally simple OSR exit implementation that generates 153 // fairly poor code but is very easy to hack. In particular, it dumps all state that 154 // needs conversion into a scratch buffer so that in step 6, where we actually do the 155 // conversions, we know that all temp registers are free to use and the variable is 156 // definitely in a well-known spot in the scratch buffer regardless of whether it had 157 // originally been in a register or spilled. This allows us to decouple "where was 158 // the variable" from "how was it represented". Consider that the 159 // Int32DisplacedInJSStack recovery: it tells us that the value is in a 160 // particular place and that that place holds an unboxed int32. We have two different 161 // places that a value could be (displaced, register) and a bunch of different 162 // ways of representing a value. The number of recoveries is two * a bunch. The code 163 // below means that we have to have two + a bunch cases rather than two * a bunch. 164 // Once we have loaded the value from wherever it was, the reboxing is the same 165 // regardless of its location. Likewise, before we do the reboxing, the way we get to 166 // the value (i.e. where we load it from) is the same regardless of its type. Because 167 // the code below always dumps everything into a scratch buffer first, the two 168 // questions become orthogonal, which simplifies adding new types and adding new 169 // locations. 170 // 171 // This raises the question: does using such a suboptimal implementation of OSR exit, 172 // where we always emit code to dump all state into a scratch buffer only to then 173 // dump it right back into the stack, hurt us in any way? The asnwer is that OSR exits 174 // are rare. Our tiering strategy ensures this. This is because if an OSR exit is 175 // taken more than ~100 times, we jettison the DFG code block along with all of its 176 // exits. It is impossible for an OSR exit - i.e. the code we compile below - to 177 // execute frequently enough for the codegen to matter that much. It probably matters 178 // enough that we don't want to turn this into some super-slow function call, but so 179 // long as we're generating straight-line code, that code can be pretty bad. Also 180 // because we tend to exit only along one OSR exit from any DFG code block - that's an 181 // empirical result that we're extremely confident about - the code size of this 182 // doesn't matter much. Hence any attempt to optimize the codegen here is just purely 183 // harmful to the system: it probably won't reduce either net memory usage or net 184 // execution time. It will only prevent us from cleanly decoupling "where was the 185 // variable" from "how was it represented", which will make it more difficult to add 186 // features in the future and it will make it harder to reason about bugs. 187 188 // 4) Save all state from GPRs into the scratch buffer. 189 190 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(sizeof(EncodedJSValue) * operands.size()); 191 EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0; 192 193 for (size_t index = 0; index < operands.size(); ++index) { 194 const ValueRecovery& recovery = operands[index]; 195 196 switch (recovery.technique()) { 197 case InGPR: 198 case UnboxedInt32InGPR: 199 case UnboxedInt52InGPR: 200 case UnboxedStrictInt52InGPR: 201 case UnboxedCellInGPR: 202 m_jit.store64(recovery.gpr(), scratch + index); 203 break; 204 205 default: 206 break; 207 } 208 } 209 210 // And voila, all GPRs are free to reuse. 211 212 // 5) Save all state from FPRs into the scratch buffer. 213 214 for (size_t index = 0; index < operands.size(); ++index) { 215 const ValueRecovery& recovery = operands[index]; 216 217 switch (recovery.technique()) { 218 case InFPR: 219 m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0); 220 m_jit.storeDouble(recovery.fpr(), MacroAssembler::Address(GPRInfo::regT0)); 221 break; 222 223 default: 224 break; 225 } 226 } 227 228 // Now, all FPRs are also free. 229 230 // 6) Save all state from the stack into the scratch buffer. For simplicity we 231 // do this even for state that's already in the right place on the stack. 232 // It makes things simpler later. 233 234 for (size_t index = 0; index < operands.size(); ++index) { 235 const ValueRecovery& recovery = operands[index]; 236 237 switch (recovery.technique()) { 238 case DisplacedInJSStack: 239 case CellDisplacedInJSStack: 240 case BooleanDisplacedInJSStack: 241 case Int32DisplacedInJSStack: 242 case DoubleDisplacedInJSStack: 243 case Int52DisplacedInJSStack: 244 case StrictInt52DisplacedInJSStack: 245 m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0); 246 m_jit.store64(GPRInfo::regT0, scratch + index); 247 break; 248 249 default: 250 break; 251 } 252 } 253 254 // 7) Do all data format conversions and store the results into the stack. 255 256 bool haveArguments = false; 257 258 for (size_t index = 0; index < operands.size(); ++index) { 259 const ValueRecovery& recovery = operands[index]; 260 int operand = operands.operandForIndex(index); 261 262 switch (recovery.technique()) { 263 case InGPR: 264 case UnboxedCellInGPR: 265 case DisplacedInJSStack: 266 case CellDisplacedInJSStack: 267 case BooleanDisplacedInJSStack: 268 m_jit.load64(scratch + index, GPRInfo::regT0); 269 m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); 270 break; 271 272 case UnboxedInt32InGPR: 273 case Int32DisplacedInJSStack: 274 m_jit.load64(scratch + index, GPRInfo::regT0); 275 m_jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0); 276 m_jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0); 277 m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); 278 break; 279 280 case UnboxedInt52InGPR: 281 case Int52DisplacedInJSStack: 282 m_jit.load64(scratch + index, GPRInfo::regT0); 283 m_jit.rshift64( 284 AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0); 285 m_jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0); 286 m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); 287 break; 288 289 case UnboxedStrictInt52InGPR: 290 case StrictInt52DisplacedInJSStack: 291 m_jit.load64(scratch + index, GPRInfo::regT0); 292 m_jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0); 293 m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); 294 break; 295 296 case InFPR: 297 case DoubleDisplacedInJSStack: 298 m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0); 299 m_jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::fpRegT0); 300 m_jit.purifyNaN(FPRInfo::fpRegT0); 301 m_jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0); 302 m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand)); 303 break; 304 305 case Constant: 306 m_jit.store64( 307 AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())), 308 AssemblyHelpers::addressFor(operand)); 309 break; 310 311 case ArgumentsThatWereNotCreated: 312 haveArguments = true; 313 // We can't restore this yet but we can make sure that the stack appears 314 // sane. 315 m_jit.store64( 316 AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue())), 317 AssemblyHelpers::addressFor(operand)); 318 break; 319 320 default: 321 break; 322 } 323 } 324 325 // 8) Adjust the old JIT's execute counter. Since we are exiting OSR, we know 326 // that all new calls into this code will go to the new JIT, so the execute 327 // counter only affects call frames that performed OSR exit and call frames 328 // that were still executing the old JIT at the time of another call frame's 329 // OSR exit. We want to ensure that the following is true: 330 // 331 // (a) Code the performs an OSR exit gets a chance to reenter optimized 332 // code eventually, since optimized code is faster. But we don't 333 // want to do such reentery too aggressively (see (c) below). 334 // 335 // (b) If there is code on the call stack that is still running the old 336 // JIT's code and has never OSR'd, then it should get a chance to 337 // perform OSR entry despite the fact that we've exited. 338 // 339 // (c) Code the performs an OSR exit should not immediately retry OSR 340 // entry, since both forms of OSR are expensive. OSR entry is 341 // particularly expensive. 342 // 343 // (d) Frequent OSR failures, even those that do not result in the code 344 // running in a hot loop, result in recompilation getting triggered. 345 // 346 // To ensure (c), we'd like to set the execute counter to 347 // counterValueForOptimizeAfterWarmUp(). This seems like it would endanger 348 // (a) and (b), since then every OSR exit would delay the opportunity for 349 // every call frame to perform OSR entry. Essentially, if OSR exit happens 350 // frequently and the function has few loops, then the counter will never 351 // become non-negative and OSR entry will never be triggered. OSR entry 352 // will only happen if a loop gets hot in the old JIT, which does a pretty 353 // good job of ensuring (a) and (b). But that doesn't take care of (d), 354 // since each speculation failure would reset the execute counter. 355 // So we check here if the number of speculation failures is significantly 356 // larger than the number of successes (we want 90% success rate), and if 357 // there have been a large enough number of failures. If so, we set the 358 // counter to 0; otherwise we set the counter to 359 // counterValueForOptimizeAfterWarmUp(). 360 361 handleExitCounts(m_jit, exit); 362 363 // 9) Reify inlined call frames. 364 365 reifyInlinedCallFrames(m_jit, exit); 366 367 // 10) Create arguments if necessary and place them into the appropriate aliased 368 // registers. 369 370 if (haveArguments) { 371 ArgumentsRecoveryGenerator argumentsRecovery; 372 373 for (size_t index = 0; index < operands.size(); ++index) { 374 const ValueRecovery& recovery = operands[index]; 375 if (recovery.technique() != ArgumentsThatWereNotCreated) 376 continue; 377 argumentsRecovery.generateFor( 378 operands.operandForIndex(index), exit.m_codeOrigin, m_jit); 379 } 380 } 381 382 // 12) And finish. 383 adjustAndJumpToTarget(m_jit, exit); 384} 385 386} } // namespace JSC::DFG 387 388#endif // ENABLE(DFG_JIT) && USE(JSVALUE64) 389