1/*
2 * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "FTLOSRExitCompiler.h"
28
29#if ENABLE(FTL_JIT)
30
31#include "DFGOSRExitCompilerCommon.h"
32#include "DFGOSRExitPreparation.h"
33#include "FTLExitArgumentForOperand.h"
34#include "FTLJITCode.h"
35#include "FTLOSRExit.h"
36#include "FTLState.h"
37#include "FTLSaveRestore.h"
38#include "LinkBuffer.h"
39#include "MaxFrameExtentForSlowPathCall.h"
40#include "OperandsInlines.h"
41#include "JSCInlines.h"
42#include "RegisterPreservationWrapperGenerator.h"
43#include "RepatchBuffer.h"
44
45namespace JSC { namespace FTL {
46
47using namespace DFG;
48
49static void compileStub(
50    unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
51{
52    StackMaps::Record* record = nullptr;
53
54    for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
55        record = &jitCode->stackmaps.records[i];
56        if (record->patchpointID == exit.m_stackmapID)
57            break;
58    }
59
60    RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
61
62    // This code requires framePointerRegister is the same as callFrameRegister
63    static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
64
65    CCallHelpers jit(vm, codeBlock);
66
67    // We need scratch space to save all registers and to build up the JSStack.
68    // Use a scratch buffer to transfer all values.
69    ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes() + jitCode->unwindInfo.m_registers.size() * sizeof(uint64_t));
70    EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
71    char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size());
72    uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
73
74    // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
75    // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
76    // that slot for saveAllRegisters().
77
78    saveAllRegisters(jit, registerScratch);
79
80    // Bring the stack back into a sane form and assert that it's sane.
81    jit.popToRestore(GPRInfo::regT0);
82    jit.checkStackPointerAlignment();
83
84    if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
85        Profiler::Database& database = *vm->m_perBytecodeProfiler;
86        Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
87
88        Profiler::OSRExit* profilerExit = compilation->addOSRExit(
89            exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
90            exit.m_kind, isWatchpoint(exit.m_kind));
91        jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
92    }
93
94    // The remaining code assumes that SP/FP are in the same state that they were in the FTL's
95    // call frame.
96
97    // Get the call frame and tag thingies.
98    // Restore the exiting function's callFrame value into a regT4
99    jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
100    jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
101
102    // Do some value profiling.
103    if (exit.m_profileValueFormat != InvalidValueFormat) {
104        record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
105        reboxAccordingToFormat(
106            exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
107
108        if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
109            CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
110            if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
111                jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
112                jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
113                jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
114                jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
115                jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
116                jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
117            }
118        }
119
120        if (!!exit.m_valueProfile)
121            jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
122    }
123
124    // Save all state from wherever the exit data tells us it was, into the appropriate place in
125    // the scratch buffer. This doesn't rebox any values yet.
126
127    for (unsigned index = exit.m_values.size(); index--;) {
128        ExitValue value = exit.m_values[index];
129
130        switch (value.kind()) {
131        case ExitValueDead:
132            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
133            break;
134
135        case ExitValueConstant:
136            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
137            break;
138
139        case ExitValueArgument:
140            record->locations[value.exitArgument().argument()].restoreInto(
141                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
142            break;
143
144        case ExitValueInJSStack:
145        case ExitValueInJSStackAsInt32:
146        case ExitValueInJSStackAsInt52:
147        case ExitValueInJSStackAsDouble:
148            jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
149            break;
150
151        case ExitValueArgumentsObjectThatWasNotCreated:
152            // We can't actually recover this yet, but we can make the stack look sane. This is
153            // a prerequisite to running the actual arguments recovery.
154            jit.move(MacroAssembler::TrustedImm64(JSValue::encode(JSValue())), GPRInfo::regT0);
155            break;
156
157        case ExitValueRecovery:
158            record->locations[value.rightRecoveryArgument()].restoreInto(
159                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT1);
160            record->locations[value.leftRecoveryArgument()].restoreInto(
161                jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
162            switch (value.recoveryOpcode()) {
163            case AddRecovery:
164                switch (value.recoveryFormat()) {
165                case ValueFormatInt32:
166                    jit.add32(GPRInfo::regT1, GPRInfo::regT0);
167                    break;
168                case ValueFormatInt52:
169                    jit.add64(GPRInfo::regT1, GPRInfo::regT0);
170                    break;
171                default:
172                    RELEASE_ASSERT_NOT_REACHED();
173                    break;
174                }
175                break;
176            case SubRecovery:
177                switch (value.recoveryFormat()) {
178                case ValueFormatInt32:
179                    jit.sub32(GPRInfo::regT1, GPRInfo::regT0);
180                    break;
181                case ValueFormatInt52:
182                    jit.sub64(GPRInfo::regT1, GPRInfo::regT0);
183                    break;
184                default:
185                    RELEASE_ASSERT_NOT_REACHED();
186                    break;
187                }
188                break;
189            default:
190                RELEASE_ASSERT_NOT_REACHED();
191                break;
192            }
193            break;
194
195        default:
196            RELEASE_ASSERT_NOT_REACHED();
197            break;
198        }
199
200        jit.store64(GPRInfo::regT0, scratch + index);
201    }
202
203    // Henceforth we make it look like the exiting function was called through a register
204    // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
205    // we restore the various things according to either exit.m_values or by copying from the
206    // old frame, and finally we save the various callee-save registers into where the
207    // restoration thunk would restore them from.
208
209    ptrdiff_t offset = registerPreservationOffset();
210    RegisterSet toSave = registersToPreserve();
211
212    // Before we start messing with the frame, we need to set aside any registers that the
213    // FTL code was preserving.
214    for (unsigned i = jitCode->unwindInfo.m_registers.size(); i--;) {
215        RegisterAtOffset entry = jitCode->unwindInfo.m_registers[i];
216        jit.load64(
217            MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
218            GPRInfo::regT0);
219        jit.store64(GPRInfo::regT0, unwindScratch + i);
220    }
221
222    jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
223
224    // Let's say that the FTL function had failed its arity check. In that case, the stack will
225    // contain some extra stuff.
226    //
227    // First we compute the padded stack space:
228    //
229    //     paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
230    //
231    // The stack will have regT2 + CallFrameHeaderSize stuff, but above it there will be
232    // paddedStackSpace gunk used by the arity check fail restoration thunk. When that happens
233    // we want to make the stack look like this, from higher addresses down:
234    //
235    //     - register preservation return PC
236    //     - preserved registers
237    //     - arity check fail return PC
238    //     - argument padding
239    //     - actual arguments
240    //     - call frame header
241    //
242    // So that the actual call frame header appears to return to the arity check fail return
243    // PC, and that then returns to the register preservation thunk. The arity check thunk that
244    // we return to will have the padding size encoded into it. It will then know to return
245    // into the register preservation thunk, which uses the argument count to figure out where
246    // registers are preserved.
247
248    // This code assumes that we're dealing with FunctionCode.
249    RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
250
251    jit.add32(
252        MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
253        GPRInfo::regT3);
254    MacroAssembler::Jump arityIntact = jit.branch32(
255        MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
256    jit.neg32(GPRInfo::regT3);
257    jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
258    jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
259    jit.add32(GPRInfo::regT3, GPRInfo::regT2);
260    arityIntact.link(&jit);
261
262    // First set up SP so that our data doesn't get clobbered by signals.
263    unsigned conservativeStackDelta =
264        registerPreservationOffset() +
265        exit.m_values.numberOfLocals() * sizeof(Register) +
266        maxFrameExtentForSlowPathCall;
267    conservativeStackDelta = WTF::roundUpToMultipleOf(
268        stackAlignmentBytes(), conservativeStackDelta);
269    jit.addPtr(
270        MacroAssembler::TrustedImm32(-conservativeStackDelta),
271        MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
272    jit.checkStackPointerAlignment();
273
274    jit.subPtr(
275        MacroAssembler::TrustedImm32(registerPreservationOffset()),
276        MacroAssembler::framePointerRegister);
277
278    // Copy the old frame data into its new location.
279    jit.add32(MacroAssembler::TrustedImm32(JSStack::CallFrameHeaderSize), GPRInfo::regT2);
280    jit.move(MacroAssembler::framePointerRegister, GPRInfo::regT1);
281    MacroAssembler::Label loop = jit.label();
282    jit.sub32(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
283    jit.load64(MacroAssembler::Address(GPRInfo::regT1, offset), GPRInfo::regT0);
284    jit.store64(GPRInfo::regT0, GPRInfo::regT1);
285    jit.addPtr(MacroAssembler::TrustedImm32(sizeof(Register)), GPRInfo::regT1);
286    jit.branchTest32(MacroAssembler::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
287
288    // At this point regT1 points to where we would save our registers. Save them here.
289    ptrdiff_t currentOffset = 0;
290    for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
291        if (!toSave.get(reg))
292            continue;
293        currentOffset += sizeof(Register);
294        unsigned unwindIndex = jitCode->unwindInfo.indexOf(reg);
295        if (unwindIndex == UINT_MAX) {
296            // The FTL compilation didn't preserve this register. This means that it also
297            // didn't use the register. So its value at the beginning of OSR exit should be
298            // preserved by the thunk. Luckily, we saved all registers into the register
299            // scratch buffer, so we can restore them from there.
300            jit.load64(registerScratch + offsetOfReg(reg), GPRInfo::regT0);
301        } else {
302            // The FTL compilation preserved the register. Its new value is therefore
303            // irrelevant, but we can get the value that was preserved by using the unwind
304            // data. We've already copied all unwind-able preserved registers into the unwind
305            // scratch buffer, so we can get it from there.
306            jit.load64(unwindScratch + unwindIndex, GPRInfo::regT0);
307        }
308        jit.store64(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT1, currentOffset));
309    }
310
311    // We need to make sure that we return into the register restoration thunk. This works
312    // differently depending on whether or not we had arity issues.
313    MacroAssembler::Jump arityIntactForReturnPC = jit.branch32(
314        MacroAssembler::GreaterThanOrEqual,
315        CCallHelpers::payloadFor(JSStack::ArgumentCount),
316        MacroAssembler::TrustedImm32(codeBlock->numParameters()));
317
318    // The return PC in the call frame header points at exactly the right arity restoration
319    // thunk. We don't want to change that. But the arity restoration thunk's frame has a
320    // return PC and we want to reroute that to our register restoration thunk. The arity
321    // restoration's return PC just just below regT1, and the register restoration's return PC
322    // is right at regT1.
323    jit.loadPtr(MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))), GPRInfo::regT0);
324    jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
325    jit.storePtr(
326        MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
327        MacroAssembler::Address(GPRInfo::regT1, -static_cast<ptrdiff_t>(sizeof(Register))));
328
329    MacroAssembler::Jump arityReturnPCReady = jit.jump();
330
331    arityIntactForReturnPC.link(&jit);
332
333    jit.loadPtr(MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()), GPRInfo::regT0);
334    jit.storePtr(GPRInfo::regT0, GPRInfo::regT1);
335    jit.storePtr(
336        MacroAssembler::TrustedImmPtr(vm->getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
337        MacroAssembler::Address(MacroAssembler::framePointerRegister, CallFrame::returnPCOffset()));
338
339    arityReturnPCReady.link(&jit);
340
341    // Now get state out of the scratch buffer and place it back into the stack. This part does
342    // all reboxing.
343    for (unsigned index = exit.m_values.size(); index--;) {
344        int operand = exit.m_values.operandForIndex(index);
345        ExitValue value = exit.m_values[index];
346
347        jit.load64(scratch + index, GPRInfo::regT0);
348        reboxAccordingToFormat(
349            value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
350        jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(static_cast<VirtualRegister>(operand)));
351    }
352
353    handleExitCounts(jit, exit);
354    reifyInlinedCallFrames(jit, exit);
355
356    ArgumentsRecoveryGenerator argumentsRecovery;
357    for (unsigned index = exit.m_values.size(); index--;) {
358        if (!exit.m_values[index].isArgumentsObjectThatWasNotCreated())
359            continue;
360        int operand = exit.m_values.operandForIndex(index);
361        argumentsRecovery.generateFor(operand, exit.m_codeOrigin, jit);
362    }
363
364    adjustAndJumpToTarget(jit, exit);
365
366    LinkBuffer patchBuffer(*vm, jit, codeBlock);
367    exit.m_code = FINALIZE_CODE_IF(
368        shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
369        patchBuffer,
370        ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
371            exitID, toCString(exit.m_codeOrigin).data(),
372            exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
373            toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
374            toCString(*record).data()));
375}
376
377extern "C" void* compileFTLOSRExit(ExecState* exec, unsigned exitID)
378{
379    SamplingRegion samplingRegion("FTL OSR Exit Compilation");
380
381    if (shouldShowDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit())
382        dataLog("Compiling OSR exit with exitID = ", exitID, "\n");
383
384    CodeBlock* codeBlock = exec->codeBlock();
385
386    ASSERT(codeBlock);
387    ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
388
389    VM* vm = &exec->vm();
390
391    // It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
392    // really be profitable.
393    DeferGCForAWhile deferGC(vm->heap);
394
395    JITCode* jitCode = codeBlock->jitCode()->ftl();
396    OSRExit& exit = jitCode->osrExit[exitID];
397
398    prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
399
400    compileStub(exitID, jitCode, exit, vm, codeBlock);
401
402    RepatchBuffer repatchBuffer(codeBlock);
403    repatchBuffer.relink(
404        exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
405
406    return exit.m_code.code().executableAddress();
407}
408
409} } // namespace JSC::FTL
410
411#endif // ENABLE(FTL_JIT)
412
413