1/*
2 * Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "DFGSpeculativeJIT.h"
28
29#if ENABLE(DFG_JIT)
30
31#include "Arguments.h"
32#include "ArrayPrototype.h"
33#include "DFGAbstractInterpreterInlines.h"
34#include "DFGCallArrayAllocatorSlowPathGenerator.h"
35#include "DFGOperations.h"
36#include "DFGSlowPathGenerator.h"
37#include "Debugger.h"
38#include "JSCInlines.h"
39#include "ObjectPrototype.h"
40#include "SpillRegistersMode.h"
41
42namespace JSC { namespace DFG {
43
44#if USE(JSVALUE64)
45
46void SpeculativeJIT::boxInt52(GPRReg sourceGPR, GPRReg targetGPR, DataFormat format)
47{
48    GPRReg tempGPR;
49    if (sourceGPR == targetGPR)
50        tempGPR = allocate();
51    else
52        tempGPR = targetGPR;
53
54    FPRReg fpr = fprAllocate();
55
56    if (format == DataFormatInt52)
57        m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), sourceGPR);
58    else
59        ASSERT(format == DataFormatStrictInt52);
60
61    m_jit.boxInt52(sourceGPR, targetGPR, tempGPR, fpr);
62
63    if (format == DataFormatInt52 && sourceGPR != targetGPR)
64        m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), sourceGPR);
65
66    if (tempGPR != targetGPR)
67        unlock(tempGPR);
68
69    unlock(fpr);
70}
71
72GPRReg SpeculativeJIT::fillJSValue(Edge edge)
73{
74    VirtualRegister virtualRegister = edge->virtualRegister();
75    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
76
77    switch (info.registerFormat()) {
78    case DataFormatNone: {
79        GPRReg gpr = allocate();
80
81        if (edge->hasConstant()) {
82            if (isInt32Constant(edge.node())) {
83                info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
84                JSValue jsValue = jsNumber(valueOfInt32Constant(edge.node()));
85                m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr);
86            } else if (isNumberConstant(edge.node())) {
87                info.fillJSValue(*m_stream, gpr, DataFormatJSDouble);
88                JSValue jsValue(JSValue::EncodeAsDouble, valueOfNumberConstant(edge.node()));
89                m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr);
90            } else {
91                ASSERT(isJSConstant(edge.node()));
92                JSValue jsValue = valueOfJSConstant(edge.node());
93                m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
94                info.fillJSValue(*m_stream, gpr, DataFormatJS);
95            }
96
97            m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
98        } else {
99            DataFormat spillFormat = info.spillFormat();
100            m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
101            switch (spillFormat) {
102            case DataFormatInt32: {
103                m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
104                m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
105                spillFormat = DataFormatJSInt32;
106                break;
107            }
108
109            default:
110                m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
111                RELEASE_ASSERT(spillFormat & DataFormatJS);
112                break;
113            }
114            info.fillJSValue(*m_stream, gpr, spillFormat);
115        }
116        return gpr;
117    }
118
119    case DataFormatInt32: {
120        GPRReg gpr = info.gpr();
121        // If the register has already been locked we need to take a copy.
122        // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32.
123        if (m_gprs.isLocked(gpr)) {
124            GPRReg result = allocate();
125            m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr, result);
126            return result;
127        }
128        m_gprs.lock(gpr);
129        m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
130        info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
131        return gpr;
132    }
133
134    case DataFormatCell:
135        // No retag required on JSVALUE64!
136    case DataFormatJS:
137    case DataFormatJSInt32:
138    case DataFormatJSDouble:
139    case DataFormatJSCell:
140    case DataFormatJSBoolean: {
141        GPRReg gpr = info.gpr();
142        m_gprs.lock(gpr);
143        return gpr;
144    }
145
146    case DataFormatBoolean:
147    case DataFormatStorage:
148    case DataFormatDouble:
149    case DataFormatInt52:
150        // this type currently never occurs
151        RELEASE_ASSERT_NOT_REACHED();
152
153    default:
154        RELEASE_ASSERT_NOT_REACHED();
155        return InvalidGPRReg;
156    }
157}
158
159void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
160{
161    JITGetByIdGenerator gen(
162        m_jit.codeBlock(), codeOrigin, usedRegisters(), JSValueRegs(baseGPR),
163        JSValueRegs(resultGPR), spillMode);
164    gen.generateFastPath(m_jit);
165
166    JITCompiler::JumpList slowCases;
167    if (slowPathTarget.isSet())
168        slowCases.append(slowPathTarget);
169    slowCases.append(gen.slowPathJump());
170
171    OwnPtr<SlowPathGenerator> slowPath = slowPathCall(
172        slowCases, this, operationGetByIdOptimize, resultGPR, gen.stubInfo(), baseGPR,
173        identifierUID(identifierNumber), spillMode);
174
175    m_jit.addGetById(gen, slowPath.get());
176    addSlowPathGenerator(slowPath.release());
177}
178
179void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
180{
181    JITPutByIdGenerator gen(
182        m_jit.codeBlock(), codeOrigin, usedRegisters(), JSValueRegs(baseGPR),
183        JSValueRegs(valueGPR), scratchGPR, spillMode, m_jit.ecmaModeFor(codeOrigin), putKind);
184
185    gen.generateFastPath(m_jit);
186
187    JITCompiler::JumpList slowCases;
188    if (slowPathTarget.isSet())
189        slowCases.append(slowPathTarget);
190    slowCases.append(gen.slowPathJump());
191
192    OwnPtr<SlowPathGenerator> slowPath = slowPathCall(
193        slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), valueGPR, baseGPR,
194        identifierUID(identifierNumber));
195
196    m_jit.addPutById(gen, slowPath.get());
197    addSlowPathGenerator(slowPath.release());
198}
199
200void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert)
201{
202    JSValueOperand arg(this, operand);
203    GPRReg argGPR = arg.gpr();
204
205    GPRTemporary result(this, Reuse, arg);
206    GPRReg resultGPR = result.gpr();
207
208    JITCompiler::Jump notCell;
209
210    JITCompiler::Jump notMasqueradesAsUndefined;
211    if (masqueradesAsUndefinedWatchpointIsStillValid()) {
212        if (!isKnownCell(operand.node()))
213            notCell = branchNotCell(JSValueRegs(argGPR));
214
215        m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR);
216        notMasqueradesAsUndefined = m_jit.jump();
217    } else {
218        GPRTemporary localGlobalObject(this);
219        GPRTemporary remoteGlobalObject(this);
220        GPRTemporary scratch(this);
221
222        if (!isKnownCell(operand.node()))
223            notCell = branchNotCell(JSValueRegs(argGPR));
224
225        JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(
226            JITCompiler::NonZero,
227            JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()),
228            JITCompiler::TrustedImm32(MasqueradesAsUndefined));
229
230        m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR);
231        notMasqueradesAsUndefined = m_jit.jump();
232
233        isMasqueradesAsUndefined.link(&m_jit);
234        GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
235        GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
236        m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR);
237        m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr());
238        m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
239        m_jit.comparePtr(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultGPR);
240    }
241
242    if (!isKnownCell(operand.node())) {
243        JITCompiler::Jump done = m_jit.jump();
244
245        notCell.link(&m_jit);
246
247        m_jit.move(argGPR, resultGPR);
248        m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR);
249        m_jit.compare64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(ValueNull), resultGPR);
250
251        done.link(&m_jit);
252    }
253
254    notMasqueradesAsUndefined.link(&m_jit);
255
256    m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
257    jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
258}
259
260void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert)
261{
262    BasicBlock* taken = branchNode->branchData()->taken.block;
263    BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
264
265    if (taken == nextBlock()) {
266        invert = !invert;
267        BasicBlock* tmp = taken;
268        taken = notTaken;
269        notTaken = tmp;
270    }
271
272    JSValueOperand arg(this, operand);
273    GPRReg argGPR = arg.gpr();
274
275    GPRTemporary result(this, Reuse, arg);
276    GPRReg resultGPR = result.gpr();
277
278    JITCompiler::Jump notCell;
279
280    if (masqueradesAsUndefinedWatchpointIsStillValid()) {
281        if (!isKnownCell(operand.node()))
282            notCell = branchNotCell(JSValueRegs(argGPR));
283
284        jump(invert ? taken : notTaken, ForceJump);
285    } else {
286        GPRTemporary localGlobalObject(this);
287        GPRTemporary remoteGlobalObject(this);
288        GPRTemporary scratch(this);
289
290        if (!isKnownCell(operand.node()))
291            notCell = branchNotCell(JSValueRegs(argGPR));
292
293        branchTest8(JITCompiler::Zero,
294            JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()),
295            JITCompiler::TrustedImm32(MasqueradesAsUndefined),
296            invert ? taken : notTaken);
297
298        GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
299        GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
300        m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR);
301        m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr());
302        m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
303        branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken);
304    }
305
306    if (!isKnownCell(operand.node())) {
307        jump(notTaken, ForceJump);
308
309        notCell.link(&m_jit);
310
311        m_jit.move(argGPR, resultGPR);
312        m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR);
313        branch64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm64(ValueNull), taken);
314    }
315
316    jump(notTaken);
317}
318
319bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert)
320{
321    unsigned branchIndexInBlock = detectPeepHoleBranch();
322    if (branchIndexInBlock != UINT_MAX) {
323        Node* branchNode = m_block->at(branchIndexInBlock);
324
325        RELEASE_ASSERT(node->adjustedRefCount() == 1);
326
327        nonSpeculativePeepholeBranchNull(operand, branchNode, invert);
328
329        use(node->child1());
330        use(node->child2());
331        m_indexInBlock = branchIndexInBlock;
332        m_currentNode = branchNode;
333
334        return true;
335    }
336
337    nonSpeculativeNonPeepholeCompareNull(operand, invert);
338
339    return false;
340}
341
342void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
343{
344    BasicBlock* taken = branchNode->branchData()->taken.block;
345    BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
346
347    JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero;
348
349    // The branch instruction will branch to the taken block.
350    // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
351    if (taken == nextBlock()) {
352        cond = JITCompiler::invert(cond);
353        callResultCondition = JITCompiler::Zero;
354        BasicBlock* tmp = taken;
355        taken = notTaken;
356        notTaken = tmp;
357    }
358
359    JSValueOperand arg1(this, node->child1());
360    JSValueOperand arg2(this, node->child2());
361    GPRReg arg1GPR = arg1.gpr();
362    GPRReg arg2GPR = arg2.gpr();
363
364    JITCompiler::JumpList slowPath;
365
366    if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
367        GPRResult result(this);
368        GPRReg resultGPR = result.gpr();
369
370        arg1.use();
371        arg2.use();
372
373        flushRegisters();
374        callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
375
376        branchTest32(callResultCondition, resultGPR, taken);
377    } else {
378        GPRTemporary result(this, Reuse, arg2);
379        GPRReg resultGPR = result.gpr();
380
381        arg1.use();
382        arg2.use();
383
384        if (!isKnownInteger(node->child1().node()))
385            slowPath.append(m_jit.branch64(MacroAssembler::Below, arg1GPR, GPRInfo::tagTypeNumberRegister));
386        if (!isKnownInteger(node->child2().node()))
387            slowPath.append(m_jit.branch64(MacroAssembler::Below, arg2GPR, GPRInfo::tagTypeNumberRegister));
388
389        branch32(cond, arg1GPR, arg2GPR, taken);
390
391        if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
392            jump(notTaken, ForceJump);
393
394            slowPath.link(&m_jit);
395
396            silentSpillAllRegisters(resultGPR);
397            callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
398            silentFillAllRegisters(resultGPR);
399
400            branchTest32(callResultCondition, resultGPR, taken);
401        }
402    }
403
404    jump(notTaken);
405
406    m_indexInBlock = m_block->size() - 1;
407    m_currentNode = branchNode;
408}
409
410template<typename JumpType>
411class CompareAndBoxBooleanSlowPathGenerator
412    : public CallSlowPathGenerator<JumpType, S_JITOperation_EJJ, GPRReg> {
413public:
414    CompareAndBoxBooleanSlowPathGenerator(
415        JumpType from, SpeculativeJIT* jit,
416        S_JITOperation_EJJ function, GPRReg result, GPRReg arg1, GPRReg arg2)
417        : CallSlowPathGenerator<JumpType, S_JITOperation_EJJ, GPRReg>(
418            from, jit, function, NeedToSpill, result)
419        , m_arg1(arg1)
420        , m_arg2(arg2)
421    {
422    }
423
424protected:
425    virtual void generateInternal(SpeculativeJIT* jit) override
426    {
427        this->setUp(jit);
428        this->recordCall(jit->callOperation(this->m_function, this->m_result, m_arg1, m_arg2));
429        jit->m_jit.and32(JITCompiler::TrustedImm32(1), this->m_result);
430        jit->m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), this->m_result);
431        this->tearDown(jit);
432    }
433
434private:
435    GPRReg m_arg1;
436    GPRReg m_arg2;
437};
438
439void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
440{
441    ASSERT(node->isBinaryUseKind(UntypedUse));
442    JSValueOperand arg1(this, node->child1());
443    JSValueOperand arg2(this, node->child2());
444    GPRReg arg1GPR = arg1.gpr();
445    GPRReg arg2GPR = arg2.gpr();
446
447    JITCompiler::JumpList slowPath;
448
449    if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
450        GPRResult result(this);
451        GPRReg resultGPR = result.gpr();
452
453        arg1.use();
454        arg2.use();
455
456        flushRegisters();
457        callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR);
458
459        m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
460        jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
461    } else {
462        GPRTemporary result(this, Reuse, arg2);
463        GPRReg resultGPR = result.gpr();
464
465        arg1.use();
466        arg2.use();
467
468        if (!isKnownInteger(node->child1().node()))
469            slowPath.append(m_jit.branch64(MacroAssembler::Below, arg1GPR, GPRInfo::tagTypeNumberRegister));
470        if (!isKnownInteger(node->child2().node()))
471            slowPath.append(m_jit.branch64(MacroAssembler::Below, arg2GPR, GPRInfo::tagTypeNumberRegister));
472
473        m_jit.compare32(cond, arg1GPR, arg2GPR, resultGPR);
474        m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
475
476        if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
477            addSlowPathGenerator(adoptPtr(
478                new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>(
479                    slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR)));
480        }
481
482        jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
483    }
484}
485
486void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert)
487{
488    BasicBlock* taken = branchNode->branchData()->taken.block;
489    BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
490
491    // The branch instruction will branch to the taken block.
492    // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
493    if (taken == nextBlock()) {
494        invert = !invert;
495        BasicBlock* tmp = taken;
496        taken = notTaken;
497        notTaken = tmp;
498    }
499
500    JSValueOperand arg1(this, node->child1());
501    JSValueOperand arg2(this, node->child2());
502    GPRReg arg1GPR = arg1.gpr();
503    GPRReg arg2GPR = arg2.gpr();
504
505    GPRTemporary result(this);
506    GPRReg resultGPR = result.gpr();
507
508    arg1.use();
509    arg2.use();
510
511    if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
512        // see if we get lucky: if the arguments are cells and they reference the same
513        // cell, then they must be strictly equal.
514        branch64(JITCompiler::Equal, arg1GPR, arg2GPR, invert ? notTaken : taken);
515
516        silentSpillAllRegisters(resultGPR);
517        callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR);
518        silentFillAllRegisters(resultGPR);
519
520        branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken);
521    } else {
522        m_jit.or64(arg1GPR, arg2GPR, resultGPR);
523
524        JITCompiler::Jump twoCellsCase = m_jit.branchTest64(JITCompiler::Zero, resultGPR, GPRInfo::tagMaskRegister);
525
526        JITCompiler::Jump leftOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg1GPR, GPRInfo::tagTypeNumberRegister);
527        JITCompiler::Jump leftDouble = m_jit.branchTest64(JITCompiler::NonZero, arg1GPR, GPRInfo::tagTypeNumberRegister);
528        leftOK.link(&m_jit);
529        JITCompiler::Jump rightOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg2GPR, GPRInfo::tagTypeNumberRegister);
530        JITCompiler::Jump rightDouble = m_jit.branchTest64(JITCompiler::NonZero, arg2GPR, GPRInfo::tagTypeNumberRegister);
531        rightOK.link(&m_jit);
532
533        branch64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, arg1GPR, arg2GPR, taken);
534        jump(notTaken, ForceJump);
535
536        twoCellsCase.link(&m_jit);
537        branch64(JITCompiler::Equal, arg1GPR, arg2GPR, invert ? notTaken : taken);
538
539        leftDouble.link(&m_jit);
540        rightDouble.link(&m_jit);
541
542        silentSpillAllRegisters(resultGPR);
543        callOperation(operationCompareStrictEq, resultGPR, arg1GPR, arg2GPR);
544        silentFillAllRegisters(resultGPR);
545
546        branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken);
547    }
548
549    jump(notTaken);
550}
551
552void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert)
553{
554    JSValueOperand arg1(this, node->child1());
555    JSValueOperand arg2(this, node->child2());
556    GPRReg arg1GPR = arg1.gpr();
557    GPRReg arg2GPR = arg2.gpr();
558
559    GPRTemporary result(this);
560    GPRReg resultGPR = result.gpr();
561
562    arg1.use();
563    arg2.use();
564
565    if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
566        // see if we get lucky: if the arguments are cells and they reference the same
567        // cell, then they must be strictly equal.
568        // FIXME: this should flush registers instead of silent spill/fill.
569        JITCompiler::Jump notEqualCase = m_jit.branch64(JITCompiler::NotEqual, arg1GPR, arg2GPR);
570
571        m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR);
572
573        JITCompiler::Jump done = m_jit.jump();
574
575        notEqualCase.link(&m_jit);
576
577        silentSpillAllRegisters(resultGPR);
578        callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR);
579        silentFillAllRegisters(resultGPR);
580
581        m_jit.and64(JITCompiler::TrustedImm32(1), resultGPR);
582        m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), resultGPR);
583
584        done.link(&m_jit);
585    } else {
586        m_jit.or64(arg1GPR, arg2GPR, resultGPR);
587
588        JITCompiler::JumpList slowPathCases;
589
590        JITCompiler::Jump twoCellsCase = m_jit.branchTest64(JITCompiler::Zero, resultGPR, GPRInfo::tagMaskRegister);
591
592        JITCompiler::Jump leftOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg1GPR, GPRInfo::tagTypeNumberRegister);
593        slowPathCases.append(m_jit.branchTest64(JITCompiler::NonZero, arg1GPR, GPRInfo::tagTypeNumberRegister));
594        leftOK.link(&m_jit);
595        JITCompiler::Jump rightOK = m_jit.branch64(JITCompiler::AboveOrEqual, arg2GPR, GPRInfo::tagTypeNumberRegister);
596        slowPathCases.append(m_jit.branchTest64(JITCompiler::NonZero, arg2GPR, GPRInfo::tagTypeNumberRegister));
597        rightOK.link(&m_jit);
598
599        m_jit.compare64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, arg1GPR, arg2GPR, resultGPR);
600        m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), resultGPR);
601
602        JITCompiler::Jump done = m_jit.jump();
603
604        twoCellsCase.link(&m_jit);
605        slowPathCases.append(m_jit.branch64(JITCompiler::NotEqual, arg1GPR, arg2GPR));
606
607        m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR);
608
609        addSlowPathGenerator(
610            adoptPtr(
611                new CompareAndBoxBooleanSlowPathGenerator<MacroAssembler::JumpList>(
612                    slowPathCases, this, operationCompareStrictEq, resultGPR, arg1GPR,
613                    arg2GPR)));
614
615        done.link(&m_jit);
616    }
617
618    jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly);
619}
620
621void SpeculativeJIT::compileMiscStrictEq(Node* node)
622{
623    JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
624    JSValueOperand op2(this, node->child2(), ManualOperandSpeculation);
625    GPRTemporary result(this);
626
627    if (node->child1().useKind() == MiscUse)
628        speculateMisc(node->child1(), op1.jsValueRegs());
629    if (node->child2().useKind() == MiscUse)
630        speculateMisc(node->child2(), op2.jsValueRegs());
631
632    m_jit.compare64(JITCompiler::Equal, op1.gpr(), op2.gpr(), result.gpr());
633    m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
634    jsValueResult(result.gpr(), node, DataFormatJSBoolean);
635}
636
637void SpeculativeJIT::emitCall(Node* node)
638{
639    if (node->op() != Call)
640        RELEASE_ASSERT(node->op() == Construct);
641
642    // For constructors, the this argument is not passed but we have to make space
643    // for it.
644    int dummyThisArgument = node->op() == Call ? 0 : 1;
645
646    CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct;
647
648    Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()];
649    JSValueOperand callee(this, calleeEdge);
650    GPRReg calleeGPR = callee.gpr();
651    use(calleeEdge);
652
653    // The call instruction's first child is the function; the subsequent children are the
654    // arguments.
655    int numPassedArgs = node->numChildren() - 1;
656
657    int numArgs = numPassedArgs + dummyThisArgument;
658
659    m_jit.store32(MacroAssembler::TrustedImm32(numArgs), calleeFramePayloadSlot(JSStack::ArgumentCount));
660    m_jit.store64(calleeGPR, calleeFrameSlot(JSStack::Callee));
661
662    for (int i = 0; i < numPassedArgs; i++) {
663        Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i];
664        JSValueOperand arg(this, argEdge);
665        GPRReg argGPR = arg.gpr();
666        use(argEdge);
667
668        m_jit.store64(argGPR, calleeArgumentSlot(i + dummyThisArgument));
669    }
670
671    flushRegisters();
672
673    GPRResult result(this);
674    GPRReg resultGPR = result.gpr();
675
676    JITCompiler::DataLabelPtr targetToCheck;
677    JITCompiler::Jump slowPath;
678
679    m_jit.emitStoreCodeOrigin(node->origin.semantic);
680
681    slowPath = m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, MacroAssembler::TrustedImmPtr(0));
682
683    m_jit.loadPtr(MacroAssembler::Address(calleeGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultGPR);
684    m_jit.store64(resultGPR, calleeFrameSlot(JSStack::ScopeChain));
685
686    JITCompiler::Call fastCall = m_jit.nearCall();
687
688    JITCompiler::Jump done = m_jit.jump();
689
690    slowPath.link(&m_jit);
691
692    m_jit.move(calleeGPR, GPRInfo::regT0); // Callee needs to be in regT0
693    CallLinkInfo* callLinkInfo = m_jit.codeBlock()->addCallLinkInfo();
694    m_jit.move(MacroAssembler::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); // Link info needs to be in regT2
695    JITCompiler::Call slowCall = m_jit.nearCall();
696
697    done.link(&m_jit);
698
699    m_jit.move(GPRInfo::returnValueGPR, resultGPR);
700
701    jsValueResult(resultGPR, m_currentNode, DataFormatJS, UseChildrenCalledExplicitly);
702
703    callLinkInfo->callType = callType;
704    callLinkInfo->codeOrigin = m_currentNode->origin.semantic;
705    callLinkInfo->calleeGPR = calleeGPR;
706
707    m_jit.addJSCall(fastCall, slowCall, targetToCheck, callLinkInfo);
708}
709
710// Clang should allow unreachable [[clang::fallthrough]] in template functions if any template expansion uses it
711// http://llvm.org/bugs/show_bug.cgi?id=18619
712#if COMPILER(CLANG) && defined(__has_warning)
713#pragma clang diagnostic push
714#if __has_warning("-Wimplicit-fallthrough")
715#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
716#endif
717#endif
718template<bool strict>
719GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnFormat)
720{
721    AbstractValue& value = m_state.forNode(edge);
722    SpeculatedType type = value.m_type;
723    ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32));
724    m_interpreter.filter(value, SpecInt32);
725    VirtualRegister virtualRegister = edge->virtualRegister();
726    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
727
728    if (edge->hasConstant() && !isInt32Constant(edge.node())) {
729        // Protect the silent spill/fill logic by failing early. If we "speculate" on
730        // the constant then the silent filler may think that we have an int32 and a
731        // constant, so it will try to fill this as an int32 constant. Bad things will
732        // happen.
733        terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
734        returnFormat = DataFormatInt32;
735        return allocate();
736    }
737
738    switch (info.registerFormat()) {
739    case DataFormatNone: {
740        GPRReg gpr = allocate();
741
742        if (edge->hasConstant()) {
743            m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
744            ASSERT(isInt32Constant(edge.node()));
745            m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
746            info.fillInt32(*m_stream, gpr);
747            returnFormat = DataFormatInt32;
748            return gpr;
749        }
750
751        DataFormat spillFormat = info.spillFormat();
752
753        RELEASE_ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInt32);
754
755        m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
756
757        if (spillFormat == DataFormatJSInt32 || spillFormat == DataFormatInt32) {
758            // If we know this was spilled as an integer we can fill without checking.
759            if (strict) {
760                m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
761                info.fillInt32(*m_stream, gpr);
762                returnFormat = DataFormatInt32;
763                return gpr;
764            }
765            if (spillFormat == DataFormatInt32) {
766                m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr);
767                m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr);
768            } else
769                m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
770            info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
771            returnFormat = DataFormatJSInt32;
772            return gpr;
773        }
774        m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
775
776        // Fill as JSValue, and fall through.
777        info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
778        m_gprs.unlock(gpr);
779        FALLTHROUGH;
780    }
781
782    case DataFormatJS: {
783        RELEASE_ASSERT(!(type & SpecInt52));
784        // Check the value is an integer.
785        GPRReg gpr = info.gpr();
786        m_gprs.lock(gpr);
787        if (type & ~SpecInt32)
788            speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branch64(MacroAssembler::Below, gpr, GPRInfo::tagTypeNumberRegister));
789        info.fillJSValue(*m_stream, gpr, DataFormatJSInt32);
790        // If !strict we're done, return.
791        if (!strict) {
792            returnFormat = DataFormatJSInt32;
793            return gpr;
794        }
795        // else fall through & handle as DataFormatJSInt32.
796        m_gprs.unlock(gpr);
797        FALLTHROUGH;
798    }
799
800    case DataFormatJSInt32: {
801        // In a strict fill we need to strip off the value tag.
802        if (strict) {
803            GPRReg gpr = info.gpr();
804            GPRReg result;
805            // If the register has already been locked we need to take a copy.
806            // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32.
807            if (m_gprs.isLocked(gpr))
808                result = allocate();
809            else {
810                m_gprs.lock(gpr);
811                info.fillInt32(*m_stream, gpr);
812                result = gpr;
813            }
814            m_jit.zeroExtend32ToPtr(gpr, result);
815            returnFormat = DataFormatInt32;
816            return result;
817        }
818
819        GPRReg gpr = info.gpr();
820        m_gprs.lock(gpr);
821        returnFormat = DataFormatJSInt32;
822        return gpr;
823    }
824
825    case DataFormatInt32: {
826        GPRReg gpr = info.gpr();
827        m_gprs.lock(gpr);
828        returnFormat = DataFormatInt32;
829        return gpr;
830    }
831
832    case DataFormatJSDouble:
833    case DataFormatCell:
834    case DataFormatBoolean:
835    case DataFormatJSCell:
836    case DataFormatJSBoolean: {
837        terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
838        returnFormat = DataFormatInt32;
839        return allocate();
840    }
841
842    case DataFormatDouble:
843    case DataFormatStorage:
844    case DataFormatInt52:
845    case DataFormatStrictInt52:
846        RELEASE_ASSERT_NOT_REACHED();
847
848    default:
849        RELEASE_ASSERT_NOT_REACHED();
850        return InvalidGPRReg;
851    }
852}
853#if COMPILER(CLANG) && defined(__has_warning)
854#pragma clang diagnostic pop
855#endif
856
857GPRReg SpeculativeJIT::fillSpeculateInt32(Edge edge, DataFormat& returnFormat)
858{
859    return fillSpeculateInt32Internal<false>(edge, returnFormat);
860}
861
862GPRReg SpeculativeJIT::fillSpeculateInt32Strict(Edge edge)
863{
864    DataFormat mustBeDataFormatInt32;
865    GPRReg result = fillSpeculateInt32Internal<true>(edge, mustBeDataFormatInt32);
866    RELEASE_ASSERT(mustBeDataFormatInt32 == DataFormatInt32);
867    return result;
868}
869
870GPRReg SpeculativeJIT::fillSpeculateInt52(Edge edge, DataFormat desiredFormat)
871{
872    ASSERT(desiredFormat == DataFormatInt52 || desiredFormat == DataFormatStrictInt52);
873    AbstractValue& value = m_state.forNode(edge);
874    m_interpreter.filter(value, SpecMachineInt);
875    VirtualRegister virtualRegister = edge->virtualRegister();
876    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
877
878    switch (info.registerFormat()) {
879    case DataFormatNone: {
880        if ((edge->hasConstant() && !valueOfJSConstant(edge.node()).isMachineInt())) {
881            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
882            return allocate();
883        }
884
885        GPRReg gpr = allocate();
886
887        if (edge->hasConstant()) {
888            JSValue jsValue = valueOfJSConstant(edge.node());
889            ASSERT(jsValue.isMachineInt());
890            m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
891            int64_t value = jsValue.asMachineInt();
892            if (desiredFormat == DataFormatInt52)
893                value = value << JSValue::int52ShiftAmount;
894            m_jit.move(MacroAssembler::Imm64(value), gpr);
895            info.fillGPR(*m_stream, gpr, desiredFormat);
896            return gpr;
897        }
898
899        DataFormat spillFormat = info.spillFormat();
900
901        RELEASE_ASSERT(spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52);
902
903        m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
904
905        m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
906        if (desiredFormat == DataFormatStrictInt52) {
907            if (spillFormat == DataFormatInt52)
908                m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr);
909            info.fillStrictInt52(*m_stream, gpr);
910            return gpr;
911        }
912        if (spillFormat == DataFormatStrictInt52)
913            m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr);
914        info.fillInt52(*m_stream, gpr);
915        return gpr;
916    }
917
918    case DataFormatStrictInt52: {
919        GPRReg gpr = info.gpr();
920        bool wasLocked = m_gprs.isLocked(gpr);
921        lock(gpr);
922        if (desiredFormat == DataFormatStrictInt52)
923            return gpr;
924        if (wasLocked) {
925            GPRReg result = allocate();
926            m_jit.move(gpr, result);
927            unlock(gpr);
928            gpr = result;
929        } else
930            info.fillInt52(*m_stream, gpr);
931        m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr);
932        return gpr;
933    }
934
935    case DataFormatInt52: {
936        GPRReg gpr = info.gpr();
937        bool wasLocked = m_gprs.isLocked(gpr);
938        lock(gpr);
939        if (desiredFormat == DataFormatInt52)
940            return gpr;
941        if (wasLocked) {
942            GPRReg result = allocate();
943            m_jit.move(gpr, result);
944            unlock(gpr);
945            gpr = result;
946        } else
947            info.fillStrictInt52(*m_stream, gpr);
948        m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr);
949        return gpr;
950    }
951
952    default:
953        RELEASE_ASSERT_NOT_REACHED();
954        return InvalidGPRReg;
955    }
956}
957
958FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge)
959{
960    ASSERT(edge.useKind() == DoubleRepUse || edge.useKind() == DoubleRepRealUse || edge.useKind() == DoubleRepMachineIntUse);
961    ASSERT(edge->hasDoubleResult());
962    VirtualRegister virtualRegister = edge->virtualRegister();
963    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
964
965    if (info.registerFormat() == DataFormatNone) {
966        if (edge->hasConstant()) {
967            GPRReg gpr = allocate();
968
969            if (isNumberConstant(edge.node())) {
970                FPRReg fpr = fprAllocate();
971                m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(edge.node()))), gpr);
972                m_jit.move64ToDouble(gpr, fpr);
973                unlock(gpr);
974
975                m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
976                info.fillDouble(*m_stream, fpr);
977                return fpr;
978            }
979            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
980            return fprAllocate();
981        }
982
983        DataFormat spillFormat = info.spillFormat();
984        RELEASE_ASSERT(spillFormat == DataFormatDouble);
985        FPRReg fpr = fprAllocate();
986        m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
987        m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
988        info.fillDouble(*m_stream, fpr);
989        return fpr;
990    }
991
992    RELEASE_ASSERT(info.registerFormat() == DataFormatDouble);
993    FPRReg fpr = info.fpr();
994    m_fprs.lock(fpr);
995    return fpr;
996}
997
998GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge)
999{
1000    AbstractValue& value = m_state.forNode(edge);
1001    SpeculatedType type = value.m_type;
1002    ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell));
1003    m_interpreter.filter(value, SpecCell);
1004    VirtualRegister virtualRegister = edge->virtualRegister();
1005    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1006
1007    switch (info.registerFormat()) {
1008    case DataFormatNone: {
1009        GPRReg gpr = allocate();
1010
1011        if (edge->hasConstant()) {
1012            JSValue jsValue = valueOfJSConstant(edge.node());
1013            if (jsValue.isCell()) {
1014                m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1015                m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
1016                info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1017                return gpr;
1018            }
1019            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1020            return gpr;
1021        }
1022
1023        if (!(info.spillFormat() & DataFormatJS)) {
1024            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1025            return gpr;
1026        }
1027
1028        m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1029        m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
1030
1031        info.fillJSValue(*m_stream, gpr, DataFormatJS);
1032        if (type & ~SpecCell)
1033            speculationCheck(BadType, JSValueRegs(gpr), edge, branchNotCell(JSValueRegs(gpr)));
1034        info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1035        return gpr;
1036    }
1037
1038    case DataFormatCell:
1039    case DataFormatJSCell: {
1040        GPRReg gpr = info.gpr();
1041        m_gprs.lock(gpr);
1042        if (!ASSERT_DISABLED) {
1043            MacroAssembler::Jump checkCell = branchIsCell(JSValueRegs(gpr));
1044            m_jit.abortWithReason(DFGIsNotCell);
1045            checkCell.link(&m_jit);
1046        }
1047        return gpr;
1048    }
1049
1050    case DataFormatJS: {
1051        GPRReg gpr = info.gpr();
1052        m_gprs.lock(gpr);
1053        if (type & ~SpecCell)
1054            speculationCheck(BadType, JSValueRegs(gpr), edge, branchNotCell(JSValueRegs(gpr)));
1055        info.fillJSValue(*m_stream, gpr, DataFormatJSCell);
1056        return gpr;
1057    }
1058
1059    case DataFormatJSInt32:
1060    case DataFormatInt32:
1061    case DataFormatJSDouble:
1062    case DataFormatJSBoolean:
1063    case DataFormatBoolean: {
1064        terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1065        return allocate();
1066    }
1067
1068    case DataFormatDouble:
1069    case DataFormatStorage:
1070    case DataFormatInt52:
1071    case DataFormatStrictInt52:
1072        RELEASE_ASSERT_NOT_REACHED();
1073
1074    default:
1075        RELEASE_ASSERT_NOT_REACHED();
1076        return InvalidGPRReg;
1077    }
1078}
1079
1080GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge)
1081{
1082    AbstractValue& value = m_state.forNode(edge);
1083    SpeculatedType type = value.m_type;
1084    m_interpreter.filter(value, SpecBoolean);
1085    VirtualRegister virtualRegister = edge->virtualRegister();
1086    GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1087
1088    switch (info.registerFormat()) {
1089    case DataFormatNone: {
1090        if (info.spillFormat() == DataFormatInt32) {
1091            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1092            return allocate();
1093        }
1094
1095        GPRReg gpr = allocate();
1096
1097        if (edge->hasConstant()) {
1098            JSValue jsValue = valueOfJSConstant(edge.node());
1099            if (jsValue.isBoolean()) {
1100                m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1101                m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr);
1102                info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1103                return gpr;
1104            }
1105            terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1106            return gpr;
1107        }
1108        RELEASE_ASSERT(info.spillFormat() & DataFormatJS);
1109        m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1110        m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr);
1111
1112        info.fillJSValue(*m_stream, gpr, DataFormatJS);
1113        if (type & ~SpecBoolean) {
1114            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1115            speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, TrustedImm32(static_cast<int32_t>(~1))), SpeculationRecovery(BooleanSpeculationCheck, gpr, InvalidGPRReg));
1116            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1117        }
1118        info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1119        return gpr;
1120    }
1121
1122    case DataFormatBoolean:
1123    case DataFormatJSBoolean: {
1124        GPRReg gpr = info.gpr();
1125        m_gprs.lock(gpr);
1126        return gpr;
1127    }
1128
1129    case DataFormatJS: {
1130        GPRReg gpr = info.gpr();
1131        m_gprs.lock(gpr);
1132        if (type & ~SpecBoolean) {
1133            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1134            speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, TrustedImm32(static_cast<int32_t>(~1))), SpeculationRecovery(BooleanSpeculationCheck, gpr, InvalidGPRReg));
1135            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), gpr);
1136        }
1137        info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean);
1138        return gpr;
1139    }
1140
1141    case DataFormatJSInt32:
1142    case DataFormatInt32:
1143    case DataFormatJSDouble:
1144    case DataFormatJSCell:
1145    case DataFormatCell:
1146        terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1147        return allocate();
1148
1149    case DataFormatDouble:
1150    case DataFormatStorage:
1151    case DataFormatInt52:
1152    case DataFormatStrictInt52:
1153        RELEASE_ASSERT_NOT_REACHED();
1154
1155    default:
1156        RELEASE_ASSERT_NOT_REACHED();
1157        return InvalidGPRReg;
1158    }
1159}
1160
1161void SpeculativeJIT::compileBaseValueStoreBarrier(Edge& baseEdge, Edge& valueEdge)
1162{
1163#if ENABLE(GGC)
1164    ASSERT(!isKnownNotCell(valueEdge.node()));
1165
1166    SpeculateCellOperand base(this, baseEdge);
1167    JSValueOperand value(this, valueEdge);
1168    GPRTemporary scratch1(this);
1169    GPRTemporary scratch2(this);
1170
1171    writeBarrier(base.gpr(), value.gpr(), valueEdge, scratch1.gpr(), scratch2.gpr());
1172#else
1173    UNUSED_PARAM(baseEdge);
1174    UNUSED_PARAM(valueEdge);
1175#endif
1176}
1177
1178void SpeculativeJIT::compileObjectEquality(Node* node)
1179{
1180    SpeculateCellOperand op1(this, node->child1());
1181    SpeculateCellOperand op2(this, node->child2());
1182    GPRTemporary result(this, Reuse, op1);
1183
1184    GPRReg op1GPR = op1.gpr();
1185    GPRReg op2GPR = op2.gpr();
1186    GPRReg resultGPR = result.gpr();
1187
1188    if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1189        DFG_TYPE_CHECK(
1190            JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchStructurePtr(
1191                MacroAssembler::Equal,
1192                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1193                m_jit.vm()->stringStructure.get()));
1194        DFG_TYPE_CHECK(
1195            JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchStructurePtr(
1196                MacroAssembler::Equal,
1197                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1198                m_jit.vm()->stringStructure.get()));
1199    } else {
1200        DFG_TYPE_CHECK(
1201            JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchStructurePtr(
1202                MacroAssembler::Equal,
1203                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1204                m_jit.vm()->stringStructure.get()));
1205        speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1206            m_jit.branchTest8(
1207                MacroAssembler::NonZero,
1208                MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
1209                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1210
1211        DFG_TYPE_CHECK(
1212            JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchStructurePtr(
1213                MacroAssembler::Equal,
1214                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1215                m_jit.vm()->stringStructure.get()));
1216        speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1217            m_jit.branchTest8(
1218                MacroAssembler::NonZero,
1219                MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
1220                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1221    }
1222
1223    MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR);
1224    m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1225    MacroAssembler::Jump done = m_jit.jump();
1226    falseCase.link(&m_jit);
1227    m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1228    done.link(&m_jit);
1229
1230    jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1231}
1232
1233void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild)
1234{
1235    SpeculateCellOperand op1(this, leftChild);
1236    JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1237    GPRTemporary result(this);
1238
1239    GPRReg op1GPR = op1.gpr();
1240    GPRReg op2GPR = op2.gpr();
1241    GPRReg resultGPR = result.gpr();
1242
1243    bool masqueradesAsUndefinedWatchpointValid =
1244        masqueradesAsUndefinedWatchpointIsStillValid();
1245
1246    if (masqueradesAsUndefinedWatchpointValid) {
1247        DFG_TYPE_CHECK(
1248            JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchStructurePtr(
1249                MacroAssembler::Equal,
1250                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1251                m_jit.vm()->stringStructure.get()));
1252    } else {
1253        DFG_TYPE_CHECK(
1254            JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchStructurePtr(
1255                MacroAssembler::Equal,
1256                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1257                m_jit.vm()->stringStructure.get()));
1258        speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1259            m_jit.branchTest8(
1260                MacroAssembler::NonZero,
1261                MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
1262                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1263    }
1264
1265    // It seems that most of the time when programs do a == b where b may be either null/undefined
1266    // or an object, b is usually an object. Balance the branches to make that case fast.
1267    MacroAssembler::Jump rightNotCell = branchNotCell(JSValueRegs(op2GPR));
1268
1269    // We know that within this branch, rightChild must be a cell.
1270    if (masqueradesAsUndefinedWatchpointValid) {
1271        DFG_TYPE_CHECK(
1272            JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1273                MacroAssembler::Equal,
1274                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1275                m_jit.vm()->stringStructure.get()));
1276    } else {
1277        DFG_TYPE_CHECK(
1278            JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1279                MacroAssembler::Equal,
1280                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1281                m_jit.vm()->stringStructure.get()));
1282        speculationCheck(BadType, JSValueRegs(op2GPR), rightChild,
1283            m_jit.branchTest8(
1284                MacroAssembler::NonZero,
1285                MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
1286                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1287    }
1288
1289    // At this point we know that we can perform a straight-forward equality comparison on pointer
1290    // values because both left and right are pointers to objects that have no special equality
1291    // protocols.
1292    MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR);
1293    MacroAssembler::Jump trueCase = m_jit.jump();
1294
1295    rightNotCell.link(&m_jit);
1296
1297    // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1298    // prove that it is either null or undefined.
1299    if (needsTypeCheck(rightChild, SpecCell | SpecOther)) {
1300        m_jit.move(op2GPR, resultGPR);
1301        m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1302
1303        typeCheck(
1304            JSValueRegs(op2GPR), rightChild, SpecCell | SpecOther,
1305            m_jit.branch64(
1306                MacroAssembler::NotEqual, resultGPR,
1307                MacroAssembler::TrustedImm64(ValueNull)));
1308    }
1309
1310    falseCase.link(&m_jit);
1311    m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1312    MacroAssembler::Jump done = m_jit.jump();
1313    trueCase.link(&m_jit);
1314    m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1315    done.link(&m_jit);
1316
1317    jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1318}
1319
1320void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode)
1321{
1322    BasicBlock* taken = branchNode->branchData()->taken.block;
1323    BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1324
1325    SpeculateCellOperand op1(this, leftChild);
1326    JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1327    GPRTemporary result(this);
1328
1329    GPRReg op1GPR = op1.gpr();
1330    GPRReg op2GPR = op2.gpr();
1331    GPRReg resultGPR = result.gpr();
1332
1333    bool masqueradesAsUndefinedWatchpointValid =
1334        masqueradesAsUndefinedWatchpointIsStillValid();
1335
1336    if (masqueradesAsUndefinedWatchpointValid) {
1337        DFG_TYPE_CHECK(
1338            JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchStructurePtr(
1339                MacroAssembler::Equal,
1340                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1341                m_jit.vm()->stringStructure.get()));
1342    } else {
1343        DFG_TYPE_CHECK(
1344            JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchStructurePtr(
1345                MacroAssembler::Equal,
1346                MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
1347                m_jit.vm()->stringStructure.get()));
1348        speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1349            m_jit.branchTest8(
1350                MacroAssembler::NonZero,
1351                MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
1352                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1353    }
1354
1355    // It seems that most of the time when programs do a == b where b may be either null/undefined
1356    // or an object, b is usually an object. Balance the branches to make that case fast.
1357    MacroAssembler::Jump rightNotCell = branchNotCell(JSValueRegs(op2GPR));
1358
1359    // We know that within this branch, rightChild must be a cell.
1360    if (masqueradesAsUndefinedWatchpointValid) {
1361        DFG_TYPE_CHECK(
1362            JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1363                MacroAssembler::Equal,
1364                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1365                m_jit.vm()->stringStructure.get()));
1366    } else {
1367        DFG_TYPE_CHECK(
1368            JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1369                MacroAssembler::Equal,
1370                MacroAssembler::Address(op2GPR, JSCell::structureIDOffset()),
1371                m_jit.vm()->stringStructure.get()));
1372        speculationCheck(BadType, JSValueRegs(op2GPR), rightChild,
1373            m_jit.branchTest8(
1374                MacroAssembler::NonZero,
1375                MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
1376                MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1377    }
1378
1379    // At this point we know that we can perform a straight-forward equality comparison on pointer
1380    // values because both left and right are pointers to objects that have no special equality
1381    // protocols.
1382    branch64(MacroAssembler::Equal, op1GPR, op2GPR, taken);
1383
1384    // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1385    // prove that it is either null or undefined.
1386    if (!needsTypeCheck(rightChild, SpecCell | SpecOther))
1387        rightNotCell.link(&m_jit);
1388    else {
1389        jump(notTaken, ForceJump);
1390
1391        rightNotCell.link(&m_jit);
1392        m_jit.move(op2GPR, resultGPR);
1393        m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1394
1395        typeCheck(
1396            JSValueRegs(op2GPR), rightChild, SpecCell | SpecOther, m_jit.branch64(
1397                MacroAssembler::NotEqual, resultGPR,
1398                MacroAssembler::TrustedImm64(ValueNull)));
1399    }
1400
1401    jump(notTaken);
1402}
1403
1404void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition)
1405{
1406    SpeculateInt32Operand op1(this, node->child1());
1407    SpeculateInt32Operand op2(this, node->child2());
1408    GPRTemporary result(this, Reuse, op1, op2);
1409
1410    m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr());
1411
1412    // If we add a DataFormatBool, we should use it here.
1413    m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
1414    jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean);
1415}
1416
1417void SpeculativeJIT::compileInt52Compare(Node* node, MacroAssembler::RelationalCondition condition)
1418{
1419    SpeculateWhicheverInt52Operand op1(this, node->child1());
1420    SpeculateWhicheverInt52Operand op2(this, node->child2(), op1);
1421    GPRTemporary result(this, Reuse, op1, op2);
1422
1423    m_jit.compare64(condition, op1.gpr(), op2.gpr(), result.gpr());
1424
1425    // If we add a DataFormatBool, we should use it here.
1426    m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
1427    jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean);
1428}
1429
1430void SpeculativeJIT::compilePeepHoleInt52Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1431{
1432    BasicBlock* taken = branchNode->branchData()->taken.block;
1433    BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1434
1435    // The branch instruction will branch to the taken block.
1436    // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1437    if (taken == nextBlock()) {
1438        condition = JITCompiler::invert(condition);
1439        BasicBlock* tmp = taken;
1440        taken = notTaken;
1441        notTaken = tmp;
1442    }
1443
1444    SpeculateWhicheverInt52Operand op1(this, node->child1());
1445    SpeculateWhicheverInt52Operand op2(this, node->child2(), op1);
1446
1447    branch64(condition, op1.gpr(), op2.gpr(), taken);
1448    jump(notTaken);
1449}
1450
1451void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition)
1452{
1453    SpeculateDoubleOperand op1(this, node->child1());
1454    SpeculateDoubleOperand op2(this, node->child2());
1455    GPRTemporary result(this);
1456
1457    m_jit.move(TrustedImm32(ValueTrue), result.gpr());
1458    MacroAssembler::Jump trueCase = m_jit.branchDouble(condition, op1.fpr(), op2.fpr());
1459    m_jit.xor64(TrustedImm32(true), result.gpr());
1460    trueCase.link(&m_jit);
1461
1462    jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1463}
1464
1465void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse)
1466{
1467    JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1468    GPRTemporary result(this);
1469    GPRReg valueGPR = value.gpr();
1470    GPRReg resultGPR = result.gpr();
1471    GPRTemporary structure;
1472    GPRReg structureGPR = InvalidGPRReg;
1473    GPRTemporary scratch;
1474    GPRReg scratchGPR = InvalidGPRReg;
1475
1476    bool masqueradesAsUndefinedWatchpointValid =
1477        masqueradesAsUndefinedWatchpointIsStillValid();
1478
1479    if (!masqueradesAsUndefinedWatchpointValid) {
1480        // The masquerades as undefined case will use the structure register, so allocate it here.
1481        // Do this at the top of the function to avoid branching around a register allocation.
1482        GPRTemporary realStructure(this);
1483        GPRTemporary realScratch(this);
1484        structure.adopt(realStructure);
1485        scratch.adopt(realScratch);
1486        structureGPR = structure.gpr();
1487        scratchGPR = scratch.gpr();
1488    }
1489
1490    MacroAssembler::Jump notCell = branchNotCell(JSValueRegs(valueGPR));
1491    if (masqueradesAsUndefinedWatchpointValid) {
1492        DFG_TYPE_CHECK(
1493            JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1494                MacroAssembler::Equal,
1495                MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()),
1496                m_jit.vm()->stringStructure.get()));
1497    } else {
1498        DFG_TYPE_CHECK(
1499            JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1500                MacroAssembler::Equal,
1501                MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()),
1502                m_jit.vm()->stringStructure.get()));
1503
1504        MacroAssembler::Jump isNotMasqueradesAsUndefined =
1505            m_jit.branchTest8(
1506                MacroAssembler::Zero,
1507                MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()),
1508                MacroAssembler::TrustedImm32(MasqueradesAsUndefined));
1509
1510        m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR);
1511        speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse,
1512            m_jit.branchPtr(
1513                MacroAssembler::Equal,
1514                MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()),
1515                MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic))));
1516
1517        isNotMasqueradesAsUndefined.link(&m_jit);
1518    }
1519    m_jit.move(TrustedImm32(ValueFalse), resultGPR);
1520    MacroAssembler::Jump done = m_jit.jump();
1521
1522    notCell.link(&m_jit);
1523
1524    if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1525        m_jit.move(valueGPR, resultGPR);
1526        m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), resultGPR);
1527        typeCheck(
1528            JSValueRegs(valueGPR), nodeUse, SpecCell | SpecOther, m_jit.branch64(
1529                MacroAssembler::NotEqual,
1530                resultGPR,
1531                MacroAssembler::TrustedImm64(ValueNull)));
1532    }
1533    m_jit.move(TrustedImm32(ValueTrue), resultGPR);
1534
1535    done.link(&m_jit);
1536
1537    jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean);
1538}
1539
1540void SpeculativeJIT::compileLogicalNot(Node* node)
1541{
1542    switch (node->child1().useKind()) {
1543    case ObjectOrOtherUse: {
1544        compileObjectOrOtherLogicalNot(node->child1());
1545        return;
1546    }
1547
1548    case Int32Use: {
1549        SpeculateInt32Operand value(this, node->child1());
1550        GPRTemporary result(this, Reuse, value);
1551        m_jit.compare32(MacroAssembler::Equal, value.gpr(), MacroAssembler::TrustedImm32(0), result.gpr());
1552        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
1553        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1554        return;
1555    }
1556
1557    case DoubleRepUse: {
1558        SpeculateDoubleOperand value(this, node->child1());
1559        FPRTemporary scratch(this);
1560        GPRTemporary result(this);
1561        m_jit.move(TrustedImm32(ValueFalse), result.gpr());
1562        MacroAssembler::Jump nonZero = m_jit.branchDoubleNonZero(value.fpr(), scratch.fpr());
1563        m_jit.xor32(TrustedImm32(true), result.gpr());
1564        nonZero.link(&m_jit);
1565        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1566        return;
1567    }
1568
1569    case BooleanUse: {
1570        if (!needsTypeCheck(node->child1(), SpecBoolean)) {
1571            SpeculateBooleanOperand value(this, node->child1());
1572            GPRTemporary result(this, Reuse, value);
1573
1574            m_jit.move(value.gpr(), result.gpr());
1575            m_jit.xor64(TrustedImm32(true), result.gpr());
1576
1577            jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1578            return;
1579        }
1580
1581        JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
1582        GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add).
1583
1584        m_jit.move(value.gpr(), result.gpr());
1585        m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr());
1586        typeCheck(
1587            JSValueRegs(value.gpr()), node->child1(), SpecBoolean, m_jit.branchTest64(
1588                JITCompiler::NonZero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1589        m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueTrue)), result.gpr());
1590
1591        // If we add a DataFormatBool, we should use it here.
1592        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
1593        return;
1594    }
1595
1596    case UntypedUse: {
1597        JSValueOperand arg1(this, node->child1());
1598        GPRTemporary result(this);
1599
1600        GPRReg arg1GPR = arg1.gpr();
1601        GPRReg resultGPR = result.gpr();
1602
1603        arg1.use();
1604
1605        m_jit.move(arg1GPR, resultGPR);
1606        m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), resultGPR);
1607        JITCompiler::Jump slowCase = m_jit.branchTest64(JITCompiler::NonZero, resultGPR, TrustedImm32(static_cast<int32_t>(~1)));
1608
1609        addSlowPathGenerator(
1610            slowPathCall(slowCase, this, operationConvertJSValueToBoolean, resultGPR, arg1GPR));
1611
1612        m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueTrue)), resultGPR);
1613        jsValueResult(resultGPR, node, DataFormatJSBoolean, UseChildrenCalledExplicitly);
1614        return;
1615    }
1616    case StringUse:
1617        return compileStringZeroLength(node);
1618
1619    default:
1620        RELEASE_ASSERT_NOT_REACHED();
1621        break;
1622    }
1623}
1624
1625void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, BasicBlock* notTaken)
1626{
1627    JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1628    GPRTemporary scratch(this);
1629    GPRTemporary structure;
1630    GPRReg valueGPR = value.gpr();
1631    GPRReg scratchGPR = scratch.gpr();
1632    GPRReg structureGPR = InvalidGPRReg;
1633
1634    if (!masqueradesAsUndefinedWatchpointIsStillValid()) {
1635        GPRTemporary realStructure(this);
1636        structure.adopt(realStructure);
1637        structureGPR = structure.gpr();
1638    }
1639
1640    MacroAssembler::Jump notCell = branchNotCell(JSValueRegs(valueGPR));
1641    if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1642        DFG_TYPE_CHECK(
1643            JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1644                MacroAssembler::Equal,
1645                MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()),
1646                m_jit.vm()->stringStructure.get()));
1647    } else {
1648        DFG_TYPE_CHECK(
1649            JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchStructurePtr(
1650                MacroAssembler::Equal,
1651                MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()),
1652                m_jit.vm()->stringStructure.get()));
1653
1654        JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(
1655            JITCompiler::Zero,
1656            MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()),
1657            TrustedImm32(MasqueradesAsUndefined));
1658
1659        m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR);
1660        speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse,
1661            m_jit.branchPtr(
1662                MacroAssembler::Equal,
1663                MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()),
1664                MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic))));
1665
1666        isNotMasqueradesAsUndefined.link(&m_jit);
1667    }
1668    jump(taken, ForceJump);
1669
1670    notCell.link(&m_jit);
1671
1672    if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1673        m_jit.move(valueGPR, scratchGPR);
1674        m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), scratchGPR);
1675        typeCheck(
1676            JSValueRegs(valueGPR), nodeUse, SpecCell | SpecOther, m_jit.branch64(
1677                MacroAssembler::NotEqual, scratchGPR, MacroAssembler::TrustedImm64(ValueNull)));
1678    }
1679    jump(notTaken);
1680
1681    noResult(m_currentNode);
1682}
1683
1684void SpeculativeJIT::emitBranch(Node* node)
1685{
1686    BasicBlock* taken = node->branchData()->taken.block;
1687    BasicBlock* notTaken = node->branchData()->notTaken.block;
1688
1689    switch (node->child1().useKind()) {
1690    case ObjectOrOtherUse: {
1691        emitObjectOrOtherBranch(node->child1(), taken, notTaken);
1692        return;
1693    }
1694
1695    case Int32Use:
1696    case DoubleRepUse: {
1697        if (node->child1().useKind() == Int32Use) {
1698            bool invert = false;
1699
1700            if (taken == nextBlock()) {
1701                invert = true;
1702                BasicBlock* tmp = taken;
1703                taken = notTaken;
1704                notTaken = tmp;
1705            }
1706
1707            SpeculateInt32Operand value(this, node->child1());
1708            branchTest32(invert ? MacroAssembler::Zero : MacroAssembler::NonZero, value.gpr(), taken);
1709        } else {
1710            SpeculateDoubleOperand value(this, node->child1());
1711            FPRTemporary scratch(this);
1712            branchDoubleNonZero(value.fpr(), scratch.fpr(), taken);
1713        }
1714
1715        jump(notTaken);
1716
1717        noResult(node);
1718        return;
1719    }
1720
1721    case UntypedUse:
1722    case BooleanUse: {
1723        JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
1724        GPRReg valueGPR = value.gpr();
1725
1726        if (node->child1().useKind() == BooleanUse) {
1727            if (!needsTypeCheck(node->child1(), SpecBoolean)) {
1728                MacroAssembler::ResultCondition condition = MacroAssembler::NonZero;
1729
1730                if (taken == nextBlock()) {
1731                    condition = MacroAssembler::Zero;
1732                    BasicBlock* tmp = taken;
1733                    taken = notTaken;
1734                    notTaken = tmp;
1735                }
1736
1737                branchTest32(condition, valueGPR, TrustedImm32(true), taken);
1738                jump(notTaken);
1739            } else {
1740                branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), notTaken);
1741                branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), taken);
1742
1743                typeCheck(JSValueRegs(valueGPR), node->child1(), SpecBoolean, m_jit.jump());
1744            }
1745            value.use();
1746        } else {
1747            GPRTemporary result(this);
1748            GPRReg resultGPR = result.gpr();
1749
1750            if (node->child1()->prediction() & SpecInt32) {
1751                branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsNumber(0))), notTaken);
1752                branch64(MacroAssembler::AboveOrEqual, valueGPR, GPRInfo::tagTypeNumberRegister, taken);
1753            }
1754
1755            if (node->child1()->prediction() & SpecBoolean) {
1756                branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(false))), notTaken);
1757                branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsBoolean(true))), taken);
1758            }
1759
1760            value.use();
1761
1762            silentSpillAllRegisters(resultGPR);
1763            callOperation(operationConvertJSValueToBoolean, resultGPR, valueGPR);
1764            silentFillAllRegisters(resultGPR);
1765
1766            branchTest32(MacroAssembler::NonZero, resultGPR, taken);
1767            jump(notTaken);
1768        }
1769
1770        noResult(node, UseChildrenCalledExplicitly);
1771        return;
1772    }
1773
1774    default:
1775        RELEASE_ASSERT_NOT_REACHED();
1776    }
1777}
1778
1779void SpeculativeJIT::compile(Node* node)
1780{
1781    NodeType op = node->op();
1782
1783#if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1784    m_jit.clearRegisterAllocationOffsets();
1785#endif
1786
1787    switch (op) {
1788    case JSConstant:
1789    case DoubleConstant:
1790    case Int52Constant:
1791        initConstantInfo(node);
1792        break;
1793
1794    case PhantomArguments:
1795        initConstantInfo(node);
1796        break;
1797
1798    case WeakJSConstant:
1799        m_jit.addWeakReference(node->weakConstant());
1800        initConstantInfo(node);
1801        break;
1802
1803    case Identity: {
1804        // CSE should always eliminate this.
1805        RELEASE_ASSERT_NOT_REACHED();
1806        break;
1807    }
1808
1809    case GetLocal: {
1810        AbstractValue& value = m_state.variables().operand(node->local());
1811
1812        // If the CFA is tracking this variable and it found that the variable
1813        // cannot have been assigned, then don't attempt to proceed.
1814        if (value.isClear()) {
1815            // FIXME: We should trap instead.
1816            // https://bugs.webkit.org/show_bug.cgi?id=110383
1817            terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1818            break;
1819        }
1820
1821        switch (node->variableAccessData()->flushFormat()) {
1822        case FlushedDouble: {
1823            FPRTemporary result(this);
1824            m_jit.loadDouble(JITCompiler::addressFor(node->machineLocal()), result.fpr());
1825            VirtualRegister virtualRegister = node->virtualRegister();
1826            m_fprs.retain(result.fpr(), virtualRegister, SpillOrderDouble);
1827            generationInfoFromVirtualRegister(virtualRegister).initDouble(node, node->refCount(), result.fpr());
1828            break;
1829        }
1830
1831        case FlushedInt32: {
1832            GPRTemporary result(this);
1833            m_jit.load32(JITCompiler::payloadFor(node->machineLocal()), result.gpr());
1834
1835            // Like int32Result, but don't useChildren - our children are phi nodes,
1836            // and don't represent values within this dataflow with virtual registers.
1837            VirtualRegister virtualRegister = node->virtualRegister();
1838            m_gprs.retain(result.gpr(), virtualRegister, SpillOrderInteger);
1839            generationInfoFromVirtualRegister(virtualRegister).initInt32(node, node->refCount(), result.gpr());
1840            break;
1841        }
1842
1843        case FlushedInt52: {
1844            GPRTemporary result(this);
1845            m_jit.load64(JITCompiler::addressFor(node->machineLocal()), result.gpr());
1846
1847            VirtualRegister virtualRegister = node->virtualRegister();
1848            m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS);
1849            generationInfoFromVirtualRegister(virtualRegister).initInt52(node, node->refCount(), result.gpr());
1850            break;
1851        }
1852
1853        default:
1854            GPRTemporary result(this);
1855            m_jit.load64(JITCompiler::addressFor(node->machineLocal()), result.gpr());
1856
1857            // Like jsValueResult, but don't useChildren - our children are phi nodes,
1858            // and don't represent values within this dataflow with virtual registers.
1859            VirtualRegister virtualRegister = node->virtualRegister();
1860            m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS);
1861
1862            DataFormat format;
1863            if (isCellSpeculation(value.m_type))
1864                format = DataFormatJSCell;
1865            else if (isBooleanSpeculation(value.m_type))
1866                format = DataFormatJSBoolean;
1867            else
1868                format = DataFormatJS;
1869
1870            generationInfoFromVirtualRegister(virtualRegister).initJSValue(node, node->refCount(), result.gpr(), format);
1871            break;
1872        }
1873        break;
1874    }
1875
1876    case GetLocalUnlinked: {
1877        GPRTemporary result(this);
1878
1879        m_jit.load64(JITCompiler::addressFor(node->unlinkedMachineLocal()), result.gpr());
1880
1881        jsValueResult(result.gpr(), node);
1882        break;
1883    }
1884
1885    case MovHint:
1886    case ZombieHint:
1887    case Check: {
1888        RELEASE_ASSERT_NOT_REACHED();
1889        break;
1890    }
1891
1892    case SetLocal: {
1893        switch (node->variableAccessData()->flushFormat()) {
1894        case FlushedDouble: {
1895            SpeculateDoubleOperand value(this, node->child1());
1896            m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->machineLocal()));
1897            noResult(node);
1898            // Indicate that it's no longer necessary to retrieve the value of
1899            // this bytecode variable from registers or other locations in the stack,
1900            // but that it is stored as a double.
1901            recordSetLocal(DataFormatDouble);
1902            break;
1903        }
1904
1905        case FlushedInt32: {
1906            SpeculateInt32Operand value(this, node->child1());
1907            m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->machineLocal()));
1908            noResult(node);
1909            recordSetLocal(DataFormatInt32);
1910            break;
1911        }
1912
1913        case FlushedInt52: {
1914            SpeculateInt52Operand value(this, node->child1());
1915            m_jit.store64(value.gpr(), JITCompiler::addressFor(node->machineLocal()));
1916            noResult(node);
1917            recordSetLocal(DataFormatInt52);
1918            break;
1919        }
1920
1921        case FlushedCell: {
1922            SpeculateCellOperand cell(this, node->child1());
1923            GPRReg cellGPR = cell.gpr();
1924            m_jit.store64(cellGPR, JITCompiler::addressFor(node->machineLocal()));
1925            noResult(node);
1926            recordSetLocal(DataFormatCell);
1927            break;
1928        }
1929
1930        case FlushedBoolean: {
1931            SpeculateBooleanOperand boolean(this, node->child1());
1932            m_jit.store64(boolean.gpr(), JITCompiler::addressFor(node->machineLocal()));
1933            noResult(node);
1934            recordSetLocal(DataFormatBoolean);
1935            break;
1936        }
1937
1938        case FlushedJSValue:
1939        case FlushedArguments: {
1940            JSValueOperand value(this, node->child1());
1941            m_jit.store64(value.gpr(), JITCompiler::addressFor(node->machineLocal()));
1942            noResult(node);
1943            recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat()));
1944            break;
1945        }
1946
1947        default:
1948            RELEASE_ASSERT_NOT_REACHED();
1949            break;
1950        }
1951
1952        break;
1953    }
1954
1955    case SetArgument:
1956        // This is a no-op; it just marks the fact that the argument is being used.
1957        // But it may be profitable to use this as a hook to run speculation checks
1958        // on arguments, thereby allowing us to trivially eliminate such checks if
1959        // the argument is not used.
1960        recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat()));
1961        break;
1962
1963    case BitAnd:
1964    case BitOr:
1965    case BitXor:
1966        if (isInt32Constant(node->child1().node())) {
1967            SpeculateInt32Operand op2(this, node->child2());
1968            GPRTemporary result(this, Reuse, op2);
1969
1970            bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr());
1971
1972            int32Result(result.gpr(), node);
1973        } else if (isInt32Constant(node->child2().node())) {
1974            SpeculateInt32Operand op1(this, node->child1());
1975            GPRTemporary result(this, Reuse, op1);
1976
1977            bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr());
1978
1979            int32Result(result.gpr(), node);
1980        } else {
1981            SpeculateInt32Operand op1(this, node->child1());
1982            SpeculateInt32Operand op2(this, node->child2());
1983            GPRTemporary result(this, Reuse, op1, op2);
1984
1985            GPRReg reg1 = op1.gpr();
1986            GPRReg reg2 = op2.gpr();
1987            bitOp(op, reg1, reg2, result.gpr());
1988
1989            int32Result(result.gpr(), node);
1990        }
1991        break;
1992
1993    case BitRShift:
1994    case BitLShift:
1995    case BitURShift:
1996        if (isInt32Constant(node->child2().node())) {
1997            SpeculateInt32Operand op1(this, node->child1());
1998            GPRTemporary result(this, Reuse, op1);
1999
2000            shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr());
2001
2002            int32Result(result.gpr(), node);
2003        } else {
2004            // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
2005            SpeculateInt32Operand op1(this, node->child1());
2006            SpeculateInt32Operand op2(this, node->child2());
2007            GPRTemporary result(this, Reuse, op1);
2008
2009            GPRReg reg1 = op1.gpr();
2010            GPRReg reg2 = op2.gpr();
2011            shiftOp(op, reg1, reg2, result.gpr());
2012
2013            int32Result(result.gpr(), node);
2014        }
2015        break;
2016
2017    case UInt32ToNumber: {
2018        compileUInt32ToNumber(node);
2019        break;
2020    }
2021
2022    case DoubleAsInt32: {
2023        compileDoubleAsInt32(node);
2024        break;
2025    }
2026
2027    case ValueToInt32: {
2028        compileValueToInt32(node);
2029        break;
2030    }
2031
2032    case DoubleRep: {
2033        compileDoubleRep(node);
2034        break;
2035    }
2036
2037    case ValueRep: {
2038        compileValueRep(node);
2039        break;
2040    }
2041
2042    case Int52Rep: {
2043        switch (node->child1().useKind()) {
2044        case Int32Use: {
2045            SpeculateInt32Operand operand(this, node->child1());
2046            GPRTemporary result(this, Reuse, operand);
2047
2048            m_jit.signExtend32ToPtr(operand.gpr(), result.gpr());
2049
2050            strictInt52Result(result.gpr(), node);
2051            break;
2052        }
2053
2054        case MachineIntUse: {
2055            GPRResult result(this);
2056            GPRReg resultGPR = result.gpr();
2057
2058            convertMachineInt(node->child1(), resultGPR);
2059
2060            strictInt52Result(resultGPR, node);
2061            break;
2062        }
2063
2064        case DoubleRepMachineIntUse: {
2065            SpeculateDoubleOperand value(this, node->child1());
2066            FPRReg valueFPR = value.fpr();
2067
2068            GPRResult result(this);
2069            GPRReg resultGPR = result.gpr();
2070
2071            flushRegisters();
2072
2073            callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR);
2074
2075            DFG_TYPE_CHECK(
2076                JSValueRegs(), node->child1(), SpecInt52AsDouble,
2077                m_jit.branch64(
2078                    JITCompiler::Equal, resultGPR,
2079                    JITCompiler::TrustedImm64(JSValue::notInt52)));
2080
2081            strictInt52Result(resultGPR, node);
2082            break;
2083        }
2084
2085        default:
2086            RELEASE_ASSERT_NOT_REACHED();
2087        }
2088        break;
2089    }
2090
2091    case ValueAdd: {
2092        JSValueOperand op1(this, node->child1());
2093        JSValueOperand op2(this, node->child2());
2094
2095        GPRReg op1GPR = op1.gpr();
2096        GPRReg op2GPR = op2.gpr();
2097
2098        flushRegisters();
2099
2100        GPRResult result(this);
2101        if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node()))
2102            callOperation(operationValueAddNotNumber, result.gpr(), op1GPR, op2GPR);
2103        else
2104            callOperation(operationValueAdd, result.gpr(), op1GPR, op2GPR);
2105
2106        jsValueResult(result.gpr(), node);
2107        break;
2108    }
2109
2110    case ArithAdd:
2111        compileAdd(node);
2112        break;
2113
2114    case MakeRope:
2115        compileMakeRope(node);
2116        break;
2117
2118    case ArithSub:
2119        compileArithSub(node);
2120        break;
2121
2122    case ArithNegate:
2123        compileArithNegate(node);
2124        break;
2125
2126    case ArithMul:
2127        compileArithMul(node);
2128        break;
2129
2130    case ArithDiv: {
2131        compileArithDiv(node);
2132        break;
2133    }
2134
2135    case ArithMod: {
2136        compileArithMod(node);
2137        break;
2138    }
2139
2140    case ArithAbs: {
2141        switch (node->child1().useKind()) {
2142        case Int32Use: {
2143            SpeculateStrictInt32Operand op1(this, node->child1());
2144            GPRTemporary result(this);
2145            GPRTemporary scratch(this);
2146
2147            m_jit.move(op1.gpr(), result.gpr());
2148            m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr());
2149            m_jit.add32(scratch.gpr(), result.gpr());
2150            m_jit.xor32(scratch.gpr(), result.gpr());
2151            speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31)));
2152            int32Result(result.gpr(), node);
2153            break;
2154        }
2155
2156        case DoubleRepUse: {
2157            SpeculateDoubleOperand op1(this, node->child1());
2158            FPRTemporary result(this);
2159
2160            m_jit.absDouble(op1.fpr(), result.fpr());
2161            doubleResult(result.fpr(), node);
2162            break;
2163        }
2164
2165        default:
2166            RELEASE_ASSERT_NOT_REACHED();
2167            break;
2168        }
2169        break;
2170    }
2171
2172    case ArithMin:
2173    case ArithMax: {
2174        switch (node->binaryUseKind()) {
2175        case Int32Use: {
2176            SpeculateStrictInt32Operand op1(this, node->child1());
2177            SpeculateStrictInt32Operand op2(this, node->child2());
2178            GPRTemporary result(this, Reuse, op1);
2179
2180            MacroAssembler::Jump op1Less = m_jit.branch32(op == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1.gpr(), op2.gpr());
2181            m_jit.move(op2.gpr(), result.gpr());
2182            if (op1.gpr() != result.gpr()) {
2183                MacroAssembler::Jump done = m_jit.jump();
2184                op1Less.link(&m_jit);
2185                m_jit.move(op1.gpr(), result.gpr());
2186                done.link(&m_jit);
2187            } else
2188                op1Less.link(&m_jit);
2189
2190            int32Result(result.gpr(), node);
2191            break;
2192        }
2193
2194        case DoubleRepUse: {
2195            SpeculateDoubleOperand op1(this, node->child1());
2196            SpeculateDoubleOperand op2(this, node->child2());
2197            FPRTemporary result(this, op1);
2198
2199            FPRReg op1FPR = op1.fpr();
2200            FPRReg op2FPR = op2.fpr();
2201            FPRReg resultFPR = result.fpr();
2202
2203            MacroAssembler::JumpList done;
2204
2205            MacroAssembler::Jump op1Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleLessThan : MacroAssembler::DoubleGreaterThan, op1FPR, op2FPR);
2206
2207            // op2 is eather the lesser one or one of then is NaN
2208            MacroAssembler::Jump op2Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleGreaterThanOrEqual : MacroAssembler::DoubleLessThanOrEqual, op1FPR, op2FPR);
2209
2210            // Unordered case. We don't know which of op1, op2 is NaN. Manufacture NaN by adding
2211            // op1 + op2 and putting it into result.
2212            m_jit.addDouble(op1FPR, op2FPR, resultFPR);
2213            done.append(m_jit.jump());
2214
2215            op2Less.link(&m_jit);
2216            m_jit.moveDouble(op2FPR, resultFPR);
2217
2218            if (op1FPR != resultFPR) {
2219                done.append(m_jit.jump());
2220
2221                op1Less.link(&m_jit);
2222                m_jit.moveDouble(op1FPR, resultFPR);
2223            } else
2224                op1Less.link(&m_jit);
2225
2226            done.link(&m_jit);
2227
2228            doubleResult(resultFPR, node);
2229            break;
2230        }
2231
2232        default:
2233            RELEASE_ASSERT_NOT_REACHED();
2234            break;
2235        }
2236        break;
2237    }
2238
2239    case ArithSqrt: {
2240        SpeculateDoubleOperand op1(this, node->child1());
2241        FPRTemporary result(this, op1);
2242
2243        m_jit.sqrtDouble(op1.fpr(), result.fpr());
2244
2245        doubleResult(result.fpr(), node);
2246        break;
2247    }
2248
2249    case ArithFRound: {
2250        SpeculateDoubleOperand op1(this, node->child1());
2251        FPRTemporary result(this, op1);
2252
2253        m_jit.convertDoubleToFloat(op1.fpr(), result.fpr());
2254        m_jit.convertFloatToDouble(result.fpr(), result.fpr());
2255
2256        doubleResult(result.fpr(), node);
2257        break;
2258    }
2259
2260    case ArithSin: {
2261        SpeculateDoubleOperand op1(this, node->child1());
2262        FPRReg op1FPR = op1.fpr();
2263
2264        flushRegisters();
2265
2266        FPRResult result(this);
2267        callOperation(sin, result.fpr(), op1FPR);
2268        doubleResult(result.fpr(), node);
2269        break;
2270    }
2271
2272    case ArithCos: {
2273        SpeculateDoubleOperand op1(this, node->child1());
2274        FPRReg op1FPR = op1.fpr();
2275
2276        flushRegisters();
2277
2278        FPRResult result(this);
2279        callOperation(cos, result.fpr(), op1FPR);
2280        doubleResult(result.fpr(), node);
2281        break;
2282    }
2283
2284    case LogicalNot:
2285        compileLogicalNot(node);
2286        break;
2287
2288    case CompareLess:
2289        if (compare(node, JITCompiler::LessThan, JITCompiler::DoubleLessThan, operationCompareLess))
2290            return;
2291        break;
2292
2293    case CompareLessEq:
2294        if (compare(node, JITCompiler::LessThanOrEqual, JITCompiler::DoubleLessThanOrEqual, operationCompareLessEq))
2295            return;
2296        break;
2297
2298    case CompareGreater:
2299        if (compare(node, JITCompiler::GreaterThan, JITCompiler::DoubleGreaterThan, operationCompareGreater))
2300            return;
2301        break;
2302
2303    case CompareGreaterEq:
2304        if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq))
2305            return;
2306        break;
2307
2308    case CompareEqConstant:
2309        ASSERT(isNullConstant(node->child2().node()));
2310        if (nonSpeculativeCompareNull(node, node->child1()))
2311            return;
2312        break;
2313
2314    case CompareEq:
2315        if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq))
2316            return;
2317        break;
2318
2319    case CompareStrictEq:
2320        if (compileStrictEq(node))
2321            return;
2322        break;
2323
2324    case StringCharCodeAt: {
2325        compileGetCharCodeAt(node);
2326        break;
2327    }
2328
2329    case StringCharAt: {
2330        // Relies on StringCharAt node having same basic layout as GetByVal
2331        compileGetByValOnString(node);
2332        break;
2333    }
2334
2335    case StringFromCharCode: {
2336        compileFromCharCode(node);
2337        break;
2338    }
2339
2340    case CheckArray: {
2341        checkArray(node);
2342        break;
2343    }
2344
2345    case Arrayify:
2346    case ArrayifyToStructure: {
2347        arrayify(node);
2348        break;
2349    }
2350
2351    case GetByVal: {
2352        switch (node->arrayMode().type()) {
2353        case Array::SelectUsingPredictions:
2354        case Array::ForceExit:
2355            RELEASE_ASSERT_NOT_REACHED();
2356            terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2357            break;
2358        case Array::Generic: {
2359            JSValueOperand base(this, node->child1());
2360            JSValueOperand property(this, node->child2());
2361            GPRReg baseGPR = base.gpr();
2362            GPRReg propertyGPR = property.gpr();
2363
2364            flushRegisters();
2365            GPRResult result(this);
2366            callOperation(operationGetByVal, result.gpr(), baseGPR, propertyGPR);
2367
2368            jsValueResult(result.gpr(), node);
2369            break;
2370        }
2371        case Array::Int32:
2372        case Array::Contiguous: {
2373            if (node->arrayMode().isInBounds()) {
2374                SpeculateStrictInt32Operand property(this, node->child2());
2375                StorageOperand storage(this, node->child3());
2376
2377                GPRReg propertyReg = property.gpr();
2378                GPRReg storageReg = storage.gpr();
2379
2380                if (!m_compileOkay)
2381                    return;
2382
2383                speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2384
2385                GPRTemporary result(this);
2386                m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.gpr());
2387                speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr()));
2388                jsValueResult(result.gpr(), node, node->arrayMode().type() == Array::Int32 ? DataFormatJSInt32 : DataFormatJS);
2389                break;
2390            }
2391
2392            SpeculateCellOperand base(this, node->child1());
2393            SpeculateStrictInt32Operand property(this, node->child2());
2394            StorageOperand storage(this, node->child3());
2395
2396            GPRReg baseReg = base.gpr();
2397            GPRReg propertyReg = property.gpr();
2398            GPRReg storageReg = storage.gpr();
2399
2400            if (!m_compileOkay)
2401                return;
2402
2403            GPRTemporary result(this);
2404            GPRReg resultReg = result.gpr();
2405
2406            MacroAssembler::JumpList slowCases;
2407
2408            slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2409
2410            m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
2411            slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, resultReg));
2412
2413            addSlowPathGenerator(
2414                slowPathCall(
2415                    slowCases, this, operationGetByValArrayInt,
2416                    result.gpr(), baseReg, propertyReg));
2417
2418            jsValueResult(resultReg, node);
2419            break;
2420        }
2421
2422        case Array::Double: {
2423            if (node->arrayMode().isInBounds()) {
2424                SpeculateStrictInt32Operand property(this, node->child2());
2425                StorageOperand storage(this, node->child3());
2426
2427                GPRReg propertyReg = property.gpr();
2428                GPRReg storageReg = storage.gpr();
2429
2430                if (!m_compileOkay)
2431                    return;
2432
2433                speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2434
2435                FPRTemporary result(this);
2436                m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.fpr());
2437                if (!node->arrayMode().isSaneChain())
2438                    speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, result.fpr(), result.fpr()));
2439                doubleResult(result.fpr(), node);
2440                break;
2441            }
2442
2443            SpeculateCellOperand base(this, node->child1());
2444            SpeculateStrictInt32Operand property(this, node->child2());
2445            StorageOperand storage(this, node->child3());
2446
2447            GPRReg baseReg = base.gpr();
2448            GPRReg propertyReg = property.gpr();
2449            GPRReg storageReg = storage.gpr();
2450
2451            if (!m_compileOkay)
2452                return;
2453
2454            GPRTemporary result(this);
2455            FPRTemporary temp(this);
2456            GPRReg resultReg = result.gpr();
2457            FPRReg tempReg = temp.fpr();
2458
2459            MacroAssembler::JumpList slowCases;
2460
2461            slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2462
2463            m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), tempReg);
2464            slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempReg, tempReg));
2465            boxDouble(tempReg, resultReg);
2466
2467            addSlowPathGenerator(
2468                slowPathCall(
2469                    slowCases, this, operationGetByValArrayInt,
2470                    result.gpr(), baseReg, propertyReg));
2471
2472            jsValueResult(resultReg, node);
2473            break;
2474        }
2475
2476        case Array::ArrayStorage:
2477        case Array::SlowPutArrayStorage: {
2478            if (node->arrayMode().isInBounds()) {
2479                SpeculateStrictInt32Operand property(this, node->child2());
2480                StorageOperand storage(this, node->child3());
2481
2482                GPRReg propertyReg = property.gpr();
2483                GPRReg storageReg = storage.gpr();
2484
2485                if (!m_compileOkay)
2486                    return;
2487
2488                speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2489
2490                GPRTemporary result(this);
2491                m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), result.gpr());
2492                speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr()));
2493
2494                jsValueResult(result.gpr(), node);
2495                break;
2496            }
2497
2498            SpeculateCellOperand base(this, node->child1());
2499            SpeculateStrictInt32Operand property(this, node->child2());
2500            StorageOperand storage(this, node->child3());
2501
2502            GPRReg baseReg = base.gpr();
2503            GPRReg propertyReg = property.gpr();
2504            GPRReg storageReg = storage.gpr();
2505
2506            if (!m_compileOkay)
2507                return;
2508
2509            GPRTemporary result(this);
2510            GPRReg resultReg = result.gpr();
2511
2512            MacroAssembler::JumpList slowCases;
2513
2514            slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2515
2516            m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), resultReg);
2517            slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, resultReg));
2518
2519            addSlowPathGenerator(
2520                slowPathCall(
2521                    slowCases, this, operationGetByValArrayInt,
2522                    result.gpr(), baseReg, propertyReg));
2523
2524            jsValueResult(resultReg, node);
2525            break;
2526        }
2527        case Array::String:
2528            compileGetByValOnString(node);
2529            break;
2530        case Array::Arguments:
2531            compileGetByValOnArguments(node);
2532            break;
2533        default: {
2534            TypedArrayType type = node->arrayMode().typedArrayType();
2535            if (isInt(type))
2536                compileGetByValOnIntTypedArray(node, type);
2537            else
2538                compileGetByValOnFloatTypedArray(node, type);
2539        } }
2540        break;
2541    }
2542
2543    case PutByValDirect:
2544    case PutByVal:
2545    case PutByValAlias: {
2546        Edge child1 = m_jit.graph().varArgChild(node, 0);
2547        Edge child2 = m_jit.graph().varArgChild(node, 1);
2548        Edge child3 = m_jit.graph().varArgChild(node, 2);
2549        Edge child4 = m_jit.graph().varArgChild(node, 3);
2550
2551        ArrayMode arrayMode = node->arrayMode().modeForPut();
2552        bool alreadyHandled = false;
2553
2554        switch (arrayMode.type()) {
2555        case Array::SelectUsingPredictions:
2556        case Array::ForceExit:
2557            RELEASE_ASSERT_NOT_REACHED();
2558            terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2559            alreadyHandled = true;
2560            break;
2561        case Array::Generic: {
2562            RELEASE_ASSERT(node->op() == PutByVal);
2563
2564            JSValueOperand arg1(this, child1);
2565            JSValueOperand arg2(this, child2);
2566            JSValueOperand arg3(this, child3);
2567            GPRReg arg1GPR = arg1.gpr();
2568            GPRReg arg2GPR = arg2.gpr();
2569            GPRReg arg3GPR = arg3.gpr();
2570            flushRegisters();
2571            if (node->op() == PutByValDirect)
2572                callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict, arg1GPR, arg2GPR, arg3GPR);
2573            else
2574                callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR);
2575
2576            noResult(node);
2577            alreadyHandled = true;
2578            break;
2579        }
2580        default:
2581            break;
2582        }
2583
2584        if (alreadyHandled)
2585            break;
2586
2587        // FIXME: the base may not be necessary for some array access modes. But we have to
2588        // keep it alive to this point, so it's likely to be in a register anyway. Likely
2589        // no harm in locking it here.
2590        SpeculateCellOperand base(this, child1);
2591        SpeculateStrictInt32Operand property(this, child2);
2592
2593        GPRReg baseReg = base.gpr();
2594        GPRReg propertyReg = property.gpr();
2595
2596        switch (arrayMode.type()) {
2597        case Array::Int32:
2598        case Array::Contiguous: {
2599            JSValueOperand value(this, child3, ManualOperandSpeculation);
2600
2601            GPRReg valueReg = value.gpr();
2602
2603            if (!m_compileOkay)
2604                return;
2605
2606            if (arrayMode.type() == Array::Int32) {
2607                DFG_TYPE_CHECK(
2608                    JSValueRegs(valueReg), child3, SpecInt32,
2609                    m_jit.branch64(
2610                        MacroAssembler::Below, valueReg, GPRInfo::tagTypeNumberRegister));
2611            }
2612
2613            StorageOperand storage(this, child4);
2614            GPRReg storageReg = storage.gpr();
2615
2616            if (node->op() == PutByValAlias) {
2617                // Store the value to the array.
2618                GPRReg propertyReg = property.gpr();
2619                GPRReg valueReg = value.gpr();
2620                m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2621
2622                noResult(node);
2623                break;
2624            }
2625
2626            GPRTemporary temporary;
2627            GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2628
2629            MacroAssembler::Jump slowCase;
2630
2631            if (arrayMode.isInBounds()) {
2632                speculationCheck(
2633                    OutOfBounds, JSValueRegs(), 0,
2634                    m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2635            } else {
2636                MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2637
2638                slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2639
2640                if (!arrayMode.isOutOfBounds())
2641                    speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2642
2643                m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2644                m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2645
2646                inBounds.link(&m_jit);
2647            }
2648
2649            m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2650
2651            base.use();
2652            property.use();
2653            value.use();
2654            storage.use();
2655
2656            if (arrayMode.isOutOfBounds()) {
2657                if (node->op() == PutByValDirect) {
2658                    addSlowPathGenerator(slowPathCall(
2659                        slowCase, this,
2660                        m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValDirectBeyondArrayBoundsNonStrict,
2661                        NoResult, baseReg, propertyReg, valueReg));
2662                } else {
2663                    addSlowPathGenerator(slowPathCall(
2664                        slowCase, this,
2665                        m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2666                        NoResult, baseReg, propertyReg, valueReg));
2667                }
2668            }
2669
2670            noResult(node, UseChildrenCalledExplicitly);
2671            break;
2672        }
2673
2674        case Array::Double: {
2675            compileDoublePutByVal(node, base, property);
2676            break;
2677        }
2678
2679        case Array::ArrayStorage:
2680        case Array::SlowPutArrayStorage: {
2681            JSValueOperand value(this, child3);
2682
2683            GPRReg valueReg = value.gpr();
2684
2685            if (!m_compileOkay)
2686                return;
2687
2688            StorageOperand storage(this, child4);
2689            GPRReg storageReg = storage.gpr();
2690
2691            if (node->op() == PutByValAlias) {
2692                // Store the value to the array.
2693                GPRReg propertyReg = property.gpr();
2694                GPRReg valueReg = value.gpr();
2695                m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2696
2697                noResult(node);
2698                break;
2699            }
2700
2701            GPRTemporary temporary;
2702            GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2703
2704            MacroAssembler::JumpList slowCases;
2705
2706            MacroAssembler::Jump beyondArrayBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2707            if (!arrayMode.isOutOfBounds())
2708                speculationCheck(OutOfBounds, JSValueRegs(), 0, beyondArrayBounds);
2709            else
2710                slowCases.append(beyondArrayBounds);
2711
2712            // Check if we're writing to a hole; if so increment m_numValuesInVector.
2713            if (arrayMode.isInBounds()) {
2714                speculationCheck(
2715                    StoreToHole, JSValueRegs(), 0,
2716                    m_jit.branchTest64(MacroAssembler::Zero, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]))));
2717            } else {
2718                MacroAssembler::Jump notHoleValue = m_jit.branchTest64(MacroAssembler::NonZero, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2719                if (arrayMode.isSlowPut()) {
2720                    // This is sort of strange. If we wanted to optimize this code path, we would invert
2721                    // the above branch. But it's simply not worth it since this only happens if we're
2722                    // already having a bad time.
2723                    slowCases.append(m_jit.jump());
2724                } else {
2725                    m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageReg, ArrayStorage::numValuesInVectorOffset()));
2726
2727                    // If we're writing to a hole we might be growing the array;
2728                    MacroAssembler::Jump lengthDoesNotNeedUpdate = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2729                    m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2730                    m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2731
2732                    lengthDoesNotNeedUpdate.link(&m_jit);
2733                }
2734                notHoleValue.link(&m_jit);
2735            }
2736
2737            // Store the value to the array.
2738            m_jit.store64(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2739
2740            base.use();
2741            property.use();
2742            value.use();
2743            storage.use();
2744
2745            if (!slowCases.empty()) {
2746                if (node->op() == PutByValDirect) {
2747                    addSlowPathGenerator(slowPathCall(
2748                        slowCases, this,
2749                        m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValDirectBeyondArrayBoundsNonStrict,
2750                        NoResult, baseReg, propertyReg, valueReg));
2751                } else {
2752                    addSlowPathGenerator(slowPathCall(
2753                        slowCases, this,
2754                        m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2755                        NoResult, baseReg, propertyReg, valueReg));
2756                }
2757            }
2758
2759            noResult(node, UseChildrenCalledExplicitly);
2760            break;
2761        }
2762
2763        case Array::Arguments: {
2764            JSValueOperand value(this, child3);
2765            GPRTemporary scratch(this);
2766            GPRTemporary scratch2(this);
2767
2768            GPRReg valueReg = value.gpr();
2769            GPRReg scratchReg = scratch.gpr();
2770            GPRReg scratch2Reg = scratch2.gpr();
2771
2772            if (!m_compileOkay)
2773                return;
2774
2775            // Two really lame checks.
2776            speculationCheck(
2777                Uncountable, JSValueSource(), 0,
2778                m_jit.branch32(
2779                    MacroAssembler::AboveOrEqual, propertyReg,
2780                    MacroAssembler::Address(baseReg, Arguments::offsetOfNumArguments())));
2781            speculationCheck(
2782                Uncountable, JSValueSource(), 0,
2783                m_jit.branchTestPtr(
2784                    MacroAssembler::NonZero,
2785                    MacroAssembler::Address(
2786                        baseReg, Arguments::offsetOfSlowArgumentData())));
2787
2788            m_jit.move(propertyReg, scratch2Reg);
2789            m_jit.signExtend32ToPtr(scratch2Reg, scratch2Reg);
2790            m_jit.loadPtr(
2791                MacroAssembler::Address(baseReg, Arguments::offsetOfRegisters()),
2792                scratchReg);
2793
2794            m_jit.store64(
2795                valueReg,
2796                MacroAssembler::BaseIndex(
2797                    scratchReg, scratch2Reg, MacroAssembler::TimesEight,
2798                    CallFrame::thisArgumentOffset() * sizeof(Register) + sizeof(Register)));
2799
2800            noResult(node);
2801            break;
2802        }
2803
2804        default: {
2805            TypedArrayType type = arrayMode.typedArrayType();
2806            if (isInt(type))
2807                compilePutByValForIntTypedArray(base.gpr(), property.gpr(), node, type);
2808            else
2809                compilePutByValForFloatTypedArray(base.gpr(), property.gpr(), node, type);
2810        } }
2811
2812        break;
2813    }
2814
2815    case RegExpExec: {
2816        if (compileRegExpExec(node))
2817            return;
2818        if (!node->adjustedRefCount()) {
2819            SpeculateCellOperand base(this, node->child1());
2820            SpeculateCellOperand argument(this, node->child2());
2821            GPRReg baseGPR = base.gpr();
2822            GPRReg argumentGPR = argument.gpr();
2823
2824            flushRegisters();
2825            GPRResult result(this);
2826            callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2827
2828            // Must use jsValueResult because otherwise we screw up register
2829            // allocation, which thinks that this node has a result.
2830            jsValueResult(result.gpr(), node);
2831            break;
2832        }
2833
2834        SpeculateCellOperand base(this, node->child1());
2835        SpeculateCellOperand argument(this, node->child2());
2836        GPRReg baseGPR = base.gpr();
2837        GPRReg argumentGPR = argument.gpr();
2838
2839        flushRegisters();
2840        GPRResult result(this);
2841        callOperation(operationRegExpExec, result.gpr(), baseGPR, argumentGPR);
2842
2843        jsValueResult(result.gpr(), node);
2844        break;
2845    }
2846
2847    case RegExpTest: {
2848        SpeculateCellOperand base(this, node->child1());
2849        SpeculateCellOperand argument(this, node->child2());
2850        GPRReg baseGPR = base.gpr();
2851        GPRReg argumentGPR = argument.gpr();
2852
2853        flushRegisters();
2854        GPRResult result(this);
2855        callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2856
2857        // If we add a DataFormatBool, we should use it here.
2858        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
2859        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
2860        break;
2861    }
2862
2863    case ArrayPush: {
2864        ASSERT(node->arrayMode().isJSArray());
2865
2866        SpeculateCellOperand base(this, node->child1());
2867        GPRTemporary storageLength(this);
2868
2869        GPRReg baseGPR = base.gpr();
2870        GPRReg storageLengthGPR = storageLength.gpr();
2871
2872        StorageOperand storage(this, node->child3());
2873        GPRReg storageGPR = storage.gpr();
2874
2875        switch (node->arrayMode().type()) {
2876        case Array::Int32:
2877        case Array::Contiguous: {
2878            JSValueOperand value(this, node->child2(), ManualOperandSpeculation);
2879            GPRReg valueGPR = value.gpr();
2880
2881            if (node->arrayMode().type() == Array::Int32) {
2882                DFG_TYPE_CHECK(
2883                    JSValueRegs(valueGPR), node->child2(), SpecInt32,
2884                    m_jit.branch64(
2885                        MacroAssembler::Below, valueGPR, GPRInfo::tagTypeNumberRegister));
2886            }
2887
2888            m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2889            MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
2890            m_jit.store64(valueGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2891            m_jit.add32(TrustedImm32(1), storageLengthGPR);
2892            m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2893            m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2894
2895            addSlowPathGenerator(
2896                slowPathCall(
2897                    slowPath, this, operationArrayPush, storageLengthGPR,
2898                    valueGPR, baseGPR));
2899
2900            jsValueResult(storageLengthGPR, node);
2901            break;
2902        }
2903
2904        case Array::Double: {
2905            SpeculateDoubleOperand value(this, node->child2());
2906            FPRReg valueFPR = value.fpr();
2907
2908            DFG_TYPE_CHECK(
2909                JSValueRegs(), node->child2(), SpecDoubleReal,
2910                m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR));
2911
2912            m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2913            MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
2914            m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2915            m_jit.add32(TrustedImm32(1), storageLengthGPR);
2916            m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2917            m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2918
2919            addSlowPathGenerator(
2920                slowPathCall(
2921                    slowPath, this, operationArrayPushDouble, storageLengthGPR,
2922                    valueFPR, baseGPR));
2923
2924            jsValueResult(storageLengthGPR, node);
2925            break;
2926        }
2927
2928        case Array::ArrayStorage: {
2929            JSValueOperand value(this, node->child2());
2930            GPRReg valueGPR = value.gpr();
2931
2932            m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
2933
2934            // Refuse to handle bizarre lengths.
2935            speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(0x7ffffffe)));
2936
2937            MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
2938
2939            m_jit.store64(valueGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
2940
2941            m_jit.add32(TrustedImm32(1), storageLengthGPR);
2942            m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
2943            m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
2944            m_jit.or64(GPRInfo::tagTypeNumberRegister, storageLengthGPR);
2945
2946            addSlowPathGenerator(
2947                slowPathCall(
2948                    slowPath, this, operationArrayPush, NoResult, storageLengthGPR,
2949                    valueGPR, baseGPR));
2950
2951            jsValueResult(storageLengthGPR, node);
2952            break;
2953        }
2954
2955        default:
2956            CRASH();
2957            break;
2958        }
2959        break;
2960    }
2961
2962    case ArrayPop: {
2963        ASSERT(node->arrayMode().isJSArray());
2964
2965        SpeculateCellOperand base(this, node->child1());
2966        StorageOperand storage(this, node->child2());
2967        GPRTemporary value(this);
2968        GPRTemporary storageLength(this);
2969        FPRTemporary temp(this); // This is kind of lame, since we don't always need it. I'm relying on the fact that we don't have FPR pressure, especially in code that uses pop().
2970
2971        GPRReg baseGPR = base.gpr();
2972        GPRReg storageGPR = storage.gpr();
2973        GPRReg valueGPR = value.gpr();
2974        GPRReg storageLengthGPR = storageLength.gpr();
2975        FPRReg tempFPR = temp.fpr();
2976
2977        switch (node->arrayMode().type()) {
2978        case Array::Int32:
2979        case Array::Double:
2980        case Array::Contiguous: {
2981            m_jit.load32(
2982                MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
2983            MacroAssembler::Jump undefinedCase =
2984                m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR);
2985            m_jit.sub32(TrustedImm32(1), storageLengthGPR);
2986            m_jit.store32(
2987                storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
2988            MacroAssembler::Jump slowCase;
2989            if (node->arrayMode().type() == Array::Double) {
2990                m_jit.loadDouble(
2991                    MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight),
2992                    tempFPR);
2993                // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old
2994                // length and the new length.
2995                m_jit.store64(
2996                    MacroAssembler::TrustedImm64(bitwise_cast<int64_t>(PNaN)), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
2997                slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR);
2998                boxDouble(tempFPR, valueGPR);
2999            } else {
3000                m_jit.load64(
3001                    MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight),
3002                    valueGPR);
3003                // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old
3004                // length and the new length.
3005                m_jit.store64(
3006                MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
3007                slowCase = m_jit.branchTest64(MacroAssembler::Zero, valueGPR);
3008            }
3009
3010            addSlowPathGenerator(
3011                slowPathMove(
3012                    undefinedCase, this,
3013                    MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), valueGPR));
3014            addSlowPathGenerator(
3015                slowPathCall(
3016                    slowCase, this, operationArrayPopAndRecoverLength, valueGPR, baseGPR));
3017
3018            // We can't know for sure that the result is an int because of the slow paths. :-/
3019            jsValueResult(valueGPR, node);
3020            break;
3021        }
3022
3023        case Array::ArrayStorage: {
3024            m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3025
3026            JITCompiler::Jump undefinedCase =
3027                m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR);
3028
3029            m_jit.sub32(TrustedImm32(1), storageLengthGPR);
3030
3031            JITCompiler::JumpList slowCases;
3032            slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset())));
3033
3034            m_jit.load64(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])), valueGPR);
3035            slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, valueGPR));
3036
3037            m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3038
3039            m_jit.store64(MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0])));
3040            m_jit.sub32(MacroAssembler::TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3041
3042            addSlowPathGenerator(
3043                slowPathMove(
3044                    undefinedCase, this,
3045                    MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), valueGPR));
3046
3047            addSlowPathGenerator(
3048                slowPathCall(
3049                    slowCases, this, operationArrayPop, valueGPR, baseGPR));
3050
3051            jsValueResult(valueGPR, node);
3052            break;
3053        }
3054
3055        default:
3056            CRASH();
3057            break;
3058        }
3059        break;
3060    }
3061
3062    case DFG::Jump: {
3063        jump(node->targetBlock());
3064        noResult(node);
3065        break;
3066    }
3067
3068    case Branch:
3069        emitBranch(node);
3070        break;
3071
3072    case Switch:
3073        emitSwitch(node);
3074        break;
3075
3076    case Return: {
3077        ASSERT(GPRInfo::callFrameRegister != GPRInfo::regT1);
3078        ASSERT(GPRInfo::regT1 != GPRInfo::returnValueGPR);
3079        ASSERT(GPRInfo::returnValueGPR != GPRInfo::callFrameRegister);
3080
3081        // Return the result in returnValueGPR.
3082        JSValueOperand op1(this, node->child1());
3083        m_jit.move(op1.gpr(), GPRInfo::returnValueGPR);
3084
3085        m_jit.emitFunctionEpilogue();
3086        m_jit.ret();
3087
3088        noResult(node);
3089        break;
3090    }
3091
3092    case Throw:
3093    case ThrowReferenceError: {
3094        // We expect that throw statements are rare and are intended to exit the code block
3095        // anyway, so we just OSR back to the old JIT for now.
3096        terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
3097        break;
3098    }
3099
3100    case BooleanToNumber: {
3101        switch (node->child1().useKind()) {
3102        case BooleanUse: {
3103            JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
3104            GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add).
3105
3106            m_jit.move(value.gpr(), result.gpr());
3107            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr());
3108            DFG_TYPE_CHECK(
3109                JSValueRegs(value.gpr()), node->child1(), SpecBoolean, m_jit.branchTest64(
3110                    JITCompiler::NonZero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
3111
3112            int32Result(result.gpr(), node);
3113            break;
3114        }
3115
3116        case UntypedUse: {
3117            JSValueOperand value(this, node->child1());
3118            GPRTemporary result(this);
3119
3120            m_jit.move(value.gpr(), result.gpr());
3121            m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr());
3122            JITCompiler::Jump isBoolean = m_jit.branchTest64(
3123                JITCompiler::Zero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1)));
3124            m_jit.move(value.gpr(), result.gpr());
3125            JITCompiler::Jump done = m_jit.jump();
3126            isBoolean.link(&m_jit);
3127            m_jit.or64(GPRInfo::tagTypeNumberRegister, result.gpr());
3128            done.link(&m_jit);
3129
3130            jsValueResult(result.gpr(), node);
3131            break;
3132        }
3133
3134        default:
3135            RELEASE_ASSERT_NOT_REACHED();
3136            break;
3137        }
3138        break;
3139    }
3140
3141    case ToPrimitive: {
3142        RELEASE_ASSERT(node->child1().useKind() == UntypedUse);
3143        JSValueOperand op1(this, node->child1());
3144        GPRTemporary result(this, Reuse, op1);
3145
3146        GPRReg op1GPR = op1.gpr();
3147        GPRReg resultGPR = result.gpr();
3148
3149        op1.use();
3150
3151        MacroAssembler::Jump alreadyPrimitive = branchNotCell(JSValueRegs(op1GPR));
3152        MacroAssembler::Jump notPrimitive = m_jit.branchStructurePtr(
3153            MacroAssembler::NotEqual,
3154            MacroAssembler::Address(op1GPR, JSCell::structureIDOffset()),
3155            m_jit.vm()->stringStructure.get());
3156
3157        alreadyPrimitive.link(&m_jit);
3158        m_jit.move(op1GPR, resultGPR);
3159
3160        addSlowPathGenerator(
3161            slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, op1GPR));
3162
3163        jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3164        break;
3165    }
3166
3167    case ToString: {
3168        if (node->child1().useKind() == UntypedUse) {
3169            JSValueOperand op1(this, node->child1());
3170            GPRReg op1GPR = op1.gpr();
3171
3172            GPRResult result(this);
3173            GPRReg resultGPR = result.gpr();
3174
3175            flushRegisters();
3176
3177            JITCompiler::Jump done;
3178            if (node->child1()->prediction() & SpecString) {
3179                JITCompiler::Jump slowPath1 = branchNotCell(JSValueRegs(op1GPR));
3180                JITCompiler::Jump slowPath2 = m_jit.branchStructurePtr(
3181                    JITCompiler::NotEqual,
3182                    JITCompiler::Address(op1GPR, JSCell::structureIDOffset()),
3183                    m_jit.vm()->stringStructure.get());
3184                m_jit.move(op1GPR, resultGPR);
3185                done = m_jit.jump();
3186                slowPath1.link(&m_jit);
3187                slowPath2.link(&m_jit);
3188            }
3189            callOperation(operationToString, resultGPR, op1GPR);
3190            if (done.isSet())
3191                done.link(&m_jit);
3192            cellResult(resultGPR, node);
3193            break;
3194        }
3195
3196        compileToStringOnCell(node);
3197        break;
3198    }
3199
3200    case NewStringObject: {
3201        compileNewStringObject(node);
3202        break;
3203    }
3204
3205    case NewArray: {
3206        JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
3207        if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) {
3208            Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3209            RELEASE_ASSERT(structure->indexingType() == node->indexingType());
3210            ASSERT(
3211                hasUndecided(structure->indexingType())
3212                || hasInt32(structure->indexingType())
3213                || hasDouble(structure->indexingType())
3214                || hasContiguous(structure->indexingType()));
3215
3216            unsigned numElements = node->numChildren();
3217
3218            GPRTemporary result(this);
3219            GPRTemporary storage(this);
3220
3221            GPRReg resultGPR = result.gpr();
3222            GPRReg storageGPR = storage.gpr();
3223
3224            emitAllocateJSArray(resultGPR, structure, storageGPR, numElements);
3225
3226            // At this point, one way or another, resultGPR and storageGPR have pointers to
3227            // the JSArray and the Butterfly, respectively.
3228
3229            ASSERT(!hasUndecided(structure->indexingType()) || !node->numChildren());
3230
3231            for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3232                Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3233                switch (node->indexingType()) {
3234                case ALL_BLANK_INDEXING_TYPES:
3235                case ALL_UNDECIDED_INDEXING_TYPES:
3236                    CRASH();
3237                    break;
3238                case ALL_DOUBLE_INDEXING_TYPES: {
3239                    SpeculateDoubleOperand operand(this, use);
3240                    FPRReg opFPR = operand.fpr();
3241                    DFG_TYPE_CHECK(
3242                        JSValueRegs(), use, SpecDoubleReal,
3243                        m_jit.branchDouble(
3244                            MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3245                    m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx));
3246                    break;
3247                }
3248                case ALL_INT32_INDEXING_TYPES:
3249                case ALL_CONTIGUOUS_INDEXING_TYPES: {
3250                    JSValueOperand operand(this, use, ManualOperandSpeculation);
3251                    GPRReg opGPR = operand.gpr();
3252                    if (hasInt32(node->indexingType())) {
3253                        DFG_TYPE_CHECK(
3254                            JSValueRegs(opGPR), use, SpecInt32,
3255                            m_jit.branch64(
3256                                MacroAssembler::Below, opGPR, GPRInfo::tagTypeNumberRegister));
3257                    }
3258                    m_jit.store64(opGPR, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx));
3259                    break;
3260                }
3261                default:
3262                    CRASH();
3263                    break;
3264                }
3265            }
3266
3267            // Yuck, we should *really* have a way of also returning the storageGPR. But
3268            // that's the least of what's wrong with this code. We really shouldn't be
3269            // allocating the array after having computed - and probably spilled to the
3270            // stack - all of the things that will go into the array. The solution to that
3271            // bigger problem will also likely fix the redundancy in reloading the storage
3272            // pointer that we currently have.
3273
3274            cellResult(resultGPR, node);
3275            break;
3276        }
3277
3278        if (!node->numChildren()) {
3279            flushRegisters();
3280            GPRResult result(this);
3281            callOperation(operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()));
3282            cellResult(result.gpr(), node);
3283            break;
3284        }
3285
3286        size_t scratchSize = sizeof(EncodedJSValue) * node->numChildren();
3287        ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
3288        EncodedJSValue* buffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
3289
3290        for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3291            // Need to perform the speculations that this node promises to perform. If we're
3292            // emitting code here and the indexing type is not array storage then there is
3293            // probably something hilarious going on and we're already failing at all the
3294            // things, but at least we're going to be sound.
3295            Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3296            switch (node->indexingType()) {
3297            case ALL_BLANK_INDEXING_TYPES:
3298            case ALL_UNDECIDED_INDEXING_TYPES:
3299                CRASH();
3300                break;
3301            case ALL_DOUBLE_INDEXING_TYPES: {
3302                SpeculateDoubleOperand operand(this, use);
3303                GPRTemporary scratch(this);
3304                FPRReg opFPR = operand.fpr();
3305                GPRReg scratchGPR = scratch.gpr();
3306                DFG_TYPE_CHECK(
3307                    JSValueRegs(), use, SpecDoubleReal,
3308                    m_jit.branchDouble(
3309                        MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3310                m_jit.boxDouble(opFPR, scratchGPR);
3311                m_jit.store64(scratchGPR, buffer + operandIdx);
3312                break;
3313            }
3314            case ALL_INT32_INDEXING_TYPES: {
3315                JSValueOperand operand(this, use, ManualOperandSpeculation);
3316                GPRReg opGPR = operand.gpr();
3317                if (hasInt32(node->indexingType())) {
3318                    DFG_TYPE_CHECK(
3319                        JSValueRegs(opGPR), use, SpecInt32,
3320                        m_jit.branch64(
3321                            MacroAssembler::Below, opGPR, GPRInfo::tagTypeNumberRegister));
3322                }
3323                m_jit.store64(opGPR, buffer + operandIdx);
3324                break;
3325            }
3326            case ALL_CONTIGUOUS_INDEXING_TYPES:
3327            case ALL_ARRAY_STORAGE_INDEXING_TYPES: {
3328                JSValueOperand operand(this, use);
3329                GPRReg opGPR = operand.gpr();
3330                m_jit.store64(opGPR, buffer + operandIdx);
3331                operand.use();
3332                break;
3333            }
3334            default:
3335                CRASH();
3336                break;
3337            }
3338        }
3339
3340        switch (node->indexingType()) {
3341        case ALL_DOUBLE_INDEXING_TYPES:
3342        case ALL_INT32_INDEXING_TYPES:
3343            useChildren(node);
3344            break;
3345        default:
3346            break;
3347        }
3348
3349        flushRegisters();
3350
3351        if (scratchSize) {
3352            GPRTemporary scratch(this);
3353
3354            // Tell GC mark phase how much of the scratch buffer is active during call.
3355            m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3356            m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr());
3357        }
3358
3359        GPRResult result(this);
3360
3361        callOperation(
3362            operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3363            static_cast<void*>(buffer), node->numChildren());
3364
3365        if (scratchSize) {
3366            GPRTemporary scratch(this);
3367
3368            m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3369            m_jit.storePtr(TrustedImmPtr(0), scratch.gpr());
3370        }
3371
3372        cellResult(result.gpr(), node, UseChildrenCalledExplicitly);
3373        break;
3374    }
3375
3376    case NewArrayWithSize: {
3377        JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
3378        if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) {
3379            SpeculateStrictInt32Operand size(this, node->child1());
3380            GPRTemporary result(this);
3381            GPRTemporary storage(this);
3382            GPRTemporary scratch(this);
3383            GPRTemporary scratch2(this);
3384
3385            GPRReg sizeGPR = size.gpr();
3386            GPRReg resultGPR = result.gpr();
3387            GPRReg storageGPR = storage.gpr();
3388            GPRReg scratchGPR = scratch.gpr();
3389            GPRReg scratch2GPR = scratch2.gpr();
3390
3391            MacroAssembler::JumpList slowCases;
3392            slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)));
3393
3394            ASSERT((1 << 3) == sizeof(JSValue));
3395            m_jit.move(sizeGPR, scratchGPR);
3396            m_jit.lshift32(TrustedImm32(3), scratchGPR);
3397            m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratchGPR, resultGPR);
3398            slowCases.append(
3399                emitAllocateBasicStorage(resultGPR, storageGPR));
3400            m_jit.subPtr(scratchGPR, storageGPR);
3401            Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3402            emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases);
3403
3404            m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3405            m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3406
3407            if (hasDouble(node->indexingType())) {
3408                m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
3409                m_jit.move(sizeGPR, scratch2GPR);
3410                MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratch2GPR);
3411                MacroAssembler::Label loop = m_jit.label();
3412                m_jit.sub32(TrustedImm32(1), scratch2GPR);
3413                m_jit.store64(scratchGPR, MacroAssembler::BaseIndex(storageGPR, scratch2GPR, MacroAssembler::TimesEight));
3414                m_jit.branchTest32(MacroAssembler::NonZero, scratch2GPR).linkTo(loop, &m_jit);
3415                done.link(&m_jit);
3416            }
3417
3418            addSlowPathGenerator(adoptPtr(
3419                new CallArrayAllocatorWithVariableSizeSlowPathGenerator(
3420                    slowCases, this, operationNewArrayWithSize, resultGPR,
3421                    globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3422                    globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage),
3423                    sizeGPR)));
3424
3425            cellResult(resultGPR, node);
3426            break;
3427        }
3428
3429        SpeculateStrictInt32Operand size(this, node->child1());
3430        GPRReg sizeGPR = size.gpr();
3431        flushRegisters();
3432        GPRResult result(this);
3433        GPRReg resultGPR = result.gpr();
3434        GPRReg structureGPR = selectScratchGPR(sizeGPR);
3435        MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX));
3436        m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR);
3437        MacroAssembler::Jump done = m_jit.jump();
3438        bigLength.link(&m_jit);
3439        m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), structureGPR);
3440        done.link(&m_jit);
3441        callOperation(operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR);
3442        cellResult(resultGPR, node);
3443        break;
3444    }
3445
3446    case NewArrayBuffer: {
3447        JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
3448        IndexingType indexingType = node->indexingType();
3449        if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(indexingType)) {
3450            unsigned numElements = node->numConstants();
3451
3452            GPRTemporary result(this);
3453            GPRTemporary storage(this);
3454
3455            GPRReg resultGPR = result.gpr();
3456            GPRReg storageGPR = storage.gpr();
3457
3458            emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements);
3459
3460            RELEASE_ASSERT(indexingType & IsArray);
3461            JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant());
3462            if (indexingType == ArrayWithDouble) {
3463                for (unsigned index = 0; index < node->numConstants(); ++index) {
3464                    double value = data[index].asNumber();
3465                    m_jit.store64(
3466                        Imm64(bitwise_cast<int64_t>(value)),
3467                        MacroAssembler::Address(storageGPR, sizeof(double) * index));
3468                }
3469            } else {
3470                for (unsigned index = 0; index < node->numConstants(); ++index) {
3471                    m_jit.store64(
3472                        Imm64(JSValue::encode(data[index])),
3473                        MacroAssembler::Address(storageGPR, sizeof(JSValue) * index));
3474                }
3475            }
3476
3477            cellResult(resultGPR, node);
3478            break;
3479        }
3480
3481        flushRegisters();
3482        GPRResult result(this);
3483
3484        callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants());
3485
3486        cellResult(result.gpr(), node);
3487        break;
3488    }
3489
3490    case NewTypedArray: {
3491        switch (node->child1().useKind()) {
3492        case Int32Use:
3493            compileNewTypedArray(node);
3494            break;
3495        case UntypedUse: {
3496            JSValueOperand argument(this, node->child1());
3497            GPRReg argumentGPR = argument.gpr();
3498
3499            flushRegisters();
3500
3501            GPRResult result(this);
3502            GPRReg resultGPR = result.gpr();
3503
3504            JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
3505            callOperation(
3506                operationNewTypedArrayWithOneArgumentForType(node->typedArrayType()),
3507                resultGPR, globalObject->typedArrayStructure(node->typedArrayType()),
3508                argumentGPR);
3509
3510            cellResult(resultGPR, node);
3511            break;
3512        }
3513        default:
3514            RELEASE_ASSERT_NOT_REACHED();
3515            break;
3516        }
3517        break;
3518    }
3519
3520    case NewRegexp: {
3521        flushRegisters();
3522        GPRResult result(this);
3523
3524        callOperation(operationNewRegexp, result.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex()));
3525
3526        cellResult(result.gpr(), node);
3527        break;
3528    }
3529
3530    case ToThis: {
3531        ASSERT(node->child1().useKind() == UntypedUse);
3532        JSValueOperand thisValue(this, node->child1());
3533        GPRTemporary temp(this);
3534        GPRReg thisValueGPR = thisValue.gpr();
3535        GPRReg tempGPR = temp.gpr();
3536
3537        MacroAssembler::JumpList slowCases;
3538        slowCases.append(branchNotCell(JSValueRegs(thisValueGPR)));
3539        slowCases.append(m_jit.branch8(
3540            MacroAssembler::NotEqual,
3541            MacroAssembler::Address(thisValueGPR, JSCell::typeInfoTypeOffset()),
3542            TrustedImm32(FinalObjectType)));
3543        m_jit.move(thisValueGPR, tempGPR);
3544        J_JITOperation_EJ function;
3545        if (m_jit.graph().executableFor(node->origin.semantic)->isStrictMode())
3546            function = operationToThisStrict;
3547        else
3548            function = operationToThis;
3549        addSlowPathGenerator(
3550            slowPathCall(slowCases, this, function, tempGPR, thisValueGPR));
3551
3552        jsValueResult(tempGPR, node);
3553        break;
3554    }
3555
3556    case CreateThis: {
3557        // Note that there is not so much profit to speculate here. The only things we
3558        // speculate on are (1) that it's a cell, since that eliminates cell checks
3559        // later if the proto is reused, and (2) if we have a FinalObject prediction
3560        // then we speculate because we want to get recompiled if it isn't (since
3561        // otherwise we'd start taking slow path a lot).
3562
3563        SpeculateCellOperand callee(this, node->child1());
3564        GPRTemporary result(this);
3565        GPRTemporary allocator(this);
3566        GPRTemporary structure(this);
3567        GPRTemporary scratch(this);
3568
3569        GPRReg calleeGPR = callee.gpr();
3570        GPRReg resultGPR = result.gpr();
3571        GPRReg allocatorGPR = allocator.gpr();
3572        GPRReg structureGPR = structure.gpr();
3573        GPRReg scratchGPR = scratch.gpr();
3574
3575        MacroAssembler::JumpList slowPath;
3576
3577        m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR);
3578        m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR);
3579        slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR));
3580        emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath);
3581
3582        addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity()));
3583
3584        cellResult(resultGPR, node);
3585        break;
3586    }
3587
3588    case AllocationProfileWatchpoint:
3589    case TypedArrayWatchpoint: {
3590        noResult(node);
3591        break;
3592    }
3593
3594    case NewObject: {
3595        GPRTemporary result(this);
3596        GPRTemporary allocator(this);
3597        GPRTemporary scratch(this);
3598
3599        GPRReg resultGPR = result.gpr();
3600        GPRReg allocatorGPR = allocator.gpr();
3601        GPRReg scratchGPR = scratch.gpr();
3602
3603        MacroAssembler::JumpList slowPath;
3604
3605        Structure* structure = node->structure();
3606        size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
3607        MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize);
3608
3609        m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR);
3610        emitAllocateJSObject(resultGPR, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath);
3611
3612        addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure));
3613
3614        cellResult(resultGPR, node);
3615        break;
3616    }
3617
3618    case GetCallee: {
3619        GPRTemporary result(this);
3620        m_jit.loadPtr(JITCompiler::addressFor(JSStack::Callee), result.gpr());
3621        cellResult(result.gpr(), node);
3622        break;
3623    }
3624
3625    case GetScope: {
3626        SpeculateCellOperand function(this, node->child1());
3627        GPRTemporary result(this, Reuse, function);
3628        m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr());
3629        cellResult(result.gpr(), node);
3630        break;
3631    }
3632
3633    case GetMyScope: {
3634        GPRTemporary result(this);
3635        GPRReg resultGPR = result.gpr();
3636
3637        m_jit.loadPtr(JITCompiler::addressFor(JSStack::ScopeChain), resultGPR);
3638        cellResult(resultGPR, node);
3639        break;
3640    }
3641
3642    case SkipTopScope: {
3643        SpeculateCellOperand scope(this, node->child1());
3644        GPRTemporary result(this, Reuse, scope);
3645        GPRReg resultGPR = result.gpr();
3646        m_jit.move(scope.gpr(), resultGPR);
3647        JITCompiler::Jump activationNotCreated =
3648            m_jit.branchTest64(
3649                JITCompiler::Zero,
3650                JITCompiler::addressFor(
3651                    static_cast<VirtualRegister>(m_jit.graph().machineActivationRegister())));
3652        m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR);
3653        activationNotCreated.link(&m_jit);
3654        cellResult(resultGPR, node);
3655        break;
3656    }
3657
3658    case SkipScope: {
3659        SpeculateCellOperand scope(this, node->child1());
3660        GPRTemporary result(this, Reuse, scope);
3661        m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr());
3662        cellResult(result.gpr(), node);
3663        break;
3664    }
3665
3666    case GetClosureRegisters: {
3667        if (WriteBarrierBase<Unknown>* registers = m_jit.graph().tryGetRegisters(node->child1().node())) {
3668            GPRTemporary result(this);
3669            GPRReg resultGPR = result.gpr();
3670            m_jit.move(TrustedImmPtr(registers), resultGPR);
3671            storageResult(resultGPR, node);
3672            break;
3673        }
3674
3675        SpeculateCellOperand scope(this, node->child1());
3676        GPRTemporary result(this);
3677        GPRReg scopeGPR = scope.gpr();
3678        GPRReg resultGPR = result.gpr();
3679
3680        m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR);
3681        storageResult(resultGPR, node);
3682        break;
3683    }
3684    case GetClosureVar: {
3685        StorageOperand registers(this, node->child1());
3686        GPRTemporary result(this);
3687        GPRReg registersGPR = registers.gpr();
3688        GPRReg resultGPR = result.gpr();
3689
3690        m_jit.load64(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register)), resultGPR);
3691        jsValueResult(resultGPR, node);
3692        break;
3693    }
3694    case PutClosureVar: {
3695        StorageOperand registers(this, node->child2());
3696        JSValueOperand value(this, node->child3());
3697
3698        GPRReg registersGPR = registers.gpr();
3699        GPRReg valueGPR = value.gpr();
3700
3701        speculate(node, node->child1());
3702
3703        m_jit.store64(valueGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register)));
3704        noResult(node);
3705        break;
3706    }
3707    case GetById: {
3708        ASSERT(node->prediction());
3709
3710        switch (node->child1().useKind()) {
3711        case CellUse: {
3712            SpeculateCellOperand base(this, node->child1());
3713            GPRTemporary result(this, Reuse, base);
3714
3715            GPRReg baseGPR = base.gpr();
3716            GPRReg resultGPR = result.gpr();
3717
3718            base.use();
3719
3720            cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber());
3721
3722            jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3723            break;
3724        }
3725
3726        case UntypedUse: {
3727            JSValueOperand base(this, node->child1());
3728            GPRTemporary result(this, Reuse, base);
3729
3730            GPRReg baseGPR = base.gpr();
3731            GPRReg resultGPR = result.gpr();
3732
3733            base.use();
3734
3735            JITCompiler::Jump notCell = branchNotCell(JSValueRegs(baseGPR));
3736
3737            cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell);
3738
3739            jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3740            break;
3741        }
3742
3743        default:
3744            RELEASE_ASSERT_NOT_REACHED();
3745            break;
3746        }
3747        break;
3748    }
3749
3750    case GetByIdFlush: {
3751        if (!node->prediction()) {
3752            terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
3753            break;
3754        }
3755
3756        switch (node->child1().useKind()) {
3757        case CellUse: {
3758            SpeculateCellOperand base(this, node->child1());
3759            GPRReg baseGPR = base.gpr();
3760
3761            GPRResult result(this);
3762
3763            GPRReg resultGPR = result.gpr();
3764
3765            base.use();
3766
3767            flushRegisters();
3768
3769            cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill);
3770
3771            jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3772            break;
3773        }
3774
3775        case UntypedUse: {
3776            JSValueOperand base(this, node->child1());
3777            GPRReg baseGPR = base.gpr();
3778
3779            GPRResult result(this);
3780            GPRReg resultGPR = result.gpr();
3781
3782            base.use();
3783            flushRegisters();
3784
3785            JITCompiler::Jump notCell = branchNotCell(JSValueRegs(baseGPR));
3786
3787            cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell, DontSpill);
3788
3789            jsValueResult(resultGPR, node, UseChildrenCalledExplicitly);
3790            break;
3791        }
3792
3793        default:
3794            RELEASE_ASSERT_NOT_REACHED();
3795            break;
3796        }
3797        break;
3798    }
3799
3800    case GetArrayLength:
3801        compileGetArrayLength(node);
3802        break;
3803
3804    case CheckFunction: {
3805        SpeculateCellOperand function(this, node->child1());
3806        speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function()));
3807        noResult(node);
3808        break;
3809    }
3810
3811    case CheckExecutable: {
3812        SpeculateCellOperand function(this, node->child1());
3813        speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable()));
3814        noResult(node);
3815        break;
3816    }
3817
3818    case CheckStructure: {
3819        SpeculateCellOperand base(this, node->child1());
3820
3821        ASSERT(node->structureSet().size());
3822
3823        ExitKind exitKind;
3824        if (node->child1()->op() == WeakJSConstant)
3825            exitKind = BadWeakConstantCache;
3826        else
3827            exitKind = BadCache;
3828
3829        if (node->structureSet().size() == 1) {
3830            speculationCheck(
3831                exitKind, JSValueSource::unboxedCell(base.gpr()), 0,
3832                m_jit.branchWeakStructure(
3833                    JITCompiler::NotEqual,
3834                    JITCompiler::Address(base.gpr(), JSCell::structureIDOffset()),
3835                    node->structureSet()[0]));
3836        } else {
3837            JITCompiler::JumpList done;
3838
3839            for (size_t i = 0; i < node->structureSet().size() - 1; ++i)
3840                done.append(m_jit.branchWeakStructure(JITCompiler::Equal, MacroAssembler::Address(base.gpr(), JSCell::structureIDOffset()), node->structureSet()[i]));
3841
3842            speculationCheck(
3843                exitKind, JSValueSource::unboxedCell(base.gpr()), 0,
3844                m_jit.branchWeakStructure(
3845                    JITCompiler::NotEqual, MacroAssembler::Address(base.gpr(), JSCell::structureIDOffset()), node->structureSet().last()));
3846
3847            done.link(&m_jit);
3848        }
3849
3850        noResult(node);
3851        break;
3852    }
3853
3854    case StructureTransitionWatchpoint: {
3855        // There is a fascinating question here of what to do about array profiling.
3856        // We *could* try to tell the OSR exit about where the base of the access is.
3857        // The DFG will have kept it alive, though it may not be in a register, and
3858        // we shouldn't really load it since that could be a waste. For now though,
3859        // we'll just rely on the fact that when a watchpoint fires then that's
3860        // quite a hint already.
3861
3862        m_jit.addWeakReference(node->structure());
3863
3864#if !ASSERT_DISABLED
3865        SpeculateCellOperand op1(this, node->child1());
3866        JITCompiler::Jump isOK = m_jit.branchStructurePtr(
3867            JITCompiler::Equal,
3868            JITCompiler::Address(op1.gpr(), JSCell::structureIDOffset()),
3869            node->structure());
3870        m_jit.abortWithReason(DFGIneffectiveWatchpoint);
3871        isOK.link(&m_jit);
3872#else
3873        speculateCell(node->child1());
3874#endif
3875
3876        noResult(node);
3877        break;
3878    }
3879
3880    case PhantomPutStructure: {
3881        ASSERT(isKnownCell(node->child1().node()));
3882        m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node);
3883        noResult(node);
3884        break;
3885    }
3886
3887    case PutStructure: {
3888        Structure* oldStructure = node->structureTransitionData().previousStructure;
3889        Structure* newStructure = node->structureTransitionData().newStructure;
3890
3891        m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node);
3892
3893        SpeculateCellOperand base(this, node->child1());
3894        GPRReg baseGPR = base.gpr();
3895
3896        ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
3897        ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
3898        ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
3899        m_jit.store32(MacroAssembler::TrustedImm32(newStructure->id()), MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()));
3900
3901        noResult(node);
3902        break;
3903    }
3904
3905    case AllocatePropertyStorage:
3906        compileAllocatePropertyStorage(node);
3907        break;
3908
3909    case ReallocatePropertyStorage:
3910        compileReallocatePropertyStorage(node);
3911        break;
3912
3913    case GetButterfly: {
3914        SpeculateCellOperand base(this, node->child1());
3915        GPRTemporary result(this, Reuse, base);
3916
3917        GPRReg baseGPR = base.gpr();
3918        GPRReg resultGPR = result.gpr();
3919
3920        m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
3921
3922        storageResult(resultGPR, node);
3923        break;
3924    }
3925
3926    case GetIndexedPropertyStorage: {
3927        compileGetIndexedPropertyStorage(node);
3928        break;
3929    }
3930
3931    case ConstantStoragePointer: {
3932        compileConstantStoragePointer(node);
3933        break;
3934    }
3935
3936    case GetTypedArrayByteOffset: {
3937        compileGetTypedArrayByteOffset(node);
3938        break;
3939    }
3940
3941    case GetByOffset: {
3942        StorageOperand storage(this, node->child1());
3943        GPRTemporary result(this, Reuse, storage);
3944
3945        GPRReg storageGPR = storage.gpr();
3946        GPRReg resultGPR = result.gpr();
3947
3948        StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
3949
3950        m_jit.load64(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)), resultGPR);
3951
3952        jsValueResult(resultGPR, node);
3953        break;
3954    }
3955
3956    case PutByOffset: {
3957        StorageOperand storage(this, node->child1());
3958        JSValueOperand value(this, node->child3());
3959        GPRTemporary scratch1(this);
3960        GPRTemporary scratch2(this);
3961
3962        GPRReg storageGPR = storage.gpr();
3963        GPRReg valueGPR = value.gpr();
3964
3965        speculate(node, node->child2());
3966
3967        StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
3968
3969        m_jit.store64(valueGPR, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)));
3970
3971        noResult(node);
3972        break;
3973    }
3974
3975    case PutByIdFlush: {
3976        SpeculateCellOperand base(this, node->child1());
3977        JSValueOperand value(this, node->child2());
3978        GPRTemporary scratch(this);
3979
3980        GPRReg baseGPR = base.gpr();
3981        GPRReg valueGPR = value.gpr();
3982        GPRReg scratchGPR = scratch.gpr();
3983        flushRegisters();
3984
3985        cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect, MacroAssembler::Jump(), DontSpill);
3986
3987        noResult(node);
3988        break;
3989    }
3990
3991    case PutById: {
3992        SpeculateCellOperand base(this, node->child1());
3993        JSValueOperand value(this, node->child2());
3994        GPRTemporary scratch(this);
3995
3996        GPRReg baseGPR = base.gpr();
3997        GPRReg valueGPR = value.gpr();
3998        GPRReg scratchGPR = scratch.gpr();
3999
4000        cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect);
4001
4002        noResult(node);
4003        break;
4004    }
4005
4006    case PutByIdDirect: {
4007        SpeculateCellOperand base(this, node->child1());
4008        JSValueOperand value(this, node->child2());
4009        GPRTemporary scratch(this);
4010
4011        GPRReg baseGPR = base.gpr();
4012        GPRReg valueGPR = value.gpr();
4013        GPRReg scratchGPR = scratch.gpr();
4014
4015        cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), Direct);
4016
4017        noResult(node);
4018        break;
4019    }
4020
4021    case GetGlobalVar: {
4022        GPRTemporary result(this);
4023
4024        m_jit.load64(node->registerPointer(), result.gpr());
4025
4026        jsValueResult(result.gpr(), node);
4027        break;
4028    }
4029
4030    case PutGlobalVar: {
4031        JSValueOperand value(this, node->child1());
4032
4033        m_jit.store64(value.gpr(), node->registerPointer());
4034
4035        noResult(node);
4036        break;
4037    }
4038
4039    case NotifyWrite: {
4040        VariableWatchpointSet* set = node->variableWatchpointSet();
4041
4042        JSValueOperand value(this, node->child1());
4043        GPRReg valueGPR = value.gpr();
4044
4045        GPRTemporary temp(this);
4046        GPRReg tempGPR = temp.gpr();
4047
4048        m_jit.load8(set->addressOfState(), tempGPR);
4049
4050        JITCompiler::Jump isDone =
4051            m_jit.branch32(JITCompiler::Equal, tempGPR, TrustedImm32(IsInvalidated));
4052        JITCompiler::Jump slowCase = m_jit.branch64(JITCompiler::NotEqual,
4053            JITCompiler::AbsoluteAddress(set->addressOfInferredValue()), valueGPR);
4054        isDone.link(&m_jit);
4055
4056        addSlowPathGenerator(
4057            slowPathCall(slowCase, this, operationNotifyWrite, NoResult, set, valueGPR));
4058
4059        noResult(node);
4060        break;
4061    }
4062
4063    case VarInjectionWatchpoint:
4064    case VariableWatchpoint: {
4065        noResult(node);
4066        break;
4067    }
4068
4069    case CheckHasInstance: {
4070        SpeculateCellOperand base(this, node->child1());
4071        GPRTemporary structure(this);
4072
4073        // Speculate that base 'ImplementsDefaultHasInstance'.
4074        speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(
4075            MacroAssembler::Zero,
4076            MacroAssembler::Address(base.gpr(), JSCell::typeInfoFlagsOffset()),
4077            MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance)));
4078
4079        noResult(node);
4080        break;
4081    }
4082
4083    case InstanceOf: {
4084        compileInstanceOf(node);
4085        break;
4086    }
4087
4088    case IsUndefined: {
4089        JSValueOperand value(this, node->child1());
4090        GPRTemporary result(this);
4091        GPRTemporary localGlobalObject(this);
4092        GPRTemporary remoteGlobalObject(this);
4093        GPRTemporary scratch(this);
4094
4095        JITCompiler::Jump isCell = branchIsCell(value.jsValueRegs());
4096
4097        m_jit.compare64(JITCompiler::Equal, value.gpr(), TrustedImm32(ValueUndefined), result.gpr());
4098        JITCompiler::Jump done = m_jit.jump();
4099
4100        isCell.link(&m_jit);
4101        JITCompiler::Jump notMasqueradesAsUndefined;
4102        if (masqueradesAsUndefinedWatchpointIsStillValid()) {
4103            m_jit.move(TrustedImm32(0), result.gpr());
4104            notMasqueradesAsUndefined = m_jit.jump();
4105        } else {
4106            JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(
4107                JITCompiler::NonZero,
4108                JITCompiler::Address(value.gpr(), JSCell::typeInfoFlagsOffset()),
4109                TrustedImm32(MasqueradesAsUndefined));
4110            m_jit.move(TrustedImm32(0), result.gpr());
4111            notMasqueradesAsUndefined = m_jit.jump();
4112
4113            isMasqueradesAsUndefined.link(&m_jit);
4114            GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
4115            GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
4116            m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->origin.semantic)), localGlobalObjectGPR);
4117            m_jit.emitLoadStructure(value.gpr(), result.gpr(), scratch.gpr());
4118            m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR);
4119            m_jit.comparePtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr());
4120        }
4121
4122        notMasqueradesAsUndefined.link(&m_jit);
4123        done.link(&m_jit);
4124        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
4125        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4126        break;
4127    }
4128
4129    case IsBoolean: {
4130        JSValueOperand value(this, node->child1());
4131        GPRTemporary result(this, Reuse, value);
4132
4133        m_jit.move(value.gpr(), result.gpr());
4134        m_jit.xor64(JITCompiler::TrustedImm32(ValueFalse), result.gpr());
4135        m_jit.test64(JITCompiler::Zero, result.gpr(), JITCompiler::TrustedImm32(static_cast<int32_t>(~1)), result.gpr());
4136        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
4137        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4138        break;
4139    }
4140
4141    case IsNumber: {
4142        JSValueOperand value(this, node->child1());
4143        GPRTemporary result(this, Reuse, value);
4144
4145        m_jit.test64(JITCompiler::NonZero, value.gpr(), GPRInfo::tagTypeNumberRegister, result.gpr());
4146        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
4147        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4148        break;
4149    }
4150
4151    case IsString: {
4152        JSValueOperand value(this, node->child1());
4153        GPRTemporary result(this, Reuse, value);
4154
4155        JITCompiler::Jump isNotCell = branchNotCell(value.jsValueRegs());
4156
4157        m_jit.compare8(JITCompiler::Equal,
4158            JITCompiler::Address(value.gpr(), JSCell::typeInfoTypeOffset()),
4159            TrustedImm32(StringType),
4160            result.gpr());
4161        m_jit.or32(TrustedImm32(ValueFalse), result.gpr());
4162        JITCompiler::Jump done = m_jit.jump();
4163
4164        isNotCell.link(&m_jit);
4165        m_jit.move(TrustedImm32(ValueFalse), result.gpr());
4166
4167        done.link(&m_jit);
4168        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4169        break;
4170    }
4171
4172    case IsObject: {
4173        JSValueOperand value(this, node->child1());
4174        GPRReg valueGPR = value.gpr();
4175        GPRResult result(this);
4176        GPRReg resultGPR = result.gpr();
4177        flushRegisters();
4178        callOperation(operationIsObject, resultGPR, valueGPR);
4179        m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
4180        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4181        break;
4182    }
4183
4184    case IsFunction: {
4185        JSValueOperand value(this, node->child1());
4186        GPRReg valueGPR = value.gpr();
4187        GPRResult result(this);
4188        GPRReg resultGPR = result.gpr();
4189        flushRegisters();
4190        callOperation(operationIsFunction, resultGPR, valueGPR);
4191        m_jit.or32(TrustedImm32(ValueFalse), resultGPR);
4192        jsValueResult(result.gpr(), node, DataFormatJSBoolean);
4193        break;
4194    }
4195
4196    case TypeOf: {
4197        JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
4198        GPRReg valueGPR = value.gpr();
4199        GPRResult result(this);
4200        GPRReg resultGPR = result.gpr();
4201        JITCompiler::JumpList doneJumps;
4202
4203        flushRegisters();
4204
4205        ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse);
4206
4207        JITCompiler::Jump isNotCell = branchNotCell(JSValueRegs(valueGPR));
4208        if (node->child1().useKind() != UntypedUse)
4209            DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecCell, isNotCell);
4210
4211        if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) {
4212            JITCompiler::Jump notString = m_jit.branch8(
4213                JITCompiler::NotEqual,
4214                JITCompiler::Address(valueGPR, JSCell::typeInfoTypeOffset()),
4215                TrustedImm32(StringType));
4216            if (node->child1().useKind() == StringUse)
4217                DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecString, notString);
4218            m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR);
4219            doneJumps.append(m_jit.jump());
4220            if (node->child1().useKind() != StringUse) {
4221                notString.link(&m_jit);
4222                callOperation(operationTypeOf, resultGPR, valueGPR);
4223                doneJumps.append(m_jit.jump());
4224            }
4225        } else {
4226            callOperation(operationTypeOf, resultGPR, valueGPR);
4227            doneJumps.append(m_jit.jump());
4228        }
4229
4230        if (node->child1().useKind() == UntypedUse) {
4231            isNotCell.link(&m_jit);
4232            JITCompiler::Jump notNumber = m_jit.branchTest64(JITCompiler::Zero, valueGPR, GPRInfo::tagTypeNumberRegister);
4233            m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR);
4234            doneJumps.append(m_jit.jump());
4235            notNumber.link(&m_jit);
4236
4237            JITCompiler::Jump notUndefined = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueUndefined));
4238            m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR);
4239            doneJumps.append(m_jit.jump());
4240            notUndefined.link(&m_jit);
4241
4242            JITCompiler::Jump notNull = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueNull));
4243            m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR);
4244            doneJumps.append(m_jit.jump());
4245            notNull.link(&m_jit);
4246
4247            // Only boolean left
4248            m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR);
4249        }
4250        doneJumps.link(&m_jit);
4251        cellResult(resultGPR, node);
4252        break;
4253    }
4254
4255    case Flush:
4256        break;
4257
4258    case Call:
4259    case Construct:
4260        emitCall(node);
4261        break;
4262
4263    case CreateActivation: {
4264        RELEASE_ASSERT(!node->origin.semantic.inlineCallFrame);
4265
4266        JSValueOperand value(this, node->child1());
4267        GPRTemporary result(this, Reuse, value);
4268
4269        GPRReg valueGPR = value.gpr();
4270        GPRReg resultGPR = result.gpr();
4271
4272        m_jit.move(valueGPR, resultGPR);
4273
4274        JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR);
4275
4276        addSlowPathGenerator(
4277            slowPathCall(
4278                notCreated, this, operationCreateActivation, resultGPR,
4279                framePointerOffsetToGetActivationRegisters()));
4280
4281        cellResult(resultGPR, node);
4282        break;
4283    }
4284
4285    case FunctionReentryWatchpoint: {
4286        noResult(node);
4287        break;
4288    }
4289
4290    case CreateArguments: {
4291        JSValueOperand value(this, node->child1());
4292        GPRTemporary scratch1(this);
4293        GPRTemporary scratch2(this);
4294        GPRTemporary result(this, Reuse, value);
4295
4296        GPRReg valueGPR = value.gpr();
4297        GPRReg scratchGPR1 = scratch1.gpr();
4298        GPRReg scratchGPR2 = scratch2.gpr();
4299        GPRReg resultGPR = result.gpr();
4300
4301        m_jit.move(valueGPR, resultGPR);
4302
4303        if (node->origin.semantic.inlineCallFrame) {
4304            JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR);
4305            addSlowPathGenerator(
4306                slowPathCall(
4307                    notCreated, this, operationCreateInlinedArguments, resultGPR,
4308                    node->origin.semantic.inlineCallFrame));
4309            cellResult(resultGPR, node);
4310            break;
4311        }
4312
4313        FunctionExecutable* executable = jsCast<FunctionExecutable*>(m_jit.graph().executableFor(node->origin.semantic));
4314        if (m_jit.codeBlock()->hasSlowArguments()
4315            || executable->isStrictMode()
4316            || !executable->parameterCount()) {
4317            JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR);
4318            addSlowPathGenerator(
4319                slowPathCall(notCreated, this, operationCreateArguments, resultGPR));
4320            cellResult(resultGPR, node);
4321            break;
4322        }
4323
4324        JITCompiler::Jump alreadyCreated = m_jit.branchTest64(JITCompiler::NonZero, resultGPR);
4325
4326        MacroAssembler::JumpList slowPaths;
4327        emitAllocateArguments(resultGPR, scratchGPR1, scratchGPR2, slowPaths);
4328        addSlowPathGenerator(
4329            slowPathCall(slowPaths, this, operationCreateArguments, resultGPR));
4330
4331        alreadyCreated.link(&m_jit);
4332        cellResult(resultGPR, node);
4333        break;
4334    }
4335
4336    case TearOffActivation: {
4337        RELEASE_ASSERT(!node->origin.semantic.inlineCallFrame);
4338
4339        JSValueOperand activationValue(this, node->child1());
4340        GPRTemporary scratch(this);
4341        GPRReg activationValueGPR = activationValue.gpr();
4342        GPRReg scratchGPR = scratch.gpr();
4343
4344        JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, activationValueGPR);
4345
4346        SymbolTable* symbolTable = m_jit.symbolTableFor(node->origin.semantic);
4347        int registersOffset = JSActivation::registersOffset(symbolTable);
4348
4349        int bytecodeCaptureStart = symbolTable->captureStart();
4350        int machineCaptureStart = m_jit.graph().m_machineCaptureStart;
4351        for (int i = symbolTable->captureCount(); i--;) {
4352            m_jit.load64(
4353                JITCompiler::Address(
4354                    GPRInfo::callFrameRegister,
4355                    (machineCaptureStart - i) * sizeof(Register)),
4356                scratchGPR);
4357            m_jit.store64(
4358                scratchGPR,
4359                JITCompiler::Address(
4360                    activationValueGPR,
4361                    registersOffset + (bytecodeCaptureStart - i) * sizeof(Register)));
4362        }
4363        m_jit.addPtr(TrustedImm32(registersOffset), activationValueGPR, scratchGPR);
4364        m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValueGPR, JSActivation::offsetOfRegisters()));
4365
4366        notCreated.link(&m_jit);
4367        noResult(node);
4368        break;
4369    }
4370
4371    case TearOffArguments: {
4372        JSValueOperand unmodifiedArgumentsValue(this, node->child1());
4373        JSValueOperand activationValue(this, node->child2());
4374        GPRReg unmodifiedArgumentsValueGPR = unmodifiedArgumentsValue.gpr();
4375        GPRReg activationValueGPR = activationValue.gpr();
4376
4377        JITCompiler::Jump created = m_jit.branchTest64(JITCompiler::NonZero, unmodifiedArgumentsValueGPR);
4378
4379        if (node->origin.semantic.inlineCallFrame) {
4380            addSlowPathGenerator(
4381                slowPathCall(
4382                    created, this, operationTearOffInlinedArguments, NoResult,
4383                    unmodifiedArgumentsValueGPR, activationValueGPR, node->origin.semantic.inlineCallFrame));
4384        } else {
4385            addSlowPathGenerator(
4386                slowPathCall(
4387                    created, this, operationTearOffArguments, NoResult, unmodifiedArgumentsValueGPR, activationValueGPR));
4388        }
4389
4390        noResult(node);
4391        break;
4392    }
4393
4394    case GetMyArgumentsLength: {
4395        GPRTemporary result(this);
4396        GPRReg resultGPR = result.gpr();
4397
4398        if (!isEmptySpeculation(
4399                m_state.variables().operand(
4400                    m_jit.graph().argumentsRegisterFor(node->origin.semantic)).m_type)) {
4401            speculationCheck(
4402                ArgumentsEscaped, JSValueRegs(), 0,
4403                m_jit.branchTest64(
4404                    JITCompiler::NonZero,
4405                    JITCompiler::addressFor(
4406                        m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic))));
4407        }
4408
4409        RELEASE_ASSERT(!node->origin.semantic.inlineCallFrame);
4410        m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR);
4411        m_jit.sub32(TrustedImm32(1), resultGPR);
4412        int32Result(resultGPR, node);
4413        break;
4414    }
4415
4416    case GetMyArgumentsLengthSafe: {
4417        GPRTemporary result(this);
4418        GPRReg resultGPR = result.gpr();
4419
4420        JITCompiler::Jump created = m_jit.branchTest64(
4421            JITCompiler::NonZero,
4422            JITCompiler::addressFor(
4423                m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic)));
4424
4425        if (node->origin.semantic.inlineCallFrame) {
4426            m_jit.move(
4427                Imm64(JSValue::encode(jsNumber(node->origin.semantic.inlineCallFrame->arguments.size() - 1))),
4428                resultGPR);
4429        } else {
4430            m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR);
4431            m_jit.sub32(TrustedImm32(1), resultGPR);
4432            m_jit.or64(GPRInfo::tagTypeNumberRegister, resultGPR);
4433        }
4434
4435        // FIXME: the slow path generator should perform a forward speculation that the
4436        // result is an integer. For now we postpone the speculation by having this return
4437        // a JSValue.
4438
4439        addSlowPathGenerator(
4440            slowPathCall(
4441                created, this, operationGetArgumentsLength, resultGPR,
4442                m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic).offset()));
4443
4444        jsValueResult(resultGPR, node);
4445        break;
4446    }
4447
4448    case GetMyArgumentByVal: {
4449        SpeculateStrictInt32Operand index(this, node->child1());
4450        GPRTemporary result(this);
4451        GPRReg indexGPR = index.gpr();
4452        GPRReg resultGPR = result.gpr();
4453
4454        if (!isEmptySpeculation(
4455                m_state.variables().operand(
4456                    m_jit.graph().argumentsRegisterFor(node->origin.semantic)).m_type)) {
4457            speculationCheck(
4458                ArgumentsEscaped, JSValueRegs(), 0,
4459                m_jit.branchTest64(
4460                    JITCompiler::NonZero,
4461                    JITCompiler::addressFor(
4462                        m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic))));
4463        }
4464
4465        m_jit.add32(TrustedImm32(1), indexGPR, resultGPR);
4466        if (node->origin.semantic.inlineCallFrame) {
4467            speculationCheck(
4468                Uncountable, JSValueRegs(), 0,
4469                m_jit.branch32(
4470                    JITCompiler::AboveOrEqual,
4471                    resultGPR,
4472                    Imm32(node->origin.semantic.inlineCallFrame->arguments.size())));
4473        } else {
4474            speculationCheck(
4475                Uncountable, JSValueRegs(), 0,
4476                m_jit.branch32(
4477                    JITCompiler::AboveOrEqual,
4478                    resultGPR,
4479                    JITCompiler::payloadFor(JSStack::ArgumentCount)));
4480        }
4481
4482        JITCompiler::JumpList slowArgument;
4483        JITCompiler::JumpList slowArgumentOutOfBounds;
4484        if (m_jit.symbolTableFor(node->origin.semantic)->slowArguments()) {
4485            RELEASE_ASSERT(!node->origin.semantic.inlineCallFrame);
4486            const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get();
4487
4488            slowArgumentOutOfBounds.append(
4489                m_jit.branch32(
4490                    JITCompiler::AboveOrEqual, indexGPR,
4491                    Imm32(m_jit.symbolTableFor(node->origin.semantic)->parameterCount())));
4492
4493            COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4494            m_jit.move(ImmPtr(slowArguments), resultGPR);
4495            m_jit.load32(
4496                JITCompiler::BaseIndex(
4497                    resultGPR, indexGPR, JITCompiler::TimesEight,
4498                    OBJECT_OFFSETOF(SlowArgument, index)),
4499                resultGPR);
4500            m_jit.signExtend32ToPtr(resultGPR, resultGPR);
4501            m_jit.load64(
4502                JITCompiler::BaseIndex(
4503                    GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight),
4504                resultGPR);
4505            slowArgument.append(m_jit.jump());
4506        }
4507        slowArgumentOutOfBounds.link(&m_jit);
4508
4509        m_jit.signExtend32ToPtr(resultGPR, resultGPR);
4510
4511        m_jit.load64(
4512            JITCompiler::BaseIndex(
4513                GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->origin.semantic)),
4514            resultGPR);
4515
4516        slowArgument.link(&m_jit);
4517        jsValueResult(resultGPR, node);
4518        break;
4519    }
4520
4521    case GetMyArgumentByValSafe: {
4522        SpeculateStrictInt32Operand index(this, node->child1());
4523        GPRTemporary result(this);
4524        GPRReg indexGPR = index.gpr();
4525        GPRReg resultGPR = result.gpr();
4526
4527        JITCompiler::JumpList slowPath;
4528        slowPath.append(
4529            m_jit.branchTest64(
4530                JITCompiler::NonZero,
4531                JITCompiler::addressFor(
4532                    m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic))));
4533
4534        m_jit.add32(TrustedImm32(1), indexGPR, resultGPR);
4535        if (node->origin.semantic.inlineCallFrame) {
4536            slowPath.append(
4537                m_jit.branch32(
4538                    JITCompiler::AboveOrEqual,
4539                    resultGPR,
4540                    Imm32(node->origin.semantic.inlineCallFrame->arguments.size())));
4541        } else {
4542            slowPath.append(
4543                m_jit.branch32(
4544                    JITCompiler::AboveOrEqual,
4545                    resultGPR,
4546                    JITCompiler::payloadFor(JSStack::ArgumentCount)));
4547        }
4548
4549        JITCompiler::JumpList slowArgument;
4550        JITCompiler::JumpList slowArgumentOutOfBounds;
4551        if (m_jit.symbolTableFor(node->origin.semantic)->slowArguments()) {
4552            RELEASE_ASSERT(!node->origin.semantic.inlineCallFrame);
4553            const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get();
4554
4555            slowArgumentOutOfBounds.append(
4556                m_jit.branch32(
4557                    JITCompiler::AboveOrEqual, indexGPR,
4558                    Imm32(m_jit.symbolTableFor(node->origin.semantic)->parameterCount())));
4559
4560            COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4561            m_jit.move(ImmPtr(slowArguments), resultGPR);
4562            m_jit.load32(
4563                JITCompiler::BaseIndex(
4564                    resultGPR, indexGPR, JITCompiler::TimesEight,
4565                    OBJECT_OFFSETOF(SlowArgument, index)),
4566                resultGPR);
4567            m_jit.signExtend32ToPtr(resultGPR, resultGPR);
4568            m_jit.load64(
4569                JITCompiler::BaseIndex(
4570                    GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight),
4571                resultGPR);
4572            slowArgument.append(m_jit.jump());
4573        }
4574        slowArgumentOutOfBounds.link(&m_jit);
4575
4576        m_jit.signExtend32ToPtr(resultGPR, resultGPR);
4577
4578        m_jit.load64(
4579            JITCompiler::BaseIndex(
4580                GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->origin.semantic)),
4581            resultGPR);
4582
4583        if (node->origin.semantic.inlineCallFrame) {
4584            addSlowPathGenerator(
4585                slowPathCall(
4586                    slowPath, this, operationGetInlinedArgumentByVal, resultGPR,
4587                    m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic).offset(),
4588                    node->origin.semantic.inlineCallFrame,
4589                    indexGPR));
4590        } else {
4591            addSlowPathGenerator(
4592                slowPathCall(
4593                    slowPath, this, operationGetArgumentByVal, resultGPR,
4594                    m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic).offset(),
4595                    indexGPR));
4596        }
4597
4598        slowArgument.link(&m_jit);
4599        jsValueResult(resultGPR, node);
4600        break;
4601    }
4602
4603    case CheckArgumentsNotCreated: {
4604        ASSERT(!isEmptySpeculation(
4605            m_state.variables().operand(
4606                m_jit.graph().argumentsRegisterFor(node->origin.semantic)).m_type));
4607        speculationCheck(
4608            ArgumentsEscaped, JSValueRegs(), 0,
4609            m_jit.branchTest64(
4610                JITCompiler::NonZero,
4611                JITCompiler::addressFor(
4612                    m_jit.graph().machineArgumentsRegisterFor(node->origin.semantic))));
4613        noResult(node);
4614        break;
4615    }
4616
4617    case NewFunctionNoCheck:
4618        compileNewFunctionNoCheck(node);
4619        break;
4620
4621    case NewFunction: {
4622        JSValueOperand value(this, node->child1());
4623        GPRTemporary result(this, Reuse, value);
4624
4625        GPRReg valueGPR = value.gpr();
4626        GPRReg resultGPR = result.gpr();
4627
4628        m_jit.move(valueGPR, resultGPR);
4629
4630        JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR);
4631
4632        addSlowPathGenerator(
4633            slowPathCall(
4634                notCreated, this, operationNewFunction,
4635                resultGPR, m_jit.codeBlock()->functionDecl(node->functionDeclIndex())));
4636
4637        jsValueResult(resultGPR, node);
4638        break;
4639    }
4640
4641    case NewFunctionExpression:
4642        compileNewFunctionExpression(node);
4643        break;
4644
4645    case In:
4646        compileIn(node);
4647        break;
4648
4649    case CountExecution:
4650        m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address()));
4651        break;
4652
4653    case ForceOSRExit: {
4654        terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
4655        break;
4656    }
4657
4658    case InvalidationPoint:
4659        emitInvalidationPoint(node);
4660        break;
4661
4662    case CheckWatchdogTimer:
4663        ASSERT(m_jit.vm()->watchdog);
4664        speculationCheck(
4665            WatchdogTimerFired, JSValueRegs(), 0,
4666            m_jit.branchTest8(
4667                JITCompiler::NonZero,
4668                JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog->timerDidFireAddress())));
4669        break;
4670
4671    case Phantom:
4672    case HardPhantom:
4673        DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate);
4674        noResult(node);
4675        break;
4676
4677    case Breakpoint:
4678    case ProfileWillCall:
4679    case ProfileDidCall:
4680    case PhantomLocal:
4681    case LoopHint:
4682        // This is a no-op.
4683        noResult(node);
4684        break;
4685
4686    case Unreachable:
4687        RELEASE_ASSERT_NOT_REACHED();
4688        break;
4689
4690    case StoreBarrier:
4691    case StoreBarrierWithNullCheck: {
4692        compileStoreBarrier(node);
4693        break;
4694    }
4695
4696#if ENABLE(FTL_JIT)
4697    case CheckTierUpInLoop: {
4698        MacroAssembler::Jump done = m_jit.branchAdd32(
4699            MacroAssembler::Signed,
4700            TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()),
4701            MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter));
4702
4703        silentSpillAllRegisters(InvalidGPRReg);
4704        m_jit.setupArgumentsExecState();
4705        appendCall(triggerTierUpNow);
4706        silentFillAllRegisters(InvalidGPRReg);
4707
4708        done.link(&m_jit);
4709        break;
4710    }
4711
4712    case CheckTierUpAtReturn: {
4713        MacroAssembler::Jump done = m_jit.branchAdd32(
4714            MacroAssembler::Signed,
4715            TrustedImm32(Options::ftlTierUpCounterIncrementForReturn()),
4716            MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter));
4717
4718        silentSpillAllRegisters(InvalidGPRReg);
4719        m_jit.setupArgumentsExecState();
4720        appendCall(triggerTierUpNow);
4721        silentFillAllRegisters(InvalidGPRReg);
4722
4723        done.link(&m_jit);
4724        break;
4725    }
4726
4727    case CheckTierUpAndOSREnter: {
4728        ASSERT(!node->origin.semantic.inlineCallFrame);
4729
4730        GPRTemporary temp(this);
4731        GPRReg tempGPR = temp.gpr();
4732
4733        MacroAssembler::Jump done = m_jit.branchAdd32(
4734            MacroAssembler::Signed,
4735            TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()),
4736            MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter));
4737
4738        silentSpillAllRegisters(tempGPR);
4739        m_jit.setupArgumentsWithExecState(
4740            TrustedImm32(node->origin.semantic.bytecodeIndex),
4741            TrustedImm32(m_stream->size()));
4742        appendCallSetResult(triggerOSREntryNow, tempGPR);
4743        MacroAssembler::Jump dontEnter = m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR);
4744        m_jit.jump(tempGPR);
4745        dontEnter.link(&m_jit);
4746        silentFillAllRegisters(tempGPR);
4747
4748        done.link(&m_jit);
4749        break;
4750    }
4751#else // ENABLE(FTL_JIT)
4752    case CheckTierUpInLoop:
4753    case CheckTierUpAtReturn:
4754    case CheckTierUpAndOSREnter:
4755        RELEASE_ASSERT_NOT_REACHED();
4756        break;
4757#endif // ENABLE(FTL_JIT)
4758
4759    case LastNodeType:
4760    case Phi:
4761    case Upsilon:
4762    case GetArgument:
4763    case ExtractOSREntryLocal:
4764    case CheckInBounds:
4765    case ArithIMul:
4766    case MultiGetByOffset:
4767    case MultiPutByOffset:
4768    case FiatInt52:
4769        RELEASE_ASSERT_NOT_REACHED();
4770        break;
4771    }
4772
4773    if (!m_compileOkay)
4774        return;
4775
4776    if (node->hasResult() && node->mustGenerate())
4777        use(node);
4778}
4779
4780#if ENABLE(GGC)
4781void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2)
4782{
4783    JITCompiler::Jump isNotCell;
4784    if (!isKnownCell(valueUse.node()))
4785        isNotCell = branchNotCell(JSValueRegs(valueGPR));
4786
4787    JITCompiler::Jump ownerNotMarkedOrAlreadyRemembered = m_jit.checkMarkByte(ownerGPR);
4788    storeToWriteBarrierBuffer(ownerGPR, scratch1, scratch2);
4789    ownerNotMarkedOrAlreadyRemembered.link(&m_jit);
4790
4791    if (!isKnownCell(valueUse.node()))
4792        isNotCell.link(&m_jit);
4793}
4794
4795void SpeculativeJIT::writeBarrier(JSCell* owner, GPRReg valueGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2)
4796{
4797    JITCompiler::Jump isNotCell;
4798    if (!isKnownCell(valueUse.node()))
4799        isNotCell = branchNotCell(JSValueRegs(valueGPR));
4800
4801    JITCompiler::Jump ownerNotMarkedOrAlreadyRemembered = m_jit.checkMarkByte(owner);
4802    storeToWriteBarrierBuffer(owner, scratch1, scratch2);
4803    ownerNotMarkedOrAlreadyRemembered.link(&m_jit);
4804
4805    if (!isKnownCell(valueUse.node()))
4806        isNotCell.link(&m_jit);
4807}
4808#endif // ENABLE(GGC)
4809
4810JITCompiler::Jump SpeculativeJIT::branchIsCell(JSValueRegs regs)
4811{
4812    return m_jit.branchTest64(MacroAssembler::Zero, regs.gpr(), GPRInfo::tagMaskRegister);
4813}
4814
4815JITCompiler::Jump SpeculativeJIT::branchNotCell(JSValueRegs regs)
4816{
4817    return m_jit.branchTest64(MacroAssembler::NonZero, regs.gpr(), GPRInfo::tagMaskRegister);
4818}
4819
4820JITCompiler::Jump SpeculativeJIT::branchIsOther(JSValueRegs regs, GPRReg tempGPR)
4821{
4822    m_jit.move(regs.gpr(), tempGPR);
4823    m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), tempGPR);
4824    return m_jit.branch64(
4825        MacroAssembler::Equal, tempGPR,
4826        MacroAssembler::TrustedImm64(ValueNull));
4827}
4828
4829JITCompiler::Jump SpeculativeJIT::branchNotOther(JSValueRegs regs, GPRReg tempGPR)
4830{
4831    m_jit.move(regs.gpr(), tempGPR);
4832    m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), tempGPR);
4833    return m_jit.branch64(
4834        MacroAssembler::NotEqual, tempGPR,
4835        MacroAssembler::TrustedImm64(ValueNull));
4836}
4837
4838void SpeculativeJIT::moveTrueTo(GPRReg gpr)
4839{
4840    m_jit.move(TrustedImm32(ValueTrue), gpr);
4841}
4842
4843void SpeculativeJIT::moveFalseTo(GPRReg gpr)
4844{
4845    m_jit.move(TrustedImm32(ValueFalse), gpr);
4846}
4847
4848void SpeculativeJIT::blessBoolean(GPRReg gpr)
4849{
4850    m_jit.or32(TrustedImm32(ValueFalse), gpr);
4851}
4852
4853void SpeculativeJIT::convertMachineInt(Edge valueEdge, GPRReg resultGPR)
4854{
4855    JSValueOperand value(this, valueEdge, ManualOperandSpeculation);
4856    GPRReg valueGPR = value.gpr();
4857
4858    JITCompiler::Jump notInt32 =
4859        m_jit.branch64(JITCompiler::Below, valueGPR, GPRInfo::tagTypeNumberRegister);
4860
4861    m_jit.signExtend32ToPtr(valueGPR, resultGPR);
4862    JITCompiler::Jump done = m_jit.jump();
4863
4864    notInt32.link(&m_jit);
4865    silentSpillAllRegisters(resultGPR);
4866    callOperation(operationConvertBoxedDoubleToInt52, resultGPR, valueGPR);
4867    silentFillAllRegisters(resultGPR);
4868
4869    DFG_TYPE_CHECK(
4870        JSValueRegs(valueGPR), valueEdge, SpecInt32 | SpecInt52AsDouble,
4871        m_jit.branch64(
4872            JITCompiler::Equal, resultGPR,
4873            JITCompiler::TrustedImm64(JSValue::notInt52)));
4874    done.link(&m_jit);
4875}
4876
4877void SpeculativeJIT::speculateMachineInt(Edge edge)
4878{
4879    if (!needsTypeCheck(edge, SpecInt32 | SpecInt52AsDouble))
4880        return;
4881
4882    GPRTemporary temp(this);
4883    convertMachineInt(edge, temp.gpr());
4884}
4885
4886void SpeculativeJIT::speculateDoubleRepMachineInt(Edge edge)
4887{
4888    if (!needsTypeCheck(edge, SpecInt52AsDouble))
4889        return;
4890
4891    SpeculateDoubleOperand value(this, edge);
4892    FPRReg valueFPR = value.fpr();
4893
4894    GPRResult result(this);
4895    GPRReg resultGPR = result.gpr();
4896
4897    flushRegisters();
4898
4899    callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR);
4900
4901    DFG_TYPE_CHECK(
4902        JSValueRegs(), edge, SpecInt52AsDouble,
4903        m_jit.branch64(
4904            JITCompiler::Equal, resultGPR,
4905            JITCompiler::TrustedImm64(JSValue::notInt52)));
4906}
4907
4908#endif
4909
4910} } // namespace JSC::DFG
4911
4912#endif
4913