{

JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);

proc->dump(WTF::dataFile());

static constexpr uintptr_t isMaterializedMask = 0x1;

OMGExpression() = default;

OMGExpression(Value* value)

: m_storage(reinterpret_cast<uintptr_t>(value))

{

ASSERT(!isEmpty() && !isMaterialized() && b3Value() == value);

}

bool NODELETE isEmpty() const

{

return !m_storage;

}

bool NODELETE isMaterialized() const

{

return m_storage & isMaterializedMask;

}

void NODELETE setMaterialized(B3::Variable* var)

{

ASSERT(!isMaterialized());

m_storage = reinterpret_cast<uintptr_t>(var) | isMaterializedMask;

ASSERT(isMaterialized() && b3Variable() == var);

}

// Not named value() to disambiguate with TypedExpression::value()

B3::Value* NODELETE b3Value() const

{

ASSERT(!isMaterialized());

// No need to mask in this common case since the isMaterialized bit will be 0

return reinterpret_cast<B3::Value*>(m_storage);

}

B3::Variable* NODELETE b3Variable() const

{

ASSERT(isMaterialized());

return reinterpret_cast<B3::Variable*>(m_storage & ~isMaterializedMask);

}

B3::Type NODELETE type() const

{

if (isMaterialized()) [[unlikely]]

return b3Variable()->type();

return b3Value()->type();

}

void dump(PrintStream& out) const

{

if (isMaterialized())

out.print(*b3Variable());

else

out.print(*b3Value());

}

uintptr_t m_storage;

WTF_MAKE_TZONE_ALLOCATED(OMGIRGenerator);

using ExpressionType = OMGExpression;

using ResultList = Vector<ExpressionType, 8>;

using CallType = CallLinkInfo::CallType;

using CallPatchpointData = std::tuple<B3::PatchpointValue*, RefPtr<PatchpointExceptionHandle>, RefPtr<B3::StackmapGenerator>>;

using WasmConstRefValue = Const64Value;

static constexpr bool shouldFuseBranchCompare = false;

static constexpr bool NODELETE tierSupportsSIMD() { return true; }

static constexpr bool validateFunctionBodySize = true;

struct ControlData {

ControlData(Procedure& proc, Origin origin, BlockSignature&& signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)

: controlBlockType(type)

, m_signature(WTF::move(signature))

, continuation(continuation)

, special(special)

{

ASSERT(type != BlockType::Try && type != BlockType::Catch);

if (type == BlockType::Loop) {

for (unsigned i = 0; i < m_signature.argumentCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.argumentType(i)), origin));

} else {

for (unsigned i = 0; i < m_signature.returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.returnType(i)), origin));

}

}

ControlData(Procedure& proc, Origin origin, BlockSignature&& signature, BlockType type, BasicBlock* continuation, unsigned tryStart, unsigned tryDepth)

: controlBlockType(type)

, m_signature(WTF::move(signature))

, continuation(continuation)

, special(nullptr)

, m_tryStart(tryStart)

, m_tryCatchDepth(tryDepth)

{

ASSERT(type == BlockType::Try || type == BlockType::TryTable);

for (unsigned i = 0; i < m_signature.returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.returnType(i)), origin));

}

ControlData()

{

}

static bool NODELETE isIf(const ControlData& control) { return control.blockType() == BlockType::If; }

static bool NODELETE isElse(const ControlData& control) { return control.blockType() == BlockType::Else; }

static bool NODELETE isTry(const ControlData& control) { return control.blockType() == BlockType::Try; }

static bool NODELETE isTryTable(const ControlData& control) { return control.blockType() == BlockType::TryTable; }

static bool NODELETE isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; }

static bool NODELETE isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }

static bool NODELETE isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; }

static bool NODELETE isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; }

static bool NODELETE isCatch(const ControlData& control)

{

if (control.blockType() != BlockType::Catch)

return false;

return control.catchKind() == CatchKind::Catch;

}

void dump(PrintStream& out) const

{

switch (blockType()) {

case BlockType::If:

out.print("If: ");

break;

case BlockType::Else:

out.print("Else: ");

break;

case BlockType::Block:

out.print("Block: ");

break;

case BlockType::Loop:

out.print("Loop: ");

break;

case BlockType::TopLevel:

out.print("TopLevel: ");

break;

case BlockType::Try:

out.print("Try: ");

break;

case BlockType::TryTable:

out.print("TryTable: ");

break;

case BlockType::Catch:

out.print("Catch: ");

break;

}

out.print("Continuation: ", *continuation, ", Special: ");

if (special)

out.print(*special);

else

out.print("None");

}

BlockType NODELETE blockType() const { return controlBlockType; }

const BlockSignature& NODELETE signature() const { return m_signature; }

bool hasNonVoidresult() const { return m_signature.returnCount() > 0; }

BasicBlock* NODELETE targetBlockForBranch()

{

if (blockType() == BlockType::Loop)

return special;

return continuation;

}

void NODELETE convertIfToBlock()

{

ASSERT(blockType() == BlockType::If);

controlBlockType = BlockType::Block;

special = nullptr;

}

void NODELETE convertTryToCatch(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::Catch;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

void NODELETE convertTryToCatchAll(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::CatchAll;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

struct TryTableTarget {

CatchKind type;

uint32_t tag;

const RTT* exceptionSignature;

ControlRef target;

};

using TargetList = Vector<TryTableTarget>;

void setTryTableTargets(TargetList&& targets)

{

m_tryTableTargets = WTF::move(targets);

}

void NODELETE endTryTable(unsigned tryEndCallSiteIndex)

{

ASSERT(blockType() == BlockType::TryTable);

m_tryEnd = tryEndCallSiteIndex;

}

FunctionArgCount branchTargetArity() const

{

if (blockType() == BlockType::Loop)

return m_signature.argumentCount();

return m_signature.returnCount();

}

Type branchTargetType(unsigned i) const

{

ASSERT(i < branchTargetArity());

if (blockType() == BlockType::Loop)

return m_signature.argumentType(i);

return m_signature.returnType(i);

}

unsigned NODELETE tryStart() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::TryTable || controlBlockType == BlockType::Catch);

return m_tryStart;

}

unsigned NODELETE tryEnd() const

{

ASSERT(controlBlockType == BlockType::Catch || controlBlockType == BlockType::TryTable);

return m_tryEnd;

}

unsigned NODELETE tryDepth() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::TryTable || controlBlockType == BlockType::Catch);

return m_tryCatchDepth;

}

CatchKind NODELETE catchKind() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_catchKind;

}

Variable* NODELETE exception() const

{

ASSERT(controlBlockType == BlockType::Catch || controlBlockType == BlockType::TryTable);

return m_exception;

}

private:

// FIXME: Compress OMGIRGenerator::ControlData fields using an union

// https://bugs.webkit.org/show_bug.cgi?id=231212

friend class OMGIRGenerator;

BlockType controlBlockType;

BlockSignature m_signature;

BasicBlock* continuation;

BasicBlock* special;

Vector<Value*> phis;

unsigned m_tryStart;

unsigned m_tryEnd;

unsigned m_tryCatchDepth;

CatchKind m_catchKind;

Variable* m_exception;

TargetList m_tryTableTargets;

};

using ControlType = ControlData;

using ExpressionList = Vector<ExpressionType, 1>;

using ControlEntry = FunctionParser<OMGIRGenerator>::ControlEntry;

using ControlStack = FunctionParser<OMGIRGenerator>::ControlStack;

using Stack = FunctionParser<OMGIRGenerator>::Stack;

using TypedExpression = FunctionParser<OMGIRGenerator>::TypedExpression;

using CatchHandler = FunctionParser<OMGIRGenerator>::CatchHandler;

using ArgumentList = FunctionParser<OMGIRGenerator>::ArgumentList;

static_assert(std::is_same_v<ResultList, FunctionParser<OMGIRGenerator>::ResultList>);

typedef String ErrorType;

typedef std::unexpected<ErrorType> UnexpectedResult;

typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;

typedef Expected<void, ErrorType> PartialResult;

static ExpressionType NODELETE emptyExpression() { return { }; };

enum class CastKind { Cast, Test };

template <typename ...Args>

[[nodiscard]] NEVER_INLINE UnexpectedResult fail(Args... args) const

{

using namespace FailureHelper; // See ADL comment in WasmParser.h.

return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));

}

#define WASM_COMPILE_FAIL_IF(condition, ...) do { \

if (condition) [[unlikely]] \

return fail(__VA_ARGS__); \

} while (0)

unsigned NODELETE advanceCallSiteIndex()

{

if (m_inlineParent)

return m_inlineRoot->advanceCallSiteIndex();

return ++m_callSiteIndex;

}

unsigned NODELETE callSiteIndex() const

{

if (m_inlineParent)

return m_inlineRoot->callSiteIndex();

return m_callSiteIndex;

}

OMGIRGenerator(AbstractHeapRepository&, CompilationContext&, Module&, CalleeGroup&, const ModuleInformation&, IPIntCallee&, InliningNode*, OptimizingJITCallee&, Procedure&, Vector<UnlinkedWasmToWasmCall>&, FixedBitVector& outgoingDirectCallees, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry);

OMGIRGenerator(AbstractHeapRepository&, CompilationContext&, OMGIRGenerator& inlineCaller, OMGIRGenerator& inlineRoot, Module&, CalleeGroup&, unsigned functionIndex, IPIntCallee&, InliningNode*, BasicBlock* returnContinuation, Vector<Value*> args);

void computeStackCheckSize(bool& needsOverflowCheck, int32_t& checkSize);

Value* NODELETE wasmRefOfCell(Value* cell)

{

return cell;

}

Value* NODELETE pointerOfWasmRef(Value* ref)

{

return ref;

}

Value* pointerOfInt32(Value* value)

{

return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);

}

Value* int32OfPointer(Value* value)

{

return m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);

}

// A runtime operation takes an address operand -- an index, offset, count, delta or length -- as

// 64 bits, so that a 64-bit address memory or table can pass one whole, and it bounds checks all

// 64 bits. The upper half of an i32 operand is don't-care everywhere else, so zero it here.

Value* addressOperand(bool is64Bit, ExpressionType operand)

{

if (is64Bit)

return get(operand);

return pointerOfInt32(get(operand));

}

// SIMD

void NODELETE notifyFunctionUsesSIMD() { ASSERT(m_info.usesSIMD(m_functionIndex)); }

[[nodiscard]] PartialResult addSIMDLoad(ExpressionType pointer, uint64_t offset, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDStore(ExpressionType value, ExpressionType pointer, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDSplat(SIMDLane, ExpressionType scalar, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDShuffle(v128_t imm, ExpressionType a, ExpressionType b, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDShift(SIMDLaneOperation, SIMDInfo, ExpressionType v, ExpressionType shift, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDExtmul(SIMDLaneOperation, SIMDInfo, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDLoadSplat(SIMDLaneOperation, ExpressionType pointer, uint64_t offset, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDLoadLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint64_t offset, uint8_t laneIndex, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDStoreLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint64_t offset, uint8_t laneIndex, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDLoadExtend(SIMDLaneOperation, ExpressionType pointer, uint64_t offset, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addSIMDLoadPad(SIMDLaneOperation, ExpressionType pointer, uint64_t offset, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] ExpressionType addSIMDConstant(v128_t value)

{

return push(constant(B3::V128, value));

}

// SIMD generated

#define B3_OP_CASE(OP) \

else if (op == SIMDLaneOperation::OP) b3Op = B3::Vector##OP;

#define B3_OP_CASES() \

B3::Opcode b3Op = B3::Oops; \

if (false) { }

auto addSIMDExtractLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorExtractLane, toB3Type(simdScalarType(info.lane)), info,

lane,

get(v)));

return { };

}

auto addSIMDReplaceLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType s, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorReplaceLane, B3::V128, info,

lane,

get(v),

get(s)));

return { };

}

auto addSIMDI_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Bitmask)

B3_OP_CASE(AnyTrue)

B3_OP_CASE(AllTrue)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::Int32, info,

get(v)));

return { };

}

auto addSIMDV_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Demote)

B3_OP_CASE(Promote)

B3_OP_CASE(Abs)

B3_OP_CASE(Popcnt)

B3_OP_CASE(Ceil)

B3_OP_CASE(Floor)

B3_OP_CASE(Trunc)

B3_OP_CASE(Nearest)

B3_OP_CASE(Sqrt)

B3_OP_CASE(ExtaddPairwise)

B3_OP_CASE(Convert)

B3_OP_CASE(ConvertLow)

B3_OP_CASE(ExtendHigh)

B3_OP_CASE(ExtendLow)

B3_OP_CASE(TruncSat)

B3_OP_CASE(RelaxedTruncSat)

B3_OP_CASE(Not)

B3_OP_CASE(Neg)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(v)));

return { };

}

auto addSIMDBitwiseSelect(ExpressionType v1, ExpressionType v2, ExpressionType c, ExpressionType& result) -> PartialResult

{

auto b3Op = B3::VectorBitwiseSelect;

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, SIMDInfo { SIMDLane::v128, SIMDSignMode::None },

get(v1), get(v2), get(c)));

return { };

}

auto addSIMDRelOp(SIMDLaneOperation, SIMDInfo info, ExpressionType lhs, ExpressionType rhs, Air::Arg relOp, ExpressionType& result) -> PartialResult

{

B3::Opcode b3Op = Oops;

if (scalarTypeIsIntegral(info.lane)) {

switch (relOp.asRelationalCondition()) {

case MacroAssembler::Equal:

b3Op = VectorEqual;

break;

case MacroAssembler::NotEqual:

b3Op = VectorNotEqual;

break;

case MacroAssembler::LessThan:

b3Op = VectorLessThan;

break;

case MacroAssembler::LessThanOrEqual:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::Below:

b3Op = VectorBelow;

break;

case MacroAssembler::BelowOrEqual:

b3Op = VectorBelowOrEqual;

break;

case MacroAssembler::GreaterThan:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::GreaterThanOrEqual:

b3Op = VectorGreaterThanOrEqual;

break;

case MacroAssembler::Above:

b3Op = VectorAbove;

break;

case MacroAssembler::AboveOrEqual:

b3Op = VectorAboveOrEqual;

break;

}

} else {

switch (relOp.asDoubleCondition()) {

case MacroAssembler::DoubleEqualAndOrdered:

b3Op = VectorEqual;

break;

case MacroAssembler::DoubleNotEqualOrUnordered:

b3Op = VectorNotEqual;

break;

case MacroAssembler::DoubleLessThanAndOrdered:

b3Op = VectorLessThan;

break;

case MacroAssembler::DoubleLessThanOrEqualAndOrdered:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::DoubleGreaterThanAndOrdered:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::DoubleGreaterThanOrEqualAndOrdered:

b3Op = VectorGreaterThanOrEqual;

break;

default:

RELEASE_ASSERT_NOT_REACHED();

break;

}

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(lhs), get(rhs)));

return { };

}

Value* fixupOutOfBoundsIndicesForSwizzle(Value* input, Value* indexes)

{

// The intel version of the swizzle instruction does not handle OOB indices properly,

// so we need to fix them up.

ASSERT(isX86());

// Let each byte mask be 112 (0x70) then after VectorAddSat

// each index > 15 would set the saturated index's bit 7 to 1,

// whose corresponding byte will be zero cleared in VectorSwizzle.

// https://github.com/WebAssembly/simd/issues/93

v128_t mask;

mask.u64x2[0] = 0x7070707070707070;

mask.u64x2[1] = 0x7070707070707070;

auto saturatingMask = m_currentBlock->appendNew<Const128Value>(m_proc, origin(), mask);

auto saturatedIndexes = m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorAddSat, B3::V128, SIMDLane::i8x16, SIMDSignMode::Unsigned, saturatingMask, indexes);

return m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorSwizzle, B3::V128, SIMDLane::i8x16, SIMDSignMode::None, input, saturatedIndexes);

}

auto addSIMDV_VV(SIMDLaneOperation op, SIMDInfo info, ExpressionType a, ExpressionType b, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(And)

B3_OP_CASE(Andnot)

B3_OP_CASE(AvgRound)

B3_OP_CASE(DotProduct)

B3_OP_CASE(Add)

B3_OP_CASE(Mul)

B3_OP_CASE(MulSat)

B3_OP_CASE(Sub)

B3_OP_CASE(Div)

B3_OP_CASE(Pmax)

B3_OP_CASE(Pmin)

B3_OP_CASE(Or)

B3_OP_CASE(Swizzle)

B3_OP_CASE(RelaxedSwizzle)

B3_OP_CASE(Xor)

B3_OP_CASE(Narrow)

B3_OP_CASE(AddSat)

B3_OP_CASE(SubSat)

B3_OP_CASE(Max)

B3_OP_CASE(Min)

B3_OP_CASE(RelaxedMin)

B3_OP_CASE(RelaxedMax)

B3_OP_CASE(RelaxedQ15Mulr)

B3_OP_CASE(RelaxedDotI8x16I7x16)

if (isX86() && b3Op == B3::VectorSwizzle) {

result = push(fixupOutOfBoundsIndicesForSwizzle(get(a), get(b)));

return { };

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(a), get(b)));

return { };

}

auto addSIMDRelaxedFMA(SIMDLaneOperation op, SIMDInfo info, ExpressionType m1, ExpressionType m2, ExpressionType add, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(RelaxedMAdd)

B3_OP_CASE(RelaxedNMAdd)

B3_OP_CASE(RelaxedDotI8x16I7x16Add)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(m1), get(m2), get(add)));

return { };

}

[[nodiscard]] PartialResult addDrop(ExpressionType);

[[nodiscard]] PartialResult addInlinedArguments(const RTT&);

[[nodiscard]] PartialResult addArguments(const RTT&);

[[nodiscard]] PartialResult addLocal(Type, uint32_t);

ExpressionType addConstant(Type, uint64_t);

// References

[[nodiscard]] PartialResult addRefIsNull(ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addRefFunc(FunctionSpaceIndex index, ExpressionType& result);

[[nodiscard]] PartialResult addRefAsNonNull(TypedExpression, ExpressionType&);

[[nodiscard]] PartialResult addRefEq(ExpressionType, ExpressionType, ExpressionType&);

// Tables

[[nodiscard]] PartialResult addTableGet(unsigned, ExpressionType index, ExpressionType& result);

[[nodiscard]] PartialResult addTableSet(unsigned, ExpressionType index, ExpressionType value);

[[nodiscard]] PartialResult addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

[[nodiscard]] PartialResult addElemDrop(unsigned);

[[nodiscard]] PartialResult addTableSize(unsigned, ExpressionType& result);

[[nodiscard]] PartialResult addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);

[[nodiscard]] PartialResult addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);

[[nodiscard]] PartialResult addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

// Locals

[[nodiscard]] PartialResult getLocal(uint32_t index, ExpressionType& result);

[[nodiscard]] PartialResult setLocal(uint32_t index, ExpressionType value);

[[nodiscard]] PartialResult teeLocal(uint32_t, ExpressionType, ExpressionType& result);

// Globals

[[nodiscard]] PartialResult getGlobal(uint32_t index, ExpressionType& result);

[[nodiscard]] PartialResult setGlobal(uint32_t index, ExpressionType value);

// Memory

[[nodiscard]] PartialResult load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult store(StoreOpType, ExpressionType pointer, ExpressionType value, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult addGrowMemory(ExpressionType delta, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addCurrentMemory(ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count, uint8_t memoryIndex);

[[nodiscard]] PartialResult addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count, uint8_t dstMemoryIndex, uint8_t srcMemoryIndex);

void emitMemoryFill(Value* dstAddress, Value* targetValue, Value* count, uint8_t memoryIndex);

void emitMemoryCopy(Value* dstAddress, Value* srcAddress, Value* count, uint8_t dstMemoryIndex, uint8_t srcMemoryIndex);

[[nodiscard]] PartialResult addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length, uint8_t memoryIndex);

[[nodiscard]] PartialResult addDataDrop(unsigned);

// Atomics

[[nodiscard]] PartialResult atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint64_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult atomicFence(ExtAtomicOpType, uint8_t flags);

// Saturated truncation.

[[nodiscard]] PartialResult truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type returnType, Type operandType);

// Wide arithmetic.

[[nodiscard]] PartialResult addI64Add128(ExpressionType lhsLo, ExpressionType lhsHi, ExpressionType rhsLo, ExpressionType rhsHi, ExpressionType& resultLo, ExpressionType& resultHi);

[[nodiscard]] PartialResult addI64Sub128(ExpressionType lhsLo, ExpressionType lhsHi, ExpressionType rhsLo, ExpressionType rhsHi, ExpressionType& resultLo, ExpressionType& resultHi);

[[nodiscard]] PartialResult addI64MulWideS(ExpressionType lhs, ExpressionType rhs, ExpressionType& resultLo, ExpressionType& resultHi);

[[nodiscard]] PartialResult addI64MulWideU(ExpressionType lhs, ExpressionType rhs, ExpressionType& resultLo, ExpressionType& resultHi);

B3::Type int64PairTupleType();

// GC

[[nodiscard]] PartialResult addRefI31(ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addI31GetS(TypedExpression ref, ExpressionType& result);

[[nodiscard]] PartialResult addI31GetU(TypedExpression ref, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNew(TypeSignatureIndex index, ExpressionType size, ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewDefault(TypeSignatureIndex index, ExpressionType size, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewFixed(TypeSignatureIndex typeIndex, ArgumentList& args, ExpressionType& result);

[[nodiscard]] PartialResult addArrayGet(ExtGCOpType arrayGetKind, TypeSignatureIndex typeIndex, TypedExpression arrayref, ExpressionType index, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewData(TypeSignatureIndex typeIndex, uint32_t dataIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewElem(TypeSignatureIndex typeIndex, uint32_t elemSegmentIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

[[nodiscard]] PartialResult addArraySet(TypeSignatureIndex typeIndex, TypedExpression arrayref, ExpressionType index, ExpressionType value);

[[nodiscard]] PartialResult addArrayLen(TypedExpression arrayref, ExpressionType& result);

[[nodiscard]] PartialResult addArrayFill(TypeSignatureIndex, TypedExpression, ExpressionType, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayCopy(TypeSignatureIndex, TypedExpression, ExpressionType, TypeSignatureIndex, TypedExpression, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayInitElem(TypeSignatureIndex, TypedExpression, ExpressionType, uint32_t, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayInitData(TypeSignatureIndex, TypedExpression, ExpressionType, uint32_t, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addStructNew(TypeSignatureIndex typeIndex, ArgumentList& args, ExpressionType& result);

[[nodiscard]] PartialResult addStructNewDefault(TypeSignatureIndex index, ExpressionType& result);

[[nodiscard]] PartialResult addStructGet(ExtGCOpType structGetKind, TypedExpression structReference, const RTT&, uint32_t fieldIndex, ExpressionType& result);

[[nodiscard]] PartialResult addStructSet(TypedExpression structReference, const RTT&, uint32_t fieldIndex, ExpressionType value);

[[nodiscard]] PartialResult addRefTest(TypedExpression reference, bool allowNull, int32_t heapType, bool shouldNegate, ExpressionType& result);

[[nodiscard]] PartialResult addRefCast(TypedExpression reference, bool allowNull, int32_t heapType, ExpressionType& result);

[[nodiscard]] PartialResult addAnyConvertExtern(ExpressionType reference, ExpressionType& result);

[[nodiscard]] PartialResult addExternConvertAny(ExpressionType reference, ExpressionType& result);

// Basic operators

#define X(name, opcode, short, idx, ...) \

[[nodiscard]] PartialResult add##name(ExpressionType arg, ExpressionType& result);

FOR_EACH_WASM_UNARY_OP(X)

#undef X

#define X(name, opcode, short, idx, ...) \

[[nodiscard]] PartialResult add##name(ExpressionType left, ExpressionType right, ExpressionType& result);

FOR_EACH_WASM_BINARY_OP(X)

#undef X

[[nodiscard]] PartialResult addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);

// Control flow

[[nodiscard]] ControlData addTopLevel(BlockSignature&&);

[[nodiscard]] PartialResult addBlock(BlockSignature&&, std::span<TypedExpression> args, ControlType& newBlock);

[[nodiscard]] PartialResult addLoop(BlockSignature&&, std::span<TypedExpression> args, ControlType& block, uint32_t loopIndex);

std::optional<ByteLoopIdiom> matchByteLoopIdiom(const BlockSignature&);

void emitByteLoopIdiom(const ByteLoopIdiom&, ControlType& loop, BasicBlock* body);

[[nodiscard]] PartialResult addIf(ExpressionType condition, BlockSignature&&, std::span<TypedExpression> args, ControlType& result);

[[nodiscard]] PartialResult addElse(ControlData&, std::span<const TypedExpression>);

[[nodiscard]] PartialResult addElseToUnreachable(ControlData&);

[[nodiscard]] PartialResult addTry(BlockSignature&&, std::span<TypedExpression> args, ControlType& result);

[[nodiscard]] PartialResult addTryTable(BlockSignature&&, std::span<TypedExpression> args, const Vector<CatchHandler>& targets, ControlType& result);

[[nodiscard]] PartialResult addCatch(unsigned exceptionIndex, const RTT&, std::span<const TypedExpression>, ControlType&, ResultList&);

[[nodiscard]] PartialResult addCatchToUnreachable(unsigned exceptionIndex, const RTT&, ControlType&, ResultList&);

[[nodiscard]] PartialResult addCatchAll(std::span<const TypedExpression>, ControlType&);

[[nodiscard]] PartialResult addCatchAllToUnreachable(ControlType&);

[[nodiscard]] PartialResult addDelegate(ControlType&, ControlType&);

[[nodiscard]] PartialResult addDelegateToUnreachable(ControlType&, ControlType&);

[[nodiscard]] PartialResult addThrow(unsigned exceptionIndex, ArgumentList& args, std::span<const TypedExpression>);

[[nodiscard]] PartialResult addRethrow(unsigned, ControlType&);

[[nodiscard]] PartialResult addThrowRef(TypedExpression exception, std::span<const TypedExpression>);

[[nodiscard]] PartialResult addInlinedReturn(const auto& returnValues);

[[nodiscard]] PartialResult addReturn(const ControlData&, std::span<const TypedExpression> returnValues);

[[nodiscard]] PartialResult addBranch(ControlData&, ExpressionType condition, std::span<const TypedExpression> returnValues);

[[nodiscard]] PartialResult addBranchNull(ControlType&, ExpressionType, std::span<const TypedExpression>, bool, ExpressionType&);

[[nodiscard]] PartialResult addBranchCast(ControlType&, TypedExpression, std::span<const TypedExpression>, bool, int32_t, bool);

[[nodiscard]] PartialResult addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, std::span<const TypedExpression> expressionStack);

[[nodiscard]] PartialResult endBlock(ControlEntry&, std::span<TypedExpression> enclosedStack);

[[nodiscard]] PartialResult addEndToUnreachable(ControlEntry&, std::span<TypedExpression> enclosedStack);

[[nodiscard]] PartialResult NODELETE endTopLevel(std::span<const TypedExpression>) { return { }; }

// Fused comparison stubs (B3 will do this for us later).

[[nodiscard]] PartialResult NODELETE addFusedBranchCompare(OpType, ControlType&, ExpressionType, std::span<const TypedExpression>) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult NODELETE addFusedBranchCompare(OpType, ControlType&, ExpressionType, ExpressionType, std::span<const TypedExpression>) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult NODELETE addFusedIfCompare(OpType, ExpressionType, BlockSignature&&, std::span<TypedExpression>, ControlType&) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult NODELETE addFusedIfCompare(OpType, ExpressionType, ExpressionType, BlockSignature&&, std::span<TypedExpression>, ControlType&) { RELEASE_ASSERT_NOT_REACHED(); }

// Calls

[[nodiscard]] PartialResult addCall(unsigned, FunctionSpaceIndex functionIndexSpace, const RTT&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addCallIndirect(unsigned, unsigned tableIndex, const RTT&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addCallRef(unsigned, const RTT&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addUnreachable();

[[nodiscard]] PartialResult addCrash();

using ValueResults = Vector<Value*, 16>;

void fillCallResults(Value* callResult, const RTT& signature, ValueResults&);

[[nodiscard]] PartialResult emitDirectCall(unsigned, FunctionSpaceIndex functionIndexSpace, const RTT&, const ArgumentList& args, ValueResults&, CallType = CallType::Call);

[[nodiscard]] PartialResult emitIndirectCall(Value* calleeInstance, Value* calleeCode, Value* boxedCalleeCallee, const RTT&, const ArgumentList& args, ValueResults&, CallType = CallType::Call);

Vector<ConstrainedValue> createCallConstrainedArgs(BasicBlock*, const CallInformation& wasmCalleeInfo, const ArgumentList&);

auto createCallPatchpoint(BasicBlock*, const RTT&, const CallInformation&, const ArgumentList& tmpArgs) -> CallPatchpointData;

auto createTailCallPatchpoint(BasicBlock*, const RTT&, const CallInformation& wasmCallerInfoAsCallee, const CallInformation& wasmCalleeInfoAsCallee, const ArgumentList& tmpArgSourceLocations, Value* boxedCalleeCallee = nullptr) -> CallPatchpointData;

InliningNode* canInline(FunctionSpaceIndex functionIndexSpace, unsigned callProfileIndex) const;

[[nodiscard]] PartialResult emitInlineDirectCall(InliningNode*, FunctionCodeIndex calleeIndex, const RTT&, const ArgumentList& args, ValueResults&);

void dump(const ControlStack&, const Stack* expressionStack);

void NODELETE setParser(FunctionParser<OMGIRGenerator>* parser) { m_parser = parser; };

ALWAYS_INLINE void willParseOpcode() { }

ALWAYS_INLINE void willParseExtendedOpcode() { }

ALWAYS_INLINE void didParseOpcode() { }

void NODELETE didFinishParsingLocals() { }

void didPopValueFromStack(ExpressionType expr, ASCIILiteral message)

{

TRACE_VALUE(Wasm::Types::Void, get(expr), "pop at height: ", m_parser->expressionStack().size() + 1, " site: [", message, "], ", expr);

}

const RTT* getArrayRTT(TypeSignatureIndex);

void getArrayElementType(TypeSignatureIndex, StorageType&);

void getArrayRefType(TypeSignatureIndex, Type&);

Value* constant(B3::Type, uint64_t bits, std::optional<Origin> = std::nullopt);

Value* constant(B3::Type, v128_t bits, std::optional<Origin> = std::nullopt);

Value* framePointer();

void insertEntrySwitch();

void insertConstants();

B3::Type toB3ResultType(const RTT*);

void addStackMap(unsigned callSiteIndex, StackMap&& stackmap)

{

if (m_inlineParent) {

m_inlineRoot->addStackMap(callSiteIndex, WTF::move(stackmap));

return;

}

m_stackmaps.add(CallSiteIndex(callSiteIndex), WTF::move(stackmap));

}

StackMaps&& NODELETE takeStackmaps()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTF::move(m_stackmaps);

}

Vector<UnlinkedHandlerInfo>&& NODELETE takeExceptionHandlers()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTF::move(m_exceptionHandlers);

}

void NODELETE emitPrepareWasmOperation(BasicBlock*)

{

}

template<typename OperationType, typename ...Args>

Value* callWasmOperation(BasicBlock* block, B3::Type resultType, OperationType operation, Args&&... args)

{

emitPrepareWasmOperation(block);

static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Sanity check");

Value* operationValue = block->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operation));

return block->appendNew<CCallValue>(m_proc, resultType, origin(), operationValue, std::forward<Args>(args)...);

}

void emitExceptionCheck(CCallHelpers&, Origin, ExceptionType);

void emitWriteBarrierForJSWrapper();

void emitWriteBarrier(Value* cell);

Value* emitCheckAndPreparePointer(Value* pointer, uint64_t offset, uint32_t sizeOfOp, uint8_t memoryIndex);

B3::Kind memoryKind(B3::Opcode memoryOp, uint8_t memoryIndex);

Value* emitLoadOp(LoadOpType, Value* pointer, uint64_t offset, uint8_t memoryIndex);

void emitStoreOp(StoreOpType, Value* pointer, Value*, uint64_t offset, uint8_t memoryIndex);

Value* sanitizeAtomicResult(ExtAtomicOpType, Type, Value* result);

Value* emitAtomicLoadOp(ExtAtomicOpType, Type, Value* pointer, uint64_t offset, uint8_t memoryIndex);

void emitAtomicStoreOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint64_t offset, uint8_t memoryIndex);

Value* emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint64_t offset, uint8_t memoryIndex);

Value* emitAtomicCompareExchange(ExtAtomicOpType, Type, Value* pointer, Value* expected, Value*, uint64_t offset, uint8_t memoryIndex);

void mutatorFence();

Value* emitGetArraySize(Value* array, bool canTrap);

Value* emitGetArraySizeWithNullCheck(Type arrayType, Value*);

void emitArrayRangeCheck(Value* arraySize, Value* offset, Value* size, ExceptionType);

Value* emitArrayElementAddress(StorageType elementType, Value* array, Value* index);

void emitArrayFillRange(StorageType elementType, Value* array, Value* offset, Value* fillValue, Value* size);

void emitNullCheck(Value*, ExceptionType);

bool emitNullCheckBeforeAccess(Value*, ptrdiff_t offset);

void emitArraySetUnchecked(TypeSignatureIndex, Value*, Value*, Value*);

bool emitArraySetUncheckedWithoutWriteBarrier(TypeSignatureIndex, Value*, Value*, Value*);

// Returns true if a writeBarrier/mutatorFence is needed.

[[nodiscard]] bool emitStructSet(bool canTrap, Value*, uint32_t, const RTT&, Value*);

using ArraySegmentOperation = EncodedJSValue SYSV_ABI (&)(JSC::JSWebAssemblyInstance*, uint32_t, uint32_t, uint32_t, uint32_t);

[[nodiscard]] ExpressionType pushArrayNewFromSegment(ArraySegmentOperation, TypeSignatureIndex typeIndex, uint32_t segmentIndex, ExpressionType arraySize, ExpressionType offset, ExceptionType);

void emitRefTestOrCast(CastKind, TypedExpression, bool, int32_t, bool, ExpressionType&);

MemoryValue* loadGCObjectStructureID(TypeSignatureIndex typeIndex)

{

auto* structureID = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), instanceValue(), safeCast<int32_t>(JSWebAssemblyInstance::offsetOfGCObjectStructureID(m_info, typeIndex.rawIndex())));

m_heaps.decorateMemory(&m_heaps.JSWebAssemblyInstance_gcObjectStructureIDs[typeIndex.rawIndex()], structureID);

structureID->setReadsMutability(B3::Mutability::Immutable);

structureID->setControlDependent(false);

return structureID;

}

const B3::AbstractHeap* structFieldHeap(const RTT& rtt, uint32_t fieldIndex)

{

return &m_heaps.JSWebAssemblyStruct_fields[rtt.fieldHeapKey(fieldIndex)];

}

const B3::AbstractHeap* arrayElementHeap(const Wasm::StorageType& elementType, Value* indexValue)

{

B3::NumberedAbstractHeap* heap;

if (elementType.is<Wasm::PackedType>()) {

switch (elementType.as<Wasm::PackedType>()) {

case Wasm::PackedType::I8:

heap = &m_heaps.JSWebAssemblyArray_i8;

break;

case Wasm::PackedType::I16:

heap = &m_heaps.JSWebAssemblyArray_i16;

break;

default:

RELEASE_ASSERT_NOT_REACHED();

}

} else {