diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index d6badce75704c0..89e70bc6edd97d 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -667,6 +667,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableHWIntrinsic, W("EnableHWIntri #if defined(TARGET_AMD64) || defined(TARGET_X86) RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX, W("EnableAVX"), 1, "Allows AVX and dependent hardware intrinsics to be disabled") +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableADX, W("EnableADX"), 1, "Allows ADX carry-chain instructions to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX2, W("EnableAVX2"), 1, "Allows AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512, W("EnableAVX512"), 1, "Allows AVX512 F+BW+CD+DQ+VL and dependent hardware intrinsics to be disabled") RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512BMM, W("EnableAVX512BMM"), 1, "Allows AVX512BMM and dependent hardware intrinsics to be disabled") diff --git a/src/coreclr/inc/corinfoinstructionset.h b/src/coreclr/inc/corinfoinstructionset.h index 2f347572ee34d4..0e86fed61c92c0 100644 --- a/src/coreclr/inc/corinfoinstructionset.h +++ b/src/coreclr/inc/corinfoinstructionset.h @@ -118,6 +118,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_AMD64 #ifdef TARGET_X86 InstructionSet_X86Base=1, @@ -166,6 +167,7 @@ enum CORINFO_InstructionSet InstructionSet_SHA_X64=44, InstructionSet_WAITPKG_X64=45, InstructionSet_X86Serialize_X64=46, + InstructionSet_ADX=47, #endif // TARGET_X86 }; @@ -879,6 +881,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_AMD64 #ifdef TARGET_X86 case InstructionSet_X86Base : @@ -941,6 +945,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet) return "AVXVNNIINT"; case InstructionSet_AVXVNNIINT_V512 : return "AVXVNNIINT_V512"; + case InstructionSet_ADX : + return "ADX"; #endif // TARGET_X86 default: @@ -1063,6 +1069,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_AMD64 #ifdef TARGET_X86 case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base; @@ -1133,6 +1140,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512; case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT; case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512; + case READYTORUN_INSTRUCTION_Adx: return InstructionSet_ADX; #endif // TARGET_X86 default: diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index a1559f43510c20..847663cada9877 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* fbbaf45f-5b0e-4767-b962-5084c8caae77 */ - 0xfbbaf45f, - 0x5b0e, - 0x4767, - {0xb9, 0x62, 0x50, 0x84, 0xc8, 0xca, 0xae, 0x77} +constexpr GUID JITEEVersionIdentifier = { /* 539ba29b-25d8-4b34-895a-01f513a3683a */ + 0x539ba29b, + 0x25d8, + 0x4b34, + {0x89, 0x5a, 0x01, 0xf5, 0x13, 0xa3, 0x68, 0x3a} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/readytoruninstructionset.h b/src/coreclr/inc/readytoruninstructionset.h index d2851e91577f1e..ddcfb2af193445 100644 --- a/src/coreclr/inc/readytoruninstructionset.h +++ b/src/coreclr/inc/readytoruninstructionset.h @@ -103,6 +103,7 @@ enum ReadyToRunInstructionSet READYTORUN_INSTRUCTION_Cssc=93, READYTORUN_INSTRUCTION_Zicond=94, READYTORUN_INSTRUCTION_Fp16=95, + READYTORUN_INSTRUCTION_Adx=96, }; diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 588afa49dadfb5..69f07c93be9d0e 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2691,6 +2691,25 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) var_types targetType = tree->TypeGet(); emitter* emit = GetEmitter(); + if (oper == GT_ADD_BORROW) + { + assert(!tree->gtSetFlags() && tree->gtGetOp2()->IsIntegralConst(0)); + emit->emitIns_R_R_COND(INS_cinc, emitActualTypeSize(tree), targetReg, tree->gtGetOp1()->GetRegNum(), + INS_COND_LO); + genProduceReg(tree); + return; + } + if (oper == GT_ADD_CARRY || oper == GT_SUB_BORROW) + { + instruction ins = oper == GT_SUB_BORROW ? (tree->gtSetFlags() ? INS_sbcs : INS_sbc) + : (tree->gtSetFlags() ? INS_adcs : INS_adc); + emit->emitIns_R_R_R(ins, emitActualTypeSize(tree), targetReg, + tree->gtGetOp1()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp1()->GetRegNum(), + tree->gtGetOp2()->isContainedIntOrIImmed() ? REG_ZR : tree->gtGetOp2()->GetRegNum()); + genProduceReg(tree); + return; + } + assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR, GT_XOR_NOT)); @@ -2700,6 +2719,16 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) // The arithmetic node must be sitting in a register (since it's not contained) assert(targetReg != REG_NA); + if (tree->IsFunnelShift()) + { + assert(!tree->gtSetFlags()); + emit->emitIns_R_R_R_I(INS_extr, emitTypeSize(tree), targetReg, op2->gtGetOp1()->GetRegNum(), + op1->gtGetOp1()->GetRegNum(), + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(tree); + return; + } + // Handles combined operations: 'madd', 'msub' if (op2->OperIs(GT_MUL) && op2->isContained()) { diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 37cca461351f49..3e845f985f2b84 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -232,6 +232,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_ARM64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif case GT_ADD: case GT_SUB: case GT_MUL: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 36c4bcfd21f72f..59baa9c7deefd4 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -921,6 +921,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* treeNode) emit->emitInsBinary(ins, size, treeNode, divisor); // DIV/IDIV instructions always store the quotient in RAX and the remainder in RDX. + assert(!treeNode->IsDivRemPair() || (targetReg == REG_RAX)); // Move the result to the desired register, if necessary if (oper == GT_DIV || oper == GT_UDIV) { @@ -951,6 +952,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) { #ifdef DEBUG bool isValidOper = treeNode->OperIs(GT_ADD, GT_SUB); +#ifdef TARGET_AMD64 + isValidOper |= treeNode->OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW, GT_ADCX, GT_ADOX); +#endif if (varTypeIsFloating(treeNode->TypeGet())) { isValidOper |= treeNode->OperIs(GT_MUL, GT_DIV); @@ -975,6 +979,18 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) GenTree* op1 = treeNode->gtGetOp1(); GenTree* op2 = treeNode->gtGetOp2(); + if (treeNode->IsFunnelShift()) + { + regNumber lo = op1->gtGetOp1()->GetRegNum(); + regNumber hi = op2->gtGetOp1()->GetRegNum(); + assert(targetReg != hi); + inst_Mov(targetType, targetReg, lo, /* canSkip */ true); + inst_RV_RV_IV(INS_shrd, emitTypeSize(targetType), targetReg, hi, + static_cast(op1->gtGetOp2()->AsIntCon()->IconValue())); + genProduceReg(treeNode); + return; + } + bool eligibleForNDD = false; // Commutative operations can mark op1 as contained or reg-optional to generate "op reg, memop/immed" @@ -1098,7 +1114,8 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) assert(!varTypeIsFloating(treeNode)); // try to use an inc or dec - if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx()) + if (oper == GT_ADD && src->isContainedIntOrIImmed() && !treeNode->gtOverflowEx() && + ((treeNode->gtFlags & GTF_ADD_CARRY_FLAGS) == 0)) { if (src->IsIntegralConst(1)) { @@ -1940,6 +1957,13 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SUB_HI: #endif // !defined(TARGET_64BIT) +#ifdef TARGET_AMD64 + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + case GT_ADCX: + case GT_ADOX: +#endif case GT_ADD: case GT_SUB: genCodeForBinary(treeNode->AsOp()); @@ -2080,6 +2104,58 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForJTrue(treeNode->AsOp()); break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + { + // TEST clears CF and OF without a temporary register or changing SP. + GetEmitter()->emitIns_R_R(INS_test, EA_4BYTE, REG_ESP, REG_ESP); + break; + } + case GT_ADX_DRAIN: + { + genConsumeOperands(treeNode->AsOp()); + regNumber target = treeNode->GetRegNum(); + regNumber zero = internalRegisters.GetSingle(treeNode); + inst_Mov(TYP_LONG, target, treeNode->gtGetOp1()->GetRegNum(), true); + // A MOV is required: XOR would destroy the carries being drained. + GetEmitter()->emitIns_R_I(INS_mov, EA_4BYTE, zero, 0); + GetEmitter()->emitIns_R_R(INS_adcx, EA_8BYTE, target, zero); + GetEmitter()->emitIns_R_R(INS_adox, EA_8BYTE, target, zero); + genProduceReg(treeNode); + break; + } + case GT_JCMP: + { + genConsumeOperands(treeNode->AsOp()); + GenTree* count = treeNode->gtGetOp1(); + // Enregistered local uses can retain their source register here. + // Copy the count to RCX when necessary. The countdown's 32-bit LEA + // (or its 32-bit reload/copy) has already cleared the upper half when + // this move is skipped; bare JRCXZ tests all of RCX, not just ECX. + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(count), REG_ECX, count->GetRegNum(), true); + assert(treeNode->gtGetOp2()->IsIntegralConst(0) && treeNode->gtGetOp2()->isContained()); + // JRCXZ preserves both carry flags but has only a rel8 encoding. + // Skip a JMP to reach the loop: two branch instructions per backedge. + GenCondition::Code condition = treeNode->AsOpCC()->gtCondition.GetCode(); + assert(condition == GenCondition::EQ || condition == GenCondition::NE); + BasicBlock* done = genCreateTempLabel(); + if (condition == GenCondition::NE) + { + GetEmitter()->emitIns_J(INS_jrcxz, done, true); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + else + { + BasicBlock* taken = genCreateTempLabel(); + GetEmitter()->emitIns_J(INS_jrcxz, taken, true); + GetEmitter()->emitIns_J(INS_jmp, done, true); + genDefineTempLabel(taken); + GetEmitter()->emitIns_J(INS_jmp, m_compiler->compCurBB->GetTrueTarget()); + } + genDefineTempLabel(done); + break; + } +#endif case GT_JCC: genCodeForJcc(treeNode->AsCC()); break; @@ -4496,6 +4572,7 @@ void CodeGen::genRangeCheck(GenTree* oper) void CodeGen::genCodeForPhysReg(GenTreePhysReg* tree) { assert(tree->OperIs(GT_PHYSREG)); + assert(!tree->IsDivRemPair() || ((tree->GetRegNum() == REG_RDX) && (tree->gtSrcReg == REG_RDX))); var_types targetType = tree->TypeGet(); regNumber targetReg = tree->GetRegNum(); @@ -4537,6 +4614,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type) switch (oper) { +#ifdef TARGET_AMD64 + case GT_ADCX: + return INS_adcx; + case GT_ADOX: + return INS_adox; + case GT_SUB_BORROW: + ins = INS_sbb; + break; + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ins = INS_adc; + break; +#endif case GT_ADD: ins = INS_add; break; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index ffacc9f0671743..3fcf4f5d2b504f 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -8353,6 +8353,9 @@ class Compiler bool optMakeLoopDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, PerLoopInfo* loopLocals); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + void optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block); +#endif bool optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, BasicBlock* exiting, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index df354db94fe56e..ecf21930c94ae2 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -516,7 +516,9 @@ void emitter::emitInsSanityCheck(instrDesc* id) } else { - assert(isGeneralRegister(id->idReg3())); + assert(isGeneralRegister(id->idReg3()) || (((id->idIns() == INS_adc) || (id->idIns() == INS_adcs) || + (id->idIns() == INS_sbc) || (id->idIns() == INS_sbcs)) && + (id->idReg3() == REG_ZR))); } assert(insOptsNone(id->idInsOpt())); break; @@ -6138,10 +6140,6 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_lsr: case INS_asr: case INS_ror: - case INS_adc: - case INS_adcs: - case INS_sbc: - case INS_sbcs: case INS_udiv: case INS_sdiv: case INS_mneg: @@ -6174,6 +6172,18 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_DR_3A; break; + case INS_adc: + case INS_adcs: + case INS_sbc: + case INS_sbcs: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + fmt = IF_DR_3A; + break; + case INS_add: case INS_sub: if (isVectorRegister(reg1)) diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index fa33e803f4273d..ee1b6e5b66875b 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -2691,7 +2691,7 @@ bool emitter::TakesRexWPrefix(const instrDesc* id) const // so we never need it if ((ins != INS_push) && (ins != INS_pop) && (ins != INS_movq) && (ins != INS_movzx) && (ins != INS_push_hide) && (ins != INS_pop_hide) && (ins != INS_ret) && (ins != INS_call) && (ins != INS_tail_i_jmp) && - !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) + (ins != INS_jrcxz) && !((ins >= INS_i_jmp) && (ins <= INS_l_jg))) { return true; } @@ -3912,9 +3912,9 @@ unsigned emitter::emitGetAdjustedSize(instrDesc* id, code_t code) const #endif // TARGET_AMD64 else { - if (ins == INS_crc32) + if (IsLegacyMap38Instruction(ins)) { - // Adjust code size for CRC32 that has 4-byte opcode but does not use SSE38 or EES3A encoding. + // Account for the mandatory prefix of legacy map-38 instructions. adjustedSize++; } @@ -5269,7 +5269,10 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id) if ((code & 0xFF00) != 0) { - sz += (IsSimdInstruction(ins) || TakesEvexPrefix(id)) ? emitInsSize(id, code, includeRexPrefixSize) : 5; + // Legacy map-38 instructions have their mandatory prefix counted above. + sz += (IsSimdInstruction(ins) || IsLegacyMap38Instruction(ins) || TakesEvexPrefix(id)) + ? emitInsSize(id, code, includeRexPrefixSize) + : 5; } else { @@ -7870,6 +7873,54 @@ bool emitter::IsRedundantMov( #endif // TARGET_64BIT } +#ifdef TARGET_AMD64 + // LSRA may keep a MULX row multiplier in a home register and insert the + // same RDX copy at every lane. MULX and the carry arithmetic preserve RDX. + // Look back only through known integer instructions in a safe peephole + // region; any write to either register (including a partial write) ends it. + if ((ins == INS_mov) && (fmt == IF_RWR_RRD) && (size == EA_8BYTE) && (dst == REG_RDX) && + genIsValidIntReg(src) && emitCanPeepholeLastIns()) + { + bool sawMulx = false; + bool redundant = false; + emitPeepholeIterateLastInstrs([&](instrDesc* id) { + if ((id->idIns() == INS_mov) && (id->idInsFmt() == IF_RWR_RRD) && + (id->idOpSize() == EA_8BYTE) && (id->idReg1() == dst) && (id->idReg2() == src)) + { + redundant = sawMulx; + return PEEPHOLE_ABORT; + } + switch (id->idIns()) + { + case INS_mulx: + sawMulx = true; + break; + case INS_mov: + case INS_add: + case INS_sub: + case INS_adc: + case INS_sbb: + case INS_adcx: + case INS_adox: + case INS_test: + break; + default: + return PEEPHOLE_ABORT; + } + if (emitIsInstrWritingToReg(id, dst) || emitIsInstrWritingToReg(id, src)) + { + return PEEPHOLE_ABORT; + } + return PEEPHOLE_CONTINUE; + }); + if (redundant) + { + JITDUMP("\n -- suppressing repeated MULX multiplier copy to RDX.\n"); + return true; + } + } +#endif // TARGET_AMD64 + // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in // functionality even if their actual identifier differs and we should optimize these @@ -10846,6 +10897,12 @@ void emitter::emitIns_J(instruction ins, BasicBlock* dst, bool keepShort, bool i #endif } + if (ins == INS_jrcxz) + { + // Only used to skip one unconditional jump to a nearby temporary label. + noway_assert(keepShort && !id->idjKeepLong); + sz = JCC_SIZE_SMALL; + } id->idCodeSize(sz); dispIns(id); @@ -14209,7 +14266,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -14251,7 +14308,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -14466,7 +14523,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is of the form "[disp]" // On x86 - disp is relative to zero // On Amd64 - disp is relative to RIP - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14529,7 +14586,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { #ifdef TARGET_X86 - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x05); } @@ -14546,7 +14603,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) noway_assert((int)dsp == dsp); // This requires, specifying a SIB byte after ModRM byte. - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14563,7 +14620,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_EBP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -14605,7 +14662,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case REG_ESP: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -14663,7 +14720,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) default: { - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Put the register in the opcode code |= insEncodeReg012(id, reg, EA_PTRSIZE, nullptr); @@ -14742,7 +14799,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -14802,7 +14859,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) regByte = insEncodeReg012(id, REG_EBP, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { dst += emitOutputByte(dst, code | 0x04); } @@ -14831,7 +14888,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // The address is "[reg+rgx+dsp]" regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero && (reg != REG_EBP)) { @@ -15101,7 +15158,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15130,7 +15187,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseSimdEncoding() && (ins != INS_crc32)) + if (UseSimdEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -15333,7 +15390,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) if (EBPbased) { // EBP-based variable: does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspInByte) { @@ -15384,7 +15441,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // !FEATURE_FIXED_OUT_ARGS // Does the offset fit in a byte? - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if (dspIsZero) { @@ -15673,7 +15730,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) #endif // TARGET_X86 // Special case emitting AVX instructions - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { if ((ins == INS_crc32) && (size > EA_1BYTE)) { @@ -15702,7 +15759,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) dst += emitOutputRexOrSimdPrefixIfNeeded(ins, dst, code); - if (UseVEXEncoding() && (ins != INS_crc32)) + if (UseVEXEncoding() && !IsLegacyMap38Instruction(ins)) { // Emit last opcode byte // TODO-XArch-CQ: Right now support 4-byte opcode instructions only @@ -16463,8 +16520,8 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) #endif // TARGET_AMD64 } #ifdef FEATURE_HW_INTRINSICS - else if ((ins == INS_bsf) || (ins == INS_bsr) || (ins == INS_crc32) || (ins == INS_lzcnt) || (ins == INS_popcnt) || - (ins == INS_tzcnt) + else if ((ins == INS_bsf) || (ins == INS_bsr) || IsLegacyMap38Instruction(ins) || (ins == INS_lzcnt) || + (ins == INS_popcnt) || (ins == INS_tzcnt) #ifdef TARGET_AMD64 || (ins == INS_lzcnt_apx) || (ins == INS_tzcnt_apx) || (ins == INS_popcnt_apx) || (ins == INS_crc32_apx) #endif // TARGET_AMD64 @@ -17403,6 +17460,12 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) jmp = true; break; + case INS_jrcxz: + noway_assert(id->idjShort && !id->idjKeepLong); + ssz = lsz = JCC_SIZE_SMALL; + jmp = true; + break; + case INS_jmp: ssz = JMP_SIZE_SMALL; lsz = JMP_SIZE_LARGE; @@ -18941,7 +19004,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) break; } - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19180,7 +19243,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte @@ -19436,7 +19499,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); - if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) + if (EncodedBySSE38orSSE3A(ins) || IsLegacyMap38Instruction(ins)) { // Special case 4-byte AVX instructions as the // regcode position conflicts with the opcode byte diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 51c0cca338f005..eaa1dfb110b2ff 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -116,6 +116,10 @@ code_t insEncodeOpreg(const instrDesc* id, regNumber reg, emitAttr size); unsigned insSSval(unsigned scale); static bool IsSSEInstruction(instruction ins); +static bool IsLegacyMap38Instruction(instruction ins) +{ + return ins == INS_crc32 || ins == INS_adcx || ins == INS_adox; +} static bool IsSSEOrAVXInstruction(instruction ins); static bool IsAvx512OnlyInstruction(instruction ins); static bool IsKMOVInstruction(instruction ins); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 3700c917e7d2e8..1ac666ef6485d4 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -12008,6 +12008,39 @@ void Compiler::gtUpdateNodeSideEffects(GenTree* tree) }); } +//------------------------------------------------------------------------ +// IsFunnelShift: Whether lowering contained two complementary shifts in an OR. +// +// Return Value: +// True for the canonical (lo >>> count) | (hi << (width - count)) form. +// +bool GenTree::IsFunnelShift() const +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + bool isFunnelShift = OperIs(GT_OR) && gtGetOp1()->OperIs(GT_RSZ) && gtGetOp1()->isContained() && + gtGetOp2()->OperIs(GT_LSH) && gtGetOp2()->isContained(); +#ifdef DEBUG + if (isFunnelShift) + { + // TryContainFunnelShift is the only path that contains both shifts. It validates + // their widths and complementary immediate counts before establishing this shape. + assert(TypeIs(TYP_INT, TYP_LONG)); + assert(gtGetOp1()->TypeGet() == TypeGet()); + assert(gtGetOp2()->TypeGet() == TypeGet()); + assert(gtGetOp1()->gtGetOp2()->IsCnsIntOrI()); + assert(gtGetOp2()->gtGetOp2()->IsCnsIntOrI()); + ssize_t width = genTypeSize(TypeGet()) * BITS_PER_BYTE; + ssize_t count = gtGetOp1()->gtGetOp2()->AsIntCon()->IconValue(); + assert((count > 0) && (count < width)); + assert(gtGetOp2()->gtGetOp2()->AsIntCon()->IconValue() == width - count); + } +#endif + return isFunnelShift; +#else + return false; +#endif +} + bool GenTree::gtSetFlags() const { return (gtFlags & GTF_SET_FLAGS) != 0; @@ -12061,6 +12094,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_JMP: case GT_JCC: case GT_SETCC: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_NO_OP: case GT_START_NONGC: case GT_START_PREEMPTGC: @@ -12083,6 +12119,9 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) // Standard unary operators case GT_STORE_LCL_VAR: +#ifdef TARGET_AMD64 + case GT_ADX_DRAIN: +#endif case GT_STORE_LCL_FLD: case GT_NOT: case GT_NEG: @@ -14294,6 +14333,10 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) } break; +#ifdef TARGET_AMD64 + case GT_ADX_SEED: + break; +#endif case GT_PHYSREG: printf(" %s", getRegName(tree->AsPhysReg()->gtSrcReg)); break; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 60477341da53e0..5ace23683f7519 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -498,6 +498,7 @@ enum GenTreeFlags : unsigned GTF_ADDRMODE_NO_CSE = 0x80000000, // GT_ADD/GT_MUL/GT_LSH/GT_CAST -- Do not CSE this node only, forms complex // addressing mode + GTF_ADD_CARRY_FLAGS = 0x04000000, // GT_ADD -- CF is consumed; INC/DEC cannot implement this add GTF_MUL_64RSLT = 0x40000000, // GT_MUL -- produce 64-bit result GTF_RELOP_NAN_UN = 0x80000000, // GT_ -- Is branch taken if ops are NaN? @@ -528,6 +529,8 @@ enum GenTreeFlags : unsigned GTF_DIV_MOD_NO_OVERFLOW = 0x40000000, // GT_DIV, GT_MOD -- Div or mod definitely does not overflow. + GTF_DIV_REM_PAIR = 0x08000000, // GT_[U]DIV, GT_PHYSREG -- adjacent quotient/remainder definitions on xarch. + GTF_ARRLEN_NONFAULTING = 0x20000000, // GT_ARR_LENGTH -- An array length operation that cannot fault. Same as GT_IND_NONFAULTING. GTF_MDARRLEN_NONFAULTING = 0x20000000, // GT_MDARR_LENGTH -- An MD array length operation that cannot fault. Same as GT_IND_NONFAULTING. @@ -537,6 +540,7 @@ enum GenTreeFlags : unsigned GTF_ALLOCOBJ_EMPTY_STATIC = 0x80000000, // GT_ALLOCOBJ -- allocation site is part of an empty static pattern #ifdef FEATURE_HW_INTRINSICS + GTF_HW_MULX = 0x08000000, // GT_HWINTRINSIC -- unsigned BigMul in a matched multiply-carry chain uses MULX GTF_HW_EM_OP = 0x10000000, // GT_HWINTRINSIC -- node is used as an operand to an embedded mask GTF_HW_USER_CALL = 0x20000000, // GT_HWINTRINSIC -- node is implemented via a user call #endif // FEATURE_HW_INTRINSICS @@ -1861,6 +1865,18 @@ struct GenTree bool OperConsumesFlags() const { +#ifdef TARGET_AMD64 + if (OperIs(GT_ADCX, GT_ADOX, GT_ADX_DRAIN)) + { + return true; + } +#endif +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (OperIs(GT_ADD_CARRY, GT_SUB_BORROW, GT_ADD_BORROW)) + { + return true; + } +#endif #if !defined(TARGET_64BIT) if (OperIs(GT_ADD_HI, GT_SUB_HI)) { @@ -2466,6 +2482,12 @@ struct GenTree bool gtOverflow() const; bool gtOverflowEx() const; bool gtSetFlags() const; + bool IsFunnelShift() const; + + bool IsDivRemPair() const + { + return OperIs(GT_DIV, GT_UDIV, GT_PHYSREG) && ((gtFlags & GTF_DIV_REM_PAIR) != 0); + } #ifdef DEBUG static int gtDispFlags(GenTreeFlags flags, GenTreeDebugFlags debugFlags); @@ -9528,6 +9550,8 @@ struct GenTreeOpCC : public GenTreeOp { #ifdef TARGET_ARM64 assert(OperIs(GT_SELECTCC, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)); +#elif defined(TARGET_AMD64) + assert(OperIs(GT_SELECTCC, GT_JCMP)); #else assert(OperIs(GT_SELECTCC)); #endif diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 622a9643e7db0b..9f638d92665198 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -391,6 +391,21 @@ GTNODE(RELOAD , GenTreeCopyOrReload,0,0,GTK_UNOP|DBK_NOTHIR) GTNODE(IL_OFFSET , GenTreeILOffset ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // marks an IL offset for debugging purposes GTNODE(RECORD_ASYNC_RESUME, GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // record native offset for async resumption info +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +#ifdef TARGET_AMD64 +// Independent unsigned carry chains, introduced only after whole-loop validation. +GTNODE(ADCX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADOX , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADX_SEED , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) +GTNODE(ADX_DRAIN , GenTreeOp ,0,0,GTK_UNOP|DBK_NOTHIR) +#endif +// Adds two operands and the carry flag. Only introduced during lowering. +GTNODE(ADD_CARRY , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +// Subtract op2 and borrow; add op2 and subtraction borrow to op1 (op2 is zero on arm64). +GTNODE(SUB_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(ADD_BORROW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +#endif + /*****************************************************************************/ #undef GTNODE /*****************************************************************************/ diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index d84f1dfa732107..d1150a9d1e97b6 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -973,6 +973,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { NI_Illegal, NI_Illegal }, // SHA_X64 { NI_Illegal, NI_Illegal }, // WAITPKG_X64 { NI_Illegal, NI_Illegal }, // X86Serialize_X64 + { NI_Illegal, NI_Illegal }, // ADX #elif defined (TARGET_ARM64) { FIRST_NI_ArmBase, LAST_NI_ArmBase }, // ArmBase { FIRST_NI_AdvSimd, LAST_NI_AdvSimd }, // AdvSimd diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index a2362695a6716b..f0a4ce975da303 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2596,6 +2596,15 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) emitAttr attr = emitTypeSize(baseType); + if ((node->gtFlags & GTF_HW_MULX) != 0) + { + assert(!rmOp->isUsedFromReg() || (rmOp->GetRegNum() != REG_EDX) || (regOp->GetRegNum() == REG_EDX)); + emit->emitIns_Mov(INS_mov, attr, REG_EDX, regOp->GetRegNum(), /* canSkip */ true); + inst_RV_RV_TT(INS_mulx, attr, node->GetRegByIndex(1), node->GetRegByIndex(0), rmOp, + /* isRMW */ false, INS_OPTS_NONE); + break; + } + // If rmOp is already in EAX, use that as implicit operand if (rmOp->isUsedFromReg() && rmOp->GetRegNum() == REG_EAX) { diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp index cbc9751324ebb3..55cb41c14d643e 100644 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -1187,6 +1187,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte } bool checkProfitability = !compStressCompile(STRESS_DOWNWARDS_COUNTED_LOOPS, 50); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + bool preferCarryCountdown = false; + // A countdown can keep carry or borrow flags live across arithmetic loops, + // even when their addressing IV remains useful after the loop. + // Leave trip-count legality to SCEV below and the carry proof to lowering. + if (checkProfitability && removableLocals.Height() == 0 && compHndBBtabCount == 0 && loop->GetHeader() == exiting && + exiting->GetTrueTarget() == exiting) + { + unsigned products = 0; + unsigned carries = 0; + unsigned nodes = 0; + unsigned differences = 0; + for (Statement* stmt : exiting->Statements()) + { + for (GenTree* node : stmt->TreeList()) + { + if (++nodes > 512) + { + break; + } + if (node->OperIs(GT_HWINTRINSIC) && + node->AsHWIntrinsic()->GetHWIntrinsicId() == +#ifdef TARGET_AMD64 + NI_X86Base_X64_BigMul && +#else + NI_ArmBase_Arm64_MultiplyHigh && +#endif + node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) + { + products++; + } + if (node->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) && (node->gtFlags & GTF_UNSIGNED) != 0) + { + carries++; + } + if (node->OperIs(GT_SUB) && !node->gtOverflow() && node->TypeIs(TYP_INT, TYP_LONG)) + { + differences++; + } + } + if (nodes > 512) + { + break; + } + } + bool multiply = products >= 2 && products <= 4 && carries == 2 * products; +#ifdef TARGET_AMD64 + multiply = multiply && compOpportunisticallyDependsOn(InstructionSet_ADX) && + compOpportunisticallyDependsOn(InstructionSet_AVX2); +#endif + bool subtract = products == 0 && differences == 2 && carries == 2; + if (nodes <= 512 && (multiply || subtract)) + { + JITDUMP(" Keeping a countdown for a potential carry/borrow loop\n"); + checkProfitability = false; + preferCarryCountdown = true; + } + } +#endif if (checkProfitability && (removableLocals.Height() <= 0)) { JITDUMP(" Found no potentially removable locals when making this loop downwards counted\n"); @@ -1199,8 +1258,65 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return false; } - Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); - if (backedgeCount == nullptr) + Scev* backedgeCount = scevContext.ComputeExitNotTakenCount(exiting); + GenTree* tripCountNode = nullptr; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // SCEV's general trip-count materialization currently handles only unit + // strides. Recognize the bounded four-lane case: i starts at zero, steps + // by four, and continues while i < length - 3 for a nonnegative length. + // Its trip count is length / 4; the induction variable cannot overflow. + if (backedgeCount == nullptr && preferCarryCountdown && cond->OperIs(GT_LT) && !cond->IsUnsigned() && + cond->gtGetOp1()->TypeIs(TYP_INT) && cond->gtGetOp2()->TypeIs(TYP_INT)) + { + Scev* iv = scevContext.Analyze(exiting, cond->gtGetOp1()); + if (iv != nullptr) + { + iv = scevContext.Simplify(iv); + } + Scev* bound = scevContext.Analyze(exiting, cond->gtGetOp2()); + int64_t start, step; + if (iv != nullptr && iv->OperIs(ScevOper::AddRec) && bound != nullptr && bound->IsInvariant() && + ((ScevAddRec*)iv)->Start->GetConstantValue(this, &start) && start == 4 && + ((ScevAddRec*)iv)->Step->GetConstantValue(this, &step) && step == 4) + { + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8 && node->OperIs(GT_LCL_VAR); depth++) + { + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if (def == nullptr || def->GetLclNum() != local->GetLclNum()) + { + break; + } + node = def->Data(); + } + return node; + }; + GenTree* limit = resolve(cond->gtGetOp2()); + if (limit->OperIs(GT_ADD) && limit->gtGetOp2()->IsIntegralConst(-3) && + IntegralRange::ForNode(resolve(limit->gtGetOp1()), this).IsNonNegative()) + { + ValueNum limitVN = vnStore->VNLiberalNormalValue(cond->gtGetOp2()->gtVNPair); + ValueNum positive = vnStore->VNForFunc(TYP_INT, VNF_GT, limitVN, vnStore->VNForIntCon(0)); + if (scevContext.EvaluateRelop(positive) == RelopEvaluationResult::True) + { + GenTree* limitNode = scevContext.Materialize(bound); + if (limitNode != nullptr) + { + GenTree* length = gtNewOperNode(GT_ADD, TYP_INT, limitNode, gtNewIconNode(3)); + tripCountNode = gtNewOperNode(GT_RSZ, TYP_INT, length, gtNewIconNode(2)); + optFoldFourLimbOffsets(scevContext, exiting); + } + } + } + } + } +#endif + if (backedgeCount == nullptr && tripCountNode == nullptr) { JITDUMP(" Could not compute backedge count -- not a counted loop\n"); return false; @@ -1213,9 +1329,12 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // to add one to the computed backedge count, giving us the trip count of // the loop. We do not need to worry about overflow here (even with // wraparound we have the right behavior). - Scev* tripCount = scevContext.Simplify( - scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); - GenTree* tripCountNode = scevContext.Materialize(tripCount); + if (tripCountNode == nullptr) + { + Scev* tripCount = scevContext.Simplify( + scevContext.NewBinop(ScevOper::Add, backedgeCount, scevContext.NewConstant(backedgeCount->Type, 1))); + tripCountNode = scevContext.Materialize(tripCount); + } if (tripCountNode == nullptr) { JITDUMP(" Could not materialize trip count into IR\n"); @@ -1239,9 +1358,9 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte exitOp = GT_NE; } - GenTree* negOne = tripCount->TypeIs(TYP_LONG) ? gtNewLconNode(-1) : gtNewIconNode(-1, tripCount->Type); - GenTree* decremented = - gtNewOperNode(GT_ADD, tripCount->Type, gtNewLclVarNode(tripCountLcl, tripCount->Type), negOne); + var_types countType = tripCountNode->TypeGet(); + GenTree* negOne = countType == TYP_LONG ? gtNewLconNode(-1) : gtNewIconNode(-1, countType); + GenTree* decremented = gtNewOperNode(GT_ADD, countType, gtNewLclVarNode(tripCountLcl, countType), negOne); store = gtNewTempStore(tripCountLcl, decremented); @@ -1253,8 +1372,8 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte // Update the test. cond->SetOper(exitOp); - cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, tripCount->Type); - cond->AsOp()->gtOp2 = gtNewZeroConNode(tripCount->Type); + cond->AsOp()->gtOp1 = gtNewLclVarNode(tripCountLcl, countType); + cond->AsOp()->gtOp2 = gtNewZeroConNode(countType); gtSetStmtInfo(jtrueStmt); fgSetStmtSeq(jtrueStmt); @@ -1267,6 +1386,171 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte return true; } +//------------------------------------------------------------------------ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +// optFoldFourLimbOffsets: Share a widened integer index among the lane addresses +// of a proven four-limb loop. The caller proves i starts at zero, steps by four, +// and executes only while i < length - 3, for a nonnegative signed int length. +// Thus i + k cannot wrap for k in [0, 3]. Managed bases remain unchanged. +void Compiler::optFoldFourLimbOffsets(ScalarEvolutionContext& scevContext, BasicBlock* block) +{ + auto resolve = [this](GenTree* node) { + for (unsigned depth = 0; depth < 8; depth++) + { + if (node->OperIs(GT_COMMA)) + { + node = node->gtGetOp2(); + continue; + } + if (!node->OperIs(GT_LCL_VAR)) + { + break; + } + GenTreeLclVar* local = node->AsLclVar(); + if (!local->HasSsaName()) + { + break; + } + GenTreeLclVarCommon* def = lvaGetDesc(local)->GetPerSsaData(local->GetSsaNum())->GetDefNode(); + if ((def == nullptr) || (def->GetLclNum() != local->GetLclNum()) || def->Data()->OperIs(GT_PHI)) + { + break; + } + node = def->Data(); + } + return node; + }; + + struct Address + { + GenTreeOp* Node; + Statement* Stmt; + unsigned Offset; + }; + jitstd::vector
addresses(getAllocator(CMK_LoopOpt)); + jitstd::vector stores(getAllocator(CMK_LoopOpt)); + GenTreeLclVar* index = nullptr; + Statement* firstStmt = nullptr; + for (Statement* stmt : block->Statements()) + { + if (stmt->IsPhiDefnStmt()) + { + continue; + } + if (firstStmt == nullptr) + { + firstStmt = stmt; + } + for (GenTree* node : stmt->TreeList()) + { + if (node->OperIsLocalStore()) + { + stores.push_back(node->AsLclVarCommon()->GetLclNum()); + } + if (!node->OperIs(GT_ADD) || !node->TypeIs(TYP_BYREF) || !node->gtGetOp1()->TypeIs(TYP_BYREF)) + { + continue; + } + GenTree* scaled = resolve(node->gtGetOp2()); + if (!scaled->OperIs(GT_LSH) || !scaled->TypeIs(TYP_LONG) || !scaled->gtGetOp2()->IsIntegralConst(3)) + { + continue; + } + GenTree* widened = resolve(scaled->gtGetOp1()); + if (!widened->OperIs(GT_CAST) || !widened->TypeIs(TYP_LONG) || !widened->IsUnsigned() || + widened->gtOverflow()) + { + continue; + } + GenTree* base = resolve(widened->AsCast()->CastOp()); + unsigned offset = 0; + if (base->OperIs(GT_ADD) && base->TypeIs(TYP_INT) && !base->gtOverflow() && base->gtGetOp2()->IsCnsIntOrI()) + { + ssize_t value = base->gtGetOp2()->AsIntCon()->IconValue(); + if ((value < 0) || (value > 3)) + { + continue; + } + offset = (unsigned)value; + base = base->gtGetOp1(); + } + if (!base->OperIs(GT_LCL_VAR) || !base->TypeIs(TYP_INT) || !base->AsLclVar()->HasSsaName()) + { + continue; + } + GenTreeLclVar* local = base->AsLclVar(); + if (lvaGetDesc(local)->IsAddressExposed() || lvaGetDesc(local)->lvIsStructField) + { + continue; + } + bool overwritten = false; + for (unsigned store : stores) + { + overwritten |= store == local->GetLclNum(); + } + if (overwritten || ((index != nullptr) && ((index->GetLclNum() != local->GetLclNum()) || + (index->GetSsaNum() != local->GetSsaNum())))) + { + continue; + } + Scev* recurrence = scevContext.Analyze(block, local); + int64_t start, step; + if ((recurrence == nullptr) || !recurrence->OperIs(ScevOper::AddRec) || + !((ScevAddRec*)recurrence)->Start->GetConstantValue(this, &start) || (start != 0) || + !((ScevAddRec*)recurrence)->Step->GetConstantValue(this, &step) || (step != 4)) + { + continue; + } + index = local; + addresses.push_back({node->AsOp(), stmt, offset}); + } + } + if (addresses.size() < 4) + { + return; + } + + unsigned temp = lvaGrabTemp(false DEBUGARG("Shared four-limb integer index")); + GenTree* widened = gtNewCastNode(TYP_LONG, gtCloneExpr(index), true, TYP_LONG); + Statement* initialization = fgNewStmtFromTree(gtNewTempStore(temp, widened)); + fgInsertStmtBefore(block, firstStmt, initialization); + gtSetStmtInfo(initialization); + fgSetStmtSeq(initialization); +#ifdef TARGET_ARM64 + // A64 has no base+index+displacement addressing mode. Share native-sized + // lane indices instead, avoiding a separate zero extension after each i+k. + unsigned laneTemps[4] = {temp, BAD_VAR_NUM, BAD_VAR_NUM, BAD_VAR_NUM}; + for (unsigned lane = 1; lane < 4; lane++) + { + laneTemps[lane] = lvaGrabTemp(false DEBUGARG("Four-limb lane index")); + GenTree* value = gtNewOperNode(GT_ADD, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewLconNode(lane)); + Statement* init = fgNewStmtFromTree(gtNewTempStore(laneTemps[lane], value)); + fgInsertStmtBefore(block, firstStmt, init); + gtSetStmtInfo(init); + fgSetStmtSeq(init); + } +#endif + for (const Address& address : addresses) + { +#ifdef TARGET_ARM64 + GenTree* offset = + gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(laneTemps[address.Offset], TYP_LONG), gtNewIconNode(3)); +#else + GenTree* scaled = gtNewOperNode(GT_LSH, TYP_LONG, gtNewLclvNode(temp, TYP_LONG), gtNewIconNode(3)); + GenTree* offset = gtNewOperNode(GT_ADD, TYP_LONG, scaled, gtNewLconNode(address.Offset * 8)); +#endif + // Preserve embedded CSE definitions and any other effects of the original offset. + GenTree* effects = nullptr; + gtExtractSideEffList(address.Node->gtOp2, &effects); + address.Node->gtOp2 = effects == nullptr ? offset : gtNewOperNode(GT_COMMA, TYP_LONG, effects, offset); + gtSetStmtInfo(address.Stmt); + fgSetStmtSeq(address.Stmt); + } + JITDUMP("Shared widened index V%02u for %u four-limb addresses in " FMT_BB "\n", temp, (unsigned)addresses.size(), + block->bbNum); +} +#endif // defined(TARGET_AMD64) || defined(TARGET_ARM64) + //------------------------------------------------------------------------ // optCanAndShouldChangeExitTest: // Check if the exit test can be rephrased to a downwards counted exit test diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index fcd6e9b3e06f6b..ba7d09541eeb65 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -1289,6 +1289,10 @@ INST3(sar, "sar", IUM_RW, 0x0038D2, BAD_CODE, INST3(sar_1, "sar", IUM_RW, 0x0038D0, 0x0038D0, 0x0038D0, 1C, 2X, INS_TT_NONE, Writes_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) INST3(sar_N, "sar", IUM_RW, 0x0038C0, 0x0038C0, BAD_CODE, 1C, 2X, INS_TT_NONE, Undefined_OF | Writes_SF | Writes_ZF | Undefined_AF | Writes_PF | Writes_CF | INS_FLAGS_HasWBit | Encoding_REX2 | INS_FLAGS_HasNDD | INS_FLAGS_HasNF) +// ADX carry chains. Keep these at the end of the three-encoding instruction table. +INST3(adcx, "adcx", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0x66, 0xF6), 1C, 2X, INS_TT_NONE, Reads_CF | Writes_CF) +INST3(adox, "adox", IUM_RW, BAD_CODE, BAD_CODE, PSSE38(0xF3, 0xF6), 1C, 2X, INS_TT_NONE, Reads_OF | Writes_OF) + // id nm um mr mi lat tp tt flags INST2(ret, "ret", IUM_RD, 0x0000C3, 0x0000C2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST2(loop, "loop", IUM_RD, BAD_CODE, 0x0000E2, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) @@ -1409,6 +1413,9 @@ INST0(l_jg, "jg", IUM_RD, 0x008F0F, INST0(align, "align", IUM_RD, BAD_CODE, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) INST0(data16, "data16", IUM_RD, 0x000066, ZERO, 4X, INS_TT_NONE, INS_FLAGS_None) +// Flags-preserving loop branch. +INST0(jrcxz, "jrcxz", IUM_RD, 0x0000E3, BRANCH_COND, 2X, INS_TT_NONE, INS_FLAGS_None) + /*****************************************************************************/ #undef INST0 #undef INST1 diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 9198ab6c529d0e..b54c6fb80748d9 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2461,6 +2461,9 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR case GT_STORE_BLK: case GT_JCMP: case GT_JTEST: +#ifdef TARGET_AMD64 + case GT_ADX_SEED: +#endif case GT_JCC: case GT_JTRUE: case GT_RETURN: @@ -2653,6 +2656,13 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR } assert(!node->OperIsLocal()); + // A paired divide also defines RDX for the following physical-register + // capture. That implicit use must survive even when the quotient is dead + // and the division has been proven non-throwing. + if (node->OperIs(GT_DIV, GT_UDIV) && node->IsDivRemPair()) + { + return false; + } if (!node->IsValue() || node->IsUnusedValue()) { // We are only interested in avoiding the removal of nodes with direct side effects @@ -2669,7 +2679,7 @@ bool Liveness::TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockR return GenTree::VisitResult::Continue; }); - if (node->OperConsumesFlags() && node->gtPrev->gtSetFlags()) + if (node->OperConsumesFlags() && (node->gtPrev != nullptr) && node->gtPrev->gtSetFlags()) { node->gtPrev->gtFlags &= ~GTF_SET_FLAGS; } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index bbd00eb1fa752c..29a15e47fcb72b 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -306,6 +306,10 @@ bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) void Lowering::LowerRange(GenTree* firstNode, GenTree* lastNode) { assert(lastNode != nullptr); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // A partial walk can have pre-existing flag consumers outside its range. + m_blockMayHaveSetCC = true; +#endif // Multiple possible behaviors of LowerNode are possible: // 1. The node being lowered may be removed @@ -375,6 +379,14 @@ GenTree* Lowering::LowerNode(GenTree* node) case GT_ADD: { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Morph represents subtraction of a constant as addition of its negation. + if (m_compiler->opts.OptimizationEnabled() && node->TypeIs(TYP_INT, TYP_LONG) && + node->gtGetOp2()->IsCnsIntOrI() && (node->gtGetOp2()->AsIntCon()->IconValue() < 0)) + { + m_lastSubtractionBlock = m_block; + } +#endif GenTree* next = LowerAdd(node->AsOp()); if (next != nullptr) { @@ -383,6 +395,14 @@ GenTree* Lowering::LowerNode(GenTree* node) } break; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: + ContainCheckBinary(node->AsOp()); + break; +#endif + #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -396,6 +416,17 @@ GenTree* Lowering::LowerNode(GenTree* node) { if (m_compiler->opts.OptimizationEnabled()) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (node->OperIs(GT_SUB)) + { + m_lastSubtractionBlock = m_block; + GenTree* next = node->gtNext; + if (TryLowerAddCarry(node->AsOp())) + { + return next; + } + } +#endif GenTree* nextNode = nullptr; if (node->OperIs(GT_AND) && TryLowerAndNegativeOne(node->AsOp(), &nextNode)) { @@ -722,6 +753,12 @@ GenTree* Lowering::LowerNode(GenTree* node) break; #endif // defined(TARGET_WASM) +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SETCC: + m_blockMayHaveSetCC = true; + break; +#endif + default: break; } @@ -4649,6 +4686,9 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) } } GenTreeCC* setcc = m_compiler->gtNewCC(GT_SETCC, cmp->TypeGet(), cmpCondition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(op1, setcc); use.ReplaceWith(setcc); @@ -4669,6 +4709,13 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) // GenTree* Lowering::LowerCompare(GenTree* cmp) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = cmp->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerCarryCompare(cmp->AsOp())) + { + return next; + } +#endif #if LOWER_DECOMPOSE_LONGS if (cmp->gtGetOp1()->TypeIs(TYP_LONG)) { @@ -4974,6 +5021,12 @@ bool Lowering::TryLowerConditionToFlagsNode(GenTree* parent, { assert((condition->gtPrev->gtFlags & GTF_SET_FLAGS) != 0); GenTree* flagsDef = condition->gtPrev; + // A value-producing flags definition can have intervening LIR users. Moving + // it next to the condition consumer would move its value past those uses. + if (flagsDef->IsValue() && !flagsDef->IsUnusedValue()) + { + return false; + } #if defined(TARGET_ARM64) || defined(TARGET_AMD64) && !defined(TARGET_WASM) // CCMP is a flag producing node that also consumes flags, so find the // "root" of the flags producers and move the entire range. @@ -5071,10 +5124,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (next->OperIs(GT_JTRUE)) { // If the instruction immediately following 'relop', i.e. 'next' is a conditional branch, - // it should always have 'relop' as its 'op1'. If it doesn't, then we have improperly - // constructed IL (the setting of a condition code should always immediately precede its - // use, since the JIT doesn't track dataflow for condition codes). Still, if it happens - // it's not our problem, it simply means that `node` is not used and can be removed. + // it should have 'relop' as its 'op1'. Only convert that use to JCC. + // JCC can be separated from its flags producer by flag-preserving nodes; + // the JIT does not otherwise track dataflow for condition codes. if (next->AsUnOp()->gtGetOp1() == relop) { assert(relop->OperIsCompare()); @@ -5093,6 +5145,9 @@ GenTreeCC* Lowering::LowerNodeCC(GenTree* node, GenCondition condition) if (BlockRange().TryGetUse(relop, &use)) { cc = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = true; +#endif BlockRange().InsertAfter(node, cc); use.ReplaceWith(cc); } @@ -7944,6 +7999,13 @@ bool Lowering::TryCreateAddrMode(GenTree* addr, bool isContainable, GenTree* par // GenTree* Lowering::LowerAdd(GenTreeOp* node) { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + GenTree* next = node->gtNext; + if (m_compiler->opts.OptimizationEnabled() && TryLowerAddCarry(node)) + { + return next; + } +#endif if (varTypeIsIntegralOrI(node->TypeGet())) { GenTree* op1 = node->gtGetOp1(); @@ -8765,6 +8827,9 @@ GenTree* Lowering::LowerSignedDivOrMod(GenTree* node) // void Lowering::LowerDivOrMod(GenTreeOp* divMod) { +#ifdef TARGET_XARCH + TryLowerDivRem(divMod); +#endif ContainCheckDivOrMod(divMod); } #endif // !TARGET_WASM @@ -8873,6 +8938,77 @@ void Lowering::TryRemoveShiftRotateMask(GenTreeOp* op) } } +//------------------------------------------------------------------------ +// TryContainFunnelShift: Combine complementary constant shifts of two values. +// +// Arguments: +// node - The binary node to check. +// +// Return Value: +// True if the node can be emitted as SHRD or EXTR. +// +bool Lowering::TryContainFunnelShift(GenTreeOp* node) +{ +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + if (!node->OperIs(GT_OR)) + { + return false; + } + if (node->IsFunnelShift()) + { + return true; + } + + if (!m_compiler->opts.OptimizationEnabled() || !node->TypeIs(TYP_INT, TYP_LONG) || node->gtSetFlags()) + { + return false; + } + + GenTree* right = node->gtGetOp1(); + GenTree* left = node->gtGetOp2(); + if (right->OperIs(GT_LSH)) + { + std::swap(right, left); + } + + if (!right->OperIs(GT_RSZ) || !left->OperIs(GT_LSH) || right->gtSetFlags() || left->gtSetFlags() || + !right->gtGetOp2()->IsCnsIntOrI() || !left->gtGetOp2()->IsCnsIntOrI() || + (right->TypeGet() != node->TypeGet()) || (left->TypeGet() != node->TypeGet())) + { + return false; + } + + ssize_t width = genTypeSize(node) * BITS_PER_BYTE; + ssize_t rightCount = right->gtGetOp2()->AsIntCon()->IconValue(); + ssize_t leftCount = left->gtGetOp2()->AsIntCon()->IconValue(); + if ((rightCount <= 0) || (rightCount >= width) || (leftCount != width - rightCount)) + { + return false; + } + + GenTree* lo = right->gtGetOp1(); + GenTree* hi = left->gtGetOp1(); + // Keep memory accesses at their original positions, and do not extend a local + // read past a redefinition. Both input values must be available in registers. + if (lo->isContained() || hi->isContained() || !IsInvariantInRange(lo, node) || !IsInvariantInRange(hi, node)) + { + return false; + } + + lo->ClearRegOptional(); + hi->ClearRegOptional(); + node->gtOp1 = right; + node->gtOp2 = left; + MakeSrcContained(node, right); + MakeSrcContained(node, left); + MakeSrcContained(right, right->gtGetOp2()); + MakeSrcContained(left, left->gtGetOp2()); + return true; +#else + return false; +#endif +} + //------------------------------------------------------------------------ // LowerShift: Lower shift nodes // @@ -9640,6 +9776,9 @@ void Lowering::LowerBlock(BasicBlock* block) assert(block->isEmpty() || block->IsLIR()); m_block = block; +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + m_blockMayHaveSetCC = false; +#endif #ifdef TARGET_ARM64 m_blockIndirs.Reset(); m_ffrTrashed = true; @@ -9667,6 +9806,23 @@ void Lowering::LowerBlock(BasicBlock* block) // void Lowering::AfterLowerBlocks() { +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + // Post-processing revisits blocks and inserts SETCCs before their consumers. + m_blockMayHaveSetCC = true; + if (m_compiler->opts.OptimizationEnabled() && m_hasAddCarry) + { + if (LowerFullAdders()) + { + LowerCarryChains(); + } +#ifdef TARGET_AMD64 + LowerMultiplyCarryLoops(); + LowerMaskSelects(); +#else + LowerArm64MultiplyCarryLoops(); +#endif + } +#endif } #endif // !TARGET_WASM @@ -9920,6 +10076,11 @@ void Lowering::ContainCheckNode(GenTree* node) case GT_ADD: case GT_SUB: +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) + case GT_SUB_BORROW: + case GT_ADD_BORROW: + case GT_ADD_CARRY: +#endif #if !defined(TARGET_64BIT) case GT_ADD_LO: case GT_ADD_HI: @@ -13072,6 +13233,7 @@ bool Lowering::TryLowerAndOrToCCMP(GenTreeOp* tree, GenTree** next) ContainCheckConditionalCompare(ccmp); tree->SetOper(GT_SETCC); + m_blockMayHaveSetCC = true; tree->AsCC()->gtCondition = cond2; JITDUMP("Conversion was legal. Result:\n"); @@ -13390,3 +13552,4093 @@ void Lowering::SetFramePointerFromArgSpaceSize() m_compiler->codeGen->setFramePointerRequired(true); } } + +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) +//------------------------------------------------------------------------ +// IsCarryLocalDeadOnExit: Prove that the current local value cannot be read +// after leaving block. Lowering cannot use stale bbLiveOut sets. Walk current +// successors up to a full definition, including EH paths and parent aliases. +// Results are not cached across IR/CFG mutations. Budget exhaustion declines +// optimization; implicit OSR state and JMP argument uses are excluded. +// During the main lowering walk successors may not yet be lowered. This relies +// on lowering preserving observations of existing tracked scalar locals: it +// cannot introduce an earlier read of a killed value or turn its full def into +// a partial def. Promoted parents are not scalar carry locals and are excluded. +bool Lowering::IsCarryLocalDeadOnExit(unsigned local, BasicBlock* block) +{ + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + // An OSR method can observe only root-method locals through its Tier0 frame. + if (dsc->lvPromoted || dsc->IsAddressExposed() || (m_compiler->opts.IsOSR() && dsc->lvIsOSRLocal) || + (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && (local == m_compiler->info.compThisArg))) + { + return false; + } + unsigned visitedWords = m_compiler->fgBBNumMax / (sizeof(size_t) * 8) + 1; + if (visitedWords > m_carryLocalScanBudget) + { + return false; + } + m_carryLocalScanBudget -= visitedWords; + BitVecTraits traits(m_compiler->fgBBNumMax + 1, m_compiler); + BitVec visited = BitVecOps::MakeEmpty(&traits); + jitstd::vector pending(m_compiler->getAllocator(CMK_Lower)); + bool exhausted = false; + auto enqueue = [&](BasicBlock* successor) { + if (m_carryLocalScanBudget == 0) + { + exhausted = true; + return BasicBlockVisit::Abort; + } + m_carryLocalScanBudget--; + if (BitVecOps::IsMember(&traits, visited, successor->bbNum)) + { + return BasicBlockVisit::Continue; + } + BitVecOps::AddElemD(&traits, visited, successor->bbNum); + pending.push_back(successor); + return BasicBlockVisit::Continue; + }; + block->VisitRegularSuccs(m_compiler, enqueue); + block->VisitEHSuccs(m_compiler, enqueue); + for (unsigned index = 0; index < pending.size(); index++) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + BasicBlock* current = pending[index]; + // A throw can reach the handler before any definition in this block. + current->VisitEHSuccs(m_compiler, enqueue); + bool killed = false; + for (GenTree* node : LIR::AsRange(current)) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + if (!node->OperIsLocal()) + { + continue; + } + unsigned reference = node->AsLclVarCommon()->GetLclNum(); + if ((reference != local) && !(dsc->lvIsStructField && (reference == dsc->lvParentLcl))) + { + continue; + } + if (node->OperIs(GT_STORE_LCL_VAR)) + { + killed = true; + break; + } + // Reads, addresses and partial stores cannot prove the old value dead. + return false; + } + if (!killed) + { + current->VisitRegularSuccs(m_compiler, enqueue); + } + } + return !exhausted; +} + +//------------------------------------------------------------------------ +// FindCarryLoopEntry: Find the sole predecessor other than the loop itself. +BasicBlock* Lowering::FindCarryLoopEntry(BasicBlock* block) +{ + BasicBlock* entry = nullptr; + for (FlowEdge* edge : block->PredEdges()) + { + BasicBlock* pred = edge->getSourceBlock(); + if (pred != block) + { + if (entry != nullptr) + { + return nullptr; + } + entry = pred; + } + } + return entry; +} + +//------------------------------------------------------------------------ +// FindDeadCarryConstantStore: Identify a constant/store pair that can be +// removed from a flags-live range. Prove the local unobserved throughout the +// current method, excluding aliases and implicit uses absent from the LIR scan. +GenTree* Lowering::FindDeadCarryConstantStore(GenTree* constant) +{ + assert(constant->OperIs(GT_CNS_INT) && !constant->isContained()); + LIR::Use use; + if (!BlockRange().TryGetUse(constant, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + unsigned local = use.User()->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->IsAddressExposed() || dsc->lvIsStructField || dsc->lvPromoted || + (m_compiler->opts.IsOSR() && dsc->lvIsOSRLocal) || (m_compiler->compJmpOpUsed && dsc->lvIsParam) || + (m_compiler->lvaKeepAliveAndReportThis() && local == m_compiler->info.compThisArg)) + { + return nullptr; + } + for (BasicBlock* block : m_compiler->Blocks()) + { + for (GenTree* node : LIR::AsRange(block)) + { + if (m_carryUseScanBudget == 0) + { + return nullptr; + } + m_carryUseScanBudget--; + if (node->OperIsLocal() && node->AsLclVarCommon()->GetLclNum() == local && !node->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + } + } + return use.User(); +} + +//------------------------------------------------------------------------ +// FindCarryLocalStore: Find the nearest full definition before a local read, +// within the fixed local search allowance. This does not prove single use or +// deadness on exit; callers establish those when deleting the definition. +// Keep short searches independent of the budget for extended carry-use scans. +GenTree* Lowering::FindCarryLocalStore(GenTree* read) +{ + assert(read->OperIs(GT_LCL_VAR)); + unsigned local = read->AsLclVar()->GetLclNum(); + GenTree* node = read->gtPrev; + for (unsigned remaining = 64; (node != nullptr) && (remaining != 0); node = node->gtPrev, remaining--) + { + if (node->OperIs(GT_STORE_LCL_VAR) && (node->AsLclVar()->GetLclNum() == local)) + { + return node; + } + } + return nullptr; +} + +//------------------------------------------------------------------------ +// TryLowerCarryCompare: Reuse arithmetic flags for unsigned carry/borrow comparisons. +// Match sum < operand, difference > minuend, or a < b after computing a-b. +// The arithmetic result can be forwarded or stored to a local in this block. +bool Lowering::TryLowerCarryCompare(GenTreeOp* cmp) +{ + auto match = [&](unsigned mode) -> bool { + // 0: sum comparison, 1: difference comparison, 2: subtraction operands, + // 3: negated value compared with zero (the borrow from 0 - value). + bool subtract = mode != 0; + bool negate = mode == 3; + + if (negate ? (!cmp->OperIs(GT_EQ, GT_NE) || !cmp->gtOp2->IsIntegralConst(0)) + : (!cmp->OperIs(GT_LT, GT_GT, GT_GE, GT_LE) || !cmp->IsUnsigned())) + { + return false; + } + + bool reversed = cmp->OperIs(GT_GT, GT_LE) != (mode == 1); + GenTree* sum = reversed ? cmp->gtOp2 : cmp->gtOp1; + GenTree* operand = reversed ? cmp->gtOp1 : cmp->gtOp2; + if ((!sum->OperIs(GT_LCL_VAR, negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && + !(mode == 1 && sum->OperIs(GT_ADD))) || + !sum->TypeIs(TYP_INT, TYP_LONG) || !operand->OperIs(GT_LCL_VAR, GT_CNS_INT) || + (sum->TypeGet() != operand->TypeGet())) + { + return false; + } + + GenTree* add = sum; + if (mode == 2) + { + if (!sum->OperIs(GT_LCL_VAR)) + { + return false; + } + // a < b can reuse an already-computed a-b, as in UInt128 subtraction + // and subtract-multiply. Compare the reaching operand values as well. + add = cmp->gtPrev; + unsigned budget = 64; + for (; add != nullptr && budget != 0; add = add->gtPrev, budget--) + { + if (m_carryUseScanBudget == 0) + { + return false; + } + m_carryUseScanBudget--; + if (add->OperIs(GT_SUB) && GenTree::Compare(add->gtGetOp1(), sum) && + GenTree::Compare(add->gtGetOp2(), operand) && IsInvariantInRange(add->gtGetOp1(), cmp) && + IsInvariantInRange(add->gtGetOp2(), cmp)) + { + break; + } + } + if (add == nullptr || budget == 0) + { + return false; + } + } + else if (sum->OperIs(GT_LCL_VAR)) + { + if (m_compiler->lvaGetDesc(sum->AsLclVar()->GetLclNum())->IsAddressExposed()) + { + return false; + } + GenTree* def = FindCarryLocalStore(sum); + if (def == nullptr) + { + return false; + } + add = def->gtGetOp1(); + } + bool negatedConstant = (mode == 1) && add->OperIs(GT_ADD) && add->gtGetOp2()->IsCnsIntOrI() && + !add->gtGetOp2()->AsIntCon()->IsIconHandle() && + (add->gtGetOp2()->AsIntCon()->IconValue() < 0) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT64_MIN) && + (add->gtGetOp2()->AsIntCon()->IconValue() != INT32_MIN); + if ((!add->OperIs(negate ? GT_NEG : (subtract ? GT_SUB : GT_ADD)) && !negatedConstant) || + (!negate && add->gtOverflow()) || add->gtSetFlags() || add->isContained() || + (add->TypeGet() != sum->TypeGet())) + { + return false; + } +#ifdef TARGET_ARM64 + // MNEG does not set flags, even when its containing NEG requests them. + if (negate && add->gtGetOp1()->OperIs(GT_MUL)) + { + return false; + } +#endif + + GenTree* matched = negate ? add->gtGetOp1() : nullptr; + for (GenTree* input : add->Operands()) + { + if ((mode != 1 || input == add->gtGetOp1()) && GenTree::Compare(input, operand) && + IsInvariantInRange(input, cmp)) + { + matched = input; + break; + } + } + LIR::Use use; + if ((matched == nullptr) || !BlockRange().TryGetUse(cmp, &use)) + { + return false; + } + + // A select needs flags at the select itself. Materializing them here can + // introduce SETCC and TEST when the sum is still used or flags are clobbered. + // Preserve the existing compare-to-select lowering in that case. + if (use.User()->OperIs(GT_SELECT)) + { + return false; + } + + GenCondition condition = cmp->OperIs(GT_LT, GT_GT, GT_NE) ? GenCondition::C : GenCondition::NC; +#ifdef TARGET_ARM64 + if (subtract) + { + condition = GenCondition::Reverse(condition); + } +#endif + bool branch = use.User()->OperIs(GT_JTRUE); + if (branch) + { +#ifdef TARGET_AMD64 + // Unlike a carry chain, a branch saves only a comparison. Avoid sacrificing LEA + // or perturbing allocation in register-heavy methods for that small saving. + // LAST - FIRST excludes SP from the inclusive integer-register range. + unsigned intRegCount = m_compiler->get_REG_INT_LAST() - REG_INT_FIRST; + if (negatedConstant || (m_compiler->lvaTrackedCount > 2 * intRegCount)) + { + return false; + } +#endif + if (use.User() != cmp->gtNext) + { + return false; + } + // Keep the branch at the end of the block, and only cross nodes that + // cannot overwrite flags. JCC need not be adjacent to its flags producer. + // Constants may become XOR on x64. + for (GenTree* node = add->gtNext; node != cmp; node = node->gtNext) + { + if ((node == nullptr) || + ((node != operand) && !node->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_IL_OFFSET))) + { + return false; + } + } + } + + JITDUMP("Replacing carry comparison [%06u] with flags from [%06u]\n", cmp->gtTreeID, add->gtTreeID); + if (negatedConstant) + { + // Restore SUB so its flags describe the borrow, rather than the carry + // from adding the two's-complement constant. The result is unchanged. + add->ChangeOper(GT_SUB); + GenTreeIntCon* constant = add->gtGetOp2()->AsIntCon(); + constant->SetIconValue(-constant->IconValue()); + } + add->gtFlags |= GTF_SET_FLAGS; + if (add->OperIs(GT_ADD)) + { + add->gtFlags |= GTF_ADD_CARRY_FLAGS; + } + for (GenTree* input : add->Operands()) + { + input->ClearContained(); + input->ClearRegOptional(); + } + if (negate) + { + ContainCheckNode(add); + } + else + { + ContainCheckBinary(add->AsOp()); + } + if (branch) + { + GenTree* jump = use.User(); + jump->ChangeOper(GT_JCC); + jump->AsCC()->gtCondition = condition; + } + else + { + GenTreeCC* carry = m_compiler->gtNewCC(GT_SETCC, TYP_INT, condition); + m_blockMayHaveSetCC = true; + BlockRange().InsertAfter(add, carry); + use.ReplaceWith(carry); + } + if (sum == add) + { + add->SetUnusedValue(); + } + else + { + BlockRange().Remove(sum); + } + BlockRange().Remove(operand); + BlockRange().Remove(cmp); + return true; + }; + return match(0) || ((m_lastSubtractionBlock == m_block) && (match(1) || match(2))) || match(3); +} + +#ifdef TARGET_AMD64 +//------------------------------------------------------------------------ +// ReplaceCarryAddressAdd: Replace a validated base-plus-constant ADD with LEA +// so the address calculation preserves both carry flags. +void Lowering::ReplaceCarryAddressAdd(GenTree* add) +{ + assert(add->OperIs(GT_ADD) && !add->gtOverflow() && !add->gtSetFlags()); + GenTree* offset = add->gtGetOp2(); + assert(offset->IsIntCnsFitsInI32()); + GenTree* lea = new (m_compiler, GT_LEA) + GenTreeAddrMode(add->TypeGet(), add->gtGetOp1(), nullptr, 1, (int)offset->AsIntCon()->IconValue()); + LIR::Use use; + bool found = BlockRange().TryGetUse(add, &use); + assert(found); + BlockRange().InsertBefore(add, lea); + use.ReplaceWith(lea); + BlockRange().Remove(add); + BlockRange().Remove(offset); +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoops: Keep both unsigned carry chains of scalar and +// unrolled widening multiply-accumulates in flags around counted loops. +// Memory containment checks interference; no bounds check is removed. +void Lowering::LowerMultiplyCarryLoops() +{ + if (m_compiler->compHndBBtabCount != 0 || !m_compiler->compOpportunisticallyDependsOn(InstructionSet_ADX) || + !m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + return; + } + + BasicBlock* savedBlock = m_block; + + // MULX is required while CF/OF are live: ordinary MUL overwrites both flags. + // Snapshot blocks since successful matches split edges. + jitstd::vector loops(m_compiler->getAllocator(CMK_Lower)); + for (BasicBlock* block : m_compiler->Blocks()) + { + loops.push_back(block); + } + for (BasicBlock* block : loops) + { + m_block = block; + bool hasAdx = TryLowerMultiplyCarryChain(); + bool hasRows = !hasAdx && TryLowerMultiplyCarryRows(); + hasAdx |= hasRows; + if (!hasRows && block->KindIs(BBJ_COND) && block->GetTrueTarget() == block && + block->GetFalseTarget() != block) + { + TryLowerMultiplyCarryBackedge(); + hasAdx |= TryLowerMultiplyCarryLoop(); + LowerMultiplyCarryMultiplier(); + } + if (hasAdx) + { + LowerMultiplyCarryLoads(); + } + } + m_block = savedBlock; +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryLoads: Forward single-use load temporaries into the memory +// operand of MULX and ADCX/ADOX after the carry patterns have been matched. +void Lowering::LowerMultiplyCarryLoads() +{ + for (GenTree* node : BlockRange()) + { + GenTree** operand = nullptr; + if (node->OperIs(GT_ADCX, GT_ADOX)) + { + operand = &node->AsOp()->gtOp2; + } + else if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG) && ((node->gtFlags & GTF_HW_MULX) != 0)) + { + operand = &node->AsHWIntrinsic()->Op(2); + } + if ((operand == nullptr) || !(*operand)->OperIs(GT_LCL_VAR)) + { + continue; + } + + GenTree* read = *operand; + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed()) + { + continue; + } + + GenTree* store = nullptr; + bool valid = true; + for (GenTree* reference : BlockRange()) + { + if (m_carryUseScanBudget == 0) + { + return; + } + --m_carryUseScanBudget; + // A promoted field's value can also be observed or changed through its parent. + if (dsc->lvIsStructField && reference->OperIsLocal() && + (reference->AsLclVarCommon()->GetLclNum() == dsc->lvParentLcl)) + { + valid = false; + break; + } + if ((reference == read) && (store == nullptr)) + { + valid = false; + break; + } + if (reference->OperIsLocal() && (reference->AsLclVarCommon()->GetLclNum() == local)) + { + if (reference->OperIs(GT_STORE_LCL_VAR) && (store == nullptr)) + { + store = reference; + } + else if (reference != read) + { + valid = false; + break; + } + } + } + if (!valid || (store == nullptr)) + { + continue; + } + GenTree* load = store->gtGetOp1(); + if (!load->OperIs(GT_IND) || !load->TypeIs(TYP_LONG) || ((load->gtFlags & GTF_IND_VOLATILE) != 0) || + !IsSafeToContainMem(node, store, load)) + { + continue; + } + + if (!IsCarryLocalDeadOnExit(local, m_block)) + { + continue; + } + + JITDUMP("Containing load [%06u] from V%02u in [%06u]\n", load->gtTreeID, local, node->gtTreeID); + *operand = load; + // Both removed nodes precede the current iterator. The load stays in + // LIR; containment delays its execution only across the checked range. + BlockRange().Remove(store); + BlockRange().Remove(read); + MakeSrcContained(node, load); + } +} + +//------------------------------------------------------------------------ +// LowerMultiplyCarryMultiplier: Give a flags-preserving multiply loop its own +// copy of an invariant multiplier so its fixed RDX uses can share a register +// assignment independently of the incoming argument and the remainder loop. +void Lowering::LowerMultiplyCarryMultiplier() +{ + if (!BlockRange().LastNode()->OperIs(GT_JCMP)) + { + return; + } + + BasicBlock* entry = FindCarryLoopEntry(m_block); + if ((entry == nullptr) || !entry->KindIs(BBJ_ALWAYS) || (entry->GetTarget() != m_block)) + { + return; + } + // These loop transforms create a dedicated entry block containing only the flags seed. + // Insert before the seed so even a zero-initializing copy cannot interfere with the flags. + if ((LIR::AsRange(entry).FirstNode() != LIR::AsRange(entry).LastNode()) || + (LIR::AsRange(entry).FirstNode() == nullptr) || !LIR::AsRange(entry).FirstNode()->OperIs(GT_ADX_SEED)) + { + return; + } + + unsigned multiplier = BAD_VAR_NUM; + jitstd::vector products(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_HWINTRINSIC)) + { + GenTreeHWIntrinsic* product = node->AsHWIntrinsic(); + if ((product->GetHWIntrinsicId() != NI_X86Base_X64_BigMul) || (product->GetSimdBaseType() != TYP_ULONG) || + ((product->gtFlags & GTF_HW_MULX) == 0) || !product->Op(1)->OperIs(GT_LCL_VAR) || + !product->Op(1)->TypeIs(TYP_LONG)) + { + return; + } + unsigned local = product->Op(1)->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (!dsc->lvTracked || dsc->IsAddressExposed() || dsc->lvIsStructField || + ((multiplier != BAD_VAR_NUM) && (multiplier != local))) + { + return; + } + multiplier = local; + products.push_back(product); + } + } + if (products.empty()) + { + return; + } + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD) && (node->AsLclVarCommon()->GetLclNum() == multiplier)) + { + return; + } + } + + unsigned temp = m_compiler->lvaGrabTemp(true DEBUGARG("ADX loop multiplier")); + JITDUMP("Copying invariant multiplier V%02u to V%02u on entry to " FMT_BB "\n", multiplier, temp, m_block->bbNum); + m_compiler->lvaGetDesc(temp)->lvType = TYP_LONG; + GenTree* read = m_compiler->gtNewLclvNode(multiplier, TYP_LONG); + GenTree* store = m_compiler->gtNewStoreLclVarNode(temp, read); + LIR::AsRange(entry).InsertAtBeginning(read, store); + for (GenTreeHWIntrinsic* product : products) + { + GenTree* oldRead = product->Op(1); + GenTree* newRead = m_compiler->gtNewLclvNode(temp, TYP_LONG); + BlockRange().InsertBefore(oldRead, newRead); + product->Op(1) = newRead; + BlockRange().Remove(oldRead); + } +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryChain: Keep CF and OF across a bounded sequence of +// widening multiply-accumulates in one block. Seed after any preceding checks +// and drain before the final high word is consumed, so checks and loop control +// outside the sequence need not preserve either flag. No memory access moves. +// Returns true if an ADX chain was created; a scalar MULX-only match returns false. +bool Lowering::TryLowerMultiplyCarryChain() +{ + const unsigned maxLanes = 4; + struct Lane + { + GenTree* product; + GenTree* productStore; + GenTree* adc[2]; + GenTree* add[2]; + GenTree* low; + GenTree* destination; + GenTree* partial; + GenTree* carry; + } lanes[maxLanes] = {}; + + unsigned laneCount = 0; + unsigned adcCount = 0; + unsigned nodeCount = 0; + for (GenTree* node : BlockRange()) + { + if (++nodeCount > 512) + { + return false; + } + if (node->OperIs(GT_HWINTRINSIC)) + { + if (laneCount == maxLanes || adcCount != laneCount * 2 || + node->AsHWIntrinsic()->GetHWIntrinsicId() != NI_X86Base_X64_BigMul || + node->AsHWIntrinsic()->GetSimdBaseType() != TYP_ULONG) + { + return false; + } + lanes[laneCount++].product = node; + } + if (node->OperIs(GT_ADD_CARRY)) + { + if (laneCount == 0 || adcCount >= laneCount * 2 || !node->TypeIs(TYP_LONG) || node->gtSetFlags() || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntegralConst(0)) + { + return false; + } + lanes[adcCount / 2].adc[adcCount % 2] = node; + adcCount++; + } + } + if (laneCount == 0 || adcCount != laneCount * 2) + { + return false; + } + + // Changed intermediate sums/high words may have only one reader and must + // not escape this block. Requiring the store first also proves availability. + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + LIR::Use productUse; + if (!BlockRange().TryGetUse(lane.product, &productUse) || !productUse.User()->OperIs(GT_STORE_LCL_VAR)) + { + return false; + } + lane.productStore = productUse.User(); + unsigned productLocal = lane.productStore->AsLclVar()->GetLclNum(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productLocal); + if (!fields->lvPromoted || fields->lvFieldCnt != 2) + { + return false; + } + unsigned lowLocal = fields->lvFieldLclStart; + unsigned highLocal = lowLocal + 1; + for (unsigned j = 0; j < 2; j++) + { + GenTree* add = lane.adc[j]->gtPrev; + if (add == nullptr || !add->OperIs(GT_ADD) || !add->TypeIs(TYP_LONG) || add->gtOverflow() || + !add->gtSetFlags() || !add->gtGetOp1()->OperIs(GT_LCL_VAR) || !add->gtGetOp2()->OperIs(GT_LCL_VAR)) + { + return false; + } + lane.add[j] = add; + } + if (lane.adc[0]->gtGetOp1()->AsLclVar()->GetLclNum() != highLocal) + { + return false; + } + GenTree* highStore = FindMultiplyCarryStore(lane.adc[1]->gtGetOp1()); + if (highStore == nullptr || highStore->gtGetOp1() != lane.adc[0]) + { + return false; + } + lane.low = lane.add[0]->gtGetOp1(); + lane.destination = lane.add[0]->gtGetOp2(); + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + std::swap(lane.low, lane.destination); + } + if (lane.low->AsLclVar()->GetLclNum() != lowLocal) + { + return false; + } + lane.partial = lane.add[1]->gtGetOp1(); + lane.carry = lane.add[1]->gtGetOp2(); + GenTree* partialStore = FindMultiplyCarryStore(lane.partial); + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + std::swap(lane.partial, lane.carry); + partialStore = FindMultiplyCarryStore(lane.partial); + } + if (partialStore == nullptr || partialStore->gtGetOp1() != lane.add[0]) + { + return false; + } + if (i != 0) + { + GenTree* carryStore = FindMultiplyCarryStore(lane.carry); + if (carryStore == nullptr || carryStore->gtGetOp1() != lanes[i - 1].adc[1]) + { + return false; + } + } + + // The two field reads must refer to this product, not to earlier or + // intervening assignments. A parent/field alias must not escape us. + bool defined = false; + for (GenTree* node : BlockRange()) + { + if (node == lane.productStore) + { + defined = true; + } + if ((node == lane.low || node == lane.adc[0]->gtGetOp1()) && !defined) + { + return false; + } + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if ((local == lowLocal && node != lane.low) || + (local == highLocal && node != lane.adc[0]->gtGetOp1()) || + (local == productLocal && node != lane.productStore)) + { + return false; + } + } + } + + // Move only the carry read, from the second addition to the first. + // Unlike intermediate carries, the initial value may have other users. + unsigned carryLocal = lane.carry->AsLclVar()->GetLclNum(); + if (m_compiler->lvaGetDesc(carryLocal)->IsAddressExposed()) + { + return false; + } + for (GenTree* node = lane.add[0]; node != lane.carry; node = node->gtNext) + { + if (node == nullptr || (node->OperIsLocalStore() && node->AsLclVarCommon()->GetLclNum() == carryLocal)) + { + return false; + } + } + } + + if (laneCount == 1) + { + // The product feeds both halves of this proven multiply-accumulate. + // Keep MULX's independent outputs for scalar tails without introducing + // an ADX seed/drain pair for a single lane. + lanes[0].product->gtFlags |= GTF_HW_MULX; + return false; + } + + GenTree* first = lanes[0].add[0]; + GenTree* last = lanes[laneCount - 1].adc[1]; + jitstd::vector addresses(m_compiler->getAllocator(CMK_Lower)); + for (GenTree* node = first; node != last->gtNext; node = node->gtNext) + { + bool matched = false; + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + matched |= node == lane.product || node == lane.add[0] || node == lane.add[1] || node == lane.adc[0] || + node == lane.adc[1]; + } + if (matched) + { + continue; + } + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: + case GT_IL_OFFSET: + case GT_LEA: + break; + case GT_CAST: + if (node->gtOverflow() || !node->TypeIs(TYP_LONG) || !node->gtGetOp1()->TypeIs(TYP_INT)) + { + return false; + } + break; + case GT_IND: + case GT_STOREIND: + if (!node->TypeIs(TYP_LONG) || (node->gtFlags & GTF_IND_VOLATILE) != 0) + { + return false; + } + break; + case GT_CNS_INT: + if (!node->isContained()) + { + return false; + } + break; + case GT_ADD: + if (node->gtSetFlags() || node->gtOverflow() || !node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) || + !node->gtGetOp1()->OperIs(GT_LCL_VAR) || !node->gtGetOp2()->IsIntCnsFitsInI32()) + { + return false; + } + addresses.push_back(node); + break; + default: + return false; + } + } + + JITDUMP("Keeping multiply-accumulate CF and OF chains across %u lanes in " FMT_BB "\n", laneCount, m_block->bbNum); + GenTree* initialize = new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID); + BlockRange().InsertBefore(first, initialize); + for (unsigned i = 0; i < laneCount; i++) + { + Lane& lane = lanes[i]; + // Every product in the matched chain must preserve both live carry flags. + lane.product->gtFlags |= GTF_HW_MULX; + BlockRange().Remove(lane.carry); + BlockRange().InsertBefore(lane.add[0], lane.carry); + lane.add[0]->AsOp()->gtOp1 = lane.carry; + lane.add[0]->AsOp()->gtOp2 = lane.destination; + lane.add[1]->AsOp()->gtOp1 = lane.partial; + lane.add[1]->AsOp()->gtOp2 = lane.low; + lane.add[0]->ChangeOper(GT_ADCX); + lane.add[1]->ChangeOper(GT_ADOX); + for (GenTree* add : lane.add) + { + for (GenTree* operand : add->Operands()) + { + operand->ClearContained(); + operand->ClearRegOptional(); + } + ContainCheckBinary(add->AsOp()); + } + for (GenTree* adc : lane.adc) + { + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + GenTree* replacement = adc->gtGetOp1(); + if (adc == last) + { + replacement = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, replacement); + BlockRange().InsertBefore(adc, replacement); + } + use.ReplaceWith(replacement); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + return true; +} + +//------------------------------------------------------------------------ +// TryLowerMultiplyCarryRows: Keep CF and OF across each row of a multi-limb +// multiply-accumulate when a block holds several rows, or rows wider than +// TryLowerMultiplyCarryChain accepts (for example the rows of a fully unrolled +// Montgomery multiplication). Lane j of a row computes +// (hi_j, lo_j) = x_j * m, r_j = lo_j [+ t_j] [+ c_j], c_{j+1} = hi_j + carries +// where c_j is the previous lane's high word. Two row shapes are recognized: +// dual: lane 0 adds t_0 (and possibly an incoming carry); every later lane adds +// both t_j and c_j. ADCX accumulates c_j + t_j in CF and ADOX adds lo_j in OF. +// single: lane 0 has no addition; every later lane adds c_j only (ADCX). +// Every product uses MULX, and the last high word is drained as hi + CF + OF. Each +// rewritten intermediate must be the only value its reader observes and must be dead +// after it, and only flags-preserving nodes may sit inside a row. No memory access moves. +// +// Returns: +// true if at least one row was rewritten. +// +bool Lowering::TryLowerMultiplyCarryRows() +{ + typedef JitHashTable, GenTree*> NodeToNodeMap; + + struct Lane + { + GenTree* product; + GenTree* productStore; + unsigned lowLocal; + unsigned highLocal; + GenTree* lowRead; // the read of the low word that the lane adds to + unsigned addCount; + GenTree* add[2]; + GenTree* adc[2]; + GenTree* other[2]; // the operand of add[i] that is neither the low word nor the partial sum + GenTree* highOutStore; // store of the high word passed to the next lane (null for a bare lane) + GenTree* carry; // read of the previous lane's high word, if this lane continues a row + GenTree* dest; // the remaining addend of a continuing lane + }; + + unsigned products = 0; + for (GenTree* node : BlockRange()) + { + if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG)) + { + products++; + } + } + if (products < 2) + { + return false; + } + + // Number the nodes, count every local's stores and reads, and link each local + // reference to the previous and next references to the same local. + CompAllocator alloc = m_compiler->getAllocator(CMK_Lower); + Compiler::NodeToUnsignedMap seq(alloc); + NodeToNodeMap prevRef(alloc); + NodeToNodeMap nextRef(alloc); + jitstd::vector stores(m_compiler->lvaCount, 0, alloc); + jitstd::vector reads(m_compiler->lvaCount, 0, alloc); + jitstd::vector lastRef(m_compiler->lvaCount, nullptr, alloc); + jitstd::vector storeNode(m_compiler->lvaCount, nullptr, alloc); + jitstd::vector readNode(m_compiler->lvaCount, nullptr, alloc); + jitstd::vector firstRef(m_compiler->lvaCount, nullptr, alloc); + jitstd::vector addressed(m_compiler->lvaCount, false, alloc); + jitstd::vector lanes(alloc); + unsigned index = 0; + for (GenTree* node : BlockRange()) + { + seq.Set(node, index++); + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + if (lastRef[local] != nullptr) + { + prevRef.Set(node, lastRef[local]); + nextRef.Set(lastRef[local], node); + } + else + { + firstRef[local] = node; + } + lastRef[local] = node; + if (node->OperIsLocalStore()) + { + stores[local]++; + storeNode[local] = node; + } + else + { + reads[local]++; + readNode[local] = node; + } + } + else if (node->OperIs(GT_LCL_ADDR)) + { + addressed[node->AsLclVarCommon()->GetLclNum()] = true; + } + if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG)) + { + Lane lane = {}; + lane.product = node; + lanes.push_back(lane); + } + } + + auto pos = [&](GenTree* node) -> unsigned { + unsigned value = 0; + bool found = seq.Lookup(node, &value); + assert(found); + return value; + }; + + // Whether a local is a temporary of this block: every reference is in the block and the + // first one fully defines it, so no value of it can be observed after the block, even + // around a loop. This avoids rescanning large blocks that are their own successor. The + // count of outside references shares the budget of the dead-on-exit proofs. + jitstd::vector outsideRefs(alloc); + bool outsideRefsCounted = false; + auto countOutsideRefs = [&]() -> bool { + outsideRefs.resize(m_compiler->lvaCount, 0); + for (BasicBlock* other : m_compiler->Blocks()) + { + if (other == m_block) + { + continue; + } + for (GenTree* node : LIR::AsRange(other)) + { + if (m_carryLocalScanBudget == 0) + { + return false; + } + m_carryLocalScanBudget--; + if (node->OperIsLocal() || node->OperIs(GT_LCL_ADDR)) + { + outsideRefs[node->AsLclVarCommon()->GetLclNum()]++; + } + } + } + return true; + }; + auto isBlockLocal = [&](unsigned local) -> bool { + if (outsideRefs.empty()) + { + outsideRefsCounted = countOutsideRefs(); + } + if (!outsideRefsCounted) + { + return false; + } + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + unsigned parent = dsc->lvIsStructField ? dsc->lvParentLcl : local; + // A promoted field is defined by its parent's first store and must not be read before it. + return !dsc->IsAddressExposed() && !dsc->lvIsParam && !(m_compiler->opts.IsOSR() && dsc->lvIsOSRLocal) && + !m_compiler->lvaGetDesc(parent)->lvIsParam && (outsideRefs[local] == 0) && (outsideRefs[parent] == 0) && + !addressed[local] && !addressed[parent] && (firstRef[parent] != nullptr) && + firstRef[parent]->OperIs(GT_STORE_LCL_VAR) && + ((parent == local) || + ((stores[local] == 0) && (firstRef[local] != nullptr) && (pos(firstRef[local]) > pos(firstRef[parent])))); + }; + + // Whether a read is the only reader of the value it observes: the previous reference to + // the local is a full store and the next one, if any, is a store. A value still live at + // the end of the block must be dead on exit. A promoted field is defined by the single + // store of its parent, which is not otherwise read. + auto isSingleUse = [&](GenTree* read) -> bool { + if ((read == nullptr) || !read->OperIs(GT_LCL_VAR)) + { + return false; + } + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + // Untracked locals are fine: the dead-on-exit proof walks the IR, not liveness sets. + if (dsc->IsAddressExposed() || dsc->lvPromoted) + { + return false; + } + GenTree* store = nullptr; + GenTree* next = nullptr; + if (dsc->lvIsStructField) + { + unsigned parent = dsc->lvParentLcl; + if ((stores[local] != 0) || (reads[local] != 1) || (reads[parent] != 0) || (stores[parent] != 1) || + (pos(storeNode[parent]) > pos(read))) + { + return false; + } + } + else if (!prevRef.Lookup(read, &store) || !store->OperIs(GT_STORE_LCL_VAR) || + (nextRef.Lookup(read, &next) && !next->OperIs(GT_STORE_LCL_VAR))) + { + return false; + } + return (next != nullptr) || isBlockLocal(local) || IsCarryLocalDeadOnExit(local, m_block); + }; + + // The only reader of the value a store defines, if there is exactly one. + auto singleReader = [&](GenTree* store) -> GenTree* { + GenTree* read = nullptr; + if ((store == nullptr) || !nextRef.Lookup(store, &read) || !isSingleUse(read)) + { + return nullptr; + } + return read; + }; + + auto storeOf = [&](GenTree* value) -> GenTree* { + LIR::Use use; + if (!BlockRange().TryGetUse(value, &use) || !use.User()->OperIs(GT_STORE_LCL_VAR)) + { + return nullptr; + } + return use.User(); + }; + + auto isFlagsAdd = [](GenTree* node) -> bool { + return (node != nullptr) && node->OperIs(GT_ADD) && node->TypeIs(TYP_LONG) && node->gtSetFlags() && + !node->gtOverflow() && node->gtGetOp1()->OperIs(GT_LCL_VAR) && node->gtGetOp2()->OperIs(GT_LCL_VAR); + }; + + auto isHighCarry = [](GenTree* node) -> bool { + return (node != nullptr) && node->OperIs(GT_ADD_CARRY) && node->TypeIs(TYP_LONG) && !node->gtSetFlags() && + node->gtGetOp1()->OperIs(GT_LCL_VAR) && node->gtGetOp2()->IsIntegralConst(0); + }; + + // Describe each lane: its two product halves and up to two (ADD, ADD_CARRY) pairs. The rewrite + // keeps the values of both halves, so they may have other readers; only the intermediates + // whose values change (the partial sum and the carried high words) need a single reader. + for (Lane& lane : lanes) + { + lane.productStore = storeOf(lane.product); + if (lane.productStore == nullptr) + { + JITDUMP("Rows: product [%06u] is not stored\n", lane.product->gtTreeID); + return false; + } + unsigned productLocal = lane.productStore->AsLclVar()->GetLclNum(); + LclVarDsc* fields = m_compiler->lvaGetDesc(productLocal); + if (!fields->lvPromoted || (fields->lvFieldCnt != 2) || (stores[productLocal] != 1) || + (reads[productLocal] != 0)) + { + JITDUMP("Rows: product [%06u] store is not promoted\n", lane.product->gtTreeID); + return false; + } + lane.lowLocal = fields->lvFieldLclStart; + lane.highLocal = lane.lowLocal + 1; + if ((stores[lane.lowLocal] != 0) || (stores[lane.highLocal] != 0)) + { + JITDUMP("Rows: product [%06u] halves are redefined\n", lane.product->gtTreeID); + return false; + } + + // Find the high word's (ADD, ADD_CARRY) pair, if any. + GenTree* highRead = nullptr; + for (GenTree* ref = lastRef[lane.highLocal]; ref != nullptr;) + { + LIR::Use use; + if (BlockRange().TryGetUse(ref, &use) && isHighCarry(use.User()) && (use.User()->gtGetOp1() == ref)) + { + if (highRead != nullptr) + { + JITDUMP("Rows: product [%06u] high word has two carries\n", lane.product->gtTreeID); + return false; + } + highRead = ref; + lane.adc[0] = use.User(); + } + if (!prevRef.Lookup(ref, &ref)) + { + break; + } + } + if (highRead == nullptr) + { + // No accumulation: the halves are used as they are. + continue; + } + lane.add[0] = lane.adc[0]->gtPrev; + if (!isFlagsAdd(lane.add[0]) || (pos(highRead) < pos(lane.productStore))) + { + JITDUMP("Rows: product [%06u] high word is not carried after an addition\n", lane.product->gtTreeID); + return false; + } + for (GenTree* operand : lane.add[0]->Operands()) + { + if (operand->AsLclVar()->GetLclNum() == lane.lowLocal) + { + lane.lowRead = operand; + } + } + if (lane.lowRead == nullptr) + { + JITDUMP("Rows: product [%06u] low word is not added before its carry\n", lane.product->gtTreeID); + return false; + } + lane.other[0] = + (lane.add[0]->gtGetOp1() == lane.lowRead) ? lane.add[0]->gtGetOp2() : lane.add[0]->gtGetOp1(); + lane.highOutStore = storeOf(lane.adc[0]); + lane.addCount = 1; + + // A second accumulation is the only reader of the partial sum and of the first high word, + // which it either reads from a local or consumes directly. + GenTree* partialRead = lane.add[0]->IsUnusedValue() ? nullptr : singleReader(storeOf(lane.add[0])); + GenTree* highRead1 = (lane.highOutStore != nullptr) ? singleReader(lane.highOutStore) : lane.adc[0]; + LIR::Use partialUse; + LIR::Use high1Use; + if ((partialRead == nullptr) || (highRead1 == nullptr) || !BlockRange().TryGetUse(partialRead, &partialUse) || + !isFlagsAdd(partialUse.User()) || !BlockRange().TryGetUse(highRead1, &high1Use) || + !high1Use.User()->OperIs(GT_ADD_CARRY) || !high1Use.User()->TypeIs(TYP_LONG) || + high1Use.User()->gtSetFlags() || !high1Use.User()->gtGetOp2()->IsIntegralConst(0) || + (high1Use.User()->gtPrev != partialUse.User()) || (high1Use.User()->gtGetOp1() != highRead1)) + { + if (lane.highOutStore == nullptr) + { + JITDUMP("Rows: product [%06u] carry is not stored\n", lane.product->gtTreeID); + return false; + } + continue; + } + lane.add[1] = partialUse.User(); + lane.adc[1] = high1Use.User(); + lane.other[1] = + (lane.add[1]->gtGetOp1() == partialRead) ? lane.add[1]->gtGetOp2() : lane.add[1]->gtGetOp1(); + lane.highOutStore = storeOf(lane.adc[1]); + lane.addCount = 2; + if (lane.highOutStore == nullptr) + { + JITDUMP("Rows: product [%06u] carry is not stored\n", lane.product->gtTreeID); + return false; + } + } + + bool changed = false; + for (size_t start = 0; start < lanes.size();) + { + // Extend the row while the next lane accumulates this lane's high word. + size_t end = start + 1; + for (; end < lanes.size(); end++) + { + Lane& prev = lanes[end - 1]; + Lane& lane = lanes[end]; + // A bare lane passes its raw high word, whose value the rewrite keeps. + GenTree* carry = (prev.addCount == 0) ? nullptr : singleReader(prev.highOutStore); + unsigned addend = BAD_VAR_NUM; + for (unsigned i = 0; i < lane.addCount; i++) + { + GenTree* other = lane.other[i]; + if ((prev.addCount == 0) ? ((other->AsLclVar()->GetLclNum() == prev.highLocal) && + (pos(other) > pos(prev.productStore))) + : (other == carry)) + { + addend = i; + carry = other; + } + } + if (addend == BAD_VAR_NUM) + { + break; + } + lane.carry = carry; + lane.dest = (lane.addCount == 2) ? lane.other[1 - addend] : nullptr; + } + + size_t first = start; + size_t last = end - 1; + start = end; + if (last == first) + { + JITDUMP("Rows: lane [%06u] (%u additions) starts no row\n", lanes[first].product->gtTreeID, + lanes[first].addCount); + continue; + } + + // Classify the row; every lane after the first continues it. + bool single = lanes[first].addCount == 0; + bool valid = single || (lanes[first].carry == nullptr); + for (size_t i = first + 1; valid && (i <= last); i++) + { + valid = single ? ((lanes[i].addCount == 1) && (lanes[i].dest == nullptr)) + : ((lanes[i].addCount == 2) && (lanes[i].dest != nullptr)); + } + + // The row's final high word is drained into its only reader. + Lane& tail = lanes[last]; + GenTree* lastAdc = valid ? tail.adc[tail.addCount - 1] : nullptr; + valid = valid && (singleReader(tail.highOutStore) != nullptr); + + // Moving the addends of a two-addition lane to its first ADD must not cross their stores. + for (size_t i = first; valid && !single && (i <= last); i++) + { + if (lanes[i].addCount != 2) + { + continue; + } + for (GenTree* addend : {lanes[i].other[0], lanes[i].other[1]}) + { + unsigned local = addend->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->IsAddressExposed() || dsc->lvIsStructField) + { + valid = false; + break; + } + bool later = pos(addend) > pos(lanes[i].add[0]); + GenTree* from = later ? lanes[i].add[0] : addend; + GenTree* to = later ? addend : lanes[i].add[0]; + for (GenTree* node = from; (node != nullptr) && (node != to); node = node->gtNext) + { + if (node->OperIsLocalStore() && (node->AsLclVarCommon()->GetLclNum() == local)) + { + valid = false; + break; + } + } + } + } + if (!valid) + { + JITDUMP("Rows: lanes [%06u..%06u] are not a recognized row\n", lanes[first].product->gtTreeID, + lanes[last].product->gtTreeID); + continue; + } + + // Only the row's own arithmetic, products and flags-preserving nodes may sit inside it. + GenTree* seed = lanes[single ? first + 1 : first].add[0]; + jitstd::vector addresses(alloc); + for (GenTree* node = seed; valid && (node != lastAdc); node = node->gtNext) + { + bool matched = false; + for (size_t i = first; i <= last; i++) + { + for (unsigned j = 0; j < lanes[i].addCount; j++) + { + matched |= (node == lanes[i].add[j]) || (node == lanes[i].adc[j]) || + (node == lanes[i].adc[j]->gtGetOp2()); + } + } + if (matched || IsCarryFlagsCrossing(node) || + (node->OperIs(GT_STORE_LCL_VAR) && node->TypeIs(TYP_STRUCT) && + node->gtGetOp1()->OperIs(GT_HWINTRINSIC))) + { + continue; + } + if (node->OperIs(GT_HWINTRINSIC) && + (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul) && + (node->AsHWIntrinsic()->GetSimdBaseType() == TYP_ULONG)) + { + // Every unsigned product inside the row becomes a flags-preserving MULX. + continue; + } + if (node->OperIs(GT_ADD) && !node->gtSetFlags() && !node->gtOverflow() && + node->TypeIs(TYP_INT, TYP_LONG, TYP_BYREF) && node->gtGetOp1()->OperIs(GT_LCL_VAR) && + node->gtGetOp2()->IsIntCnsFitsInI32()) + { + addresses.push_back(node); + continue; + } + JITDUMP("Rows: [%06u] %s blocks lanes [%06u..%06u]\n", node->gtTreeID, GenTree::OpName(node->OperGet()), + lanes[first].product->gtTreeID, lanes[last].product->gtTreeID); + valid = false; + } + if (!valid) + { + continue; + } + + JITDUMP("Keeping %s multiply-accumulate row of %u lanes in flags in " FMT_BB "\n", + single ? "single" : "dual", (unsigned)(last - first + 1), m_block->bbNum); + changed = true; + BlockRange().InsertBefore(seed, new (m_compiler, GT_ADX_SEED) GenTree(GT_ADX_SEED, TYP_VOID)); + for (GenTree* node = seed; node != lastAdc; node = node->gtNext) + { + if (node->OperIs(GT_HWINTRINSIC) && (node->AsHWIntrinsic()->GetHWIntrinsicId() == NI_X86Base_X64_BigMul)) + { + node->gtFlags |= GTF_HW_MULX; + } + } + // Containment may have swapped the commutative BigMul operands and + // put a row constant in RDX. Prefer the invariant multiplier common + // to all lanes as MULX's implicit operand, so it stays in RDX. + unsigned multiplierLocal = BAD_VAR_NUM; + GenTreeHWIntrinsic* firstProduct = lanes[first].product->AsHWIntrinsic(); + for (unsigned operandIndex = 1; operandIndex <= 2; operandIndex++) + { + GenTree* operand = firstProduct->Op(operandIndex); + if (!operand->OperIs(GT_LCL_VAR) || + m_compiler->lvaGetDesc(operand->AsLclVar()->GetLclNum())->IsAddressExposed() || + !IsInvariantInRange(operand, lanes[last].product)) + { + continue; + } + unsigned local = operand->AsLclVar()->GetLclNum(); + bool shared = true; + for (size_t i = first + 1; i <= last; i++) + { + GenTreeHWIntrinsic* product = lanes[i].product->AsHWIntrinsic(); + bool found = false; + for (unsigned j = 1; j <= 2; j++) + { + found |= product->Op(j)->OperIs(GT_LCL_VAR) && + product->Op(j)->AsLclVar()->GetLclNum() == local; + } + shared &= found; + } + if (shared) + { + multiplierLocal = local; + break; + } + } + for (size_t i = first; i <= last; i++) + { + Lane& lane = lanes[i]; + lane.product->gtFlags |= GTF_HW_MULX; + if (multiplierLocal != BAD_VAR_NUM) + { + GenTreeHWIntrinsic* product = lane.product->AsHWIntrinsic(); + if (product->Op(2)->OperIs(GT_LCL_VAR) && + product->Op(2)->AsLclVar()->GetLclNum() == multiplierLocal) + { + std::swap(product->Op(1), product->Op(2)); + } + product->Op(1)->ClearContained(); + product->Op(1)->ClearRegOptional(); + } + if (lane.addCount == 0) + { + continue; + } + if (single) + { + // lo + c + CF + lane.add[0]->ChangeOper(GT_ADCX); + } + else if (lane.addCount == 1) + { + // t + lo + OF; the row's CF chain starts at the next lane. + lane.add[0]->AsOp()->gtOp1 = lane.other[0]; + lane.add[0]->AsOp()->gtOp2 = lane.lowRead; + lane.add[0]->ChangeOper(GT_ADOX); + } + else + { + // (c + t + CF) + lo + OF, with both addends read at the first addition. + GenTree* carry = (i == first) ? lane.other[1] : lane.carry; + GenTree* dest = (i == first) ? lane.other[0] : lane.dest; + GenTree* partial = (lane.add[1]->gtGetOp1() == lane.other[1]) ? lane.add[1]->gtGetOp2() + : lane.add[1]->gtGetOp1(); + for (GenTree* addend : {carry, dest}) + { + BlockRange().Remove(addend); + BlockRange().InsertBefore(lane.add[0], addend); + } + lane.add[0]->AsOp()->gtOp1 = carry; + lane.add[0]->AsOp()->gtOp2 = dest; + lane.add[1]->AsOp()->gtOp1 = partial; + lane.add[1]->AsOp()->gtOp2 = lane.lowRead; + lane.add[0]->ChangeOper(GT_ADCX); + lane.add[1]->ChangeOper(GT_ADOX); + } + for (unsigned j = 0; j < lane.addCount; j++) + { + for (GenTree* operand : lane.add[j]->Operands()) + { + operand->ClearContained(); + operand->ClearRegOptional(); + } + } + for (unsigned j = 0; j < lane.addCount; j++) + { + GenTree* adc = lane.adc[j]; + LIR::Use use; + bool found = BlockRange().TryGetUse(adc, &use); + assert(found); + GenTree* replacement = adc->gtGetOp1(); + if (adc == lastAdc) + { + replacement = m_compiler->gtNewOperNode(GT_ADX_DRAIN, TYP_LONG, replacement); + BlockRange().InsertBefore(adc, replacement); + } + use.ReplaceWith(replacement); + BlockRange().Remove(adc->gtGetOp2()); + BlockRange().Remove(adc); + } + } + for (GenTree* add : addresses) + { + ReplaceCarryAddressAdd(add); + } + } + return changed; +} + +//------------------------------------------------------------------------ +// LowerMaskSelects: Turn constant-time selects written with a mask made from a +// flags bit, +// mask = 0 - bit; r_i = t_i ^ ((t_i ^ s_i) & mask) (bit from a SETCC) +// into CMOVs that read the bit's flags directly. The mask's only uses must be +// such selects and the flags must still hold the bit at every select, which is +// typical right after a carry or borrow chain. This removes the SETCC, NEG, AND +// and XOR latency that the select otherwise adds after the chain. +// +void Lowering::LowerMaskSelects() +{ + // Like the carry passes, leave methods with handlers alone: a handler can observe + // the mask and bit locals this removes. + if (m_compiler->compHndBBtabCount != 0) + { + return; + } + for (BasicBlock* block : m_compiler->Blocks()) + { + m_block = block; + bool hasMask = false; + for (GenTree* node : BlockRange()) + { + hasMask |= node->OperIs(GT_NEG); + } + // A rewrite removes nodes, so look again with fresh reference links after each one. + while (hasMask && TryLowerMaskSelect()) + { + } + } +} + +//------------------------------------------------------------------------ +// TryLowerMaskSelect: Rewrite the first eligible mask select in the current block; +// see LowerMaskSelects. +// +// Returns: +// true if a mask and its selects were rewritten. +// +bool Lowering::TryLowerMaskSelect() +{ + typedef JitHashTable, GenTree*> NodeToNodeMap; + typedef JitHashTable, GenTree*> LocalToNodeMap; + CompAllocator alloc = m_compiler->getAllocator(CMK_Lower); + + // Link each local reference to the previous and next references to the same local. + // Each rewrite rescans the block, so the scans share a budget. + NodeToNodeMap prevRef(alloc); + NodeToNodeMap nextRef(alloc); + LocalToNodeMap lastRef(alloc); + for (GenTree* node : BlockRange()) + { + if (m_maskSelectScanBudget == 0) + { + return false; + } + m_maskSelectScanBudget--; + if (node->OperIsLocal()) + { + unsigned local = node->AsLclVarCommon()->GetLclNum(); + GenTree* last = nullptr; + if (lastRef.Lookup(local, &last)) + { + prevRef.Set(node, last); + nextRef.Set(last, node); + } + lastRef.Set(local, node, LocalToNodeMap::Overwrite); + } + } + + // Whether the value a read observes has no reader after it: the local is stored + // again next, or is dead when the block ends. + auto isLastRead = [&](GenTree* read) -> bool { + unsigned local = read->AsLclVar()->GetLclNum(); + LclVarDsc* dsc = m_compiler->lvaGetDesc(local); + if (dsc->IsAddressExposed() || dsc->lvPromoted || dsc->lvIsStructField) + { + return false; + } + GenTree* next = nullptr; + return nextRef.Lookup(read, &next) ? next->OperIs(GT_STORE_LCL_VAR) : IsCarryLocalDeadOnExit(local, m_block); + }; + + struct Select + { + GenTree* outer; + GenTree* andNode; + GenTree* inner; + GenTree* maskRead; + GenTree* innerT; + GenTree* s; + GenTree* t; + }; + jitstd::vector