diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 93c9219bb03d1c..8678f886c321c0 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -4493,6 +4493,75 @@ AssertionIndex Compiler::optLocalAssertionIsEqualOrNotEqual( return NO_ASSERTION_INDEX; } +//------------------------------------------------------------------------------ +// optAssertionProp_ZeroOneCompare: Fold "x != 0" to "x" and "x == 0" to "x ^ 1" when +// "x" is known to be 0 or 1 and the compare is used as a value. Runs after the +// assertion based relop folding, which cannot fold "x ^ 1". +// +GenTree* Compiler::optAssertionProp_ZeroOneCompare(GenTreeOp* cmp, Statement* stmt) +{ + assert(cmp->OperIs(GT_EQ, GT_NE)); + +#ifdef TARGET_WASM + // "i32.eqz" is smaller than "i32.const 1; i32.xor". + if (cmp->OperIs(GT_EQ)) + { + return nullptr; + } +#endif + + GenTree* op1 = cmp->gtGetOp1(); + GenTree* op2 = cmp->gtGetOp2(); + if (op1->IsIntegralConst(0)) + { + std::swap(op1, op2); + } + + if (!cmp->TypeIs(TYP_INT) || !op2->IsIntegralConst(0) || (genActualType(op1) != TYP_INT) || + !vnStore->IsVNZeroOrOne(op1->GetVN(VNK_Conservative))) + { + return nullptr; + } + +#ifdef TARGET_XARCH + // Lowering already handles "(y & 1) != 0" and "(y & 1) == 0". + if (op1->OperIs(GT_AND) && op1->gtGetOp2()->IsIntegralConst(1)) + { + return nullptr; + } +#endif + + GenTree* const parent = gtFindLink(stmt, cmp).parent; + if ((parent == nullptr) || parent->OperIs(GT_JTRUE)) + { + return nullptr; + } + +#ifdef TARGET_ARM64 + // AND/OR of two compares is lowered to a ccmp chain. + if (parent->OperIs(GT_AND, GT_OR)) + { + GenTree* const sibling = (parent->gtGetOp1() == cmp) ? parent->gtGetOp2() : parent->gtGetOp1(); + if (sibling->OperIsCompare()) + { + return nullptr; + } + } +#endif + + GenTree* result = op1; + if (cmp->OperIs(GT_EQ)) + { + GenTree* one = gtNewIconNode(1); + one->gtVNPair.SetBoth(vnStore->VNForIntCon(1)); + + result = gtNewOperNode(GT_XOR, TYP_INT, op1, one); + result->gtVNPair = cmp->gtVNPair; + } + + return optAssertionProp_Update(result, cmp, stmt); +} + /***************************************************************************** * * Given a tree consisting of a RelOp and a set of available assertions @@ -5892,7 +5961,14 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, case GT_LE: case GT_GT: case GT_GE: - return optAssertionProp_RelOp(assertions, tree, stmt, block); + { + GenTree* newTree = optAssertionProp_RelOp(assertions, tree, stmt, block); + if ((newTree == nullptr) && !optLocalAssertionProp && tree->OperIs(GT_EQ, GT_NE)) + { + newTree = optAssertionProp_ZeroOneCompare(tree->AsOp(), stmt); + } + return newTree; + } case GT_JTRUE: if (block != nullptr) diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index cd6cd0dc4a5ebf..2148a6ca92a3bd 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2692,7 +2692,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) emitter* emit = GetEmitter(); assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR, - GT_XOR_NOT)); + GT_XOR_NOT, GT_ADD_HI, GT_SUB_HI)); GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); @@ -2700,6 +2700,19 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) // The arithmetic node must be sitting in a register (since it's not contained) assert(targetReg != REG_NA); + if (tree->OperIs(GT_ADD_HI, GT_SUB_HI)) + { + // adc/sbc have no immediate form, a contained zero is encoded as the zero register. + assert(!op1->isContained()); + assert(!op2->isContained() || op2->IsIntegralConst(0)); + + regNumber op2Reg = op2->isContained() ? REG_ZR : op2->GetRegNum(); + emit->emitIns_R_R_R(genGetInsForOper(oper, targetType), emitActualTypeSize(tree), targetReg, op1->GetRegNum(), + op2Reg); + genProduceReg(tree); + return; + } + // Handles combined operations: 'madd', 'msub' if (op2->OperIs(GT_MUL) && op2->isContained()) { @@ -4148,6 +4161,12 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type) case GT_ADD: ins = INS_add; break; + case GT_ADD_HI: + ins = INS_adc; + break; + case GT_SUB_HI: + ins = INS_sbc; + break; case GT_AND: ins = INS_and; break; diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 7486fc8f96d377..94f41921b8c277 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -227,11 +227,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if !defined(TARGET_64BIT) case GT_ADD_LO: - case GT_ADD_HI: case GT_SUB_LO: - case GT_SUB_HI: #endif // !defined(TARGET_64BIT) + case GT_ADD_HI: + case GT_SUB_HI: case GT_ADD: case GT_SUB: case GT_MUL: diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index d6166401a0f712..c89487152d5dbe 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -953,9 +953,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) } else { - isValidOper |= treeNode->OperIs(GT_AND, GT_OR, GT_XOR); + isValidOper |= treeNode->OperIs(GT_AND, GT_OR, GT_XOR, GT_ADD_HI, GT_SUB_HI); #ifndef TARGET_64BIT - isValidOper |= treeNode->OperIs(GT_ADD_LO, GT_ADD_HI, GT_SUB_LO, GT_SUB_HI); + isValidOper |= treeNode->OperIs(GT_ADD_LO, GT_SUB_LO); #endif } assert(isValidOper); @@ -1127,11 +1127,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) if (treeNode->gtOverflowEx()) { -#if !defined(TARGET_64BIT) assert(oper == GT_ADD || oper == GT_SUB || oper == GT_ADD_HI || oper == GT_SUB_HI); -#else - assert(oper == GT_ADD || oper == GT_SUB); -#endif genCheckOverflow(treeNode); } genProduceReg(treeNode); @@ -1931,11 +1927,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if !defined(TARGET_64BIT) case GT_ADD_LO: - case GT_ADD_HI: case GT_SUB_LO: - case GT_SUB_HI: #endif // !defined(TARGET_64BIT) + case GT_ADD_HI: + case GT_SUB_HI: case GT_ADD: case GT_SUB: genCodeForBinary(treeNode->AsOp()); @@ -4572,19 +4568,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type) case GT_XOR: ins = INS_xor; break; + case GT_ADD_HI: + ins = INS_adc; + break; + case GT_SUB_HI: + ins = INS_sbb; + break; #if !defined(TARGET_64BIT) case GT_ADD_LO: ins = INS_add; break; - case GT_ADD_HI: - ins = INS_adc; - break; case GT_SUB_LO: ins = INS_sub; break; - case GT_SUB_HI: - ins = INS_sbb; - break; case GT_LSH_HI: ins = INS_shld; break; diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 7bb50d17eb7aed..3ff00c5fce1144 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -9402,6 +9402,7 @@ class Compiler GenTree* optAssertionProp_Cast(ASSERT_VALARG_TP assertions, GenTreeCast* cast, Statement* stmt, BasicBlock* block); GenTree* optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call, Statement* stmt); GenTree* optAssertionProp_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block); + GenTree* optAssertionProp_ZeroOneCompare(GenTreeOp* cmp, Statement* stmt); GenTree* optAssertionProp_BndsChk(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block); GenTree* optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index d519af1065641c..6ce0bb9c61f2a5 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -516,7 +516,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) } else { - assert(isGeneralRegister(id->idReg3())); + assert(isGeneralRegisterOrZR(id->idReg3())); } assert(insOptsNone(id->idInsOpt())); break; @@ -6138,10 +6138,6 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_lsr: case INS_asr: case INS_ror: - case INS_adc: - case INS_adcs: - case INS_sbc: - case INS_sbcs: case INS_udiv: case INS_sdiv: case INS_mneg: @@ -6174,6 +6170,18 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_DR_3A; break; + case INS_adc: + case INS_adcs: + case INS_sbc: + case INS_sbcs: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + fmt = IF_DR_3A; + break; + case INS_add: case INS_sub: if (isVectorRegister(reg1)) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 20ac702b96b9ba..02dff5d74f5ba0 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -8826,10 +8826,8 @@ ExceptionSetFlags GenTree::OperExceptions(Compiler* comp) case GT_SUB: case GT_MUL: case GT_CAST: -#ifndef TARGET_64BIT case GT_ADD_HI: case GT_SUB_HI: -#endif // !TARGET_64BIT return gtOverflow() ? ExceptionSetFlags::OverflowException : ExceptionSetFlags::None; case GT_MOD: diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index ec79901011d828..97401994c9ebd6 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1648,11 +1648,7 @@ struct GenTree static bool OperMayOverflow(genTreeOps gtOper) { - return StaticOperIs(gtOper, GT_ADD, GT_SUB, GT_MUL, GT_CAST) -#if !defined(TARGET_64BIT) - || StaticOperIs(gtOper, GT_ADD_HI, GT_SUB_HI) -#endif - ; + return StaticOperIs(gtOper, GT_ADD, GT_SUB, GT_MUL, GT_CAST, GT_ADD_HI, GT_SUB_HI); } bool OperMayOverflow() const @@ -1818,12 +1814,10 @@ struct GenTree bool OperConsumesFlags() const { -#if !defined(TARGET_64BIT) if (OperIs(GT_ADD_HI, GT_SUB_HI)) { return true; } -#endif #if defined(TARGET_ARM64) if (OperIs(GT_CCMP, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC)) { @@ -2250,11 +2244,7 @@ struct GenTree void SetUnsigned() { -#if defined(TARGET_64BIT) - assert(OperIsCompare() || OperIsMul() || OperIs(GT_ADD, GT_SUB, GT_CAST)); -#else assert(OperIsCompare() || OperIsMul() || OperIs(GT_ADD, GT_SUB, GT_CAST, GT_ADD_HI, GT_SUB_HI)); -#endif gtFlags |= GTF_UNSIGNED; } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 622a9643e7db0b..6165f58bd441f3 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -163,6 +163,12 @@ GTNODE(COLON , GenTreeColon ,0,1,GTK_BINOP|DBK_NOTLIR) GTNODE(INDEX_ADDR , GenTreeIndexAddr ,0,0,GTK_BINOP|GTK_EXOP) // Address of SZ-array-element. GTNODE(LEA , GenTreeAddrMode ,0,0,GTK_BINOP|GTK_EXOP|DBK_NOTHIR) +// Add/subtract with carry: op1 + op2 + carry (ADD_HI) or op1 - op2 - borrow (SUB_HI), consuming the +// flags set by the previous node. Used for the high half of decomposed longs and for adding a compare +// result to a value. +GTNODE(ADD_HI , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(SUB_HI , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) + #if !defined(TARGET_64BIT) // A GT_LONG node simply represents the long value produced by the concatenation // of its two (lower and upper half) operands. Some GT_LONG nodes are transient, @@ -170,15 +176,12 @@ GTNODE(LEA , GenTreeAddrMode ,0,0,GTK_BINOP|GTK_EXOP|DBK_NOTHIR) // nodes such as calls, returns and stores of long lclVars. GTNODE(LONG , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) -// The following are nodes representing x86/arm32 specific long operators, including -// high operators of a 64-bit operations that requires a carry/borrow, which are -// named GT_XXX_HI for consistency, low operators of 64-bit operations that need -// to not be modified in phases post-decompose, and operators that return 64-bit -// results in one instruction. +// The following are nodes representing x86/arm32 specific long operators: low operators +// of 64-bit operations that need to not be modified in phases post-decompose (the high +// operators that require a carry/borrow are GT_ADD_HI/GT_SUB_HI above), and operators +// that return 64-bit results in one instruction. GTNODE(ADD_LO , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) -GTNODE(ADD_HI , GenTreeOp ,1,0,GTK_BINOP|DBK_NOTHIR) GTNODE(SUB_LO , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) -GTNODE(SUB_HI , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) // The following are nodes that specify shifts that take a GT_LONG op1. The GT_LONG // contains the hi and lo parts of three operand shift form where one op will be diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index 01a09fae72b480..36f601ffdb6c09 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -47,6 +47,8 @@ class OptIfConversionDsc genTreeOps m_mainOper = GT_COUNT; // The main oper of the if conversion. + bool m_onlyArithmeticSelect = false; // Only if-convert into a compare or arithmetic. + bool IfConvertCheck(); bool IfConvertCheckFlow(); bool IfConvertCheckStmts(BasicBlock* block, IfConvertOperation* foundOperation); @@ -56,6 +58,8 @@ class OptIfConversionDsc GenTree* TrySelectToCnsOpCond(GenTreeConditional* select); GenTree* TrySelectToLclOpCond(GenTreeConditional* select); GenTree* TrySelectToCondOpLcl(GenTreeConditional* select); + void TryForwardToArithmeticUse(); + bool IsCarryCompare(GenTree* cond, var_types type); #ifdef DEBUG void IfConvertDump(); @@ -224,7 +228,11 @@ bool OptIfConversionDsc::IfConvertCheckStmts(BasicBlock* block, IfConvertOperati // for some common cases that we know are always side effect free. if (((m_cond->gtFlags & GTF_ORDER_SIDEEFF) != 0) && !op1->IsInvariant() && !op1->OperIsLocal()) { - return false; + if ((op1->gtFlags & GTF_ALL_EFFECT) != 0) + { + return false; + } + m_onlyArithmeticSelect = true; } found = true; @@ -554,23 +562,24 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) if (!m_compiler->compStressCompile(Compiler::STRESS_IF_CONVERSION_INNER_LOOPS, 25)) { - // Don't optimise the block if it is inside a loop. Loop-carried - // dependencies can cause significant stalls if if-converted. + // Inside a loop, only convert into a compare or arithmetic. Loop-carried + // dependencies can cause significant stalls if if-converted into a SELECT. // Detect via the block weight as that will be high when inside a loop. - - if (m_startBlock->getBBWeight(m_compiler) > BB_UNITY_WEIGHT * 1.05) - { - JITDUMP("Skipping if-conversion inside loop (via weight)\n"); - return false; - } + bool const isHeavyBlock = m_startBlock->getBBWeight(m_compiler) > BB_UNITY_WEIGHT * 1.05; // We may be inside an unnatural loop, so do the expensive check. Compiler::ReachabilityResult reachability = - m_compiler->optReachableWithBudget(m_finalBlock, m_startBlock, nullptr, pReachabilityBudget); + isHeavyBlock ? Compiler::ReachabilityResult::Reachable + : m_compiler->optReachableWithBudget(m_finalBlock, m_startBlock, nullptr, pReachabilityBudget); if (reachability == Compiler::ReachabilityResult::Reachable) { - JITDUMP("Skipping if-conversion inside loop (via reachability)\n"); +#ifdef TARGET_X86 + // Branch-free code in loops causes spills with the few x86 registers. + JITDUMP("Skipping if-conversion inside loop\n"); return false; +#else + m_onlyArithmeticSelect = true; +#endif } else if (reachability == Compiler::ReachabilityResult::BudgetExceeded) { @@ -635,6 +644,15 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) if (select->OperIs(GT_SELECT)) { + GenTree* const op1 = select->AsConditional()->gtOp1; + GenTree* const op2 = select->AsConditional()->gtOp2; + if (m_onlyArithmeticSelect && !(op1->IsIntegralConst(0) && op2->IsIntegralConst(1)) && + !(op1->IsIntegralConst(1) && op2->IsIntegralConst(0))) + { + JITDUMP("Skipping if-conversion that would not become a compare or arithmetic\n"); + return true; + } + if (varTypeIsFloating(select)) { JITDUMP("Abort: SELECT of type float\n"); @@ -739,6 +757,11 @@ bool OptIfConversionDsc::optIfConvert(int* pReachabilityBudget) } } + if (m_mainOper == GT_STORE_LCL_VAR) + { + TryForwardToArithmeticUse(); + } + #ifdef DEBUG if (m_compiler->verbose) { @@ -843,6 +866,42 @@ static IntConstSelectOper MatchIntConstSelectValues(int64_t trueVal, int64_t fal } #endif // TARGET_RISCV64 +#ifndef TARGET_RISCV64 +//----------------------------------------------------------------------------- +// IsCarryCompare: Check if lowering can add the compare to or subtract it from a +// value of the given type without materializing it (see TryLowerAddSubOfCompare). +// +bool OptIfConversionDsc::IsCarryCompare(GenTree* cond, var_types type) +{ + if (!cond->OperIs(GT_EQ, GT_NE, GT_LT, GT_LE, GT_GE, GT_GT) || !varTypeIsIntegral(cond->gtGetOp1())) + { + return false; + } + +#ifndef TARGET_64BIT + // Long compares and adds are decomposed. + if (varTypeIsLong(cond->gtGetOp1()) || varTypeIsLong(type)) + { + return false; + } +#endif + + // HW intrinsic compares are lowered to flag checks like "ptest" that need a setcc. + if (m_onlyArithmeticSelect && cond->gtGetOp1()->OperIsHWIntrinsic()) + { + return false; + } + + // "(x & m) == 0" is lowered to a test, which does not produce a carry. + if (cond->OperIs(GT_EQ, GT_NE)) + { + return cond->gtGetOp2()->IsIntegralConst(0) && !cond->gtGetOp1()->OperIsBinary(); + } + + return cond->IsUnsigned(); +} +#endif // !TARGET_RISCV64 + //----------------------------------------------------------------------------- // TrySelectToCnsOpCond: Try to optimize: // SELECT(cond, 0, 1) -> cond @@ -904,7 +963,33 @@ GenTree* OptIfConversionDsc::TrySelectToCnsOpCond(GenTreeConditional* select) } return m_compiler->gtNewOperNode(selectOper.oper, selectOper.type, left, right); } -#endif // TARGET_RISCV64 +#else // !TARGET_RISCV64 + if (IsCarryCompare(cond, select->TypeGet())) + { + bool const isInt = select->TypeIs(TYP_INT); + auto isIncrementOf = [isInt](int64_t a, int64_t b) { + return isInt ? ((b != INT32_MAX) && (a == b + 1)) : ((b != INT64_MAX) && (a == b + 1)); + }; + bool const isCondReversed = isIncrementOf(falseVal, trueVal); + + bool const isConvertible = isCondReversed || isIncrementOf(trueVal, falseVal); +#ifdef TARGET_XARCH + // "setcc; movzx; add r, imm32" is larger than "mov; mov; cmovcc". + if (isConvertible && FitsIn(isCondReversed ? trueVal : falseVal)) +#else + if (isConvertible) +#endif + { + GenTree* base = isCondReversed ? trueInput : falseInput; + GenTree* inc = isCondReversed ? m_compiler->gtReverseCond(cond) : cond; + if (genActualType(inc) != genActualType(select)) + { + inc = m_compiler->gtNewCastNode(select->TypeGet(), inc, true, select->TypeGet()); + } + return m_compiler->gtNewOperNode(GT_ADD, select->TypeGet(), base, inc); + } + } +#endif // !TARGET_RISCV64 return nullptr; } @@ -961,8 +1046,75 @@ GenTree* OptIfConversionDsc::TrySelectToLclOpCond(GenTreeConditional* select) } } } -#endif // TARGET_RISCV64 return nullptr; +#else // !TARGET_RISCV64 + // SELECT(cond, x +/- 1, x) -> x +/- cond + GenTree* cond = select->gtCond; + GenTree* oper = select->gtOp1; + GenTree* lcl = select->gtOp2; + + if (!IsCarryCompare(cond, select->TypeGet()) || !varTypeIsIntegral(select)) + { + return nullptr; + } + +#ifdef TARGET_X86 + // "test; jcc; inc" with the 1-byte inc/dec is smaller than "cmp a, 1; adc". + if (cond->OperIs(GT_EQ, GT_NE)) + { + return nullptr; + } +#endif + + bool const isCondReversed = !lcl->OperIs(GT_LCL_VAR); + if (isCondReversed) + { + std::swap(oper, lcl); + } + + if (!lcl->OperIs(GT_LCL_VAR) || !oper->OperIs(GT_ADD) || oper->gtOverflow() || + (genActualType(oper) != genActualType(select))) + { + return nullptr; + } + + GenTree* lcl2 = oper->gtGetOp1(); + GenTree* one = oper->gtGetOp2(); + if (!one->IsIntegralConst()) + { + std::swap(lcl2, one); + } + + bool const isDecrement = one->IsIntegralConst(-1); + if ((!one->IsIntegralConst(1) && !isDecrement) || !lcl2->OperIs(GT_LCL_VAR) || + (lcl2->AsLclVar()->GetLclNum() != lcl->AsLclVar()->GetLclNum())) + { + return nullptr; + } + + // The compare may modify the local, e.g. by a call taking its address, so evaluate it first. + bool const condMayStore = (cond->gtFlags & (GTF_ASG | GTF_CALL)) != 0; + if (condMayStore && isDecrement) + { + return nullptr; + } + + GenTree* inc = isCondReversed ? m_compiler->gtReverseCond(cond) : cond; + if (genActualType(inc) != genActualType(oper)) + { + inc = m_compiler->gtNewCastNode(oper->TypeGet(), inc, true, oper->TypeGet()); + } + + oper->AsOp()->gtOp1 = condMayStore ? inc : lcl2; + oper->AsOp()->gtOp2 = condMayStore ? lcl2 : inc; + if (isDecrement) + { + oper->ChangeOper(GT_SUB); + } + + oper->gtFlags |= cond->gtFlags & GTF_ALL_EFFECT; + return oper; +#endif // !TARGET_RISCV64 } //----------------------------------------------------------------------------- @@ -1012,6 +1164,103 @@ GenTree* OptIfConversionDsc::TrySelectToCondOpLcl(GenTreeConditional* select) return nullptr; } +//----------------------------------------------------------------------------- +// TryForwardToArithmeticUse: Forward "tmp = cond" or "tmp = k + cond" into its single +// use in "x + tmp" or "x - tmp", so that lowering can combine the compare with the add. +// +void OptIfConversionDsc::TryForwardToArithmeticUse() +{ + GenTreeLclVar* const store = m_thenOperation.node->AsLclVar(); + Statement* const defStmt = m_thenOperation.stmt; + GenTree* const value = store->Data(); + unsigned const lclNum = store->GetLclNum(); + LclVarDsc* const varDsc = m_compiler->lvaGetDesc(lclNum); + + if (!varTypeIsIntegral(value) || ((value->gtFlags & GTF_ALL_EFFECT) != 0) || varDsc->lvIsParam || + varDsc->lvIsStructField || varDsc->IsAddressExposed() || varDsc->lvPinned || varTypeIsSmall(varDsc) || + (genActualType(varDsc) != genActualType(value))) + { + return; + } + + GenTree* cond = value; + if (cond->OperIs(GT_ADD) && (cond->gtGetOp1()->IsCnsIntOrI() || cond->gtGetOp2()->IsCnsIntOrI())) + { + cond = cond->gtGetOp1()->IsCnsIntOrI() ? cond->gtGetOp2() : cond->gtGetOp1(); + } + + if (cond->OperIs(GT_CAST)) + { + cond = cond->AsCast()->CastOp(); + } + + if (!cond->OperIsCompare() || (m_finalBlock == nullptr) || + (m_finalBlock->GetUniquePred(m_compiler) != m_startBlock) || + !BasicBlock::sameEHRegion(m_startBlock, m_finalBlock) || (m_startBlock->lastStmt() != defStmt) || + (m_finalBlock->firstStmt() == nullptr)) + { + return; + } + + // Embedded stores could modify the locals read by the value before the use. + Statement* const useStmt = m_finalBlock->firstStmt(); + GenTree* use = nullptr; + for (GenTree* const node : useStmt->TreeList()) + { + if (node->OperIsAnyLocal() && (node->AsLclVarCommon()->GetLclNum() == lclNum)) + { + if ((use != nullptr) || !node->OperIs(GT_LCL_VAR)) + { + return; + } + use = node; + } + else if ((node != useStmt->GetRootNode()) && node->OperIsStore()) + { + return; + } + } + + if (use == nullptr) + { + return; + } + + Compiler::FindLinkData const link = m_compiler->gtFindLink(useStmt, use); + GenTree* const user = link.parent; + if ((user == nullptr) || !user->OperIs(GT_ADD, GT_SUB) || !varTypeIsIntegral(user) || user->gtOverflow() || + (user->OperIs(GT_SUB) && (user->gtGetOp2() != use))) + { + return; + } + + for (BasicBlock* const block : m_compiler->Blocks()) + { + for (Statement* const stmt : block->Statements()) + { + if (stmt == useStmt) + { + continue; + } + + for (GenTree* const node : stmt->TreeList()) + { + if ((node != store) && node->OperIsAnyLocal() && (node->AsLclVarCommon()->GetLclNum() == lclNum)) + { + return; + } + } + } + } + + JITDUMP("Forwarding V%02u into its use in " FMT_STMT "\n", lclNum, useStmt->GetID()); + + *link.result = value; + m_compiler->fgRemoveStmt(m_startBlock, defStmt); + m_compiler->gtUpdateStmtSideEffects(useStmt); + m_compiler->gtSetEvalOrder(useStmt->GetRootNode()); + m_compiler->fgSetStmtSeq(useStmt); +} //----------------------------------------------------------------------------- // optIfConversion: If conversion // diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e62695aa572de6..2ec3611e482e3f 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -385,10 +385,10 @@ GenTree* Lowering::LowerNode(GenTree* node) #if !defined(TARGET_64BIT) case GT_ADD_LO: - case GT_ADD_HI: case GT_SUB_LO: - case GT_SUB_HI: #endif + case GT_ADD_HI: + case GT_SUB_HI: case GT_SUB: case GT_AND: case GT_OR: @@ -408,6 +408,15 @@ GenTree* Lowering::LowerNode(GenTree* node) { return nextNode; } + + if (node->OperIs(GT_SUB)) + { + GenTree* replacementNode = TryLowerAddSubOfCompare(node->AsOp()); + if (replacementNode != nullptr) + { + return replacementNode->gtNext; + } + } } return LowerBinaryArithmetic(node->AsOp()); @@ -7916,6 +7925,253 @@ bool Lowering::TryCreateAddrMode(GenTree* addr, bool isContainable, GenTree* par return true; } +//------------------------------------------------------------------------ +// FindAddSubCompareOperand: Find the compare in ADD/SUB(x, cond), ADD/SUB(x, CAST(long <- cond)) +// or ADD/SUB(x, ADD(cond, k)). +// +// Arguments: +// node - GT_ADD or GT_SUB node +// other - [out] the "x" operand +// condOp - [out] the operand of "node" that contains the compare +// addCns - [out] the "k" constant, or nullptr if there is none +// +// Return Value: +// The compare, or nullptr if "node" has none of these shapes. +// +GenTree* Lowering::FindAddSubCompareOperand(GenTreeOp* node, GenTree** other, GenTree** condOp, GenTree** addCns) +{ + assert(node->OperIs(GT_ADD, GT_SUB)); + + if (!m_compiler->opts.OptimizationEnabled() || !node->TypeIs(TYP_INT, TYP_LONG) || node->gtOverflow() || + node->gtSetFlags()) + { + return nullptr; + } + + auto findCompare = [this, node](GenTree* op, GenTree** foundCns) -> GenTree* { + *foundCns = nullptr; + + if (op->OperIs(GT_CAST) && node->TypeIs(TYP_LONG) && !op->gtOverflow() && + op->AsCast()->CastOp()->OperIsCompare() && op->AsCast()->CastOp()->TypeIs(TYP_INT)) + { + return op->AsCast()->CastOp(); + } + + if (op->OperIs(GT_ADD) && op->TypeIs(TYP_INT) && node->TypeIs(TYP_INT) && !op->gtOverflow() && + !op->gtSetFlags()) + { + GenTree* addOp1 = op->gtGetOp1(); + GenTree* addOp2 = op->gtGetOp2(); + + if (addOp1->IsCnsIntOrI()) + { + std::swap(addOp1, addOp2); + } + + if (addOp1->OperIsCompare() && addOp1->TypeIs(TYP_INT) && addOp2->IsCnsIntOrI() && + !addOp2->AsIntCon()->ImmedValNeedsReloc(m_compiler)) + { + *foundCns = addOp2; + return addOp1; + } + + return nullptr; + } + + return (op->OperIsCompare() && (genActualType(op) == genActualType(node))) ? op : nullptr; + }; + + *other = node->gtGetOp1(); + *condOp = node->gtGetOp2(); + GenTree* relop = findCompare(*condOp, addCns); + + if ((relop == nullptr) && node->OperIs(GT_ADD)) + { + std::swap(*other, *condOp); + relop = findCompare(*condOp, addCns); + } + + return relop; +} + +//------------------------------------------------------------------------ +// TryLowerAddSubOfCompare: Lower ADD/SUB(x, cond) and ADD/SUB(x, ADD(cond, k)), where "cond" is true iff +// the unsigned subtraction "a - b" borrows (or iff it does not), to the compare followed by an +// add/subtract with carry, when the target has a suitable one (see TryGetCarryOp). For example: +// +// x + ((a cmp a, b; adc x, k (xarch) +// x - (a cmp a, b; sbc x, x, zr (arm64) +// +// "a == 0", "a != 0", "a <=u c" and "a >u c" are handled as "a =u 1", +// "a =u c + 1" respectively. +// +// Arguments: +// node - GT_ADD or GT_SUB node +// +// Return Value: +// The ADD_HI/SUB_HI node if the transformation was done, otherwise nullptr. +// +GenTree* Lowering::TryLowerAddSubOfCompare(GenTreeOp* node) +{ + GenTree* other; + GenTree* condOp; + GenTree* addCns; + GenTree* relop = FindAddSubCompareOperand(node, &other, &condOp, &addCns); + + // The flags consumer needs "other" in a register and it must not be a constant that may be + // materialized with a flags-clobbering instruction. + if ((relop == nullptr) || other->IsCnsIntOrI() || !relop->OperIs(GT_EQ, GT_NE, GT_LT, GT_LE, GT_GE, GT_GT)) + { + return nullptr; + } + + GenTree* cmpOp1 = relop->gtGetOp1(); + GenTree* cmpOp2 = relop->gtGetOp2(); + + if (!varTypeIsIntegral(cmpOp1) || !varTypeIsIntegral(cmpOp2)) + { + return nullptr; + } + +#ifdef TARGET_XARCH + // Keep "*p = *p + cond" as a read-modify-write add. The store address is not lowered yet. + LIR::Use use; + if (other->OperIs(GT_IND) && BlockRange().TryGetUse(node, &use) && use.User()->OperIs(GT_STOREIND) && + (use.User()->AsStoreInd()->Data() == node)) + { + return nullptr; + } +#endif + +#ifndef TARGET_64BIT + if (varTypeIsLong(cmpOp1) || varTypeIsLong(cmpOp2)) + { + return nullptr; + } +#endif + + // After the compare, the borrow is set iff "cmpOp1 OperIs(GT_EQ, GT_NE)) + { + if (!cmpOp2->IsIntegralConst(0)) + { + return nullptr; + } + + borrowIsCond = relop->OperIs(GT_EQ); + bumpCns = true; + } + else + { + if (!relop->IsUnsigned()) + { + return nullptr; + } + + borrowIsCond = relop->OperIs(GT_LT, GT_LE); + bumpCns = relop->OperIs(GT_LE, GT_GT); + } + + ssize_t k = (addCns != nullptr) ? addCns->AsIntCon()->IconValue() : 0; + genTreeOps carryOper; + ssize_t imm; + if (!TryGetCarryOp(node->OperIs(GT_ADD), borrowIsCond, k, &carryOper, &imm)) + { + return nullptr; + } + + if (node->TypeIs(TYP_INT)) + { + imm = static_cast(imm); + } + + ssize_t bumpedCns = 0; + if (bumpCns) + { + if (!cmpOp2->IsCnsIntOrI() || cmpOp2->IsIconHandle() || cmpOp2->AsIntCon()->ImmedValNeedsReloc(m_compiler)) + { + return nullptr; + } + +#ifdef TARGET_XARCH + unsigned cmpSize = relop->AsOp()->GetCompareSize(); +#else + unsigned cmpSize = genTypeSize(genActualType(cmpOp1)); +#endif + uint64_t mask = (cmpSize == 8) ? UINT64_MAX : ((uint64_t(1) << (cmpSize * 8)) - 1); + uint64_t value = static_cast(cmpOp2->AsIntCon()->IconValue()) & mask; + + if (value == mask) + { + // "a <=u MAX" and "a >u MAX" have no "a (value) : static_cast(value); + + if (cmpOp2->isContained()) + { + ssize_t cns = cmpOp2->AsIntCon()->IconValue(); + cmpOp2->AsIntCon()->SetIconValue(bumpedCns); + bool canContain = IsContainableImmed(relop, cmpOp2); + cmpOp2->AsIntCon()->SetIconValue(cns); + + if (!canContain) + { + return nullptr; + } + } + } + + if (!IsInvariantInRange(relop, node)) + { + return nullptr; + } + + JITDUMP("Lowering ADD/SUB of a compare to a carry consuming op:\n"); + DISPTREERANGE(BlockRange(), node); + + if (bumpCns) + { + cmpOp2->AsIntCon()->SetIconValue(bumpedCns); + } + + if (condOp != relop) + { + BlockRange().Remove(condOp); + } + + if (addCns != nullptr) + { + BlockRange().Remove(addCns); + } + + relop->SetOper(GT_CMP); + relop->gtType = TYP_VOID; + relop->gtFlags |= GTF_SET_FLAGS; + BlockRange().Remove(relop); + BlockRange().InsertBefore(node, relop); + + // The flags consumer must immediately follow the flags producer. + GenTree* immNode = m_compiler->gtNewIconNode(imm, node->TypeGet()); + BlockRange().InsertBefore(relop, immNode); + + node->ChangeOper(carryOper); + node->gtOp1 = other; + node->gtOp2 = immNode; + ContainCheckBinary(node); + + JITDUMP("Result:\n"); + DISPTREERANGE(BlockRange(), node); + JITDUMP("\n"); + + return node; +} + //------------------------------------------------------------------------ // LowerAdd: turn this add into a GT_LEA if that would be profitable // @@ -8026,6 +8282,15 @@ GenTree* Lowering::LowerAdd(GenTreeOp* node) } } + if (node->OperIs(GT_ADD)) + { + GenTree* replacementNode = TryLowerAddSubOfCompare(node); + if (replacementNode != nullptr) + { + return replacementNode->gtNext; + } + } + #ifdef TARGET_XARCH if (BlockRange().TryGetUse(node, &use)) { @@ -8053,6 +8318,12 @@ GenTree* Lowering::LowerAdd(GenTreeOp* node) { return next; } + + GenTree* replacementNode = TryLowerAddOfCompareToCinc(node); + if (replacementNode != nullptr) + { + return replacementNode->gtNext; + } } #endif // TARGET_ARM64 @@ -9591,10 +9862,10 @@ void Lowering::ContainCheckNode(GenTree* node) case GT_SUB: #if !defined(TARGET_64BIT) case GT_ADD_LO: - case GT_ADD_HI: case GT_SUB_LO: - case GT_SUB_HI: #endif + case GT_ADD_HI: + case GT_SUB_HI: case GT_AND: case GT_OR: case GT_XOR: @@ -11630,9 +11901,7 @@ bool Lowering::TryMakeIndirsAdjacent(GenTreeIndir* prevIndir, GenTreeIndir* indi { // 'cur' is part of data flow of 'indir', so we will be moving the // currently recorded effects past 'cur'. - // MarkTree does not track implicit flags dependencies. - if (cur->OperConsumesFlags() || cur->gtSetFlags() || - m_scratchSideEffects.InterferesWith(m_compiler, cur, true)) + if (m_scratchSideEffects.InterferesWith(m_compiler, cur, true)) { JITDUMP("Giving up due to interference with [%06u]\n", Compiler::dspTreeID(cur)); return false; diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 5c86b89fbf117f..9eb0a0e74c213d 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -97,7 +97,8 @@ class Lowering final : public Phase bool TryContainingCselOp(GenTreeHWIntrinsic* parentNode, GenTreeHWIntrinsic* childNode); #endif #if defined(TARGET_ARM64) - bool TryLowerAndRshToBFX(GenTreeOp* tree, GenTree** next); + bool TryLowerAndRshToBFX(GenTreeOp* tree, GenTree** next); + GenTree* TryLowerAddOfCompareToCinc(GenTreeOp* node); #endif #ifdef TARGET_RISCV64 bool TryLowerShiftAddToShxadd(GenTreeOp* tree, GenTree** next); @@ -446,6 +447,10 @@ class Lowering final : public Phase GenTree* TryLowerMulWithConstant(GenTreeOp* node); #endif // TARGET_XARCH + GenTree* FindAddSubCompareOperand(GenTreeOp* node, GenTree** other, GenTree** condOp, GenTree** addCns); + GenTree* TryLowerAddSubOfCompare(GenTreeOp* node); + bool TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm); + #ifdef TARGET_WASM static void SetMultiplyUsed(GenTree* node DEBUGARG(const char* reason)); GenTreeAddrMode* GetFoldableAddrMode(GenTreeIndir* indirNode); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 329d59a57d2eff..0e438a9093cc12 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -149,6 +149,8 @@ bool Lowering::IsContainableImmed(GenTree* parentNode, GenTree* childNode) const #ifdef TARGET_ARM64 case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + case GT_ADD_HI: + case GT_SUB_HI: if (immVal == 0) return true; break; @@ -622,6 +624,41 @@ GenTree* Lowering::LowerMul(GenTreeOp* mul) return mul->gtNext; } +//------------------------------------------------------------------------ +// TryGetCarryOp: Get the add/subtract with carry that computes "x + (cond + k)" or "x - (cond + k)" +// after a compare that sets the borrow iff "cond" (or iff "!cond"). +// +// Arguments: +// isAdd - true for "x + (cond + k)", false for "x - (cond + k)" +// borrowIsCond - true if the borrow is set iff "cond", false if it is set iff "!cond" +// k - the constant added to "cond" +// carryOper - [out] GT_ADD_HI or GT_SUB_HI +// imm - [out] the second operand of the carry op +// +// Return Value: +// True if the target has such an instruction. +// +bool Lowering::TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm) +{ + // adc adds the carry, which is the inverse of the borrow, and sbc subtracts the borrow. + if (isAdd == borrowIsCond) + { + return false; + } + +#ifdef TARGET_ARM64 + // adc/sbc have no immediate form. + if (k != 0) + { + return false; + } +#endif + + *carryOper = isAdd ? GT_ADD_HI : GT_SUB_HI; + *imm = k; + return true; +} + //------------------------------------------------------------------------ // LowerBinaryArithmetic: lowers the given binary arithmetic node. // @@ -1216,9 +1253,7 @@ bool Lowering::TryMakeIndirAndStoreAdjacent(GenTreeIndir* prevIndir, GenTreeLclV { // 'cur' is part of data flow of 'store', so we will be moving the // currently recorded effects past 'cur'. - // MarkTree does not track implicit flags dependencies. - if (cur->OperConsumesFlags() || cur->gtSetFlags() || - m_scratchSideEffects.InterferesWith(m_compiler, cur, true)) + if (m_scratchSideEffects.InterferesWith(m_compiler, cur, true)) { JITDUMP("Giving up due to interference with [%06u]\n", Compiler::dspTreeID(cur)); return false; @@ -3405,6 +3440,70 @@ void Lowering::TryLowerCselToCSOp(GenTreeOp* select, GenTree* cond) #endif } +//---------------------------------------------------------------------------------------------- +// TryLowerAddOfCompareToCinc: Lower ADD(x, cond) and ADD(x, ADD(cond, k)), where "cond" is a compare, +// to a conditional increment: +// +// x + cond => cmp a, b; cinc x, x, cond +// x + (cond + k) => add t, x, #k; cmp a, b; cinc t, t, cond +// +// Arguments: +// node - GT_ADD node +// +// Return Value: +// The SELECT_INCCC node if the transformation was done, otherwise nullptr. +// +GenTree* Lowering::TryLowerAddOfCompareToCinc(GenTreeOp* node) +{ + assert(node->OperIs(GT_ADD)); + + GenTree* other; + GenTree* condOp; + GenTree* addCns; + GenTree* relop = FindAddSubCompareOperand(node, &other, &condOp, &addCns); + + GenCondition cond; + if ((relop == nullptr) || !TryLowerConditionToFlagsNode(node, relop, &cond, /* allowMultipleFlagsChecks */ false)) + { + return nullptr; + } + + // A constant is materialized with "mov", which does not change the flags. + GenTree* incSrc = other; + if (other->IsCnsIntOrI()) + { + other->ClearContained(); + } + + if (addCns != nullptr) + { + // Reuse ADD(cond, k) to compute "other + k" right before the compare. + condOp->AsOp()->gtOp1 = other; + condOp->AsOp()->gtOp2 = addCns; + addCns->ClearContained(); + BlockRange().Remove(condOp); + BlockRange().InsertBefore(relop, condOp); + ContainCheckBinary(condOp->AsOp()); + incSrc = condOp; + } + else if (condOp != relop) + { + BlockRange().Remove(condOp); + } + + node->ChangeOper(GT_SELECT_INCCC); + GenTreeOpCC* inc = node->AsOpCC(); + inc->gtCondition = cond; + inc->gtOp1 = incSrc; + inc->gtOp2 = nullptr; + + JITDUMP("Lowered ADD of a compare to SELECT_INCCC:\n"); + DISPTREERANGE(BlockRange(), node); + JITDUMP("\n"); + + return node; +} + //---------------------------------------------------------------------------------------------- // TryLowerCnsIntCselToCinc: Try converting SELECT/SELECTCC to SELECT_INC/SELECT_INCCC. // Conversion is possible if both the trueVal and falseVal are integer constants and abs(trueVal - falseVal) = 1, @@ -3444,11 +3543,6 @@ void Lowering::TryLowerCnsIntCselToCinc(GenTreeOp* select, GenTree* cond) GenTree* revCond = m_compiler->gtReverseCond(cond); assert(cond == revCond); // Ensure `gtReverseCond` did not create a new node. } - else - { - std::swap(select->gtOp1, select->gtOp2); - } - BlockRange().Remove(select->gtOp2, true); select->gtOp2 = nullptr; select->SetOper(GT_SELECT_INC); diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index e0d63a2ab5eb5a..e2322770044c79 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -200,6 +200,18 @@ GenTree* Lowering::LowerJTrue(GenTreeOp* jtrue) return jtrue->gtNext; } +//------------------------------------------------------------------------ +// TryGetCarryOp: Get the add/subtract with carry that computes "x + (cond + k)" or "x - (cond + k)" +// after a compare that sets the borrow iff "cond" (or iff "!cond"). +// +// Return Value: +// Always false, there is no flags register. +// +bool Lowering::TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm) +{ + return false; +} + //------------------------------------------------------------------------ // LowerBinaryArithmetic: lowers the given binary arithmetic node. // diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index dfe6fb65d94257..2fe42cf8b740d9 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -282,6 +282,18 @@ void Lowering::SignExtendIfNecessary(GenTree** arg) BlockRange().InsertAfter((*arg)->gtGetOp1(), *arg); } +//------------------------------------------------------------------------ +// TryGetCarryOp: Get the add/subtract with carry that computes "x + (cond + k)" or "x - (cond + k)" +// after a compare that sets the borrow iff "cond" (or iff "!cond"). +// +// Return Value: +// Always false, there is no flags register. +// +bool Lowering::TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm) +{ + return false; +} + //------------------------------------------------------------------------ // LowerBinaryArithmetic: lowers the given binary arithmetic node. // diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 2feb985d1d0435..e5a83895d533cd 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -341,6 +341,18 @@ GenTree* Lowering::LowerJTrue(GenTreeOp* jtrue) return nullptr; } +//------------------------------------------------------------------------ +// TryGetCarryOp: Get the add/subtract with carry that computes "x + (cond + k)" or "x - (cond + k)" +// after a compare that sets the borrow iff "cond" (or iff "!cond"). +// +// Return Value: +// Always false, there is no flags register. +// +bool Lowering::TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm) +{ + return false; +} + //------------------------------------------------------------------------ // LowerBinaryArithmetic: lowers the given binary arithmetic node. // diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index e56c591f414e3f..27d67cc99b7b1f 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -246,6 +246,28 @@ GenTree* Lowering::LowerMul(GenTreeOp* mul) return mul->gtNext; } +//------------------------------------------------------------------------ +// TryGetCarryOp: Get the add/subtract with carry that computes "x + (cond + k)" or "x - (cond + k)" +// after a compare that sets the borrow iff "cond" (or iff "!cond"). +// +// Arguments: +// isAdd - true for "x + (cond + k)", false for "x - (cond + k)" +// borrowIsCond - true if the borrow is set iff "cond", false if it is set iff "!cond" +// k - the constant added to "cond" +// carryOper - [out] GT_ADD_HI or GT_SUB_HI +// imm - [out] the second operand of the carry op +// +// Return Value: +// True if the target has such an instruction. +// +bool Lowering::TryGetCarryOp(bool isAdd, bool borrowIsCond, ssize_t k, genTreeOps* carryOper, ssize_t* imm) +{ + // adc adds the borrow (CF) and sbb subtracts it; "!cond" is "1 - cond". + *carryOper = (borrowIsCond == isAdd) ? GT_ADD_HI : GT_SUB_HI; + *imm = borrowIsCond ? k : ~k; + return true; +} + //------------------------------------------------------------------------ // LowerBinaryArithmetic: lowers the given binary arithmetic node. // diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 102fbc1f6549e9..0b0790b18a2dcb 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -956,6 +956,8 @@ int LinearScan::BuildNode(GenTree* tree) } FALLTHROUGH; + case GT_ADD_HI: + case GT_SUB_HI: case GT_AND: case GT_AND_NOT: case GT_OR: diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 4b170bd03912f6..2de95f56cd7c9d 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -316,10 +316,10 @@ int LinearScan::BuildNode(GenTree* tree) #if !defined(TARGET_64BIT) case GT_ADD_LO: - case GT_ADD_HI: case GT_SUB_LO: - case GT_SUB_HI: #endif + case GT_ADD_HI: + case GT_SUB_HI: case GT_ADD: case GT_SUB: case GT_AND: diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index ac8164bcc400c1..70258c5da08901 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -7526,6 +7526,61 @@ genTreeOps ValueNumStore::VNRelopToGenTreeOp(VNFunc vnf, bool* isUnsigned) } } +//------------------------------------------------------------------------ +// IsVNZeroOrOne: Check if a value number is known to be 0 or 1, e.g. a compare. +// +// Notes: +// Values read from memory, parameters and call results may hold any value, +// even when they are typed as bool. +// +bool ValueNumStore::IsVNZeroOrOne(ValueNum vn, int budget) +{ + if (budget <= 0) + { + return false; + } + + auto isZeroOrOne = [this, budget](ValueNum reachingVN) { + reachingVN = VNNormalValue(reachingVN); + + int64_t value; + if (IsVNIntegralConstant(reachingVN, &value)) + { + return (value == 0) || (value == 1); + } + + VNFuncApp funcApp; + if (!GetVNFunc(reachingVN, &funcApp)) + { + return false; + } + + if (VNFuncIsComparison(funcApp.m_func)) + { + return true; + } + + switch (funcApp.m_func) + { + case VNFunc(GT_AND): + return IsVNZeroOrOne(funcApp.m_args[0], budget - 1) || IsVNZeroOrOne(funcApp.m_args[1], budget - 1); + + case VNFunc(GT_OR): + case VNFunc(GT_XOR): + return IsVNZeroOrOne(funcApp.m_args[0], budget - 1) && IsVNZeroOrOne(funcApp.m_args[1], budget - 1); + + case VNF_Cast: + return IsVNZeroOrOne(funcApp.m_args[0], budget - 1); + + default: + return false; + } + }; + + return VNVisitReachingVNs(vn, [&isZeroOrOne](ValueNum reachingVN) { + return isZeroOrOne(reachingVN) ? VNVisit::Continue : VNVisit::Abort; + }) == VNVisit::Continue; +} //------------------------------------------------------------------------ // GetRelatedRelop: return value number for reversed/swapped comparison // diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index 22ac813977f15c..554e5a8e61d38c 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -1266,6 +1266,9 @@ class ValueNumStore // Returns "true" iff "vnf" is a comparison (and thus binary) operator. static bool VNFuncIsComparison(VNFunc vnf); + // Returns "true" iff "vn" is known to be 0 or 1. + bool IsVNZeroOrOne(ValueNum vn, int budget = 4); + // Returns "true" iff "vnf" is a signed comparison (and thus binary) operator. static bool VNFuncIsSignedComparison(VNFunc vnf);