JIT: If-convert to x +/- cond and fold 0/1 compares

src/coreclr/jit/assertionprop.cpp

@@ -4493,6 +4493,75 @@ AssertionIndex Compiler::optLocalAssertionIsEqualOrNotEqual(
     return NO_ASSERTION_INDEX;
 }
 
+//------------------------------------------------------------------------------
+// optAssertionProp_ZeroOneCompare: Fold "x != 0" to "x" and "x == 0" to "x ^ 1" when
+//    "x" is known to be 0 or 1 and the compare is used as a value. Runs after the
+//    assertion based relop folding, which cannot fold "x ^ 1".
+//
+GenTree* Compiler::optAssertionProp_ZeroOneCompare(GenTreeOp* cmp, Statement* stmt)
+{
+    assert(cmp->OperIs(GT_EQ, GT_NE));
+
+#ifdef TARGET_WASM
+    // "i32.eqz" is smaller than "i32.const 1; i32.xor".
+    if (cmp->OperIs(GT_EQ))
+    {
+        return nullptr;
+    }
+#endif
+
+    GenTree* op1 = cmp->gtGetOp1();
+    GenTree* op2 = cmp->gtGetOp2();
+    if (op1->IsIntegralConst(0))
+    {
+        std::swap(op1, op2);
+    }
+
+    if (!cmp->TypeIs(TYP_INT) || !op2->IsIntegralConst(0) || (genActualType(op1) != TYP_INT) ||
+        !vnStore->IsVNZeroOrOne(op1->GetVN(VNK_Conservative)))
+    {
+        return nullptr;
+    }
+
+#ifdef TARGET_XARCH
+    // Lowering already handles "(y & 1) != 0" and "(y & 1) == 0".
+    if (op1->OperIs(GT_AND) && op1->gtGetOp2()->IsIntegralConst(1))
+    {
+        return nullptr;
+    }
+#endif
+
+    GenTree* const parent = gtFindLink(stmt, cmp).parent;
+    if ((parent == nullptr) || parent->OperIs(GT_JTRUE))
+    {
+        return nullptr;
+    }
+
+#ifdef TARGET_ARM64
+    // AND/OR of two compares is lowered to a ccmp chain.
+    if (parent->OperIs(GT_AND, GT_OR))
+    {
+        GenTree* const sibling = (parent->gtGetOp1() == cmp) ? parent->gtGetOp2() : parent->gtGetOp1();
+        if (sibling->OperIsCompare())
+        {
+            return nullptr;
+        }
+    }
+#endif
+
+    GenTree* result = op1;
+    if (cmp->OperIs(GT_EQ))
+    {
+        GenTree* one = gtNewIconNode(1);
+        one->gtVNPair.SetBoth(vnStore->VNForIntCon(1));
+
+        result           = gtNewOperNode(GT_XOR, TYP_INT, op1, one);
+        result->gtVNPair = cmp->gtVNPair;
+    }
+
+    return optAssertionProp_Update(result, cmp, stmt);
+}
+
 /*****************************************************************************
  *
  *  Given a tree consisting of a RelOp and a set of available assertions
@@ -5892,7 +5961,14 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree,
         case GT_LE:
         case GT_GT:
         case GT_GE:
-            return optAssertionProp_RelOp(assertions, tree, stmt, block);
+        {
+            GenTree* newTree = optAssertionProp_RelOp(assertions, tree, stmt, block);
+            if ((newTree == nullptr) && !optLocalAssertionProp && tree->OperIs(GT_EQ, GT_NE))
+            {
+                newTree = optAssertionProp_ZeroOneCompare(tree->AsOp(), stmt);
+            }
+            return newTree;
+        }
 
         case GT_JTRUE:
             if (block != nullptr)

src/coreclr/jit/codegenarm64.cpp

@@ -2692,14 +2692,27 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree)
     emitter*         emit       = GetEmitter();
 
     assert(tree->OperIs(GT_ADD, GT_SUB, GT_MUL, GT_DIV, GT_UDIV, GT_AND, GT_AND_NOT, GT_OR, GT_OR_NOT, GT_XOR,
-                        GT_XOR_NOT));
+                        GT_XOR_NOT, GT_ADD_HI, GT_SUB_HI));
 
     GenTree* op1 = tree->gtGetOp1();
     GenTree* op2 = tree->gtGetOp2();
 
     // The arithmetic node must be sitting in a register (since it's not contained)
     assert(targetReg != REG_NA);
 
+    if (tree->OperIs(GT_ADD_HI, GT_SUB_HI))
+    {
+        // adc/sbc have no immediate form, a contained zero is encoded as the zero register.
+        assert(!op1->isContained());
+        assert(!op2->isContained() || op2->IsIntegralConst(0));
+
+        regNumber op2Reg = op2->isContained() ? REG_ZR : op2->GetRegNum();
+        emit->emitIns_R_R_R(genGetInsForOper(oper, targetType), emitActualTypeSize(tree), targetReg, op1->GetRegNum(),
+                            op2Reg);
+        genProduceReg(tree);
+        return;
+    }
+
     // Handles combined operations: 'madd', 'msub'
     if (op2->OperIs(GT_MUL) && op2->isContained())
     {
@@ -4148,6 +4161,12 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type)
             case GT_ADD:
                 ins = INS_add;
                 break;
+            case GT_ADD_HI:
+                ins = INS_adc;
+                break;
+            case GT_SUB_HI:
+                ins = INS_sbc;
+                break;
             case GT_AND:
                 ins = INS_and;
                 break;

src/coreclr/jit/codegenarmarch.cpp

@@ -227,11 +227,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
 
 #if !defined(TARGET_64BIT)
         case GT_ADD_LO:
-        case GT_ADD_HI:
         case GT_SUB_LO:
-        case GT_SUB_HI:
 #endif // !defined(TARGET_64BIT)
 
+        case GT_ADD_HI:
+        case GT_SUB_HI:
         case GT_ADD:
         case GT_SUB:
         case GT_MUL:

src/coreclr/jit/codegenxarch.cpp

@@ -953,9 +953,9 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
     }
     else
     {
-        isValidOper |= treeNode->OperIs(GT_AND, GT_OR, GT_XOR);
+        isValidOper |= treeNode->OperIs(GT_AND, GT_OR, GT_XOR, GT_ADD_HI, GT_SUB_HI);
 #ifndef TARGET_64BIT
-        isValidOper |= treeNode->OperIs(GT_ADD_LO, GT_ADD_HI, GT_SUB_LO, GT_SUB_HI);
+        isValidOper |= treeNode->OperIs(GT_ADD_LO, GT_SUB_LO);
 #endif
     }
     assert(isValidOper);
@@ -1127,11 +1127,7 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode)
 
     if (treeNode->gtOverflowEx())
     {
-#if !defined(TARGET_64BIT)
         assert(oper == GT_ADD || oper == GT_SUB || oper == GT_ADD_HI || oper == GT_SUB_HI);
-#else
-        assert(oper == GT_ADD || oper == GT_SUB);
-#endif
         genCheckOverflow(treeNode);
     }
     genProduceReg(treeNode);
@@ -1931,11 +1927,11 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
 
 #if !defined(TARGET_64BIT)
         case GT_ADD_LO:
-        case GT_ADD_HI:
         case GT_SUB_LO:
-        case GT_SUB_HI:
 #endif // !defined(TARGET_64BIT)
 
+        case GT_ADD_HI:
+        case GT_SUB_HI:
         case GT_ADD:
         case GT_SUB:
             genCodeForBinary(treeNode->AsOp());
@@ -4572,19 +4568,19 @@ instruction CodeGen::genGetInsForOper(genTreeOps oper, var_types type)
         case GT_XOR:
             ins = INS_xor;
             break;
+        case GT_ADD_HI:
+            ins = INS_adc;
+            break;
+        case GT_SUB_HI:
+            ins = INS_sbb;
+            break;
 #if !defined(TARGET_64BIT)
         case GT_ADD_LO:
             ins = INS_add;
             break;
-        case GT_ADD_HI:
-            ins = INS_adc;
-            break;
         case GT_SUB_LO:
             ins = INS_sub;
             break;
-        case GT_SUB_HI:
-            ins = INS_sbb;
-            break;
         case GT_LSH_HI:
             ins = INS_shld;
             break;

src/coreclr/jit/compiler.h

@@ -9402,6 +9402,7 @@ class Compiler
     GenTree* optAssertionProp_Cast(ASSERT_VALARG_TP assertions, GenTreeCast* cast, Statement* stmt, BasicBlock* block);
     GenTree* optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCall* call, Statement* stmt);
     GenTree* optAssertionProp_RelOp(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block);
+    GenTree* optAssertionProp_ZeroOneCompare(GenTreeOp* cmp, Statement* stmt);
     GenTree* optAssertionProp_BndsChk(ASSERT_VALARG_TP assertions, GenTree* tree, Statement* stmt, BasicBlock* block);
     GenTree* optAssertionPropGlobal_RelOp(ASSERT_VALARG_TP assertions,
                                           GenTree*         tree,

src/coreclr/jit/emitarm64.cpp

@@ -516,7 +516,7 @@ void emitter::emitInsSanityCheck(instrDesc* id)
             }
             else
             {
-                assert(isGeneralRegister(id->idReg3()));
+                assert(isGeneralRegisterOrZR(id->idReg3()));
             }
             assert(insOptsNone(id->idInsOpt()));
             break;
@@ -6138,10 +6138,6 @@ void emitter::emitIns_R_R_R(instruction     ins,
         case INS_lsr:
         case INS_asr:
         case INS_ror:
-        case INS_adc:
-        case INS_adcs:
-        case INS_sbc:
-        case INS_sbcs:
         case INS_udiv:
         case INS_sdiv:
         case INS_mneg:
@@ -6174,6 +6170,18 @@ void emitter::emitIns_R_R_R(instruction     ins,
             fmt = IF_DR_3A;
             break;
 
+        case INS_adc:
+        case INS_adcs:
+        case INS_sbc:
+        case INS_sbcs:
+            assert(insOptsNone(opt));
+            assert(isValidGeneralDatasize(size));
+            assert(isGeneralRegister(reg1));
+            assert(isGeneralRegisterOrZR(reg2));
+            assert(isGeneralRegisterOrZR(reg3));
+            fmt = IF_DR_3A;
+            break;
+
         case INS_add:
         case INS_sub:
             if (isVectorRegister(reg1))

src/coreclr/jit/gentree.cpp

@@ -8826,10 +8826,8 @@ ExceptionSetFlags GenTree::OperExceptions(Compiler* comp)
         case GT_SUB:
         case GT_MUL:
         case GT_CAST:
-#ifndef TARGET_64BIT
         case GT_ADD_HI:
         case GT_SUB_HI:
-#endif // !TARGET_64BIT
             return gtOverflow() ? ExceptionSetFlags::OverflowException : ExceptionSetFlags::None;
 
         case GT_MOD:

src/coreclr/jit/gentree.h

@@ -1648,11 +1648,7 @@ struct GenTree
 
     static bool OperMayOverflow(genTreeOps gtOper)
     {
-        return StaticOperIs(gtOper, GT_ADD, GT_SUB, GT_MUL, GT_CAST)
-#if !defined(TARGET_64BIT)
-               || StaticOperIs(gtOper, GT_ADD_HI, GT_SUB_HI)
-#endif
-            ;
+        return StaticOperIs(gtOper, GT_ADD, GT_SUB, GT_MUL, GT_CAST, GT_ADD_HI, GT_SUB_HI);
     }
 
     bool OperMayOverflow() const
@@ -1818,12 +1814,10 @@ struct GenTree
 
     bool OperConsumesFlags() const
     {
-#if !defined(TARGET_64BIT)
         if (OperIs(GT_ADD_HI, GT_SUB_HI))
         {
             return true;
         }
-#endif
 #if defined(TARGET_ARM64)
         if (OperIs(GT_CCMP, GT_SELECT_INCCC, GT_SELECT_INVCC, GT_SELECT_NEGCC))
         {
@@ -2250,11 +2244,7 @@ struct GenTree
 
     void SetUnsigned()
     {
-#if defined(TARGET_64BIT)
-        assert(OperIsCompare() || OperIsMul() || OperIs(GT_ADD, GT_SUB, GT_CAST));
-#else
         assert(OperIsCompare() || OperIsMul() || OperIs(GT_ADD, GT_SUB, GT_CAST, GT_ADD_HI, GT_SUB_HI));
-#endif
         gtFlags |= GTF_UNSIGNED;
     }
 

src/coreclr/jit/gtlist.h

@@ -163,22 +163,25 @@ GTNODE(COLON            , GenTreeColon       ,0,1,GTK_BINOP|DBK_NOTLIR)
 GTNODE(INDEX_ADDR       , GenTreeIndexAddr   ,0,0,GTK_BINOP|GTK_EXOP)   // Address of SZ-array-element.
 GTNODE(LEA              , GenTreeAddrMode    ,0,0,GTK_BINOP|GTK_EXOP|DBK_NOTHIR)
 
+// Add/subtract with carry: op1 + op2 + carry (ADD_HI) or op1 - op2 - borrow (SUB_HI), consuming the
+// flags set by the previous node. Used for the high half of decomposed longs and for adding a compare
+// result to a value.
+GTNODE(ADD_HI           , GenTreeOp          ,1,0,GTK_BINOP|DBK_NOTHIR)
+GTNODE(SUB_HI           , GenTreeOp          ,0,0,GTK_BINOP|DBK_NOTHIR)
+
 #if !defined(TARGET_64BIT)
 // A GT_LONG node simply represents the long value produced by the concatenation
 // of its two (lower and upper half) operands.  Some GT_LONG nodes are transient,
 // during the decomposing of longs; others are handled by codegen as operands of
 // nodes such as calls, returns and stores of long lclVars.
 GTNODE(LONG             , GenTreeOp          ,0,0,GTK_BINOP|DBK_NOTHIR)
 
-// The following are nodes representing x86/arm32 specific long operators, including
-// high operators of a 64-bit operations that requires a carry/borrow, which are
-// named GT_XXX_HI for consistency, low operators of 64-bit operations that need
-// to not be modified in phases post-decompose, and operators that return 64-bit
-// results in one instruction.
+// The following are nodes representing x86/arm32 specific long operators: low operators
+// of 64-bit operations that need to not be modified in phases post-decompose (the high
+// operators that require a carry/borrow are GT_ADD_HI/GT_SUB_HI above), and operators
+// that return 64-bit results in one instruction.
 GTNODE(ADD_LO           , GenTreeOp          ,1,0,GTK_BINOP|DBK_NOTHIR)
-GTNODE(ADD_HI           , GenTreeOp          ,1,0,GTK_BINOP|DBK_NOTHIR)
 GTNODE(SUB_LO           , GenTreeOp          ,0,0,GTK_BINOP|DBK_NOTHIR)
-GTNODE(SUB_HI           , GenTreeOp          ,0,0,GTK_BINOP|DBK_NOTHIR)
 
 // The following are nodes that specify shifts that take a GT_LONG op1. The GT_LONG
 // contains the hi and lo parts of three operand shift form where one op will be