diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index 52a57e153bb4b..9b7f2705cff81 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -1811,6 +1811,19 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) { return BinaryOperator::CreateAnd(Add, A); } + // Align-up idiom: + // X + ((-X) & (C - 1)) --> (X + C - 1) & -C, for a power-of-two C. + // Note -C == ~(C - 1), so the mask is simply the inverted low-bit mask. + { + const APInt *LowMask; + if (match(&I, + m_c_Add(m_OneUse(m_And(m_Neg(m_Value(A)), m_LowBitMask(LowMask))), + m_Deferred(A)))) { + Value *NewAdd = Builder.CreateAdd(A, ConstantInt::get(Ty, *LowMask)); + return BinaryOperator::CreateAnd(NewAdd, ConstantInt::get(Ty, ~*LowMask)); + } + } + // Canonicalize ((A & -A) - 1) --> ((A - 1) & ~A) // Forms all commutable operations, and simplifies ctpop -> cttz folds. if (match(&I, diff --git a/llvm/test/Transforms/InstCombine/add-mask-neg.ll b/llvm/test/Transforms/InstCombine/add-mask-neg.ll index cb23763249d63..a83d86da58b82 100644 --- a/llvm/test/Transforms/InstCombine/add-mask-neg.ll +++ b/llvm/test/Transforms/InstCombine/add-mask-neg.ll @@ -119,5 +119,173 @@ define <2 x i32> @dec_mask_multiuse_neg_multiuse_v2i32(<2 x i32> %X) { ret <2 x i32> %dec } +; +; X + ((-X) & (C - 1)) --> (X + C - 1) & -C, for a power-of-two C +; +define i32 @align_up(i32 %x) { +; CHECK-LABEL: @align_up( +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X:%.*]], 15 +; CHECK-NEXT: [[R:%.*]] = and i32 [[TMP1]], -16 +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 %neg, 15 + %r = add i32 %x, %and + ret i32 %r +} + +define i32 @align_up_commuted(i32 %x) { +; CHECK-LABEL: @align_up_commuted( +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X:%.*]], 15 +; CHECK-NEXT: [[R:%.*]] = and i32 [[TMP1]], -16 +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 %neg, 15 + %r = add i32 %and, %x + ret i32 %r +} + +define i32 @align_up_commuted_and(i32 %x) { +; CHECK-LABEL: @align_up_commuted_and( +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X:%.*]], 15 +; CHECK-NEXT: [[R:%.*]] = and i32 [[TMP1]], -16 +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 15, %neg + %r = add i32 %x, %and + ret i32 %r +} + +define <2 x i32> @align_up_vec(<2 x i32> %x) { +; CHECK-LABEL: @align_up_vec( +; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], splat (i32 15) +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[TMP1]], splat (i32 -16) +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %neg = sub <2 x i32> zeroinitializer, %x + %and = and <2 x i32> %neg, splat (i32 15) + %r = add <2 x i32> %x, %and + ret <2 x i32> %r +} + +; negative test - the mask is not a low-bit mask + +define i32 @align_up_not_lowbitmask(i32 %x) { +; CHECK-LABEL: @align_up_not_lowbitmask( +; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and i32 [[NEG]], 13 +; CHECK-NEXT: [[R:%.*]] = add i32 [[X]], [[AND]] +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 %neg, 13 + %r = add i32 %x, %and + ret i32 %r +} + +; negative test - extra use of the and + +define i32 @align_up_multiuse_and(i32 %x) { +; CHECK-LABEL: @align_up_multiuse_and( +; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and i32 [[NEG]], 15 +; CHECK-NEXT: [[R:%.*]] = add i32 [[X]], [[AND]] +; CHECK-NEXT: call void @use(i32 [[AND]]) +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 %neg, 15 + %r = add i32 %x, %and + call void @use(i32 %and) + ret i32 %r +} + +; negative test - a different value is negated + +define i32 @align_up_wrong_value(i32 %x, i32 %y) { +; CHECK-LABEL: @align_up_wrong_value( +; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[Y:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and i32 [[NEG]], 15 +; CHECK-NEXT: [[R:%.*]] = add i32 [[X:%.*]], [[AND]] +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %y + %and = and i32 %neg, 15 + %r = add i32 %x, %and + ret i32 %r +} + +; negative test - non-splat vector + +define <2 x i32> @align_up_nonsplat_vec(<2 x i32> %x) { +; CHECK-LABEL: @align_up_nonsplat_vec( +; CHECK-NEXT: [[NEG:%.*]] = sub <2 x i32> zeroinitializer, [[X:%.*]] +; CHECK-NEXT: [[AND:%.*]] = and <2 x i32> [[NEG]], +; CHECK-NEXT: [[R:%.*]] = add <2 x i32> [[X]], [[AND]] +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %neg = sub <2 x i32> zeroinitializer, %x + %and = and <2 x i32> %neg, + %r = add <2 x i32> %x, %and + ret <2 x i32> %r +} + +; the neg has an extra use + +define i32 @align_up_multiuse_neg(i32 %x) { +; CHECK-LABEL: @align_up_multiuse_neg( +; CHECK-NEXT: [[NEG:%.*]] = sub i32 0, [[X:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X]], 15 +; CHECK-NEXT: [[R:%.*]] = and i32 [[TMP1]], -16 +; CHECK-NEXT: call void @use(i32 [[NEG]]) +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub i32 0, %x + %and = and i32 %neg, 15 + %r = add i32 %x, %and + call void @use(i32 %neg) + ret i32 %r +} + +; nsw/nuw on the source add are dropped + +define i32 @align_up_nsw_nuw(i32 %x) { +; CHECK-LABEL: @align_up_nsw_nuw( +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[X:%.*]], 15 +; CHECK-NEXT: [[R:%.*]] = and i32 [[TMP1]], -16 +; CHECK-NEXT: ret i32 [[R]] +; + %neg = sub nsw i32 0, %x + %and = and i32 %neg, 15 + %r = add nuw nsw i32 %x, %and + ret i32 %r +} + +define @align_up_scalable_vec( %x) { +; CHECK-LABEL: @align_up_scalable_vec( +; CHECK-NEXT: [[TMP1:%.*]] = add [[X:%.*]], splat (i32 15) +; CHECK-NEXT: [[R:%.*]] = and [[TMP1]], splat (i32 -16) +; CHECK-NEXT: ret [[R]] +; + %neg = sub zeroinitializer, %x + %and = and %neg, splat (i32 15) + %r = add %x, %and + ret %r +} + +define <2 x i32> @align_up_vec_poison_elt(<2 x i32> %x) { +; CHECK-LABEL: @align_up_vec_poison_elt( +; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], splat (i32 15) +; CHECK-NEXT: [[R:%.*]] = and <2 x i32> [[TMP1]], splat (i32 -16) +; CHECK-NEXT: ret <2 x i32> [[R]] +; + %neg = sub <2 x i32> zeroinitializer, %x + %and = and <2 x i32> %neg, + %r = add <2 x i32> %x, %and + ret <2 x i32> %r +} + declare void @use(i32) declare void @usev(<2 x i32>)