[CostModel][X86] Add variable divisor div/rem costs for scalar and <=i32 vectors - #215124
Open
as4230 wants to merge 1 commit into
Open
[CostModel][X86] Add variable divisor div/rem costs for scalar and <=i32 vectors#215124as4230 wants to merge 1 commit into
as4230 wants to merge 1 commit into
Conversation
You can test this locally with the following command:git-clang-format --diff origin/main HEAD --extensions cpp -- llvm/lib/Target/X86/X86TargetTransformInfo.cpp --diff_from_common_commit
View the diff from clang-format here.diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 4818ad956..0c1d790e1 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -755,22 +755,22 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
// rem matches div because the divider returns the remainder for free.
static const CostKindTblEntry ScalarVarDivCostTable[] = {
- { ISD::SDIV, MVT::i8, { 15, 20, 2, 4 } },
- { ISD::UDIV, MVT::i8, { 15, 20, 2, 4 } },
- { ISD::SREM, MVT::i8, { 15, 20, 2, 4 } },
- { ISD::UREM, MVT::i8, { 15, 20, 2, 4 } },
- { ISD::SDIV, MVT::i16, { 17, 20, 2, 4 } },
- { ISD::UDIV, MVT::i16, { 17, 20, 2, 4 } },
- { ISD::SREM, MVT::i16, { 17, 20, 2, 4 } },
- { ISD::UREM, MVT::i16, { 17, 20, 2, 4 } },
- { ISD::SDIV, MVT::i32, { 25, 22, 2, 4 } },
- { ISD::UDIV, MVT::i32, { 25, 22, 2, 4 } },
- { ISD::SREM, MVT::i32, { 25, 22, 2, 4 } },
- { ISD::UREM, MVT::i32, { 25, 22, 2, 4 } },
- { ISD::SDIV, MVT::i64, { 41, 24, 2, 4 } },
- { ISD::UDIV, MVT::i64, { 41, 24, 2, 4 } },
- { ISD::SREM, MVT::i64, { 41, 24, 2, 4 } },
- { ISD::UREM, MVT::i64, { 41, 24, 2, 4 } },
+ {ISD::SDIV, MVT::i8, {15, 20, 2, 4}},
+ {ISD::UDIV, MVT::i8, {15, 20, 2, 4}},
+ {ISD::SREM, MVT::i8, {15, 20, 2, 4}},
+ {ISD::UREM, MVT::i8, {15, 20, 2, 4}},
+ {ISD::SDIV, MVT::i16, {17, 20, 2, 4}},
+ {ISD::UDIV, MVT::i16, {17, 20, 2, 4}},
+ {ISD::SREM, MVT::i16, {17, 20, 2, 4}},
+ {ISD::UREM, MVT::i16, {17, 20, 2, 4}},
+ {ISD::SDIV, MVT::i32, {25, 22, 2, 4}},
+ {ISD::UDIV, MVT::i32, {25, 22, 2, 4}},
+ {ISD::SREM, MVT::i32, {25, 22, 2, 4}},
+ {ISD::UREM, MVT::i32, {25, 22, 2, 4}},
+ {ISD::SDIV, MVT::i64, {41, 24, 2, 4}},
+ {ISD::UDIV, MVT::i64, {41, 24, 2, 4}},
+ {ISD::SREM, MVT::i64, {41, 24, 2, 4}},
+ {ISD::UREM, MVT::i64, {41, 24, 2, 4}},
};
if (!LT.second.isVector() && !Op2Info.isConstant())
@@ -787,42 +787,42 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
!Op2Info.isConstant() && (!IsStrictFP || ST->useAVX512Regs());
static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
- { ISD::UDIV, MVT::v16i8, { 10 } }, // unpack+cvt+divps sequence
- { ISD::SDIV, MVT::v16i8, { 10 } },
- { ISD::UREM, MVT::v16i8, { 10 } },
- { ISD::SREM, MVT::v16i8, { 10 } },
- { ISD::UDIV, MVT::v32i8, { 20 } },
- { ISD::SDIV, MVT::v32i8, { 20 } },
- { ISD::UREM, MVT::v32i8, { 20 } },
- { ISD::SREM, MVT::v32i8, { 20 } },
- { ISD::UDIV, MVT::v64i8, { 40 } },
- { ISD::SDIV, MVT::v64i8, { 40 } },
- { ISD::UREM, MVT::v64i8, { 40 } },
- { ISD::SREM, MVT::v64i8, { 40 } },
- { ISD::UDIV, MVT::v8i16, { 5 } },
- { ISD::SDIV, MVT::v8i16, { 5 } },
- { ISD::UREM, MVT::v8i16, { 5 } },
- { ISD::SREM, MVT::v8i16, { 5 } },
- { ISD::UDIV, MVT::v16i16, { 10 } },
- { ISD::SDIV, MVT::v16i16, { 10 } },
- { ISD::UREM, MVT::v16i16, { 10 } },
- { ISD::SREM, MVT::v16i16, { 10 } },
- { ISD::UDIV, MVT::v32i16, { 20 } },
- { ISD::SDIV, MVT::v32i16, { 20 } },
- { ISD::UREM, MVT::v32i16, { 20 } },
- { ISD::SREM, MVT::v32i16, { 20 } },
- { ISD::UDIV, MVT::v4i32, { 8 } }, // cvt+divpd sequence
- { ISD::SDIV, MVT::v4i32, { 8 } },
- { ISD::UREM, MVT::v4i32, { 8 } },
- { ISD::SREM, MVT::v4i32, { 8 } },
- { ISD::UDIV, MVT::v8i32, { 16 } },
- { ISD::SDIV, MVT::v8i32, { 16 } },
- { ISD::UREM, MVT::v8i32, { 16 } },
- { ISD::SREM, MVT::v8i32, { 16 } },
- { ISD::UDIV, MVT::v16i32, { 32 } },
- { ISD::SDIV, MVT::v16i32, { 32 } },
- { ISD::UREM, MVT::v16i32, { 32 } },
- { ISD::SREM, MVT::v16i32, { 32 } },
+ {ISD::UDIV, MVT::v16i8, {10}}, // unpack+cvt+divps sequence
+ {ISD::SDIV, MVT::v16i8, {10}},
+ {ISD::UREM, MVT::v16i8, {10}},
+ {ISD::SREM, MVT::v16i8, {10}},
+ {ISD::UDIV, MVT::v32i8, {20}},
+ {ISD::SDIV, MVT::v32i8, {20}},
+ {ISD::UREM, MVT::v32i8, {20}},
+ {ISD::SREM, MVT::v32i8, {20}},
+ {ISD::UDIV, MVT::v64i8, {40}},
+ {ISD::SDIV, MVT::v64i8, {40}},
+ {ISD::UREM, MVT::v64i8, {40}},
+ {ISD::SREM, MVT::v64i8, {40}},
+ {ISD::UDIV, MVT::v8i16, {5}},
+ {ISD::SDIV, MVT::v8i16, {5}},
+ {ISD::UREM, MVT::v8i16, {5}},
+ {ISD::SREM, MVT::v8i16, {5}},
+ {ISD::UDIV, MVT::v16i16, {10}},
+ {ISD::SDIV, MVT::v16i16, {10}},
+ {ISD::UREM, MVT::v16i16, {10}},
+ {ISD::SREM, MVT::v16i16, {10}},
+ {ISD::UDIV, MVT::v32i16, {20}},
+ {ISD::SDIV, MVT::v32i16, {20}},
+ {ISD::UREM, MVT::v32i16, {20}},
+ {ISD::SREM, MVT::v32i16, {20}},
+ {ISD::UDIV, MVT::v4i32, {8}}, // cvt+divpd sequence
+ {ISD::SDIV, MVT::v4i32, {8}},
+ {ISD::UREM, MVT::v4i32, {8}},
+ {ISD::SREM, MVT::v4i32, {8}},
+ {ISD::UDIV, MVT::v8i32, {16}},
+ {ISD::SDIV, MVT::v8i32, {16}},
+ {ISD::UREM, MVT::v8i32, {16}},
+ {ISD::SREM, MVT::v8i32, {16}},
+ {ISD::UDIV, MVT::v16i32, {32}},
+ {ISD::SDIV, MVT::v16i32, {32}},
+ {ISD::UREM, MVT::v16i32, {32}},
+ {ISD::SREM, MVT::v16i32, {32}},
};
if (VarDivToFP && ST->hasBWI())
@@ -832,42 +832,42 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
return LT.first * *KindCost;
static const CostKindTblEntry AVX512VarDivCostTable[] = {
- { ISD::UDIV, MVT::v16i8, { 14 } }, // unpack+cvt+divps sequence
- { ISD::SDIV, MVT::v16i8, { 14 } },
- { ISD::UREM, MVT::v16i8, { 14 } },
- { ISD::SREM, MVT::v16i8, { 14 } },
- { ISD::UDIV, MVT::v32i8, { 28 } },
- { ISD::SDIV, MVT::v32i8, { 28 } },
- { ISD::UREM, MVT::v32i8, { 28 } },
- { ISD::SREM, MVT::v32i8, { 28 } },
- { ISD::UDIV, MVT::v64i8, { 56 } },
- { ISD::SDIV, MVT::v64i8, { 56 } },
- { ISD::UREM, MVT::v64i8, { 56 } },
- { ISD::SREM, MVT::v64i8, { 56 } },
- { ISD::UDIV, MVT::v8i16, { 14 } },
- { ISD::SDIV, MVT::v8i16, { 14 } },
- { ISD::UREM, MVT::v8i16, { 14 } },
- { ISD::SREM, MVT::v8i16, { 14 } },
- { ISD::UDIV, MVT::v16i16, { 14 } },
- { ISD::SDIV, MVT::v16i16, { 14 } },
- { ISD::UREM, MVT::v16i16, { 14 } },
- { ISD::SREM, MVT::v16i16, { 14 } },
- { ISD::UDIV, MVT::v32i16, { 28 } },
- { ISD::SDIV, MVT::v32i16, { 28 } },
- { ISD::UREM, MVT::v32i16, { 28 } },
- { ISD::SREM, MVT::v32i16, { 28 } },
- { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
- { ISD::SDIV, MVT::v4i32, { 28 } },
- { ISD::UREM, MVT::v4i32, { 28 } },
- { ISD::SREM, MVT::v4i32, { 28 } },
- { ISD::UDIV, MVT::v8i32, { 28 } },
- { ISD::SDIV, MVT::v8i32, { 28 } },
- { ISD::UREM, MVT::v8i32, { 28 } },
- { ISD::SREM, MVT::v8i32, { 28 } },
- { ISD::UDIV, MVT::v16i32, { 56 } },
- { ISD::SDIV, MVT::v16i32, { 56 } },
- { ISD::UREM, MVT::v16i32, { 56 } },
- { ISD::SREM, MVT::v16i32, { 56 } },
+ {ISD::UDIV, MVT::v16i8, {14}}, // unpack+cvt+divps sequence
+ {ISD::SDIV, MVT::v16i8, {14}},
+ {ISD::UREM, MVT::v16i8, {14}},
+ {ISD::SREM, MVT::v16i8, {14}},
+ {ISD::UDIV, MVT::v32i8, {28}},
+ {ISD::SDIV, MVT::v32i8, {28}},
+ {ISD::UREM, MVT::v32i8, {28}},
+ {ISD::SREM, MVT::v32i8, {28}},
+ {ISD::UDIV, MVT::v64i8, {56}},
+ {ISD::SDIV, MVT::v64i8, {56}},
+ {ISD::UREM, MVT::v64i8, {56}},
+ {ISD::SREM, MVT::v64i8, {56}},
+ {ISD::UDIV, MVT::v8i16, {14}},
+ {ISD::SDIV, MVT::v8i16, {14}},
+ {ISD::UREM, MVT::v8i16, {14}},
+ {ISD::SREM, MVT::v8i16, {14}},
+ {ISD::UDIV, MVT::v16i16, {14}},
+ {ISD::SDIV, MVT::v16i16, {14}},
+ {ISD::UREM, MVT::v16i16, {14}},
+ {ISD::SREM, MVT::v16i16, {14}},
+ {ISD::UDIV, MVT::v32i16, {28}},
+ {ISD::SDIV, MVT::v32i16, {28}},
+ {ISD::UREM, MVT::v32i16, {28}},
+ {ISD::SREM, MVT::v32i16, {28}},
+ {ISD::UDIV, MVT::v4i32, {28}}, // cvt+divpd sequence
+ {ISD::SDIV, MVT::v4i32, {28}},
+ {ISD::UREM, MVT::v4i32, {28}},
+ {ISD::SREM, MVT::v4i32, {28}},
+ {ISD::UDIV, MVT::v8i32, {28}},
+ {ISD::SDIV, MVT::v8i32, {28}},
+ {ISD::UREM, MVT::v8i32, {28}},
+ {ISD::SREM, MVT::v8i32, {28}},
+ {ISD::UDIV, MVT::v16i32, {56}},
+ {ISD::SDIV, MVT::v16i32, {56}},
+ {ISD::UREM, MVT::v16i32, {56}},
+ {ISD::SREM, MVT::v16i32, {56}},
};
if (VarDivToFP && ST->hasAVX512())
@@ -877,30 +877,30 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
return LT.first * *KindCost;
static const CostKindTblEntry AVX2VarDivCostTable[] = {
- { ISD::UDIV, MVT::v16i8, { 28 } }, // unpack+cvt+divps sequence
- { ISD::SDIV, MVT::v16i8, { 28 } },
- { ISD::UREM, MVT::v16i8, { 28 } },
- { ISD::SREM, MVT::v16i8, { 28 } },
- { ISD::UDIV, MVT::v32i8, { 56 } },
- { ISD::SDIV, MVT::v32i8, { 56 } },
- { ISD::UREM, MVT::v32i8, { 56 } },
- { ISD::SREM, MVT::v32i8, { 56 } },
- { ISD::UDIV, MVT::v8i16, { 14 } },
- { ISD::SDIV, MVT::v8i16, { 14 } },
- { ISD::UREM, MVT::v8i16, { 14 } },
- { ISD::SREM, MVT::v8i16, { 14 } },
- { ISD::UDIV, MVT::v16i16, { 28 } },
- { ISD::SDIV, MVT::v16i16, { 28 } },
- { ISD::UREM, MVT::v16i16, { 28 } },
- { ISD::SREM, MVT::v16i16, { 28 } },
- { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
- { ISD::SDIV, MVT::v4i32, { 28 } },
- { ISD::UREM, MVT::v4i32, { 28 } },
- { ISD::SREM, MVT::v4i32, { 28 } },
- { ISD::UDIV, MVT::v8i32, { 56 } },
- { ISD::SDIV, MVT::v8i32, { 56 } },
- { ISD::UREM, MVT::v8i32, { 56 } },
- { ISD::SREM, MVT::v8i32, { 56 } },
+ {ISD::UDIV, MVT::v16i8, {28}}, // unpack+cvt+divps sequence
+ {ISD::SDIV, MVT::v16i8, {28}},
+ {ISD::UREM, MVT::v16i8, {28}},
+ {ISD::SREM, MVT::v16i8, {28}},
+ {ISD::UDIV, MVT::v32i8, {56}},
+ {ISD::SDIV, MVT::v32i8, {56}},
+ {ISD::UREM, MVT::v32i8, {56}},
+ {ISD::SREM, MVT::v32i8, {56}},
+ {ISD::UDIV, MVT::v8i16, {14}},
+ {ISD::SDIV, MVT::v8i16, {14}},
+ {ISD::UREM, MVT::v8i16, {14}},
+ {ISD::SREM, MVT::v8i16, {14}},
+ {ISD::UDIV, MVT::v16i16, {28}},
+ {ISD::SDIV, MVT::v16i16, {28}},
+ {ISD::UREM, MVT::v16i16, {28}},
+ {ISD::SREM, MVT::v16i16, {28}},
+ {ISD::UDIV, MVT::v4i32, {28}}, // cvt+divpd sequence
+ {ISD::SDIV, MVT::v4i32, {28}},
+ {ISD::UREM, MVT::v4i32, {28}},
+ {ISD::SREM, MVT::v4i32, {28}},
+ {ISD::UDIV, MVT::v8i32, {56}},
+ {ISD::SDIV, MVT::v8i32, {56}},
+ {ISD::UREM, MVT::v8i32, {56}},
+ {ISD::SREM, MVT::v8i32, {56}},
};
if (VarDivToFP && ST->hasAVX2())
@@ -912,26 +912,26 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
// No unsigned i32 entries below AVX2, where the u32 to f64 converts are
// emulated and the fold stays off.
static const CostKindTblEntry AVX1VarDivCostTable[] = {
- { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
- { ISD::SDIV, MVT::v16i8, { 56 } },
- { ISD::UREM, MVT::v16i8, { 56 } },
- { ISD::SREM, MVT::v16i8, { 56 } },
- { ISD::UDIV, MVT::v32i8, { 112 } },
- { ISD::SDIV, MVT::v32i8, { 112 } },
- { ISD::UREM, MVT::v32i8, { 112 } },
- { ISD::SREM, MVT::v32i8, { 112 } },
- { ISD::UDIV, MVT::v8i16, { 28 } },
- { ISD::SDIV, MVT::v8i16, { 28 } },
- { ISD::UREM, MVT::v8i16, { 28 } },
- { ISD::SREM, MVT::v8i16, { 28 } },
- { ISD::UDIV, MVT::v16i16, { 56 } },
- { ISD::SDIV, MVT::v16i16, { 56 } },
- { ISD::UREM, MVT::v16i16, { 56 } },
- { ISD::SREM, MVT::v16i16, { 56 } },
- { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
- { ISD::SREM, MVT::v4i32, { 44 } },
- { ISD::SDIV, MVT::v8i32, { 88 } },
- { ISD::SREM, MVT::v8i32, { 88 } },
+ {ISD::UDIV, MVT::v16i8, {56}}, // unpack+cvt+divps sequence
+ {ISD::SDIV, MVT::v16i8, {56}},
+ {ISD::UREM, MVT::v16i8, {56}},
+ {ISD::SREM, MVT::v16i8, {56}},
+ {ISD::UDIV, MVT::v32i8, {112}},
+ {ISD::SDIV, MVT::v32i8, {112}},
+ {ISD::UREM, MVT::v32i8, {112}},
+ {ISD::SREM, MVT::v32i8, {112}},
+ {ISD::UDIV, MVT::v8i16, {28}},
+ {ISD::SDIV, MVT::v8i16, {28}},
+ {ISD::UREM, MVT::v8i16, {28}},
+ {ISD::SREM, MVT::v8i16, {28}},
+ {ISD::UDIV, MVT::v16i16, {56}},
+ {ISD::SDIV, MVT::v16i16, {56}},
+ {ISD::UREM, MVT::v16i16, {56}},
+ {ISD::SREM, MVT::v16i16, {56}},
+ {ISD::SDIV, MVT::v4i32, {44}}, // cvt+divpd sequence
+ {ISD::SREM, MVT::v4i32, {44}},
+ {ISD::SDIV, MVT::v8i32, {88}},
+ {ISD::SREM, MVT::v8i32, {88}},
};
if (VarDivToFP && ST->hasAVX())
@@ -941,16 +941,16 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
return LT.first * *KindCost;
static const CostKindTblEntry SSE2VarDivCostTable[] = {
- { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
- { ISD::SDIV, MVT::v16i8, { 56 } },
- { ISD::UREM, MVT::v16i8, { 56 } },
- { ISD::SREM, MVT::v16i8, { 56 } },
- { ISD::UDIV, MVT::v8i16, { 28 } },
- { ISD::SDIV, MVT::v8i16, { 28 } },
- { ISD::UREM, MVT::v8i16, { 28 } },
- { ISD::SREM, MVT::v8i16, { 28 } },
- { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
- { ISD::SREM, MVT::v4i32, { 44 } },
+ {ISD::UDIV, MVT::v16i8, {56}}, // unpack+cvt+divps sequence
+ {ISD::SDIV, MVT::v16i8, {56}},
+ {ISD::UREM, MVT::v16i8, {56}},
+ {ISD::SREM, MVT::v16i8, {56}},
+ {ISD::UDIV, MVT::v8i16, {28}},
+ {ISD::SDIV, MVT::v8i16, {28}},
+ {ISD::UREM, MVT::v8i16, {28}},
+ {ISD::SREM, MVT::v8i16, {28}},
+ {ISD::SDIV, MVT::v4i32, {44}}, // cvt+divpd sequence
+ {ISD::SREM, MVT::v4i32, {44}},
};
if (VarDivToFP && ST->hasSSE2())
|
You can test this locally with the following command:git diff -U0 --pickaxe-regex -S '([^a-zA-Z0-9#_-]undef([^a-zA-Z0-9_-]|$)|UndefValue::get)' 'HEAD~1' HEAD llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll llvm/lib/Target/X86/X86TargetTransformInfo.cpp llvm/test/Analysis/CostModel/X86/div.ll llvm/test/Analysis/CostModel/X86/rem.ll llvm/test/Analysis/CostModel/X86/size-cost.ll llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll llvm/test/Transforms/LoopVectorize/X86/cost-model.ll llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll llvm/test/Transforms/SLPVectorizer/X86/div-possibly-extended-with-poisons.ll llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.llThe following files introduce new uses of undef:
Undef is now deprecated and should only be used in the rare cases where no replacement is possible. For example, a load of uninitialized memory yields In tests, avoid using For example, this is considered a bad practice: define void @fn() {
...
br i1 undef, ...
}Please use the following instead: define void @fn(i1 %cond) {
...
br i1 %cond, ...
}Please refer to the Undefined Behavior Manual for more information. |
Andarwinux
added
the
backend:X86 Scheduler Models
as4230
changed the title
as4230
marked this pull request as ready for review
August 10, 2026 02:43
llvmorg-github-actions
Bot
added
backend:X86
llvm:analysis
|
@llvm/pr-subscribers-backend-x86 @llvm/pr-subscribers-llvm-transforms Author: Adam Scott (as4230) ChangesDiv and rem by a variable divisor have no X86 cost entry at all, so they fall through to the BasicTTI default and cost 1. A 20 to 40 cycle unpipelined instruction ends up priced the same as an add. That mattered less while vector integer division always scalarized, since the price was wrong but the decision was usually right anyway. After #205263 the <=i32 shapes lower through a float divide, so the vector cost model is now describing a sequence that really exists and it still prices it as scalarization. This adds scalar entries plus five feature gated vector tables. Numbers are the maximum llvm-mca Block RThroughput across the CPUs that fold the shape with the low power lines excluded (atom and jaguar). Vector entries take the max within the tier a CPU's feature level selects, since the lookup takes the first matching table. Scalar takes it across all of them. rem is priced the same as div because a variable rem is the same instruction and the hardware hands back the remainder in rdx or AH. For vector the fold pays a multiply and subtract to recover it but the divide dominates the throughput so the number doesnt change. div-rem-strictfp.ll is new and pins the strictfp gate. Two SLP tests run on forced negative thresholds and needed those raised so both produce output identical to before at the new value. Patch is 208.06 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/215124.diff 15 Files Affected:
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2c57855a687a6..fa9e5ee02041d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -753,6 +753,212 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
if (auto KindCost = Entry->Cost[CostKind])
return LT.first * *KindCost;
+ // rem matches div because the divider returns the remainder for free.
+ static const CostKindTblEntry ScalarVarDivCostTable[] = {
+ { ISD::SDIV, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::UDIV, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::SREM, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::UREM, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::SDIV, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::UDIV, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::SREM, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::UREM, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::SDIV, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::UDIV, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::SREM, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::UREM, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::SDIV, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::UDIV, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::SREM, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::UREM, MVT::i64, { 41, 24, 2, 4 } },
+ };
+
+ if (!LT.second.isVector() && !Op2Info.isConstant())
+ if (const auto *Entry =
+ CostTableLookup(ScalarVarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ // Variable divisors lower through a float divide. strictfp needs SAE
+ // rounding which is 512-bit only.
+ bool IsStrictFP =
+ CxtI && CxtI->getFunction()->hasFnAttribute(Attribute::StrictFP);
+ bool VarDivToFP =
+ !Op2Info.isConstant() && (!IsStrictFP || ST->useAVX512Regs());
+
+ static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 10 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 10 } },
+ { ISD::UREM, MVT::v16i8, { 10 } },
+ { ISD::SREM, MVT::v16i8, { 10 } },
+ { ISD::UDIV, MVT::v32i8, { 20 } },
+ { ISD::SDIV, MVT::v32i8, { 20 } },
+ { ISD::UREM, MVT::v32i8, { 20 } },
+ { ISD::SREM, MVT::v32i8, { 20 } },
+ { ISD::UDIV, MVT::v64i8, { 40 } },
+ { ISD::SDIV, MVT::v64i8, { 40 } },
+ { ISD::UREM, MVT::v64i8, { 40 } },
+ { ISD::SREM, MVT::v64i8, { 40 } },
+ { ISD::UDIV, MVT::v8i16, { 5 } },
+ { ISD::SDIV, MVT::v8i16, { 5 } },
+ { ISD::UREM, MVT::v8i16, { 5 } },
+ { ISD::SREM, MVT::v8i16, { 5 } },
+ { ISD::UDIV, MVT::v16i16, { 10 } },
+ { ISD::SDIV, MVT::v16i16, { 10 } },
+ { ISD::UREM, MVT::v16i16, { 10 } },
+ { ISD::SREM, MVT::v16i16, { 10 } },
+ { ISD::UDIV, MVT::v32i16, { 20 } },
+ { ISD::SDIV, MVT::v32i16, { 20 } },
+ { ISD::UREM, MVT::v32i16, { 20 } },
+ { ISD::SREM, MVT::v32i16, { 20 } },
+ { ISD::UDIV, MVT::v4i32, { 8 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 8 } },
+ { ISD::UREM, MVT::v4i32, { 8 } },
+ { ISD::SREM, MVT::v4i32, { 8 } },
+ { ISD::UDIV, MVT::v8i32, { 16 } },
+ { ISD::SDIV, MVT::v8i32, { 16 } },
+ { ISD::UREM, MVT::v8i32, { 16 } },
+ { ISD::SREM, MVT::v8i32, { 16 } },
+ { ISD::UDIV, MVT::v16i32, { 32 } },
+ { ISD::SDIV, MVT::v16i32, { 32 } },
+ { ISD::UREM, MVT::v16i32, { 32 } },
+ { ISD::SREM, MVT::v16i32, { 32 } },
+ };
+
+ if (VarDivToFP && ST->hasBWI())
+ if (const auto *Entry =
+ CostTableLookup(AVX512BWVarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX512VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 14 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 14 } },
+ { ISD::UREM, MVT::v16i8, { 14 } },
+ { ISD::SREM, MVT::v16i8, { 14 } },
+ { ISD::UDIV, MVT::v32i8, { 28 } },
+ { ISD::SDIV, MVT::v32i8, { 28 } },
+ { ISD::UREM, MVT::v32i8, { 28 } },
+ { ISD::SREM, MVT::v32i8, { 28 } },
+ { ISD::UDIV, MVT::v64i8, { 56 } },
+ { ISD::SDIV, MVT::v64i8, { 56 } },
+ { ISD::UREM, MVT::v64i8, { 56 } },
+ { ISD::SREM, MVT::v64i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 14 } },
+ { ISD::SDIV, MVT::v8i16, { 14 } },
+ { ISD::UREM, MVT::v8i16, { 14 } },
+ { ISD::SREM, MVT::v8i16, { 14 } },
+ { ISD::UDIV, MVT::v16i16, { 14 } },
+ { ISD::SDIV, MVT::v16i16, { 14 } },
+ { ISD::UREM, MVT::v16i16, { 14 } },
+ { ISD::SREM, MVT::v16i16, { 14 } },
+ { ISD::UDIV, MVT::v32i16, { 28 } },
+ { ISD::SDIV, MVT::v32i16, { 28 } },
+ { ISD::UREM, MVT::v32i16, { 28 } },
+ { ISD::SREM, MVT::v32i16, { 28 } },
+ { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 28 } },
+ { ISD::UREM, MVT::v4i32, { 28 } },
+ { ISD::SREM, MVT::v4i32, { 28 } },
+ { ISD::UDIV, MVT::v8i32, { 28 } },
+ { ISD::SDIV, MVT::v8i32, { 28 } },
+ { ISD::UREM, MVT::v8i32, { 28 } },
+ { ISD::SREM, MVT::v8i32, { 28 } },
+ { ISD::UDIV, MVT::v16i32, { 56 } },
+ { ISD::SDIV, MVT::v16i32, { 56 } },
+ { ISD::UREM, MVT::v16i32, { 56 } },
+ { ISD::SREM, MVT::v16i32, { 56 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX512())
+ if (const auto *Entry =
+ CostTableLookup(AVX512VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX2VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 28 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 28 } },
+ { ISD::UREM, MVT::v16i8, { 28 } },
+ { ISD::SREM, MVT::v16i8, { 28 } },
+ { ISD::UDIV, MVT::v32i8, { 56 } },
+ { ISD::SDIV, MVT::v32i8, { 56 } },
+ { ISD::UREM, MVT::v32i8, { 56 } },
+ { ISD::SREM, MVT::v32i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 14 } },
+ { ISD::SDIV, MVT::v8i16, { 14 } },
+ { ISD::UREM, MVT::v8i16, { 14 } },
+ { ISD::SREM, MVT::v8i16, { 14 } },
+ { ISD::UDIV, MVT::v16i16, { 28 } },
+ { ISD::SDIV, MVT::v16i16, { 28 } },
+ { ISD::UREM, MVT::v16i16, { 28 } },
+ { ISD::SREM, MVT::v16i16, { 28 } },
+ { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 28 } },
+ { ISD::UREM, MVT::v4i32, { 28 } },
+ { ISD::SREM, MVT::v4i32, { 28 } },
+ { ISD::UDIV, MVT::v8i32, { 56 } },
+ { ISD::SDIV, MVT::v8i32, { 56 } },
+ { ISD::UREM, MVT::v8i32, { 56 } },
+ { ISD::SREM, MVT::v8i32, { 56 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX2())
+ if (const auto *Entry =
+ CostTableLookup(AVX2VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ // No unsigned i32 entries below AVX2, where the u32 to f64 converts are
+ // emulated and the fold stays off.
+ static const CostKindTblEntry AVX1VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 56 } },
+ { ISD::UREM, MVT::v16i8, { 56 } },
+ { ISD::SREM, MVT::v16i8, { 56 } },
+ { ISD::UDIV, MVT::v32i8, { 112 } },
+ { ISD::SDIV, MVT::v32i8, { 112 } },
+ { ISD::UREM, MVT::v32i8, { 112 } },
+ { ISD::SREM, MVT::v32i8, { 112 } },
+ { ISD::UDIV, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v8i16, { 28 } },
+ { ISD::UREM, MVT::v8i16, { 28 } },
+ { ISD::SREM, MVT::v8i16, { 28 } },
+ { ISD::UDIV, MVT::v16i16, { 56 } },
+ { ISD::SDIV, MVT::v16i16, { 56 } },
+ { ISD::UREM, MVT::v16i16, { 56 } },
+ { ISD::SREM, MVT::v16i16, { 56 } },
+ { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
+ { ISD::SREM, MVT::v4i32, { 44 } },
+ { ISD::SDIV, MVT::v8i32, { 88 } },
+ { ISD::SREM, MVT::v8i32, { 88 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX())
+ if (const auto *Entry =
+ CostTableLookup(AVX1VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry SSE2VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 56 } },
+ { ISD::UREM, MVT::v16i8, { 56 } },
+ { ISD::SREM, MVT::v16i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v8i16, { 28 } },
+ { ISD::UREM, MVT::v8i16, { 28 } },
+ { ISD::SREM, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
+ { ISD::SREM, MVT::v4i32, { 44 } },
+ };
+
+ if (VarDivToFP && ST->hasSSE2())
+ if (const auto *Entry =
+ CostTableLookup(SSE2VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
static const CostKindTblEntry AVX512BWUniformCostTable[] = {
{ ISD::SHL, MVT::v16i8, { 3, 5, 5, 7 } }, // psllw + pand.
{ ISD::SRL, MVT::v16i8, { 3,10, 5, 8 } }, // psrlw + pand.
diff --git a/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll b/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll
new file mode 100644
index 0000000000000..9a745c20a7fec
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll
@@ -0,0 +1,310 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw,+avx512vl,+prefer-256-bit | FileCheck %s --check-prefix=AVX512-256
+
+define void @vector_div_rem(<32 x i8> %a8, <32 x i8> %b8, <16 x i16> %a16, <16 x i16> %b16, <8 x i32> %a32, <8 x i32> %b32) {
+; SSE2-LABEL: 'vector_div_rem'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'vector_div_rem'
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'vector_div_rem'
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-256-LABEL: 'vector_div_rem'
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'vector_div_rem'
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512VL-LABEL: 'vector_div_rem'
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %sdiv8 = sdiv <32 x i8> %a8, %b8
+ %udiv8 = udiv <32 x i8> %a8, %b8
+ %srem8 = srem <32 x i8> %a8, %b8
+ %urem8 = urem <32 x i8> %a8, %b8
+ %sdiv16 = sdiv <16 x i16> %a16, %b16
+ %udiv16 = udiv <16 x i16> %a16, %b16
+ %srem16 = srem <16 x i16> %a16, %b16
+ %urem16 = urem <16 x i16> %a16, %b16
+ %sdiv32 = sdiv <8 x i32> %a32, %b32
+ %udiv32 = udiv <8 x i32> %a32, %b32
+ %srem32 = srem <8 x i32> %a32, %b32
+ %urem32 = urem <8 x i32> %a32, %b32
+ ret void
+}
+
+define void @vector_div_rem_strictfp(<32 x i8> %a8, <32 x i8> %b8, <16 x i16> %a16, <16 x i16> %b16, <8 x i32> %a32, <8 x i32> %b32) strictfp {
+; SSE2-LABEL: 'vector_div_rem_strictfp'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %urem8 = urem <32 x i8> %a8, %b...
[truncated]
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Div and rem by a variable divisor have no X86 cost entry at all, so they fall through to the BasicTTI default and cost 1. A 20 to 40 cycle unpipelined instruction ends up priced the same as an add.
That mattered less while vector integer division always scalarized, since the price was wrong but the decision was usually right anyway. After #205263 the <=i32 shapes lower through a float divide, so the vector cost model is now describing a sequence that really exists and it still prices it as scalarization.
This adds scalar entries plus five feature gated vector tables.
Numbers are the maximum llvm-mca Block RThroughput across the CPUs that fold the shape with the low power lines excluded (atom and jaguar). Vector entries take the max within the tier a CPU's feature level selects, since the lookup takes the first matching table. Scalar takes it across all of them.
rem is priced the same as div because a variable rem is the same instruction and the hardware hands back the remainder in rdx or AH. For vector the fold pays a multiply and subtract to recover it but the divide dominates the throughput so the number doesnt change.
div-rem-strictfp.ll is new and pins the strictfp gate. Two SLP tests run on forced negative thresholds and needed those raised so both produce output identical to before at the new value.