From 44f0b76832db66b2d0bf6af2b183717f8cb1de98 Mon Sep 17 00:00:00 2001 From: Satya Nikhil Date: Fri, 24 Jul 2026 20:14:23 +0000 Subject: [PATCH 1/4] move all configs --- .../gemm/basic/gemm_afp4wfp4.py | 12 +--- .../gluon/gemm/GEMM-AFP4WFP4.json} | 0 .../triton/configs/gfx1250/gluon/moe/.gitkeep | 0 .../configs/gfx1250/triton/gemm/.gitkeep | 0 .../configs/gfx1250/triton/moe/.gitkeep | 0 .../gluon/gemm/GEMM-AFP4WFP4.json} | 0 .../triton/configs/gfx950/gluon/moe/.gitkeep | 0 .../gemm/GEMM-AFP4WFP4-N=106496-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=1280-K=8192.json} | 0 .../gemm/GEMM-AFP4WFP4-N=13312-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=13312.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=2048.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=26624.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=4096.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=53248.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=6656.json} | 0 .../gemm/GEMM-AFP4WFP4-N=16384-K=8192.json} | 0 .../gemm/GEMM-AFP4WFP4-N=18432-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=2112-K=7168.json} | 0 .../gemm/GEMM-AFP4WFP4-N=2304-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=24576-K=1536.json} | 0 .../gemm/GEMM-AFP4WFP4-N=26624-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=28672-K=4096.json} | 0 .../gemm/GEMM-AFP4WFP4-N=3072-K=1536.json} | 0 .../gemm/GEMM-AFP4WFP4-N=32768-K=512.json} | 0 .../gemm/GEMM-AFP4WFP4-N=36864-K=7168.json} | 0 .../gemm/GEMM-AFP4WFP4-N=4096-K=14336.json} | 0 .../gemm/GEMM-AFP4WFP4-N=4096-K=7168.json} | 0 .../gemm/GEMM-AFP4WFP4-N=4608-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=4608-K=7168.json} | 0 .../gemm/GEMM-AFP4WFP4-N=512-K=128.json} | 0 .../gemm/GEMM-AFP4WFP4-N=512-K=7168.json} | 0 .../gemm/GEMM-AFP4WFP4-N=53248-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=57344-K=8192.json} | 0 .../gemm/GEMM-AFP4WFP4-N=7168-K=16384.json} | 0 .../gemm/GEMM-AFP4WFP4-N=7168-K=2048.json} | 0 .../gemm/GEMM-AFP4WFP4-N=7168-K=2304.json} | 0 .../gemm/GEMM-AFP4WFP4-N=7168-K=256.json} | 0 .../gemm/GEMM-AFP4WFP4-N=8192-K=28672.json} | 0 .../gemm/GEMM-AFP4WFP4-N=8192-K=8192.json} | 0 .../gemm/GEMM-AFP4WFP4-N=9216-K=16384.json} | 0 .../triton/gemm/GEMM-AFP4WFP4.json} | 0 .../triton/configs/gfx950/triton/moe/.gitkeep | 0 aiter/ops/triton/gemm/basic/gemm_afp4wfp4.py | 2 +- aiter/ops/triton/gluon/gemm_afp4wfp4.py | 2 +- aiter/ops/triton/utils/gemm_config_utils.py | 58 +++++++++++++------ 47 files changed, 45 insertions(+), 29 deletions(-) rename aiter/ops/triton/configs/{gemm/gfx1250-GEMM-AFP4WFP4.json => gfx1250/gluon/gemm/GEMM-AFP4WFP4.json} (100%) create mode 100644 aiter/ops/triton/configs/gfx1250/gluon/moe/.gitkeep create mode 100644 aiter/ops/triton/configs/gfx1250/triton/gemm/.gitkeep create mode 100644 aiter/ops/triton/configs/gfx1250/triton/moe/.gitkeep rename aiter/ops/triton/configs/{gemm/gluon/gfx950-GEMM-AFP4WFP4.json => gfx950/gluon/gemm/GEMM-AFP4WFP4.json} (100%) create mode 100644 aiter/ops/triton/configs/gfx950/gluon/moe/.gitkeep rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=106496-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=106496-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=1280-K=8192.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=1280-K=8192.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=13312-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=13312-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=13312.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=13312.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=2048.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=2048.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=26624.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=26624.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=4096.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=4096.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=53248.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=53248.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=6656.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=6656.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=8192.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=8192.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=18432-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=18432-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=2112-K=7168.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=2112-K=7168.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=2304-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=2304-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=24576-K=1536.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=24576-K=1536.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=26624-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=26624-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=28672-K=4096.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=28672-K=4096.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=3072-K=1536.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=3072-K=1536.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=32768-K=512.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=32768-K=512.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=36864-K=7168.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=36864-K=7168.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=14336.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=14336.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=7168.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=7168.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=7168.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=7168.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=512-K=128.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=128.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=512-K=7168.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=7168.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=53248-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=53248-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=57344-K=8192.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=57344-K=8192.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2048.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2048.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2304.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2304.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=256.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=256.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=28672.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=28672.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=8192.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=8192.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4-N=9216-K=16384.json => gfx950/triton/gemm/GEMM-AFP4WFP4-N=9216-K=16384.json} (100%) rename aiter/ops/triton/configs/{gemm/gfx950-GEMM-AFP4WFP4.json => gfx950/triton/gemm/GEMM-AFP4WFP4.json} (100%) create mode 100644 aiter/ops/triton/configs/gfx950/triton/moe/.gitkeep diff --git a/aiter/ops/triton/_triton_kernels/gemm/basic/gemm_afp4wfp4.py b/aiter/ops/triton/_triton_kernels/gemm/basic/gemm_afp4wfp4.py index 4fd55f2d026..0e29f25d8ac 100644 --- a/aiter/ops/triton/_triton_kernels/gemm/basic/gemm_afp4wfp4.py +++ b/aiter/ops/triton/_triton_kernels/gemm/basic/gemm_afp4wfp4.py @@ -724,7 +724,7 @@ def _get_config( # Note: Config files use K=2*K in their naming K = 2 * K if shuffle: - cfg, is_tuned = get_gemm_config( + return get_gemm_config( "GEMM-AFP4WFP4_PRESHUFFLED", M, N, @@ -732,12 +732,4 @@ def _get_config( bounds=(4, 8, 16, 31, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192), ) else: - cfg, is_tuned = get_gemm_config("GEMM-AFP4WFP4", M, N, K) - if cfg.get("NUM_KSPLIT", None) is None: - cfg["NUM_KSPLIT"] = 1 - cfg.setdefault("GROUP_SIZE_M", 8) - cfg.setdefault("num_stages", 0) - cfg.setdefault("waves_per_eu", 0) - cfg.setdefault("matrix_instr_nonkdim", 16) - cfg.setdefault("cache_modifier", ".cg") - return cfg, is_tuned + return get_gemm_config("GEMM-AFP4WFP4", M, N, K) diff --git a/aiter/ops/triton/configs/gemm/gfx1250-GEMM-AFP4WFP4.json b/aiter/ops/triton/configs/gfx1250/gluon/gemm/GEMM-AFP4WFP4.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx1250-GEMM-AFP4WFP4.json rename to aiter/ops/triton/configs/gfx1250/gluon/gemm/GEMM-AFP4WFP4.json diff --git a/aiter/ops/triton/configs/gfx1250/gluon/moe/.gitkeep b/aiter/ops/triton/configs/gfx1250/gluon/moe/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aiter/ops/triton/configs/gfx1250/triton/gemm/.gitkeep b/aiter/ops/triton/configs/gfx1250/triton/gemm/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aiter/ops/triton/configs/gfx1250/triton/moe/.gitkeep b/aiter/ops/triton/configs/gfx1250/triton/moe/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aiter/ops/triton/configs/gemm/gluon/gfx950-GEMM-AFP4WFP4.json b/aiter/ops/triton/configs/gfx950/gluon/gemm/GEMM-AFP4WFP4.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gluon/gfx950-GEMM-AFP4WFP4.json rename to aiter/ops/triton/configs/gfx950/gluon/gemm/GEMM-AFP4WFP4.json diff --git a/aiter/ops/triton/configs/gfx950/gluon/moe/.gitkeep b/aiter/ops/triton/configs/gfx950/gluon/moe/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=106496-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=106496-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=106496-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=106496-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=1280-K=8192.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=1280-K=8192.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=1280-K=8192.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=1280-K=8192.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=13312-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=13312-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=13312-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=13312-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=13312.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=13312.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=13312.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=13312.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=2048.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=2048.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=2048.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=2048.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=26624.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=26624.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=26624.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=26624.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=4096.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=4096.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=4096.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=4096.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=53248.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=53248.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=53248.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=53248.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=6656.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=6656.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=6656.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=6656.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=8192.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=8192.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=16384-K=8192.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=16384-K=8192.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=18432-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=18432-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=18432-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=18432-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=2112-K=7168.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=2112-K=7168.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=2112-K=7168.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=2112-K=7168.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=2304-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=2304-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=2304-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=2304-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=24576-K=1536.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=24576-K=1536.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=24576-K=1536.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=24576-K=1536.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=26624-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=26624-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=26624-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=26624-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=28672-K=4096.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=28672-K=4096.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=28672-K=4096.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=28672-K=4096.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=3072-K=1536.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=3072-K=1536.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=3072-K=1536.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=3072-K=1536.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=32768-K=512.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=32768-K=512.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=32768-K=512.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=32768-K=512.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=36864-K=7168.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=36864-K=7168.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=36864-K=7168.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=36864-K=7168.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=14336.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=14336.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=14336.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=14336.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=7168.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=7168.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4096-K=7168.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4096-K=7168.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=7168.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=7168.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=4608-K=7168.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=4608-K=7168.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=512-K=128.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=128.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=512-K=128.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=128.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=512-K=7168.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=7168.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=512-K=7168.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=512-K=7168.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=53248-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=53248-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=53248-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=53248-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=57344-K=8192.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=57344-K=8192.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=57344-K=8192.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=57344-K=8192.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2048.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2048.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2048.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2048.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2304.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2304.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=2304.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=2304.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=256.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=256.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=7168-K=256.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=7168-K=256.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=28672.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=28672.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=28672.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=28672.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=8192.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=8192.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=8192-K=8192.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=8192.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=9216-K=16384.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=9216-K=16384.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4-N=9216-K=16384.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=9216-K=16384.json diff --git a/aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4.json b/aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4.json similarity index 100% rename from aiter/ops/triton/configs/gemm/gfx950-GEMM-AFP4WFP4.json rename to aiter/ops/triton/configs/gfx950/triton/gemm/GEMM-AFP4WFP4.json diff --git a/aiter/ops/triton/configs/gfx950/triton/moe/.gitkeep b/aiter/ops/triton/configs/gfx950/triton/moe/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aiter/ops/triton/gemm/basic/gemm_afp4wfp4.py b/aiter/ops/triton/gemm/basic/gemm_afp4wfp4.py index 2d4379ae241..37790ee1f64 100644 --- a/aiter/ops/triton/gemm/basic/gemm_afp4wfp4.py +++ b/aiter/ops/triton/gemm/basic/gemm_afp4wfp4.py @@ -445,7 +445,7 @@ def gemm_afp4wfp4_preshuffle( """ assert arch_info.is_fp4_avail(), "MXFP4 is not available on your device" - use_gluon = False # arch_info.get_arch() == "gfx1250" TODO: (Satya) revert after upstream triton is fixed + use_gluon = arch_info.get_arch() == "gfx1250" M, K_bytes = x_fp4.shape n16, _ = w_preshuf.shape diff --git a/aiter/ops/triton/gluon/gemm_afp4wfp4.py b/aiter/ops/triton/gluon/gemm_afp4wfp4.py index 036a24def7f..c6d833e9758 100644 --- a/aiter/ops/triton/gluon/gemm_afp4wfp4.py +++ b/aiter/ops/triton/gluon/gemm_afp4wfp4.py @@ -483,7 +483,7 @@ def _get_config( dev = arch_info.get_arch() if dev not in ["gfx950", "gfx1250"]: raise ValueError("Gluon implementation is not supported on this device.") - fpath = f"{AITER_TRITON_CONFIGS_PATH}/gemm/gluon/{dev}-GEMM-AFP4WFP4.json" + fpath = f"{AITER_TRITON_CONFIGS_PATH}/{dev}/gluon/gemm/GEMM-AFP4WFP4.json" with open(fpath, "r") as file: config = json.load(file) _get_config._config_dict = config diff --git a/aiter/ops/triton/utils/gemm_config_utils.py b/aiter/ops/triton/utils/gemm_config_utils.py index 2a4435e10b7..b2632143c3c 100644 --- a/aiter/ops/triton/utils/gemm_config_utils.py +++ b/aiter/ops/triton/utils/gemm_config_utils.py @@ -59,11 +59,14 @@ def _get_gemm_config_cached( ``get_gemm_config()`` instead, which returns a defensive deep-copy so callers can freely mutate the returned dict without polluting the cache. - When ``backend`` is given, configs are looked up in the per-backend - subdirectory ``gemm//`` (e.g. ``gemm/gluon/``); if that backend - has no default config file, it falls back to the shared ``gemm/`` directory. - ``backend=None`` (the default) keeps the original ``gemm/`` behavior, so - existing callers are unaffected. + Configs are resolved from the ``//gemm/`` hierarchy first + (e.g. ``gfx950/triton/gemm/``), where the filename carries no arch prefix. + ``backend=None`` prefers the arch's ``triton/`` dir, then falls back to its + ``gluon/`` dir (covering arches that only ship a gluon-tuned file, e.g. + gfx1250); an explicit ``backend`` looks only in that backend's dir. For + configs not yet migrated, resolution falls back to the legacy flat layout + (``gemm/`` and ``gemm//``) with arch-prefixed filenames, so existing + callers are unaffected. See the TODO(migration) note in the body. """ # Input validation assert M >= 0, "M must be positive." @@ -81,20 +84,41 @@ def _get_gemm_config_cached( dev = arch_info.get_arch() cache_key = f"{dev}_{config_name}" + (f"_{backend}" if backend else "") - # Per-backend config directory, falling back to the shared gemm/ dir if the - # backend has no default config file (e.g. triton uses the shared dir). - base_dir = f"{AITER_TRITON_CONFIGS_PATH}/gemm" - cfg_dir = base_dir - if backend is not None: - backend_dir = f"{base_dir}/{backend}" - if os.path.exists(f"{backend_dir}/{dev}-{config_name}.json"): - cfg_dir = backend_dir + # Resolve the config directory + filename convention. + # New layout: configs///gemm/.json (the arch + # is in the path, so the filename has NO arch prefix). backend=None + # prefers the arch's triton dir, then falls back to its gluon dir (covers + # configs that only ship a gluon-tuned file, e.g. gfx1250). + # Legacy layout: configs/gemm[/]/-.json (kept + # as a fallback for configs not yet migrated; filenames keep the prefix). + # + # TODO(migration): remove the legacy-layout candidate dirs (and the + # arch-prefixed filename convention) once every GEMM config has been migrated + # to the //gemm/ hierarchy. End state = new-layout dirs only. + arch_prefix = f"{dev}-" + if backend is None: + candidate_dirs = [ + (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/triton/gemm", ""), + (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/gluon/gemm", ""), + (f"{AITER_TRITON_CONFIGS_PATH}/gemm", arch_prefix), # TODO(migration): legacy, remove + ] + else: + candidate_dirs = [ + (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/{backend}/gemm", ""), + (f"{AITER_TRITON_CONFIGS_PATH}/gemm/{backend}", arch_prefix), # TODO(migration): legacy, remove + (f"{AITER_TRITON_CONFIGS_PATH}/gemm", arch_prefix), # TODO(migration): legacy, remove + ] + cfg_dir, name_prefix = candidate_dirs[-1] + for _dir, _prefix in candidate_dirs: + if os.path.exists(f"{_dir}/{_prefix}{config_name}.json"): + cfg_dir, name_prefix = _dir, _prefix + break if cache_key not in _get_gemm_config_cached._config_cache: _get_gemm_config_cached._config_cache[cache_key] = {} # Load default config (must exist) - fpath = f"{cfg_dir}/{dev}-{config_name}.json" + fpath = f"{cfg_dir}/{name_prefix}{config_name}.json" _load_config_file( _get_gemm_config_cached._config_cache, cache_key, @@ -109,7 +133,7 @@ def _get_gemm_config_cached( if specialized_filename is not None: spec_key = specialized_filename if spec_key not in _get_gemm_config_cached._config_cache[cache_key]: - fpath = f"{cfg_dir}/{dev}-{config_name}-{specialized_filename}.json" + fpath = f"{cfg_dir}/{name_prefix}{config_name}-{specialized_filename}.json" if _load_config_file( _get_gemm_config_cached._config_cache, cache_key, fpath, spec_key ): @@ -122,7 +146,7 @@ def _get_gemm_config_cached( if B is not None: bnk_key = f"{B}_{N}_{K}" if bnk_key not in _get_gemm_config_cached._config_cache[cache_key]: - fpath = f"{cfg_dir}/{dev}-{config_name}-B={B}-N={N}-K={K}.json" + fpath = f"{cfg_dir}/{name_prefix}{config_name}-B={B}-N={N}-K={K}.json" if _load_config_file( _get_gemm_config_cached._config_cache, cache_key, @@ -137,7 +161,7 @@ def _get_gemm_config_cached( if config_dict_key == "default": nk_key = f"{N}_{K}" if nk_key not in _get_gemm_config_cached._config_cache[cache_key]: - fpath = f"{cfg_dir}/{dev}-{config_name}-N={N}-K={K}.json" + fpath = f"{cfg_dir}/{name_prefix}{config_name}-N={N}-K={K}.json" if _load_config_file( _get_gemm_config_cached._config_cache, cache_key, From bf85ab244d0c65ac7701affde6248b2713232315 Mon Sep 17 00:00:00 2001 From: Satya Nikhil Date: Fri, 24 Jul 2026 20:29:11 +0000 Subject: [PATCH 2/4] format --- aiter/ops/triton/utils/gemm_config_utils.py | 40 ++++++++++----------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/aiter/ops/triton/utils/gemm_config_utils.py b/aiter/ops/triton/utils/gemm_config_utils.py index b2632143c3c..e3059e5a9c6 100644 --- a/aiter/ops/triton/utils/gemm_config_utils.py +++ b/aiter/ops/triton/utils/gemm_config_utils.py @@ -59,14 +59,9 @@ def _get_gemm_config_cached( ``get_gemm_config()`` instead, which returns a defensive deep-copy so callers can freely mutate the returned dict without polluting the cache. - Configs are resolved from the ``//gemm/`` hierarchy first - (e.g. ``gfx950/triton/gemm/``), where the filename carries no arch prefix. - ``backend=None`` prefers the arch's ``triton/`` dir, then falls back to its - ``gluon/`` dir (covering arches that only ship a gluon-tuned file, e.g. - gfx1250); an explicit ``backend`` looks only in that backend's dir. For - configs not yet migrated, resolution falls back to the legacy flat layout - (``gemm/`` and ``gemm//``) with arch-prefixed filenames, so existing - callers are unaffected. See the TODO(migration) note in the body. + Resolves from ``//gemm/`` (prefix-less filenames) first; + ``backend=None`` tries triton then gluon. Falls back to the legacy flat + ``gemm/`` layout (arch-prefixed) for unmigrated configs. """ # Input validation assert M >= 0, "M must be positive." @@ -84,29 +79,30 @@ def _get_gemm_config_cached( dev = arch_info.get_arch() cache_key = f"{dev}_{config_name}" + (f"_{backend}" if backend else "") - # Resolve the config directory + filename convention. - # New layout: configs///gemm/.json (the arch - # is in the path, so the filename has NO arch prefix). backend=None - # prefers the arch's triton dir, then falls back to its gluon dir (covers - # configs that only ship a gluon-tuned file, e.g. gfx1250). - # Legacy layout: configs/gemm[/]/-.json (kept - # as a fallback for configs not yet migrated; filenames keep the prefix). - # - # TODO(migration): remove the legacy-layout candidate dirs (and the - # arch-prefixed filename convention) once every GEMM config has been migrated - # to the //gemm/ hierarchy. End state = new-layout dirs only. + # New layout //gemm/ (no prefix) first, then legacy flat + # gemm/ (arch-prefixed) for unmigrated configs. + # TODO(satya): drop the legacy dirs once all configs are migrated. arch_prefix = f"{dev}-" if backend is None: candidate_dirs = [ (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/triton/gemm", ""), (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/gluon/gemm", ""), - (f"{AITER_TRITON_CONFIGS_PATH}/gemm", arch_prefix), # TODO(migration): legacy, remove + ( + f"{AITER_TRITON_CONFIGS_PATH}/gemm", + arch_prefix, + ), # TODO(satya): legacy, remove ] else: candidate_dirs = [ (f"{AITER_TRITON_CONFIGS_PATH}/{dev}/{backend}/gemm", ""), - (f"{AITER_TRITON_CONFIGS_PATH}/gemm/{backend}", arch_prefix), # TODO(migration): legacy, remove - (f"{AITER_TRITON_CONFIGS_PATH}/gemm", arch_prefix), # TODO(migration): legacy, remove + ( + f"{AITER_TRITON_CONFIGS_PATH}/gemm/{backend}", + arch_prefix, + ), # TODO(satya): legacy, remove + ( + f"{AITER_TRITON_CONFIGS_PATH}/gemm", + arch_prefix, + ), # TODO(satya): legacy, remove ] cfg_dir, name_prefix = candidate_dirs[-1] for _dir, _prefix in candidate_dirs: From 5cd936753e7c9f8c043a3091b128b9e93b33b9e9 Mon Sep 17 00:00:00 2001 From: Satya Nikhil Kodukula Date: Mon, 27 Jul 2026 11:52:32 -0400 Subject: [PATCH 3/4] docs(configs): add Claude Code guide for GEMM/MOE config layout --- aiter/ops/triton/configs/CLAUDE.md | 365 +++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 aiter/ops/triton/configs/CLAUDE.md diff --git a/aiter/ops/triton/configs/CLAUDE.md b/aiter/ops/triton/configs/CLAUDE.md new file mode 100644 index 00000000000..ce2c9ba4fd2 --- /dev/null +++ b/aiter/ops/triton/configs/CLAUDE.md @@ -0,0 +1,365 @@ +# Triton kernel configs — rules for automated edits + +Scope: **GEMM and MOE configs only.** Read this before adding, moving, +renaming, or tuning a GEMM or MOE JSON file under +`aiter/ops/triton/configs/`, or before touching +`utils/gemm_config_utils.py` or `utils/moe_config_utils.py`. + +Out of scope, do not touch without an explicit request: `configs/conv/`, +`configs/hstu_attn/`, and the flat attention / GMM / MHC / MLA files at the top +of `configs/`. They have their own loaders and are unaffected by anything here. + +The tree is **mid-migration** from a flat, arch-prefixed layout to a nested +`///` layout. Both layouts are live, but **the legacy flat +layout is deprecated and will be removed** — treat it as read-only history, not +as a place to add things. + +Two non-negotiables: + +1. **Tuning values live in JSON, never in Python.** No `setdefault`, no inline + dict literals, no arch-conditional constants, no hardcoded fallback configs. + If a value is missing, fix the JSON. +2. **New configs go in the target layout** unless their family is still in the + legacy directory (see §6). + +`GEMM-AFP4WFP4` (gfx950 triton, gfx950/gfx1250 gluon) is the only migrated +family and the worked reference — copy its shape when in doubt. + +--- + +## 1. Layouts + +### Target layout (use for all new configs) + +``` +configs////[-].json +``` + +| Segment | Values | +| ----------- | --------------------------------------------------------- | +| `` | `gfx942`, `gfx950`, `gfx1250`, `gfx1151`, `gfx1200`, `gfx1201` | +| `` | `triton` or `gluon` | +| `` | `gemm` or `moe` | +| filename | **no arch prefix** — the arch is the directory | + +``` +configs/gfx950/triton/gemm/GEMM-AFP4WFP4.json +configs/gfx950/triton/gemm/GEMM-AFP4WFP4-N=8192-K=8192.json +configs/gfx950/gluon/gemm/GEMM-AFP4WFP4.json +configs/gfx1250/gluon/gemm/GEMM-AFP4WFP4.json +``` + +The `//moe/` directories exist but are empty, held open with +`.gitkeep`. Keep them. **No MOE config has been migrated and no MOE resolver +understands the nested layout yet** — see §5. + +### Legacy layout — deprecated, pending removal + +``` +configs/gemm/-[-].json +configs/gemm/gluon/-[-].json +configs/moe/-MOE-.json +configs/moe/-A8W4.json +configs/moe/-MOE_ROUTING_SIGMOID_TOPK1.json +``` + +Regenerate rather than trusting this listing: +`git ls-tree -r --name-only HEAD aiter/ops/triton/configs/` + +Still authoritative for every family not yet migrated. For GEMM it is reached +through the fallback chain in §2; for MOE it is the *only* path that works. +The GEMM fallback is temporary — anything left in `configs/gemm/` when the +legacy candidates are dropped from `gemm_config_utils.py` will stop resolving. + +--- + +## 2. GEMM resolution order — `get_gemm_config()` + +`utils/gemm_config_utils.py` picks a directory by probing for the *default* +config file (`.json`) in order and taking the first hit. +Specialized files are then read from that same directory. + +**`backend=None`** (what every caller uses today): + +1. `configs//triton/gemm/.json` +2. `configs//gluon/gemm/.json` +3. `configs/gemm/-.json` *(legacy)* + +**`backend="triton"|"gluon"`**: + +1. `configs///gemm/.json` +2. `configs/gemm//-.json` *(legacy)* +3. `configs/gemm/-.json` *(legacy)* + +If nothing matches, the last legacy candidate is used and the missing-default +assertion fires there — so error messages still point at `configs/gemm/`. + +The legacy candidates are marked `# TODO(satya): legacy, remove` and are +scheduled for deletion. Do not write new code that depends on them resolving. + +Consequences to keep in mind: + +- **A directory is chosen as a unit.** Splitting a config family across + `/triton/gemm/` and legacy `configs/gemm/` silently drops the + specialized files in whichever directory loses the probe. Move a family + wholesale or not at all. +- **`backend=None` prefers `triton` over `gluon`.** On an arch with only a + gluon default (currently gfx1250 `GEMM-AFP4WFP4`), lookup falls through to + gluon. Adding `configs/gfx1250/triton/gemm/GEMM-AFP4WFP4.json` later would + change which file gfx1250 resolves to — verify that is intended. +- Results are cached per `(arch, config_name, backend)` via + `functools.lru_cache` plus `_config_cache`. Adding a file at runtime after a + lookup has happened has no effect; restart the process. + +Direct-path loaders bypass all of this. Grep for +`f"{AITER_TRITON_CONFIGS_PATH}/..."` before moving anything — `gluon/gemm_a8w8.py` +and `gluon/gemm_a8w8_blockscale.py` still build legacy `gemm/gluon/` paths by +hand and must be edited when their configs move. `gluon/gemm_afp4wfp4.py` was +already updated to the nested path. + +--- + +## 3. GEMM config file contents + +Required top-level shape: + +```json +{ + "M_LEQ_64": { "...": "..." }, + "M_GEQ_4096": { "...": "..." }, + "any": { "...": "..." } +} +``` + +- `M_LEQ_x` is searched ascending over `STANDARD_M_BOUNDS = + (4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192)`, then `M_GEQ_x` + descending, then `any`. A caller may override with `bounds=(...)`, which must + be strictly increasing positive ints. +- `any` must exist unless every reachable `M` is covered by an explicit bound. +- The deprecated `{"large": ..., "small": ...}` shape must not be introduced. +- A `KeyError` at lookup time means no bound matched — usually a missing `any`. + +Each `M_*` entry carries at minimum: + +``` +BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, GROUP_SIZE_M, +num_warps, num_stages, waves_per_eu, matrix_instr_nonkdim, +cache_modifier, NUM_KSPLIT +``` + +`add_default_gemm_config_params()` backfills `NUM_KSPLIT=1` and +`cache_modifier=None` as a last resort, and `compute_splitk_params()` derives +`SPLITK_BLOCK_SIZE` and may clamp `BLOCK_SIZE_K` / `NUM_KSPLIT`. Neither is a +license to omit keys. + +`get_gemm_config()` returns `(config, is_tuned)`. `is_tuned` is `True` only when +a specialized (N/K-, B-, or `specialized_filename`-keyed) file was hit, `False` +for the default file or `any`. Do not discard it. + +### The JSON is the only place tuning values live + +A `_get_config()` should do nothing but call `get_gemm_config()` and return: + +```python +def _get_config(M: int, N: int, K: int): + return get_gemm_config("GEMM-AFP4WFP4", M, N, K) +``` + +`_triton_kernels/gemm/basic/gemm_afp4wfp4.py` carried a block of `setdefault` +calls and it was deleted — it masked incomplete config files with values nobody +had tuned, and made the effective config un-inspectable from the JSON. + +--- + +## 4. GEMM naming + +| Kind | Target layout | Legacy layout | +| ----------------- | ----------------------------------------------- | ---------------------------------------------------- | +| Default | `GEMM-A16W16.json` | `gfx950-GEMM-A16W16.json` | +| N/K specialized | `GEMM-A16W16-N=256-K=7168.json` | `gfx950-GEMM-A16W16-N=256-K=7168.json` | +| Batched (B, N, K) | `BATCHED_GEMM-A16W16-B=4-N=1024-K=4096.json` | `gfx1250-BATCHED_GEMM-A16W16-B=4-N=1024-K=4096.json` | +| Custom suffix | `FUSED-GEMM-AFP4WFP4-A16W16-N4=512-N16=256-K=7168.json` | same, arch-prefixed | + +Config-name patterns: `GEMM-A{x}W{y}`, `BATCHED_GEMM-A{x}W{y}`, +`GEMM_PREQUANT-...`, `FUSED-GEMM-{op}`, `FF-A{x}W{y}-fused`; variant suffixes +`_PRESHUFFLED`, `_BLOCKSCALE`. + +**`K` in AFP4WFP4 filenames is the logical K, i.e. `2 * K_bytes`.** The kernel +does `K = 2 * K` before calling `get_gemm_config`. Tuning output that names +files by the packed byte width will never be found. + +--- + +## 5. MOE configs + +MOE does **not** go through `get_gemm_config()`. There is no probe order, no +nested-layout support, and no `is_tuned` signal. Three independent loaders read +`configs/moe/` directly, each with its own schema: + +| Loader | File | Schema | +| ------ | ---- | ------ | +| `utils/moe_config_utils.py::get_moe_configs` | `moe/-MOE-.json` | `small_M` / `medium_M` / `large_M` | +| `moe/moe_op_gemm_a8w4.py::_get_a8w4_dispatch` | `moe/-A8W4.json` | `bm_n_k` | +| `_triton_kernels/moe/moe_routing_sigmoid_top1_fused.py` | `moe/-MOE_ROUTING_SIGMOID_TOPK1.json` | `N16` → `small` / `medium` / … | + +`` comes from `get_config_dtype_str()`: `DEFAULT`, `FP8_W8A8`, +`INT8_W8A16`, `INT8_W8A8`, `INT4_W4A16`, `MX_FP4`. + +`small_M` / `medium_M` / `large_M` split on `M_THRESHOLD_SMALL = 256` and +`M_THRESHOLD_MEDIUM = 1024`, both module constants in `moe_config_utils.py`. +This is **not** the GEMM `M_LEQ_x` / `M_GEQ_y` scheme — do not mix them. + +### MOE is the main offender for tuning values in Python + +Fix these as you touch them; do not add more: + +- `get_optimal_moe_config()` returns a hardcoded dict (`BLOCK_SIZE_M: 256`, + `BLOCK_SIZE_N: 256`, …) when no config file exists, behind a + `warnings.warn`. A missing config silently runs untuned values. +- `moe_op_gemm_a8w4.py` has a three-tier Python fallback: exact + `bm_n_k` hit → any-`block_m` proxy with matching `(N, K)` → a gfx942-gated + shape heuristic → a conservative default. Only the first tier reads tuned + numbers from JSON. + +### Planned: `get_moe_config()` — design, not yet implemented + +**Status: design only. Nothing below exists in the tree yet.** Do not assume +`get_moe_config` is importable; do not move MOE JSON files in anticipation of +it. If you are asked to implement it, follow this shape. + +The unification is **path resolution only.** The three MOE schemas stay as they +are — the loader finds and parses the file, each caller keeps interpreting its +own structure. Converging the schemas is a separate, later decision (it would +touch every MOE config file and require re-validating dispatch on every arch). + +**Step 1 — extract the shared probe.** The candidate-directory logic currently +inlined in `_get_gemm_config_cached()` becomes a helper in `utils/`, parameterised +on ``: + +```python +def resolve_config_dir(op: str, config_name: str, backend: str | None = None, + legacy_dir: str | None = None) -> tuple[str, str]: + """Return (cfg_dir, name_prefix) for the first candidate that has + /.json. Falls back to the last + candidate so the missing-file assertion names a legacy path.""" +``` + +Candidates for `op="moe"`, mirroring §2: + +- `backend=None` → `/triton/moe/`, `/gluon/moe/`, then legacy + `configs/moe/` with the `-` prefix +- `backend=...` → `//moe/`, then legacy `configs/moe/` prefixed + +`gemm_config_utils.py` is then refactored onto the same helper with `op="gemm"` +and `legacy_dir="gemm"` — behaviour-identical, and the legacy candidates stay +tagged `# TODO(satya): legacy, remove` so both ops retire together. + +**Step 2 — the loader.** + +```python +def get_moe_config(config_name: str, backend: str | None = None) -> dict | None: + """Load a MOE config by name. Returns the parsed JSON (a deep copy, safe to + mutate), or None if no file exists for this arch. Schema interpretation is + the caller's job — MOE files are not uniform.""" +``` + +- `config_name` is the unprefixed stem: `MOE-FP8_W8A8`, `MOE-DEFAULT`, `A8W4`, + `MOE_ROUTING_SIGMOID_TOPK1`. +- Cache with `functools.lru_cache` and deep-copy on return, exactly as + `get_gemm_config()` does — callers mutate configs. +- Return `None` rather than raising: unlike GEMM, a missing MOE config is + currently normal (only gfx942 and gfx950 ship any). +- No `is_tuned` flag. MOE has no default-vs-specialized distinction to report. + +**Step 3 — port the three loaders** onto it, one PR each, without changing +schemas or file locations: + +| Loader | Call becomes | +| ------ | ------------ | +| `moe_config_utils.py::get_moe_configs` | `get_moe_config(f"MOE-{dtype_str}")` | +| `moe_op_gemm_a8w4.py::_get_a8w4_dispatch` | `get_moe_config("A8W4") or {}` | +| `moe_routing_sigmoid_top1_fused.py` | `get_moe_config("MOE_ROUTING_SIGMOID_TOPK1")` | + +**Step 4 — delete the hardcoded Python fallbacks.** Blocked on shipping a +`-MOE-DEFAULT.json` for the arches that have none — today only gfx942 and +gfx950 have MOE configs at all, so gfx1250/gfx1151/gfx1200/gfx1201 hit the +hardcoded dict in `get_optimal_moe_config()`. Once every supported arch has a +file, that dict and the `warnings.warn` come out. + +**Step 5 — only now `git mv`** MOE files into `//moe/` and drop +the arch prefix, following §6. + +Moving MOE JSON before step 2 lands would silently break resolution, and the +Python fallbacks would swallow the breakage instead of surfacing it. + +--- + +## 6. Migration playbook (GEMM) + +One family = one `` × `` × ``, including every +specialized file. `GEMM-AFP4WFP4` is the worked example — diff it against its +legacy form if a step is ambiguous. + +1. **Scope it.** `ls configs/gemm/-*.json` — the default plus + every specialized file. All of them move together. +2. **Find every reader.** Grep for the config name and for + `AITER_TRITON_CONFIGS_PATH` in `aiter/ops/triton/`. Hand-built paths must be + rewritten; `get_gemm_config()` callers need no change. +3. **Move with `git mv`** so the change reviews as a rename, and strip the + `-` prefix: + ``` + git mv configs/gemm/gfx950-GEMM-FOO-N=1-K=2.json \ + configs/gfx950/triton/gemm/GEMM-FOO-N=1-K=2.json + ``` + Create `//{gemm,moe}/` with a `.gitkeep` if absent. +4. **Do not edit contents in the same commit.** Keep renames at 100% similarity; + content changes go in a follow-up commit. +5. **Update CI test selection.** `.github/scripts/select_triton_tests.py` + `resolve_gemm_config_names()` globs `configs/gemm/{dev}-{name}*.json` only — + it does **not** see the nested layout, so moved configs stop triggering their + tests. Teach it the new layout in the same PR, or note the gap explicitly. +6. **Update docs.** `aiter/ops/triton/README.md` ("How config selection works", + "Config file naming convention") and + `aiter/ops/triton/utils/_triton/tunning/README.md` (the step that says + `cp *.json .../configs/gemm/`) both still describe only the legacy layout. +7. **Pull any tuning values still hardcoded in Python into the JSON.** A + migrated family must be fully described by its config files. +8. **Verify** on the target arch: config resolves, `is_tuned` is `True` for a + shape that has a specialized file, and numerics are unchanged. +9. Leave the `# TODO(satya): legacy, remove` markers in `gemm_config_utils.py` + until `configs/gemm/` is empty. Deleting the legacy fallback is the final + step of the migration, not an intermediate one. + +### Adding a *new* tuned config (no migration) + +- **GEMM**, new arch/backend combination or a family already migrated → + target layout, unprefixed filename. +- **GEMM**, family still in `configs/gemm/` → add to `configs/gemm/` with the + arch prefix, and consider migrating the whole family in the same PR. Never + create a lone nested file for a family whose default lives in legacy; the + directory probe picks one directory and ignores the other. +- **MOE** → `configs/moe/` with the arch prefix, matching the schema of the + loader that will read it. The nested layout is not wired up for MOE. + +### Do not + +- Rename or delete `.gitkeep` placeholder directories. +- Put an arch prefix on a file inside `/...`. +- Put tuning values in `.py` files — no `setdefault`, no inline dicts, no + arch-conditional constants, no hardcoded fallback configs. +- Mix the GEMM `M_LEQ_x`/`M_GEQ_y` scheme with the MOE + `small_M`/`medium_M`/`large_M` scheme. +- Move MOE configs into `//moe/` before a resolver exists. + +### Not tuning configs + +Two AOT code paths build directories under this tree at runtime that are **not +checked in and out of scope**: + +- `configs/gemm/aot/_M=…-N=…-K=…` — `gemm/fused/fused_gemm_afp4wfp4_a16w16.py`, + `gemm/fused/fused_gemm_afp4wfp4_mul_add.py` +- `configs/paged_mqa_logits/aot/` — `attention/pa_mqa_logits.py` + +Both are guarded by `use_aot and os.path.exists(...)` and hold compiled-kernel +metadata, not tuning parameters. Do not create, migrate, or document them as +config directories. From 97f7da700622b1d915d7257966c0dee4e1314d3b Mon Sep 17 00:00:00 2001 From: Satya Nikhil Kodukula Date: Mon, 27 Jul 2026 11:56:55 -0400 Subject: [PATCH 4/4] Update CLAUDE.md --- aiter/ops/triton/configs/CLAUDE.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/aiter/ops/triton/configs/CLAUDE.md b/aiter/ops/triton/configs/CLAUDE.md index ce2c9ba4fd2..9c4d19ec728 100644 --- a/aiter/ops/triton/configs/CLAUDE.md +++ b/aiter/ops/triton/configs/CLAUDE.md @@ -314,19 +314,15 @@ legacy form if a step is ambiguous. Create `//{gemm,moe}/` with a `.gitkeep` if absent. 4. **Do not edit contents in the same commit.** Keep renames at 100% similarity; content changes go in a follow-up commit. -5. **Update CI test selection.** `.github/scripts/select_triton_tests.py` - `resolve_gemm_config_names()` globs `configs/gemm/{dev}-{name}*.json` only — - it does **not** see the nested layout, so moved configs stop triggering their - tests. Teach it the new layout in the same PR, or note the gap explicitly. -6. **Update docs.** `aiter/ops/triton/README.md` ("How config selection works", +5. **Update docs.** `aiter/ops/triton/README.md` ("How config selection works", "Config file naming convention") and `aiter/ops/triton/utils/_triton/tunning/README.md` (the step that says `cp *.json .../configs/gemm/`) both still describe only the legacy layout. -7. **Pull any tuning values still hardcoded in Python into the JSON.** A +6. **Pull any tuning values still hardcoded in Python into the JSON.** A migrated family must be fully described by its config files. -8. **Verify** on the target arch: config resolves, `is_tuned` is `True` for a +7. **Verify** on the target arch: config resolves, `is_tuned` is `True` for a shape that has a specialized file, and numerics are unchanged. -9. Leave the `# TODO(satya): legacy, remove` markers in `gemm_config_utils.py` +8. Leave the `# TODO(satya): legacy, remove` markers in `gemm_config_utils.py` until `configs/gemm/` is empty. Deleting the legacy fallback is the final step of the migration, not an intermediate one.