diff --git a/CMakeLists.txt b/CMakeLists.txt index e9dbcf96e7..e5608ee153 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,12 @@ add_subdirectory(tools) include(CTest) if(BUILD_TESTING) enable_testing() + find_package(Python3 COMPONENTS Interpreter REQUIRED) + add_test( + NAME pto_isa_v0571_contract_check + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/check_v057_pto_manifest.py + --ptoas-root ${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(tools/ptobc/tests) endif() diff --git a/docs/PTO_IR_manual.md b/docs/PTO_IR_manual.md index 0feb014f7f..2db9461552 100644 --- a/docs/PTO_IR_manual.md +++ b/docs/PTO_IR_manual.md @@ -793,47 +793,54 @@ pto.tload ins(%pv : !pto.partition_tensor_view<16x16xf16>) --- -##### `pto.tprefetch` - Prefetch Partition View into Tile +##### `pto.tprefetch` - Prefetch a Global Byte Range -**Summary:** Prefetches a GM-backed partition view into a temporary local tile buffer. This maps to PTO-ISA `TPREFETCH(dst, src)` and, unlike most PTO intrinsics, does not add implicit wait-event synchronization in the C++ wrapper. +**Summary:** Probes and prefetches a global byte range with TLOAD-equivalent +translation, permission, fault, restart, coherence, and ordering behavior. PTO +ISA 0.57.1 deliberately makes this operation destination-free: it changes +target cache or data-movement state without publishing a tile result. **Semantics:** ``` -TPREFETCH(dst, src) +TPREFETCH(address, byte_count) ``` -The detailed caching / hint behavior is target-defined by PTO-ISA. In PTOAS the -op is modeled as writing the prefetched data into `dst`. +The detailed cache placement is target-defined, but access faults are +architectural rather than optional hint behavior. Unlike most generated PTO +intrinsic wrappers, `TPREFETCH` does not add implicit wait-event synchronization. **Arguments:** | Name | Type | Description | |------|------|-------------| -| `src` | `pto.partition_tensor_view` or lowered GM memref | Source global view | -| `dst` | `pto.tile_buf` or lowered local memref | Destination local tile | +| `address` | `i64` | Base address of the global byte range | +| `byte_count` | `index` | Number of bytes to prefetch | -**Results:** None. Writes into `dst` via DPS pattern. +**Results:** None. The operation has no destination tile and publishes no result +queue value. **Constraints & Verification:** -- `src` must be a partition view before lowering, or the corresponding lowered ranked memref form after `PTOViewToMemref`. -- `dst` must be a tile buffer before lowering, or the corresponding lowered ranked memref form after `PTOViewToMemref`. -- `dst` must use `loc=vec` or `loc=mat`. -- Static source extents and static destination valid extents must be positive when known. -- `src` and `dst` element types must have the same element size in bytes. +- `address` must have exactly type `i64`. +- `byte_count` is an `index` operand and denotes bytes, not elements. +- `byte_count` must be in the inclusive range `0..262144`. PTOAS rejects an + out-of-range compile-time constant; a dynamic value remains legal in the IR + and must satisfy the same range when executed. +- A partition view or tile destination is not a compatibility spelling and is + rejected by the PTOAS parser/verifier. **Hardware Mapping:** -- Executes on the **DMA pipeline** (`PIPE_MTE2`, GM -> local tile) +- Executes on the **DMA pipeline** (`PIPE_MTE2`). +- EmitC lowers exactly to `TPREFETCH(address, byte_count)`. **Basic Example:** ```mlir -pto.tprefetch ins(%pv : !pto.partition_tensor_view<16x16xf16>) - outs(%tb : !pto.tile_buf) +%address = arith.constant 4096 : i64 +%bytes = arith.constant 512 : index +pto.tprefetch ins(%address, %bytes : i64, index) ``` --- @@ -3163,239 +3170,6 @@ pto.trems ins(%a, %s, %tmp : !pto.tile_buf, , : , , ) - outs( : ) -``` - -**Constraints & Verification:** - -- The implementation uses `dst valid row` / `dst valid column` as the iteration domain. - -**Hardware Mapping:** - -- Executes on the **Vector pipeline** (`PIPE_V`) -- Operates on data in the **VEC (UB)** memory space - -**Basic Example:** - -```mlir -pto.taddc ins(%a, %b, %c : !pto.tile_buf, - !pto.tile_buf, - !pto.tile_buf) - outs(%d : !pto.tile_buf) -``` - ---- - -##### `pto.tsubc` - Elementwise Ternary Subtract-Add - -**Summary:** Computes `src0 - src1 + src2` element-by-element. - -**Semantics:** - -``` -For each element (i, j): - dst[i, j] = src0[i, j] - src1[i, j] + src2[i, j] -``` - -**Arguments:** - -| Name | Type | Description | -|------|------|-------------| -| `src0` | `pto.tile_buf` | First source tile buffer | -| `src1` | `pto.tile_buf` | Subtrahend tile buffer | -| `src2` | `pto.tile_buf` | Addend tile buffer | -| `dst` | `pto.tile_buf` | Destination tile buffer | - -**Results:** None. Writes into `dst` via DPS pattern. - -**Assembly Format:** - -``` -pto.tsubc ins(, , : , , ) - outs( : ) -``` - -**Constraints & Verification:** - -- The implementation uses `dst valid row` / `dst valid column` as the iteration domain. - -**Hardware Mapping:** - -- Executes on the **Vector pipeline** (`PIPE_V`) -- Operates on data in the **VEC (UB)** memory space - -**Basic Example:** - -```mlir -pto.tsubc ins(%a, %b, %c : !pto.tile_buf, - !pto.tile_buf, - !pto.tile_buf) - outs(%d : !pto.tile_buf) -``` - ---- - -##### `pto.taddsc` - Fused Add-Scalar-Add - -**Summary:** Computes `src0 + scalar + src1` element-by-element. - -**Semantics:** - -``` -For each element (i, j): - dst[i, j] = src0[i, j] + scalar + src1[i, j] -``` - -**Arguments:** - -| Name | Type | Description | -|------|------|-------------| -| `src0` | `pto.tile_buf` | First source tile buffer | -| `scalar` | `ScalarType` (signless integer / float) | Scalar value | -| `src1` | `pto.tile_buf` | Second source tile buffer | -| `dst` | `pto.tile_buf` | Destination tile buffer | - -**Results:** None. Writes into `dst` via DPS pattern. - -**Assembly Format:** - -``` -pto.taddsc ins(, , : , , ) - outs( : ) -``` - -**Constraints & Verification:** - -- The implementation uses `dst valid row` / `dst valid column` as the iteration domain. - -**Hardware Mapping:** - -- Executes on the **Vector pipeline** (`PIPE_V`) -- Operates on data in the **VEC (UB)** memory space - -**Basic Example:** - -```mlir -pto.taddsc ins(%a, %s, %b : !pto.tile_buf, f32, - !pto.tile_buf) - outs(%c : !pto.tile_buf) -``` - ---- - -##### `pto.tsubsc` - Fused Subtract-Scalar-Add - -**Summary:** Computes `src0 - scalar + src1` element-by-element. - -**Semantics:** - -``` -For each element (i, j): - dst[i, j] = src0[i, j] - scalar + src1[i, j] -``` - -**Arguments:** - -| Name | Type | Description | -|------|------|-------------| -| `src0` | `pto.tile_buf` | First source tile buffer | -| `scalar` | `ScalarType` (signless integer / float) | Scalar value | -| `src1` | `pto.tile_buf` | Second source tile buffer | -| `dst` | `pto.tile_buf` | Destination tile buffer | - -**Results:** None. Writes into `dst` via DPS pattern. - -**Assembly Format:** - -``` -pto.tsubsc ins(, , : , , ) - outs( : ) -``` - -**Constraints & Verification:** - -- The implementation uses `dst valid row` / `dst valid column` as the iteration domain. - -**Hardware Mapping:** - -- Executes on the **Vector pipeline** (`PIPE_V`) -- Operates on data in the **VEC (UB)** memory space - -**Basic Example:** - -```mlir -pto.tsubsc ins(%a, %s, %b : !pto.tile_buf, f32, - !pto.tile_buf) - outs(%c : !pto.tile_buf) -``` - ---- - #### Unary Operations | Op | Semantics | @@ -3408,7 +3182,6 @@ pto.tsubsc ins(%a, %s, %b : !pto.tile_buf 0 ? src[i,j] : slope * src[i,j]` | --- @@ -3924,65 +3697,6 @@ pto.trelu ins(%a : !pto.tile_buf 0 ? src[i, j] : slope * src[i, j] -``` - -**Arguments:** - -| Name | Type | Description | -|------|------|-------------| -| `src` | `pto.tile_buf` | Source tile buffer | -| `slope` | `F32` | Negative slope coefficient | -| `dst` | `pto.tile_buf` | Destination tile buffer | - -**Results:** None. Writes into `dst` via DPS pattern. - -**Assembly Format:** - -``` -pto.tlrelu ins(, : , ) - outs( : ) -``` - -**Constraints & Verification:** - -- **Implementation checks (A2A3)** - - Tile element type must be one of: `f16`, `f32`. - - Tile must use `loc=vec`. - - Valid bounds: `0 < valid row <= rows` and `0 < valid column <= cols`. - - Runtime: `src` and `dst` tiles should have the same `validRow/validCol`. -- **Implementation checks (A5)** - - Tile element type must be one of: `f16`, `f32`. - - Tile must use `loc=vec`. - - Valid bounds: `valid row <= rows` and `valid column <= cols`. - - Runtime: `src` and `dst` tiles should have the same `validRow/validCol`. - -**Hardware Mapping:** - -- Executes on the **Vector pipeline** (`PIPE_V`) -- Operates on data in the **VEC (UB)** memory space - -**Basic Example:** - -```mlir -pto.tlrelu ins(%a, %slope : !pto.tile_buf, f32) - outs(%c : !pto.tile_buf) -``` - ---- - ### 4.6 Reduction Operations Reduce along rows or columns of a tile. All execute on the **Vector pipeline** (`PIPE_V`). @@ -6900,6 +6614,95 @@ pto.mgather ins(%mem, %idx : memref<...>, !pto.tile_buf<...>) --- +##### `pto.mgather_mask` - Masked Gather-Load from Global Memory + +**Summary:** Loads active lanes selected by `mask` from indexed global-memory +addresses into `dst`. Inactive lanes do not read their index or access memory. + +**Semantics:** + +``` +if mask[i] != 0: + dst[i] = mem[idx[i]] +else: + dst[i] = the destination tile's configured pad value +``` + +**Arguments:** + +| Name | Type | Description | +|------|------|-------------| +| `mem` | partition view or lowered memref | Global-memory base | +| `idx` | VEC tile of signless `i32` | Per-lane element indices | +| `mask` | VEC tile | Nonzero lanes are active | +| `dst` | VEC tile | Gathered values | + +**Results:** None. `dst` is the DPS destination. + +**Assembly Format:** + +``` +pto.mgather_mask ins(, , : , , ) + outs( : ) +``` + +**Constraints & Verification:** + +- Supported only for A5 targets. +- `idx`, `mask`, and `dst` must have matching valid shapes. +- `mem` element type must match `dst`; `idx` must use signless `i32`. + +**Hardware Mapping:** Executes on `PIPE_MTE2` and lowers exactly to +`MGATHER_MASK(dst, mem, idx, mask)`. + +--- + +##### `pto.mgather_cas` - Indexed Gather Compare-and-Swap + +**Summary:** Atomically compares indexed memory lanes with `expected`, writes +`replacement` on equality, and returns each lane's old memory value in `dst`. + +**Semantics:** + +``` +old = atomic_load(mem[idx[i]]) +dst[i] = old +if old == expected[i]: + atomic_store(mem[idx[i]], replacement[i]) +``` + +**Arguments:** + +| Name | Type | Description | +|------|------|-------------| +| `mem` | partition view or lowered memref | Global-memory base, read and conditionally written | +| `idx` | VEC tile of signless `i32` | Per-lane element indices | +| `expected` | VEC tile | Comparison values | +| `replacement` | VEC tile | Values stored on equality | +| `dst` | VEC tile | Old memory values | + +**Results:** None. `dst` is the DPS destination. + +**Assembly Format:** + +``` +pto.mgather_cas ins(, , , : + , , , ) + outs( : ) +``` + +**Constraints & Verification:** + +- Supported only for A5 targets. +- `idx`, `expected`, `replacement`, and `dst` must have matching valid shapes. +- `mem`, `expected`, `replacement`, and `dst` must have compatible element + types; `idx` must use signless `i32`. + +**Hardware Mapping:** Executes on `PIPE_MTE2` and lowers exactly to +`MGATHER_CAS(dst, mem, idx, expected, replacement)`. + +--- + ##### `pto.mscatter` - Scatter-Store to Global Memory **Summary:** Stores elements from a VEC tile into a global table using per-element indices. Supports optional A5-only atomic and out-of-bounds modes that lower to the corresponding `MSCATTER<...>` template overload family. @@ -6973,6 +6776,48 @@ pto.mscatter ins(%src, %idx : !pto.tile_buf<...>, !pto.tile_buf<...>) --- +##### `pto.mscatter_mask` - Masked Scatter-Store to Global Memory + +**Summary:** Stores active lanes selected by `mask` from `src` to indexed +global-memory addresses. Inactive lanes read neither `src` nor `idx` and issue +no memory access. + +**Semantics:** + +``` +if mask[i] != 0: + mem[idx[i]] = src[i] +``` + +**Arguments:** + +| Name | Type | Description | +|------|------|-------------| +| `src` | VEC tile | Values to store | +| `idx` | VEC tile of signless `i32` | Per-lane element indices | +| `mask` | VEC tile | Nonzero lanes are active | +| `mem` | partition view or lowered memref | Global-memory destination | + +**Results:** None. `mem` is the DPS destination. + +**Assembly Format:** + +``` +pto.mscatter_mask ins(, , : , , ) + outs( : ) +``` + +**Constraints & Verification:** + +- Supported only for A5 targets. +- `src`, `idx`, and `mask` must have matching valid shapes. +- `mem` element type must match `src`; `idx` must use signless `i32`. + +**Hardware Mapping:** Executes on `PIPE_MTE3` and lowers exactly to +`MSCATTER_MASK(mem, src, idx, mask)`. + +--- + ##### `pto.treshape` - Reinterpret Tile Shape/Layout **Summary:** Reinterprets a tile buffer with a new shape/layout (no data movement). @@ -7234,42 +7079,54 @@ pto.tfillpad_inplace ins(%tile : !pto.tile_buf` | Optional scratch tile for the tmp-taking DPS overload | -| `dst` | `pto.tile_buf` | Output tile storing sorted value-index pairs | +| `src` | `pto.tile_buf` | Source values | +| `dst` | `pto.tile_buf` | Sorted values | +| `dst_indices` | `pto.tile_buf<..., dtype=ui32, ...>` | Original index within each 32-element group | +| `descending` | `BoolAttr` (default: `false`) | Select descending order | -**Results:** None. Writes into `dst` via DPS pattern. +**Results:** None. `dst` and `dst_indices` are the two DPS destinations. **Assembly Format:** ``` -pto.tsort32 ins(, [, ] : , [, ]) - outs( : ) +pto.tsort ins( : ) + outs(, : , ) + {descending = true} ``` **Constraints & Verification:** -- **Implementation checks (A2/A3/A5)** - - `dst` element type must be `f16` or `f32`. - - `src` element type must match `dst` element type. - - `idx` element type must be `u32`. - - `src`, `dst`, and `idx` must all use `loc=vec` and `blayout=row_major`. +- `src` and `dst` must have the same element type and the same valid element + count. +- `dst_indices` must have element type `ui32` and the same valid element count + as `src`. +- `dst` and `dst_indices` must be distinct destinations. +- The valid element count must be a non-zero multiple of 32; partial trailing + groups are illegal in PTOAS. +- `src`, `dst`, and `dst_indices` must satisfy the vector-tile layout rules. +- Equal values and signed zeros retain source order. NaNs follow all numeric + values and retain source order for both ascending and descending sorts. +- Each group writes original indices `0..31`; indices restart at zero for the + next group. +- The old `pto.tsort32` name and the old `idx`/`tmp` input forms are not aliases + and are rejected. **Hardware Mapping:** @@ -7278,12 +7135,11 @@ pto.tsort32 ins(, [, ] : , [, ]) **Basic Example:** ```mlir -pto.tsort32 ins(%src, %idx : !pto.tile_buf<...>, !pto.tile_buf<...>) - outs(%dst : !pto.tile_buf<...>) - -# Optional scratch form: -pto.tsort32 ins(%src, %idx, %tmp : !pto.tile_buf<...>, !pto.tile_buf<...>, !pto.tile_buf<...>) - outs(%dst : !pto.tile_buf<...>) +pto.tsort ins(%src : !pto.tile_buf) + outs(%dst, %indices : + !pto.tile_buf, + !pto.tile_buf) + {descending = true} ``` --- @@ -7431,6 +7287,47 @@ pto.tcvt ins(%src {rmode = #pto, satmode = #pto : ) +``` + +**Constraints & Verification:** + +- `dst` must be an ordinary publishable tile/memref. `loc=acc` is illegal here: + ACC is the implicit source architectural state, not the explicit destination. +- The architectural operation requires a live accumulator whose valid shape + matches `dst`; conversion is selected by the operation and destination type, + not by a separately encoded ACC operand. +- Commit order is destination publication followed by ACC release. A failed or + retried operation therefore retains the live accumulator. + +**Hardware Mapping:** Executes on `PIPE_M` and lowers exactly to `ACCCVT(dst)`. + +--- + ### 4.13 Integer Sequence Generation Operations ##### `pto.tci` - Contiguous Integer Sequence diff --git a/include/PTO/IR/PTOOps.td b/include/PTO/IR/PTOOps.td index 9ef501dc73..22e5ad3f34 100644 --- a/include/PTO/IR/PTOOps.td +++ b/include/PTO/IR/PTOOps.td @@ -519,23 +519,19 @@ def TLoadOp : PTO_TOp<"tload", [ } def TPrefetchOp : PTO_TOp<"tprefetch", [ - PTO_DpsInitOpInterface, OpPipeInterface, DeclareOpInterfaceMethods ]> { - let summary = "Prefetch global data into a local tile buffer."; + let summary = "Prefetch a byte range from a global address."; let description = [{ - Prefetches data from a global partition view into a temporary local tile - buffer. Unlike most PTO intrinsics, TPREFETCH does not carry implicit - TSYNC(wait_events...) semantics in PTO-ISA. - - DPS form: - pto.tprefetch ins(%src) outs(%dst) + PTO ISA 0.57.1 models TPREFETCH as a destination-free TMA operation over + a base address and byte count. Unlike most PTO intrinsics, TPREFETCH does + not carry implicit TSYNC(wait_events...) semantics in PTO-ISA. }]; let arguments = (ins - PTODpsType:$src, - PTODpsType:$dst + I64:$address, + Index:$byte_count ); let results = (outs); @@ -543,14 +539,12 @@ def TPrefetchOp : PTO_TOp<"tprefetch", [ let hasVerifier = 1; let assemblyFormat = [{ - `ins` `(` $src `:` qualified(type($src)) `)` - `outs` `(` $dst `:` qualified(type($dst)) `)` + `ins` `(` $address `,` $byte_count `:` type($address) `,` type($byte_count) `)` attr-dict }]; let extraClassDeclaration = [{ ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_MTE2; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } }]; } @@ -2535,6 +2529,70 @@ def MGatherOp : PTO_TOp<"mgather", [ }]; } +def MGatherMaskOp : PTO_TOp<"mgather_mask", [ + PTO_DpsInitOpInterface, + OpPipeInterface, + DeclareOpInterfaceMethods +]> { + let summary = "Masked gather-load elements from memory into a tile using per-element indices."; + + let arguments = (ins + PTODpsType:$mem, + PTODpsType:$idx, + PTODpsType:$mask, + PTODpsType:$dst + ); + + let results = (outs); + let hasVerifier = 1; + + let assemblyFormat = [{ + `ins` `(` $mem `,` $idx `,` $mask `:` + type($mem) `,` type($idx) `,` qualified(type($mask)) `)` + `outs` `(` $dst `:` qualified(type($dst) ) `)` + attr-dict + }]; + + let extraClassDeclaration = [{ + static StringRef getIntrinsicName() { return "MGATHER_MASK"; } + ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_MTE2; } + ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } + }]; +} + +def MGatherCasOp : PTO_TOp<"mgather_cas", [ + PTO_DpsInitOpInterface, + OpPipeInterface, + DeclareOpInterfaceMethods +]> { + let summary = "Gather compare-and-swap elements using per-element indices."; + + let arguments = (ins + PTODpsType:$mem, + PTODpsType:$idx, + PTODpsType:$expected, + PTODpsType:$replacement, + PTODpsType:$dst + ); + + let results = (outs); + let hasVerifier = 1; + + let assemblyFormat = [{ + `ins` `(` $mem `,` $idx `,` $expected `,` $replacement `:` + type($mem) `,` type($idx) `,` qualified(type($expected)) `,` + qualified(type($replacement)) `)` + `outs` `(` $dst `:` qualified(type($dst) ) `)` + attr-dict + }]; + + let extraClassDeclaration = [{ + static StringRef getIntrinsicName() { return "MGATHER_CAS"; } + ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_MTE2; } + ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } + }]; +} + // ---- tile-world TOp version ---- @@ -2715,6 +2773,37 @@ def MScatterOp : PTO_TOp<"mscatter", [ }]; } +def MScatterMaskOp : PTO_TOp<"mscatter_mask", [ + PTO_DpsInitOpInterface, + OpPipeInterface, + DeclareOpInterfaceMethods +]> { + let summary = "Masked scatter-store elements from a tile into memory using per-element indices."; + + let arguments = (ins + PTODpsType:$src, + PTODpsType:$idx, + PTODpsType:$mask, + PTODpsType:$mem + ); + + let results = (outs); + let hasVerifier = 1; + + let assemblyFormat = [{ + `ins` `(` $src `,` $idx `,` $mask `:` + qualified(type($src)) `,` type($idx) `,` qualified(type($mask)) `)` + `outs` `(` $mem `:` type($mem) `)` + attr-dict + }]; + + let extraClassDeclaration = [{ + static StringRef getIntrinsicName() { return "MSCATTER_MASK"; } + ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_MTE3; } + ::mlir::MutableOperandRange getDpsInitsMutable() { return getMemMutable(); } + }]; +} + //===----------------------------------------------------------------------===// // PTO_TOPs(Tilebuffer, DPS) //===----------------------------------------------------------------------===// @@ -2788,40 +2877,6 @@ def TAddOp : PTO_TOp<"tadd", [ }]; } -def TAddCOp : PTO_TOp<"taddc", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "Elementwise ternary add of tiles "; - let description = [{ - For each element (i, j): dst[i,j] = src0[i,j] + src1[i,j] + src2[i,j]. - }]; - - let arguments = (ins - PTODpsType:$src0, - PTODpsType:$src1, - PTODpsType:$src2, - PTODpsType:$dst - ); - - let results = (outs); - - let assemblyFormat = [{ - `ins` `(` $src0 `,` $src1 `,` $src2 `:` - qualified(type($src0)) `,` qualified(type($src1)) `,` qualified(type($src2)) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let hasVerifier = 1; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; -} - def TAddSOp : PTO_TOp<"tadds", [ PTO_DpsInitOpInterface, OpPipeInterface, @@ -2884,35 +2939,6 @@ def TAxpyOp : PTO_TOp<"taxpy", [ }]; } -def TAddSCOp : PTO_TOp<"taddsc", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "dst = src0 + scalar + src1 "; - let arguments = (ins - PTODpsType:$src0, - ScalarType:$scalar, - PTODpsType:$src1, - PTODpsType:$dst - ); - let results = (outs); - - let assemblyFormat = [{ - `ins` `(` $src0 `,` $scalar `,` $src1 `:` qualified(type($src0)) `,` type($scalar) `,` qualified(type($src1)) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let hasVerifier = 1; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; -} - - def TAndOp : PTO_TOp<"tand", [ PTO_DpsInitOpInterface, OpPipeInterface, @@ -3588,36 +3614,28 @@ def TCvtOp : PTO_TOp<"tcvt", [ }]; } -def TRandomOp : PTO_TOp<"trandom", [ +def AccCvtOp : PTO_TOp<"acccvt", [ PTO_DpsInitOpInterface, OpPipeInterface, DeclareOpInterfaceMethods ]> { - let summary = "Generate random values into dst tile using key/counter words."; + let summary = "Convert the implicit accumulator into a publishable destination tile."; let arguments = (ins - AnySignlessInteger:$key0, - AnySignlessInteger:$key1, - AnySignlessInteger:$counter0, - AnySignlessInteger:$counter1, - AnySignlessInteger:$counter2, - AnySignlessInteger:$counter3, - PTODpsType:$dst, - DefaultValuedAttr:$rounds + PTODpsType:$dst ); let results = (outs); let hasVerifier = 1; let assemblyFormat = [{ - `ins` `(` $key0 `,` $key1 `,` $counter0 `,` $counter1 `,` $counter2 `,` $counter3 - attr-dict - `:` type($key0) `,` type($key1) `,` type($counter0) `,` type($counter1) `,` type($counter2) `,` type($counter3) `)` `outs` `(` $dst `:` qualified(type($dst) ) `)` + attr-dict }]; let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } + static StringRef getIntrinsicName() { return "ACCCVT"; } + ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_M; } ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } }]; } @@ -3682,64 +3700,6 @@ def TDivSOp : PTO_TOp<"tdivs", [ } -def TFModOp : PTO_TOp<"tfmod", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "Elementwise fmod/remainder of two tiles (tilebuf, DPS)"; - - let arguments = (ins - PTODpsType:$src0, - PTODpsType:$src1, - PTODpsType:$dst - ); - - let results = (outs); - - let hasVerifier = 1; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; - - let assemblyFormat = [{ - `ins` `(` $src0 `,` $src1 `:` qualified(type($src0)) `,` qualified(type($src1)) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; -} - -def TFModSOp : PTO_TOp<"tfmods", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "Elementwise fmod/remainder with a scalar (tilebuf, DPS)"; - - let arguments = (ins - PTODpsType:$src, - ScalarType:$scalar, - PTODpsType:$dst - ); - - let results = (outs); - - let hasVerifier = 1; - - let assemblyFormat = [{ - `ins` `(` $src `,` $scalar `:` qualified(type($src)) `,` type($scalar) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; -} - def TExpOp : PTO_TOp<"texp", [ PTO_DpsInitOpInterface, OpPipeInterface, @@ -4288,35 +4248,6 @@ def TLogOp : PTO_TOp<"tlog", [ }]; } -def TLReluOp : PTO_TOp<"tlrelu", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "Leaky ReLU with a scalar slope (tilebuf, DPS)"; - - let arguments = (ins - PTODpsType:$src, - F32:$slope, - PTODpsType:$dst - ); - - let results = (outs); - - let hasVerifier = 1; - - let assemblyFormat = [{ - `ins` `(` $src `,` $slope `:` qualified(type($src)) `,` type($slope) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; -} - def TMaxOp : PTO_TOp<"tmax", [ PTO_DpsInitOpInterface, OpPipeInterface, @@ -5879,37 +5810,45 @@ def TShrSOp : PTO_TOp<"tshrs", [ }]; } //===----------------------------------------------------------------------===// -// PTOOps.td (add TSORT32 TBDPS/tile buffer op) +// PTOOps.td (TSORT DPS/tile buffer op) //===----------------------------------------------------------------------===// -def TSort32Op: PTO_TOp<"tsort32", [ +def TSortOp: PTO_TOp<"tsort", [ PTO_DpsInitOpInterface, OpPipeInterface, DeclareOpInterfaceMethods ]> { - let summary = "TSORT32: Sort fixed-size 32-element blocks using explicit input indices."; + let summary = "TSORT: Stable sort of independent 32-element groups."; let description = [{ - DPS form matching pto-isa overloads: (dst, src, idx) or (dst, src, idx, tmp). - In PTO IR, `idx` is modeled as an input operand in `ins(...)`, while `dst` - remains the sole DPS init/output buffer in `outs(...)`. + Sorts each consecutive 32-element group from `src` into `dst`. Equal + values retain source order. `dst_indices` receives the U32 original index + within each group. `descending` selects descending instead of ascending + order; NaNs follow numeric values in either direction. }]; let arguments = (ins PTODpsType:$src, - PTODpsType:$idx, - Optional:$tmp, - PTODpsType:$dst + PTODpsType:$dst, + PTODpsType:$dst_indices, + DefaultValuedAttr:$descending ); let results = (outs); let hasVerifier = 1; - let hasCustomAssemblyFormat = 1; + let assemblyFormat = [{ + `ins` `(` $src `:` qualified(type($src)) `)` + `outs` `(` $dst `,` $dst_indices `:` + qualified(type($dst)) `,` qualified(type($dst_indices)) `)` + attr-dict + }]; let extraClassDeclaration = [{ ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } + ::mlir::MutableOperandRange getDpsInitsMutable() { + return ::mlir::MutableOperandRange(getOperation(), 1, 2); + } }]; } //===----------------------------------------------------------------------===// @@ -6005,45 +5944,6 @@ def TSubOp: PTO_TOp<"tsub", [ }]; } //===----------------------------------------------------------------------===// -// PTOOps.td (add TSUBC TBDPS/tile buffer op) -//===----------------------------------------------------------------------===// - -def TSubCOp: PTO_TOp<"tsubc", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "TSUBC: Elementwise ternary op: src0 - src1 + src2."; - - let arguments = (ins - PTODpsType:$src0, - PTODpsType:$src1, - PTODpsType:$src2, - PTODpsType:$dst - ); - - let results = (outs); - - let hasVerifier = 1; - - let assemblyFormat = [{ - `ins` `(` $src0 `,` $src1 `,` $src2 `:` - qualified(type($src0)) `,` qualified(type($src1)) `,` qualified(type($src2)) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; - - -} -//===----------------------------------------------------------------------===// -// PTOOps.td (add TSUBS TBDPS/tile buffer op) -//===----------------------------------------------------------------------===// - def TSubSOp: PTO_TOp<"tsubs", [ PTO_DpsInitOpInterface, OpPipeInterface, @@ -6075,41 +5975,6 @@ def TSubSOp: PTO_TOp<"tsubs", [ } //===----------------------------------------------------------------------===// -// PTOOps.td (add TSUBSC TBDPS/tile buffer op) -//===----------------------------------------------------------------------===// - -def TSubSCOp: PTO_TOp<"tsubsc", [ - PTO_DpsInitOpInterface, - OpPipeInterface, - DeclareOpInterfaceMethods -]> { - let summary = "TSUBSC: Elementwise fused op: src0 - scalar + src1."; - - let arguments = (ins - PTODpsType:$src0, - ScalarType:$scalar, - PTODpsType:$src1, - PTODpsType:$dst - ); - - let results = (outs); - - let hasVerifier = 1; - - let assemblyFormat = [{ - `ins` `(` $src0 `,` $scalar `,` $src1 `:` - qualified(type($src0)) `,` type($scalar) `,` qualified(type($src1)) `)` - `outs` `(` $dst `:` qualified(type($dst) ) `)` - attr-dict - }]; - - let extraClassDeclaration = [{ - ::mlir::pto::PIPE getPipe() { return ::mlir::pto::PIPE::PIPE_V; } - ::mlir::MutableOperandRange getDpsInitsMutable() { return getDstMutable(); } - }]; - -} - //===----------------------------------------------------------------------===// // PTOOps.td (add TXORS TBDPS/tile buffer op) //===----------------------------------------------------------------------===// diff --git a/lib/PTO/IR/PTO.cpp b/lib/PTO/IR/PTO.cpp index e33625e512..7715b9d26b 100644 --- a/lib/PTO/IR/PTO.cpp +++ b/lib/PTO/IR/PTO.cpp @@ -110,7 +110,8 @@ static bool isTileLikeType(Type ty); static SmallVector getShapeVec(Type ty); static SmallVector getValidShapeVec(Type ty); static SmallVector getValidShapeVec(Value value); -static LogicalResult verifyTileBufCommon(Operation *op, Type ty, StringRef name); +static LogicalResult verifyTileBufCommon(Operation *op, Type ty, StringRef name, + bool allowLowPrecision = false); static LogicalResult verifyTileBufSameElemType(Operation *op, Type lhs, Type rhs, StringRef lhsName, StringRef rhsName); @@ -132,17 +133,23 @@ static LogicalResult verifyAccTileCommonA2A3(Operation *op, Type ty, static LogicalResult verifyAccTileCommonA5(Operation *op, Type ty, StringRef name); static LogicalResult verifyMatTileOperands(Operation *op, Type lhsTy, Type rhsTy, - Type dstTy); + Type dstTy, + bool allowLowPrecisionInputs = false); static LogicalResult verifyMatTileOperandsA2A3(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy); + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs); static LogicalResult verifyMatTileOperandsA5(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy); + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs); static LogicalResult verifyGemvTileOperands(Operation *op, Type lhsTy, Type rhsTy, - Type dstTy); + Type dstTy, + bool allowLowPrecisionInputs = false); static LogicalResult verifyGemvTileOperandsA2A3(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy); + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs); static LogicalResult verifyGemvTileOperandsA5(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy); + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs); static LogicalResult verifyMatBiasTile(Operation *op, Type biasTy, Type dstTy, bool requireFloatBias = false); static LogicalResult verifyMatBiasTileA2A3(Operation *op, Type biasTy, Type dstTy, @@ -2297,59 +2304,12 @@ LogicalResult TLoadOp::verify() { } LogicalResult TPrefetchOp::verify() { - Type srcTy = getSrc().getType(); - Type dstTy = getDst().getType(); - - Type srcElem; - Type dstElem; - - if (auto srcPart = dyn_cast(srcTy)) { - auto srcShape = srcPart.getShape(); - for (unsigned i = 0; i < srcShape.size(); ++i) { - if (srcShape[i] != ShapedType::kDynamic && srcShape[i] <= 0) - return emitOpError() << "expects src shape[" << i << "] to be positive"; - } - srcElem = srcPart.getElementType(); - } else if (auto srcMr = dyn_cast(srcTy)) { - if (!srcMr.hasRank()) - return emitOpError("expects src memref to be ranked"); - for (int64_t dim : srcMr.getShape()) { - if (dim != ShapedType::kDynamic && dim <= 0) - return emitOpError("expects src memref shape to be positive"); - } - srcElem = srcMr.getElementType(); - } else { - return emitOpError("expects src to be !pto.partition_tensor_view or memref"); - } - - if (auto dstTile = dyn_cast(dstTy)) { - if (failed(verifyTileBufCommon(*this, dstTile, "dst"))) - return failure(); - auto dstValid = dstTile.getValidShape(); - for (unsigned i = 0; i < dstValid.size(); ++i) { - if (dstValid[i] != ShapedType::kDynamic && dstValid[i] <= 0) - return emitOpError() << "expects dst valid_shape[" << i << "] to be positive"; - } - auto dstSpace = getPTOMemorySpaceEnum(dstTile); - if (!dstSpace || (*dstSpace != pto::AddressSpace::VEC && - *dstSpace != pto::AddressSpace::MAT)) - return emitOpError("expects dst to use loc=vec or loc=mat"); - dstElem = dstTile.getElementType(); - } else if (auto dstMr = dyn_cast(dstTy)) { - auto dstSpace = getPTOMemorySpaceEnum(dstMr); - if (!dstSpace || (*dstSpace != pto::AddressSpace::VEC && - *dstSpace != pto::AddressSpace::MAT)) - return emitOpError("expects dst memref to use loc=vec or loc=mat"); - if (!dstMr.hasRank()) - return emitOpError("expects dst memref to be ranked"); - dstElem = dstMr.getElementType(); - } else { - return emitOpError("expects dst to be !pto.tile_buf or memref"); - } - - if (getElemByteSize(srcElem) != getElemByteSize(dstElem)) - return emitOpError("expects src and dst element types to have the same element size"); - + if (!getAddress().getType().isInteger(64)) + return emitOpError("expects address to have type i64"); + if (auto byteCount = getConstantIntegerValue(getByteCount()); + byteCount && (*byteCount < 0 || *byteCount > 262144)) + return emitOpError( + "expects a static byte_count in the inclusive range 0..262144"); return success(); } @@ -3082,19 +3042,20 @@ static LogicalResult verifyMGatherMScatterIdxTile(Operation *op, Type ty, return success(); } -static LogicalResult verifyTileBufCommon(Operation *op, Type ty, StringRef name) { +static LogicalResult verifyTileBufCommon(Operation *op, Type ty, StringRef name, + bool allowLowPrecision) { auto tb = dyn_cast(ty); if (tb) { if (tb.getRank() != 2) return op->emitOpError() << "expects " << name << " to be a rank-2 tile_buf"; Type elemTy = tb.getElementType(); - if (isPTOLowPrecisionType(elemTy)) + if (!allowLowPrecision && isPTOLowPrecisionType(elemTy)) return op->emitOpError() << name << ": dtype " << elemTy << " is not supported by this op yet"; } else if (auto mr = dyn_cast(ty)) { if (mr.getRank() != 2) return op->emitOpError() << "expects " << name << " to be a rank-2 memref"; - if (isPTOLowPrecisionType(mr.getElementType())) + if (!allowLowPrecision && isPTOLowPrecisionType(mr.getElementType())) return op->emitOpError() << name << ": dtype " << mr.getElementType() << " is not supported by this op yet"; } else { @@ -3557,9 +3518,10 @@ static LogicalResult verifyAccTileCommon(Operation *op, Type ty, StringRef name) } static LogicalResult verifyMatTileOperandsA2A3(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy) { - if (failed(verifyTileBufCommon(op, lhsTy, "lhs")) || - failed(verifyTileBufCommon(op, rhsTy, "rhs")) || + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs) { + if (failed(verifyTileBufCommon(op, lhsTy, "lhs", allowLowPrecisionInputs)) || + failed(verifyTileBufCommon(op, rhsTy, "rhs", allowLowPrecisionInputs)) || failed(verifyAccTileCommon(op, dstTy, "dst"))) return failure(); auto lhsSpace = getPTOMemorySpaceEnum(lhsTy); @@ -3592,8 +3554,10 @@ static LogicalResult verifyMatTileOperandsA2A3(Operation *op, Type lhsTy, } static LogicalResult verifyMatTileOperandsA5(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy) { - if (failed(verifyMatTileOperandsA2A3(op, lhsTy, rhsTy, dstTy))) + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs) { + if (failed(verifyMatTileOperandsA2A3(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs))) return failure(); auto lhsTb = mlir::dyn_cast(lhsTy); @@ -3619,20 +3583,24 @@ static LogicalResult verifyMatTileOperandsA5(Operation *op, Type lhsTy, } static LogicalResult verifyMatTileOperands(Operation *op, Type lhsTy, Type rhsTy, - Type dstTy) { + Type dstTy, + bool allowLowPrecisionInputs) { switch (getVerifierTargetArch(op)) { case VerifierTargetArch::A2A3: - return verifyMatTileOperandsA2A3(op, lhsTy, rhsTy, dstTy); + return verifyMatTileOperandsA2A3(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs); case VerifierTargetArch::A5: - return verifyMatTileOperandsA5(op, lhsTy, rhsTy, dstTy); + return verifyMatTileOperandsA5(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs); } return failure(); } static LogicalResult verifyGemvTileOperandsA2A3(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy) { - if (failed(verifyTileBufCommon(op, lhsTy, "lhs")) || - failed(verifyTileBufCommon(op, rhsTy, "rhs")) || + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs) { + if (failed(verifyTileBufCommon(op, lhsTy, "lhs", allowLowPrecisionInputs)) || + failed(verifyTileBufCommon(op, rhsTy, "rhs", allowLowPrecisionInputs)) || failed(verifyAccTileCommon(op, dstTy, "dst"))) return failure(); @@ -3666,19 +3634,25 @@ static LogicalResult verifyGemvTileOperandsA2A3(Operation *op, Type lhsTy, } static LogicalResult verifyGemvTileOperandsA5(Operation *op, Type lhsTy, - Type rhsTy, Type dstTy) { - if (failed(verifyGemvTileOperandsA2A3(op, lhsTy, rhsTy, dstTy))) + Type rhsTy, Type dstTy, + bool allowLowPrecisionInputs) { + if (failed(verifyGemvTileOperandsA2A3(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs))) return failure(); - return verifyMatTileOperandsA5(op, lhsTy, rhsTy, dstTy); + return verifyMatTileOperandsA5(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs); } static LogicalResult verifyGemvTileOperands(Operation *op, Type lhsTy, Type rhsTy, - Type dstTy) { + Type dstTy, + bool allowLowPrecisionInputs) { switch (getVerifierTargetArch(op)) { case VerifierTargetArch::A2A3: - return verifyGemvTileOperandsA2A3(op, lhsTy, rhsTy, dstTy); + return verifyGemvTileOperandsA2A3(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs); case VerifierTargetArch::A5: - return verifyGemvTileOperandsA5(op, lhsTy, rhsTy, dstTy); + return verifyGemvTileOperandsA5(op, lhsTy, rhsTy, dstTy, + allowLowPrecisionInputs); } return failure(); } @@ -3761,26 +3735,6 @@ LogicalResult pto::TAddOp::verify() { "expects A5 tadd element type to be i32/i16/i8/f16/bf16/f32"); } -LogicalResult pto::TAddCOp::verify() { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - Type t0 = getSrc0().getType(); - Type t1 = getSrc1().getType(); - Type t2 = getSrc2().getType(); - Type td = getDst().getType(); - - if (!isPTOShapedLike(t0) || !isPTOShapedLike(t1) || - !isPTOShapedLike(t2) || !isPTOShapedLike(td)) - return emitOpError("expects src0/src1/src2/dst to be memref/tile_buf types"); - - auto s0 = getShapeVec(t0); - auto s1 = getShapeVec(t1); - auto s2 = getShapeVec(t2); - auto sd = getShapeVec(td); - if (s0 != s1 || s0 != s2 || s0 != sd) - return emitOpError("expects src0/src1/src2/dst to have the same shape"); - return success(); -} LogicalResult pto::TAddSOp::verify() { return verifyArithmeticScalarTileOpWithArchDispatch( getOperation(), getSrc().getType(), getDst().getType(), getScalar().getType(), @@ -3847,23 +3801,6 @@ LogicalResult pto::TAxpyOp::verify() { return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); } -LogicalResult pto::TAddSCOp::verify() { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - Type ts0 = getSrc0().getType(); - Type ts1 = getSrc1().getType(); - Type td = getDst().getType(); - if (!isPTOShapedLike(ts0) || !isPTOShapedLike(ts1) || !isPTOShapedLike(td)) - return emitOpError("expects src0/src1/dst to be PTO shaped-like types"); - - auto s0 = getShapeVec(ts0); - auto s1 = getShapeVec(ts1); - auto sd = getShapeVec(td); - if (s0 != s1 || s0 != sd) - return emitOpError("expects src0/src1/dst to have the same shape"); - return success(); -} - LogicalResult pto::TAndOp::verify() { auto verifyCommon = [&]() -> FailureOr { return verifyMatchingRowMajorBinaryTileOpCommon( @@ -4695,45 +4632,18 @@ llvm::LogicalResult mlir::pto::TCvtOp::verify() { return mlir::success(); } -llvm::LogicalResult mlir::pto::TRandomOp::verify() { - auto verifyA2A3 = [&]() -> LogicalResult { - return emitOpError("trandom is only supported for A5 targets"); - }; - auto verifyA5 = [&]() -> LogicalResult { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - - Type dstTy = getDst().getType(); - if (failed(verifyTileBufCommon(*this, dstTy, "dst"))) - return failure(); - if (!isRowMajorTileBuf(dstTy)) - return emitOpError("expects dst to use row-major layout"); - - Type elemTy = getElemTy(dstTy); - if (!elemTy.isInteger(32)) - return emitOpError("expects dst element type to be i32 or ui32"); - - auto checkWord = [&](Value v, StringRef name) -> LogicalResult { - auto ty = dyn_cast(v.getType()); - if (!ty || ty.getWidth() != 32) - return emitOpError() << "expects " << name << " to be i32/ui32"; - return success(); - }; - if (failed(checkWord(getKey0(), "key0")) || - failed(checkWord(getKey1(), "key1")) || - failed(checkWord(getCounter0(), "counter0")) || - failed(checkWord(getCounter1(), "counter1")) || - failed(checkWord(getCounter2(), "counter2")) || - failed(checkWord(getCounter3(), "counter3"))) - return failure(); - - int32_t rounds = getRounds(); - if (rounds != 7 && rounds != 10) - return emitOpError("expects rounds to be 7 or 10"); - +llvm::LogicalResult mlir::pto::AccCvtOp::verify() { + if (shouldBypassDecodedMemrefVerifier(getOperation())) return success(); - }; - return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); + Type dstTy = getDst().getType(); + if (failed(verifyTileBufCommon(*this, dstTy, "dst"))) + return failure(); + auto dstSpace = getPTOMemorySpaceEnum(dstTy); + if (dstSpace && *dstSpace == pto::AddressSpace::ACC) + return emitOpError( + "expects explicit dst to be a publishable tile/memref, not loc=acc; " + "ACCCVT reads ACC implicitly"); + return success(); } LogicalResult mlir::pto::TDivOp::verify() { @@ -5766,45 +5676,6 @@ mlir::LogicalResult mlir::pto::TLogOp::verify() { return mlir::success(); } -mlir::LogicalResult mlir::pto::TLReluOp::verify() { - Type srcTy = getSrc().getType(); - Type dstTy = getDst().getType(); - auto verifyA2A3 = [&]() -> LogicalResult { - if (failed(verifyVecTileStorage(*this, srcTy, "src")) || - failed(verifyVecTileStorage(*this, dstTy, "dst"))) - return failure(); - if (failed(verifyTileBufSameElemType(*this, srcTy, dstTy, "src", "dst")) || - failed(verifyTileBufSameValidShape(*this, srcTy, dstTy, "src", "dst"))) - return failure(); - auto valid = getValidShapeVec(srcTy); - if (valid.size() != 2) - return emitOpError("expects src to have rank-2 valid_shape"); - if (valid[0] != ShapedType::kDynamic && valid[0] <= 0) - return emitOpError("expects src valid_shape[0] to be positive"); - if (valid[1] != ShapedType::kDynamic && valid[1] <= 0) - return emitOpError("expects src valid_shape[1] to be positive"); - Type elemTy = getElemTy(srcTy); - if (!(elemTy.isF16() || elemTy.isF32())) - return emitOpError() << "expects A2/A3 tlrelu element type to be f16 or f32"; - return success(); - }; - auto verifyA5 = [&]() -> LogicalResult { - if (failed(verifyVecTileStorage(*this, srcTy, "src")) || - failed(verifyVecTileStorage(*this, dstTy, "dst"))) - return failure(); - if (failed(verifyTileBufSameElemType(*this, srcTy, dstTy, "src", "dst")) || - failed(verifyTileBufSameValidShape(*this, srcTy, dstTy, "src", "dst"))) - return failure(); - Type elemTy = getElemTy(srcTy); - if (!(elemTy.isF16() || elemTy.isF32())) - return emitOpError() << "expects A5 tlrelu element type to be f16 or f32"; - if (!getSlope().getType().isF32()) - return emitOpError() << "expects slope to have type f32"; - return success(); - }; - return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); -} - mlir::LogicalResult mlir::pto::TMaxOp::verify() { return verifyArithmeticBinaryTileOpWithArchDispatch( getOperation(), getSrc0().getType(), getSrc1().getType(), getDst().getType(), @@ -6163,7 +6034,8 @@ LogicalResult TGemvMxOp::verify() { failed(verifyScaleTileMatchesOperand(*this, getBScale().getType(), getB().getType(), "b_scale", "b")) || failed(verifyGemvTileOperands(*this, getA().getType(), getB().getType(), - getDst().getType()))) + getDst().getType(), + /*allowLowPrecisionInputs=*/true))) return failure(); if (failed(verifyA5MxTypeTriple(*this, getA().getType(), getB().getType(), getDst().getType(), "a", "b", "dst"))) @@ -6185,7 +6057,8 @@ LogicalResult TGemvMxAccOp::verify() { failed(verifyScaleTileMatchesOperand(*this, getBScale().getType(), getB().getType(), "b_scale", "b")) || failed(verifyGemvTileOperands(*this, getA().getType(), getB().getType(), - getDst().getType()))) + getDst().getType(), + /*allowLowPrecisionInputs=*/true))) return failure(); if (failed(verifyA5MxTypeTriple(*this, getA().getType(), getB().getType(), getDst().getType(), "a", "b", "dst"))) @@ -6211,7 +6084,8 @@ LogicalResult TGemvMxBiasOp::verify() { failed(verifyScaleTileMatchesOperand(*this, getBScale().getType(), getB().getType(), "b_scale", "b")) || failed(verifyGemvTileOperands(*this, getA().getType(), getB().getType(), - getDst().getType())) || + getDst().getType(), + /*allowLowPrecisionInputs=*/true)) || failed(verifyMatBiasTile(*this, getBias().getType(), getDst().getType(), /*requireFloatBias=*/true))) return failure(); @@ -6551,6 +6425,110 @@ LogicalResult MGatherOp::verify() { return success(); } +LogicalResult MGatherMaskOp::verify() { + if (shouldBypassDecodedMemrefVerifier(getOperation())) + return success(); + if (!isTargetArchA5(getOperation())) + return emitOpError("pto.mgather_mask is only supported on A5 targets"); + Type memTy = getMem().getType(); + Type idxTy = getIdx().getType(); + Type dstTy = getDst().getType(); + Type maskTy = getMask().getType(); + if (getPTOTypeRank(memTy) == -1 || getPTOTypeRank(idxTy) == -1 || + getPTOTypeRank(dstTy) == -1 || getPTOTypeRank(maskTy) == -1) + return emitOpError("expects mem, idx, mask, and dst to use supported PTO shapes"); + if (failed(verifyNDStyleVecTile(*this, dstTy, "dst")) || + failed(verifyNDStyleVecTile(*this, maskTy, "mask")) || + failed(verifyMGatherMScatterIdxTile(getOperation(), idxTy, "idx"))) + return failure(); + Type dstElem = getElemTy(dstTy); + Type idxElem = getElemTy(idxTy); + if (!dstElem || !idxElem) + return emitOpError("failed to resolve element types for dst or idx"); + if (!isSupportedMGatherMScatterPayloadElemType(getOperation(), dstElem)) + return emitOpError("expects dst element type to be supported by mgather_mask"); + if (!isSupportedMGatherMScatterIndexElemType(idxElem)) + return emitOpError("expects idx element type to be signless i32"); + if (failed(verifyMGatherMScatterMemOperand(getOperation(), getMem(), dstElem, + "dst")) || + failed(verifyMGatherMScatterTileShape(getOperation(), dstTy, idxTy, "dst")) || + failed(verifyMGatherMScatterTileShape(getOperation(), maskTy, idxTy, "mask"))) + return failure(); + return success(); +} + +LogicalResult MGatherCasOp::verify() { + if (shouldBypassDecodedMemrefVerifier(getOperation())) + return success(); + if (!isTargetArchA5(getOperation())) + return emitOpError("pto.mgather_cas is only supported on A5 targets"); + Type memTy = getMem().getType(); + Type idxTy = getIdx().getType(); + Type dstTy = getDst().getType(); + Type expectedTy = getExpected().getType(); + Type replacementTy = getReplacement().getType(); + if (getPTOTypeRank(memTy) == -1 || getPTOTypeRank(idxTy) == -1 || + getPTOTypeRank(dstTy) == -1 || getPTOTypeRank(expectedTy) == -1 || + getPTOTypeRank(replacementTy) == -1) + return emitOpError( + "expects mem, idx, expected, replacement, and dst to use supported PTO shapes"); + if (failed(verifyNDStyleVecTile(*this, dstTy, "dst")) || + failed(verifyNDStyleVecTile(*this, expectedTy, "expected")) || + failed(verifyNDStyleVecTile(*this, replacementTy, "replacement"))) + return failure(); + Type dstElem = getElemTy(dstTy); + Type idxElem = getElemTy(idxTy); + if (!dstElem || !idxElem) + return emitOpError("failed to resolve element types for dst or idx"); + if (!isSupportedMGatherMScatterPayloadElemType(getOperation(), dstElem)) + return emitOpError("expects dst element type to be supported by mgather_cas"); + if (!isSupportedMGatherMScatterIndexElemType(idxElem)) + return emitOpError("expects idx element type to be signless i32"); + if (getElemTy(expectedTy) != dstElem || getElemTy(replacementTy) != dstElem) + return emitOpError("expects expected/replacement element types to match dst"); + if (failed(verifyMGatherMScatterMemOperand(getOperation(), getMem(), dstElem, + "dst")) || + failed(verifyMGatherMScatterTileShape(getOperation(), dstTy, idxTy, "dst")) || + failed(verifyMGatherMScatterTileShape(getOperation(), expectedTy, idxTy, + "expected")) || + failed(verifyMGatherMScatterTileShape(getOperation(), replacementTy, idxTy, + "replacement"))) + return failure(); + return success(); +} + +LogicalResult MScatterMaskOp::verify() { + if (shouldBypassDecodedMemrefVerifier(getOperation())) + return success(); + if (!isTargetArchA5(getOperation())) + return emitOpError("pto.mscatter_mask is only supported on A5 targets"); + Type srcTy = getSrc().getType(); + Type idxTy = getIdx().getType(); + Type memTy = getMem().getType(); + Type maskTy = getMask().getType(); + if (getPTOTypeRank(srcTy) == -1 || getPTOTypeRank(idxTy) == -1 || + getPTOTypeRank(memTy) == -1 || getPTOTypeRank(maskTy) == -1) + return emitOpError("expects src, idx, mask, and mem to use supported PTO shapes"); + if (failed(verifyNDStyleVecTile(*this, srcTy, "src")) || + failed(verifyNDStyleVecTile(*this, maskTy, "mask")) || + failed(verifyMGatherMScatterIdxTile(getOperation(), idxTy, "idx"))) + return failure(); + Type srcElem = getElemTy(srcTy); + Type idxElem = getElemTy(idxTy); + if (!srcElem || !idxElem) + return emitOpError("failed to resolve element types for src or idx"); + if (!isSupportedMGatherMScatterPayloadElemType(getOperation(), srcElem)) + return emitOpError("expects src element type to be supported by mscatter_mask"); + if (!isSupportedMGatherMScatterIndexElemType(idxElem)) + return emitOpError("expects idx element type to be signless i32"); + if (failed(verifyMGatherMScatterMemOperand(getOperation(), getMem(), srcElem, + "src")) || + failed(verifyMGatherMScatterTileShape(getOperation(), srcTy, idxTy, "src")) || + failed(verifyMGatherMScatterTileShape(getOperation(), maskTy, idxTy, "mask"))) + return failure(); + return success(); +} + void mlir::pto::TCvtOp::print(OpAsmPrinter &p) { p << " ins(" << getSrc(); Builder builder(getContext()); @@ -7564,14 +7542,6 @@ mlir::LogicalResult mlir::pto::TRemOp::verify() { return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); } -mlir::LogicalResult mlir::pto::TFModOp::verify() { - return verifyArithmeticBinaryTileOpWithArchDispatch( - getOperation(), getSrc0().getType(), getSrc1().getType(), getDst().getType(), - /*allowInt8OnA5=*/false, /*allowBf16OnA5=*/false, - "expects A2/A3 tfmod element type to be i32/i16/f16/f32", - "expects A5 tfmod element type to be i32/i16/f16/f32"); -} - mlir::LogicalResult mlir::pto::TRemSOp::verify() { if (shouldBypassDecodedMemrefVerifier(getOperation())) return success(); @@ -7615,40 +7585,6 @@ mlir::LogicalResult mlir::pto::TRemSOp::verify() { return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); } -mlir::LogicalResult mlir::pto::TFModSOp::verify() { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - - Type srcTy = getSrc().getType(); - Type dstTy = getDst().getType(); - Type scalarTy = getScalar().getType(); - if (failed(verifyTileBufCommon(*this, srcTy, "src")) || - failed(verifyTileBufCommon(*this, dstTy, "dst"))) - return failure(); - if (failed(verifyTileBufSameElemType(*this, srcTy, dstTy, "src", "dst")) || - failed(verifyTileBufSameValidShape(*this, srcTy, dstTy, "src", "dst"))) - return failure(); - if (!isRowMajorTileBuf(srcTy) || !isRowMajorTileBuf(dstTy)) - return emitOpError("expects src and dst to use row-major layout"); - - Type elem = getElemTy(srcTy); - if (scalarTy != elem) - return emitOpError("expects scalar type to match the tile element type"); - - auto verifyA2A3 = [&]() -> LogicalResult { - if (!(elem.isInteger(32) || elem.isInteger(16) || elem.isF16() || elem.isF32())) - return emitOpError("expects A2/A3 tfmods element type to be i32/i16/f16/f32"); - return success(); - }; - auto verifyA5 = [&]() -> LogicalResult { - if (!(elem.isInteger(32) || elem.isInteger(16) || elem.isF16() || elem.isF32())) - return emitOpError("expects A5 tfmods element type to be i32/i16/f16/f32"); - return success(); - }; - return dispatchVerifierByArch(getOperation(), verifyA2A3, verifyA5); -} - - static std::optional getStaticNumElements(ArrayRef shape) { int64_t numel = 1; for (int64_t d : shape) { @@ -7914,69 +7850,6 @@ mlir::LogicalResult mlir::pto::TRowExpandOp::verify() { } -ParseResult mlir::pto::TSort32Op::parse(OpAsmParser &parser, OperationState &result) { - OpAsmParser::UnresolvedOperand src, idx, tmp, dst; - Type srcTy, dstTy, idxTy, tmpTy; - bool hasTmp = false; - - if (parser.parseKeyword("ins") || parser.parseLParen() || parser.parseOperand(src)) - return failure(); - if (succeeded(parser.parseOptionalComma())) { - if (parser.parseOperand(idx)) - return failure(); - if (succeeded(parser.parseOptionalComma())) { - if (parser.parseOperand(tmp)) - return failure(); - hasTmp = true; - } - } else { - return failure(); - } - if (parser.parseColonType(srcTy) || parser.parseComma() || parser.parseType(idxTy)) - return failure(); - if (hasTmp) { - if (parser.parseComma() || parser.parseType(tmpTy)) - return failure(); - } - if (parser.parseRParen()) - return failure(); - - if (parser.parseKeyword("outs") || parser.parseLParen() || - parser.parseOperand(dst) || parser.parseColonType(dstTy) || - parser.parseRParen()) - return failure(); - if (parser.parseOptionalAttrDict(result.attributes)) - return failure(); - - if (parser.resolveOperand(src, srcTy, result.operands) || - parser.resolveOperand(idx, idxTy, result.operands)) - return failure(); - if (hasTmp) { - if (parser.resolveOperand(tmp, tmpTy, result.operands)) - return failure(); - } - if (parser.resolveOperand(dst, dstTy, result.operands)) - return failure(); - - result.addAttribute( - "operandSegmentSizes", - parser.getBuilder().getDenseI32ArrayAttr({1, 1, hasTmp ? 1 : 0, 1})); - return success(); -} - -void mlir::pto::TSort32Op::print(OpAsmPrinter &p) { - p << " ins(" << getSrc() << ", " << getIdx(); - if (getTmp()) { - p << ", " << getTmp(); - p << " : " << getSrc().getType() << ", " << getIdx().getType() - << ", " << getTmp().getType() << ")"; - } else { - p << " : " << getSrc().getType() << ", " << getIdx().getType() << ")"; - } - p << " outs(" << getDst() << " : " << getDst().getType() << ")"; - p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{"operandSegmentSizes"}); -} - ParseResult mlir::pto::TRsqrtOp::parse(OpAsmParser &parser, OperationState &result) { OpAsmParser::UnresolvedOperand src, tmp, dst; Type srcTy, tmpTy, dstTy; @@ -8808,31 +8681,52 @@ mlir::LogicalResult mlir::pto::TShrOp::verify() { } -mlir::LogicalResult mlir::pto::TSort32Op::verify() { +mlir::LogicalResult mlir::pto::TSortOp::verify() { if (shouldBypassDecodedMemrefVerifier(getOperation())) return success(); Type srcTy = getSrc().getType(); Type dstTy = getDst().getType(); - Type idxTy = getIdx().getType(); + Type idxTy = getDstIndices().getType(); + if (getDst() == getDstIndices()) + return emitOpError("expects dst and dst_indices to be distinct destinations"); if (failed(verifyVecTileCommon(*this, srcTy, "src")) || failed(verifyVecTileCommon(*this, dstTy, "dst")) || - failed(verifyVecTileCommon(*this, idxTy, "idx"))) - return failure(); - if (getTmp() && - failed(verifyVecTileCommon(*this, getTmp().getType(), "tmp"))) + failed(verifyVecTileCommon(*this, idxTy, "dst_indices"))) return failure(); auto srcElem = getElemTy(srcTy); auto dstElem = getElemTy(dstTy); if (!srcElem || !dstElem || srcElem != dstElem) return emitOpError() << "expects src and dst to have the same element type"; - if (!(srcElem.isF16() || srcElem.isF32())) - return emitOpError() << "expects src and dst element type to be f16 or f32"; auto idxElem = getElemTy(idxTy); auto idxInt = dyn_cast(idxElem); - if (!idxInt || idxInt.getWidth() != 32) - return emitOpError() << "expects idx element type to be i32/u32"; + if (!idxInt || !idxInt.isUnsigned() || idxInt.getWidth() != 32) + return emitOpError() << "expects dst_indices element type to be ui32"; + + auto staticElementCount = [](Type type) -> std::optional { + auto shape = getValidShapeVec(type); + if (shape.empty()) + return std::nullopt; + int64_t count = 1; + for (int64_t dim : shape) { + if (ShapedType::isDynamic(dim)) + return std::nullopt; + count *= dim; + } + return count; + }; + auto srcCount = staticElementCount(srcTy); + auto dstCount = staticElementCount(dstTy); + auto idxCount = staticElementCount(idxTy); + if (srcCount && dstCount && *srcCount != *dstCount) + return emitOpError("expects src and dst to have the same valid element count"); + if (srcCount && idxCount && *srcCount != *idxCount) + return emitOpError( + "expects src and dst_indices to have the same valid element count"); + if (srcCount && (*srcCount == 0 || *srcCount % 32 != 0)) + return emitOpError( + "expects the valid element count to contain complete 32-element groups"); return mlir::success(); } @@ -8955,23 +8849,6 @@ mlir::LogicalResult mlir::pto::TSubOp::verify() { } -mlir::LogicalResult mlir::pto::TSubCOp::verify() { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - Type src0Ty = getSrc0().getType(); - Type src1Ty = getSrc1().getType(); - Type src2Ty = getSrc2().getType(); - Type dstTy = getDst().getType(); - if (!isPTOShapedLike(src0Ty) || !isPTOShapedLike(src1Ty) || !isPTOShapedLike(src2Ty) || !isPTOShapedLike(dstTy)) - return emitOpError() << "expects PTO shaped-like src0, src1, src2, and dst"; - - auto d = getShapeVec(dstTy); - if (getShapeVec(src0Ty).size() != d.size() || getShapeVec(src1Ty).size() != d.size() || getShapeVec(src2Ty).size() != d.size()) - return emitOpError() << "expects all tensors to have the same rank"; - return mlir::success(); -} - - mlir::LogicalResult mlir::pto::TSubSOp::verify() { return verifyArithmeticScalarTileOpWithArchDispatch( getOperation(), getSrc().getType(), getDst().getType(), getScalar().getType(), @@ -8983,20 +8860,6 @@ mlir::LogicalResult mlir::pto::TSubSOp::verify() { } -mlir::LogicalResult mlir::pto::TSubSCOp::verify() { - if (shouldBypassDecodedMemrefVerifier(getOperation())) - return success(); - Type src0Ty = getSrc0().getType(); - Type src1Ty = getSrc1().getType(); - Type dstTy = getDst().getType(); - if (!isPTOShapedLike(src0Ty) || !isPTOShapedLike(src1Ty) || !isPTOShapedLike(dstTy)) - return emitOpError() << "expects PTO shaped-like src0, src1, and dst"; - - auto d = getShapeVec(dstTy); - if (getShapeVec(src0Ty).size() != d.size() || getShapeVec(src1Ty).size() != d.size()) - return emitOpError() << "expects src0, src1, and dst to have the same rank"; - return mlir::success(); -} mlir::LogicalResult mlir::pto::TTransOp::verify() { auto verifyA2A3 = [&]() -> LogicalResult { Type srcTy = getSrc().getType(); @@ -10138,8 +10001,12 @@ void TLoadOp::getEffects(SmallVectorImpl> &effects) { - addEffect(effects, &getSrcMutable(), MemoryEffects::Read::get()); - addEffect(effects, &getDstMutable(), MemoryEffects::Write::get()); + // Model the architectural cache-state mutation, not just reads of the two + // scalar operands. The conservative unbound write keeps generic DCE from + // erasing a destination-free prefetch hint. + effects.emplace_back(MemoryEffects::Write::get()); + addEffect(effects, &getAddressMutable(), MemoryEffects::Read::get()); + addEffect(effects, &getByteCountMutable(), MemoryEffects::Read::get()); } // === TAbsOp === @@ -10230,6 +10097,24 @@ void MGatherOp::getEffects( PTO_ADD_WRITE(getDstMutable()); } +void MGatherMaskOp::getEffects( + SmallVectorImpl> &effects) { + PTO_ADD_READ(getMemMutable()); + PTO_ADD_READ(getIdxMutable()); + PTO_ADD_READ(getMaskMutable()); + PTO_ADD_WRITE(getDstMutable()); +} + +void MGatherCasOp::getEffects( + SmallVectorImpl> &effects) { + PTO_ADD_READ(getMemMutable()); + PTO_ADD_WRITE(getMemMutable()); + PTO_ADD_READ(getIdxMutable()); + PTO_ADD_READ(getExpectedMutable()); + PTO_ADD_READ(getReplacementMutable()); + PTO_ADD_WRITE(getDstMutable()); +} + // MSCATTER: Read(src, idx) -> Write(mem) void MScatterOp::getEffects( SmallVectorImpl> &effects) { @@ -10238,6 +10123,14 @@ void MScatterOp::getEffects( PTO_ADD_WRITE(getMemMutable()); } +void MScatterMaskOp::getEffects( + SmallVectorImpl> &effects) { + PTO_ADD_READ(getSrcMutable()); + PTO_ADD_READ(getIdxMutable()); + PTO_ADD_READ(getMaskMutable()); + PTO_ADD_WRITE(getMemMutable()); +} + // TGETVAL: Read(src) -> scalar result void TGetValOp::getEffects( SmallVectorImpl> &effects) { @@ -10271,9 +10164,7 @@ void SetValidShapeOp::getEffects( // Elementwise + reductions: mostly PIPE_V tilebuf ops PTO_DEFINE_BINARY_EFFECTS(TAddOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) -PTO_DEFINE_TERNARY_EFFECTS(TAddCOp, getSrc0Mutable(), getSrc1Mutable(), getSrc2Mutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TAddSOp, getSrcMutable(), getDstMutable()) -PTO_DEFINE_BINARY_EFFECTS(TAddSCOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) void TAxpyOp::getEffects( SmallVectorImpl> &effects) { PTO_ADD_READ(getSrcMutable()); @@ -10342,8 +10233,12 @@ void TCvtOp::getEffects( PTO_ADD_READ(getSrcMutable()); PTO_ADD_WRITE(getDstMutable()); } -void TRandomOp::getEffects( +void AccCvtOp::getEffects( SmallVectorImpl> &effects) { + // ACC is implicit in the opcode rather than an encoded operand. Model both + // the accumulator read and the release-on-commit state transition. + effects.emplace_back(MemoryEffects::Read::get()); + effects.emplace_back(MemoryEffects::Write::get()); PTO_ADD_WRITE(getDstMutable()); } PTO_DEFINE_BINARY_EFFECTS(TDivOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) @@ -10420,8 +10315,6 @@ void TDeinterleaveOp::getEffects( } PTO_DEFINE_BINARY_EFFECTS(TInterleaveOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TLogOp, getSrcMutable(), getDstMutable()) -PTO_DEFINE_UNARY_EFFECTS(TLReluOp, getSrcMutable(), getDstMutable()) - PTO_DEFINE_BINARY_EFFECTS(TMaxOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TMaxSOp, getSrcMutable(), getDstMutable()) PTO_DEFINE_BINARY_EFFECTS(TMinOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) @@ -10501,8 +10394,6 @@ PTO_DEFINE_TERNARY_EFFECTS(TDequantOp, getSrcMutable(), getScaleMutable(), getOffsetMutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TRecipOp, getSrcMutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TReluOp, getSrcMutable(), getDstMutable()) -PTO_DEFINE_BINARY_EFFECTS(TFModOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) -PTO_DEFINE_UNARY_EFFECTS(TFModSOp, getSrcMutable(), getDstMutable()) void TRemOp::getEffects( SmallVectorImpl> &effects) { PTO_ADD_READ(getSrc0Mutable()); @@ -10666,22 +10557,17 @@ PTO_DEFINE_BINARY_EFFECTS(TShrOp, getSrc0Mutable(), getSrc1Mutable(), getDstMuta PTO_DEFINE_UNARY_EFFECTS(TShlSOp, getSrcMutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TShrSOp, getSrcMutable(), getDstMutable()) -// TSORT32: Read(src, idx) -> Write(dst [, tmp]) -void TSort32Op::getEffects( +// TSORT: Read(src) -> Write(dst, dst_indices) +void TSortOp::getEffects( SmallVectorImpl> &effects) { PTO_ADD_READ(getSrcMutable()); - PTO_ADD_READ(getIdxMutable()); - auto tmp = getTmpMutable(); - if (!tmp.empty()) - PTO_ADD_WRITE(tmp[0]); PTO_ADD_WRITE(getDstMutable()); + PTO_ADD_WRITE(getDstIndicesMutable()); } PTO_DEFINE_UNARY_EFFECTS(TSqrtOp, getSrcMutable(), getDstMutable()) PTO_DEFINE_BINARY_EFFECTS(TSubOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) -PTO_DEFINE_TERNARY_EFFECTS(TSubCOp, getSrc0Mutable(), getSrc1Mutable(), getSrc2Mutable(), getDstMutable()) PTO_DEFINE_UNARY_EFFECTS(TSubSOp, getSrcMutable(), getDstMutable()) -PTO_DEFINE_BINARY_EFFECTS(TSubSCOp, getSrc0Mutable(), getSrc1Mutable(), getDstMutable()) // TXORS: Read(src) -> Write(tmp, dst) void TXorSOp::getEffects( diff --git a/lib/PTO/Transforms/PTOMaterializeTileHandles.cpp b/lib/PTO/Transforms/PTOMaterializeTileHandles.cpp index f0f3a3d6ff..731621f0dd 100644 --- a/lib/PTO/Transforms/PTOMaterializeTileHandles.cpp +++ b/lib/PTO/Transforms/PTOMaterializeTileHandles.cpp @@ -358,7 +358,7 @@ getMaterializedTileShape(MemRefType memTy, const TileHandleMetadata &meta) { SmallVector inheritedStrides; int64_t inheritedOffset = ShapedType::kDynamic; if (failed(sourceMrTy.getStridesAndOffset(inheritedStrides, - inheritedOffset)) || + inheritedOffset)) || inheritedStrides.size() < 2) return shape; diff --git a/lib/PTO/Transforms/PTOToEmitC.cpp b/lib/PTO/Transforms/PTOToEmitC.cpp index 8989a21eba..1fe64e7373 100644 --- a/lib/PTO/Transforms/PTOToEmitC.cpp +++ b/lib/PTO/Transforms/PTOToEmitC.cpp @@ -2668,6 +2668,9 @@ struct PTOMGatherToMGATHER : public OpConversionPattern { Value memArg = maybeWrapGlobalMemrefAsGlobalTensor( rewriter, op.getLoc(), mem, op.getMem().getType(), op.getOperation()); + dst = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), dst); + memArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), memArg); + idx = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), idx); auto gatherOobTok = [&](pto::GatherOOB mode) -> StringRef { switch (mode) { @@ -2708,6 +2711,60 @@ struct PTOMGatherToMGATHER : public OpConversionPattern { } }; +struct PTOMGatherMaskToMGATHER_MASK + : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite(pto::MGatherMaskOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Value mem = peelUnrealized(adaptor.getMem()); + Value idx = peelUnrealized(adaptor.getIdx()); + Value mask = peelUnrealized(adaptor.getMask()); + Value dst = peelUnrealized(adaptor.getDst()); + Value memArg = maybeWrapGlobalMemrefAsGlobalTensor( + rewriter, op.getLoc(), mem, op.getMem().getType(), op.getOperation()); + dst = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), dst); + memArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), memArg); + idx = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), idx); + mask = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), mask); + + rewriter.create( + op.getLoc(), TypeRange{}, "MGATHER_MASK", + ArrayAttr{}, ArrayAttr{}, ValueRange{dst, memArg, idx, mask}); + rewriter.eraseOp(op); + return success(); + } +}; + +struct PTOMGatherCasToMGATHER_CAS + : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite(pto::MGatherCasOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Value mem = peelUnrealized(adaptor.getMem()); + Value idx = peelUnrealized(adaptor.getIdx()); + Value expected = peelUnrealized(adaptor.getExpected()); + Value replacement = peelUnrealized(adaptor.getReplacement()); + Value dst = peelUnrealized(adaptor.getDst()); + Value memArg = maybeWrapGlobalMemrefAsGlobalTensor( + rewriter, op.getLoc(), mem, op.getMem().getType(), op.getOperation()); + dst = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), dst); + memArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), memArg); + idx = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), idx); + expected = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), expected); + replacement = + loadEmitCLValueIfNeeded(rewriter, op.getLoc(), replacement); + + rewriter.create( + op.getLoc(), TypeRange{}, "MGATHER_CAS", + ArrayAttr{}, ArrayAttr{}, + ValueRange{dst, memArg, idx, expected, replacement}); + rewriter.eraseOp(op); + return success(); + } +}; + struct AffineApplyMulConstToEmitC : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; @@ -4156,10 +4213,12 @@ struct PointerCastConversion : public OpConversionPattern { .getResult(0); } + Value assignDst = + loadEmitCLValueIfNeeded(rewriter, loc, resultValue); rewriter.create( loc, TypeRange{}, "TASSIGN", ArrayAttr{}, ArrayAttr{}, - ValueRange{resultValue, addr}); + ValueRange{assignDst, addr}); rewriter.replaceOp(op, resultValue); return success(); @@ -4193,6 +4252,8 @@ struct PTOTLoadToTLOAD : public OpConversionPattern { srcArg = gt; } } + dst = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), dst); + srcArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), srcArg); rewriter.create( op.getLoc(), TypeRange{}, "TLOAD", @@ -4213,28 +4274,14 @@ struct PTOTPrefetchToTPREFETCH : public OpConversionPattern { LogicalResult matchAndRewrite(pto::TPrefetchOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - if (!op.getDst()) - return rewriter.notifyMatchFailure(op, "expected outs(dst) on pto.tprefetch"); - - Value src = peelUnrealized(adaptor.getSrc()); - Value dst = peelUnrealized(adaptor.getDst()); - Value srcArg = src; - if (auto srcMrTy = dyn_cast(op.getSrc().getType())) { - bool isGlobal = true; - if (auto asAttr = dyn_cast_or_null(srcMrTy.getMemorySpace())) { - auto as = asAttr.getAddressSpace(); - isGlobal = (as == pto::AddressSpace::GM || as == pto::AddressSpace::Zero); - } - if (isGlobal) { - if (Value gt = buildGlobalTensorFromMemref(rewriter, op.getLoc(), src, srcMrTy, - op.getOperation())) - srcArg = gt; - } - } + Value address = peelUnrealized(adaptor.getAddress()); + Value byteCount = peelUnrealized(adaptor.getByteCount()); + address = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), address); + byteCount = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), byteCount); rewriter.create( op.getLoc(), TypeRange{}, "TPREFETCH", - ArrayAttr{}, ArrayAttr{}, ValueRange{dst, srcArg}); + ArrayAttr{}, ArrayAttr{}, ValueRange{address, byteCount}); rewriter.eraseOp(op); return success(); } @@ -5336,6 +5383,9 @@ struct PTOMScatterToMSCATTER : public OpConversionPattern { Value memArg = maybeWrapGlobalMemrefAsGlobalTensor( rewriter, op.getLoc(), mem, op.getMem().getType(), op.getOperation()); + memArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), memArg); + src = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), src); + idx = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), idx); auto scatterAtomicTok = [&](pto::ScatterAtomicOp atomic) -> StringRef { switch (atomic) { @@ -5388,6 +5438,31 @@ struct PTOMScatterToMSCATTER : public OpConversionPattern { return success(); } }; + +struct PTOMScatterMaskToMSCATTER_MASK + : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite(pto::MScatterMaskOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Value src = peelUnrealized(adaptor.getSrc()); + Value idx = peelUnrealized(adaptor.getIdx()); + Value mask = peelUnrealized(adaptor.getMask()); + Value mem = peelUnrealized(adaptor.getMem()); + Value memArg = maybeWrapGlobalMemrefAsGlobalTensor( + rewriter, op.getLoc(), mem, op.getMem().getType(), op.getOperation()); + memArg = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), memArg); + src = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), src); + idx = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), idx); + mask = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), mask); + + rewriter.create( + op.getLoc(), TypeRange{}, "MSCATTER_MASK", + ArrayAttr{}, ArrayAttr{}, ValueRange{memArg, src, idx, mask}); + rewriter.eraseOp(op); + return success(); + } +}; struct PTOSetValToSETVAL : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; @@ -7017,36 +7092,6 @@ struct ReinterpretCastToEmitC : public OpConversionPattern TADDC(dst, src0, src1, src2) -//===----------------------------------------------------------------------===// - -struct PTOTAddCToTADDC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TAddCOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - Value src0 = peelUnrealized(adaptor.getSrc0()); - Value src1 = peelUnrealized(adaptor.getSrc1()); - Value src2 = peelUnrealized(adaptor.getSrc2()); - Value dst = peelUnrealized(adaptor.getDst()); - - // pto-isa does not provide NPU implementation for TADDC yet. - // Decompose: dst = src0 + src1 + src2 - rewriter.create( - loc, TypeRange{}, "TADD", - ArrayAttr{}, ArrayAttr{}, - ValueRange{dst, src0, src1}); - rewriter.create( - loc, TypeRange{}, "TADD", - ArrayAttr{}, ArrayAttr{}, - ValueRange{dst, dst, src2}); - - rewriter.eraseOp(op); - return success(); - } -}; -//===----------------------------------------------------------------------===// // pto.tadds lowering -> TADDS(dst, src, scalar) //===----------------------------------------------------------------------===// @@ -7068,36 +7113,6 @@ struct PTOAddSToTADDS : public OpConversionPattern { return success(); } }; -//===----------------------------------------------------------------------===// -// pto.taddsc lowering -> TADDSC(dst, src0, scalar, src1) -//===----------------------------------------------------------------------===// - -struct PTOAddSCToTADDSC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TAddSCOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - Value src0 = peelUnrealized(adaptor.getSrc0()); - Value scalar = peelUnrealized(adaptor.getScalar()); - Value src1 = peelUnrealized(adaptor.getSrc1()); - Value dst = peelUnrealized(adaptor.getDst()); - - // pto-isa does not provide NPU implementation for TADDSC yet. - // Decompose: dst = src0 + scalar + src1 - rewriter.create( - loc, TypeRange{}, "TADDS", - ArrayAttr{}, ArrayAttr{}, - ValueRange{dst, src0, scalar}); - rewriter.create( - loc, TypeRange{}, "TADD", - ArrayAttr{}, ArrayAttr{}, - ValueRange{dst, dst, src1}); - - rewriter.eraseOp(op); - return success(); - } -}; struct PTOTAndToEmitC : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; @@ -7719,30 +7734,18 @@ struct PTOCvtToEmitC : public OpConversionPattern { return success(); } }; -struct PTORandomToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(pto::TRandomOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - auto *ctx = rewriter.getContext(); +struct PTOAccCvtToEmitC : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(pto::AccCvtOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { Value dst = peelUnrealized(adaptor.getDst()); - SmallVector operands{ - dst, - peelUnrealized(adaptor.getKey0()), - peelUnrealized(adaptor.getKey1()), - peelUnrealized(adaptor.getCounter0()), - peelUnrealized(adaptor.getCounter1()), - peelUnrealized(adaptor.getCounter2()), - peelUnrealized(adaptor.getCounter3()), - }; - ArrayAttr templateArgs = rewriter.getArrayAttr( - {emitc::OpaqueAttr::get(ctx, std::to_string(op.getRounds()))}); - + dst = loadEmitCLValueIfNeeded(rewriter, op.getLoc(), dst); rewriter.create( - loc, TypeRange{}, "PTOAS__TRANDOM", - /*args=*/ArrayAttr{}, /*templateArgs=*/templateArgs, operands); + op.getLoc(), TypeRange{}, "ACCCVT", + /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, + /*operands=*/ValueRange{dst}); rewriter.eraseOp(op); return success(); } @@ -8271,33 +8274,6 @@ struct PTOLogToEmitC : public OpConversionPattern { -//===----------------------------------------------------------------------===// -// TLRELU lowering to EmitC (PTOConvert.cpp) -//===----------------------------------------------------------------------===// - - struct PTOLReluToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TLReluOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - - Value src = peelUnrealized(adaptor.getSrc()); - Value slope = peelUnrealized(adaptor.getSlope()); - Value dst = peelUnrealized(adaptor.getDst()); - - SmallVector operands{dst, src, slope}; - - rewriter.create( - loc, TypeRange{}, "TLRELU", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/operands); - - rewriter.eraseOp(op); - return success(); - } -}; - //===----------------------------------------------------------------------===// // TMAX lowering to EmitC (PTOConvert.cpp) //===----------------------------------------------------------------------===// @@ -9125,27 +9101,6 @@ struct PTORemToEmitC : public OpConversionPattern { } }; -struct PTOFModToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TFModOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - - Value src0 = peelUnrealized(adaptor.getSrc0()); - Value src1 = peelUnrealized(adaptor.getSrc1()); - Value dst = peelUnrealized(adaptor.getDst()); - - SmallVector operands{dst, src0, src1}; - rewriter.create( - loc, TypeRange{}, "TFMOD", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/operands); - - rewriter.eraseOp(op); - return success(); - } -}; //===----------------------------------------------------------------------===// // PTOConvert.cpp (add lowering + patterns.add for TREMS DPS/memref op) //===----------------------------------------------------------------------===// @@ -9172,28 +9127,6 @@ struct PTORemSToEmitC : public OpConversionPattern { } }; -struct PTOFModSToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TFModSOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - - Value src = peelUnrealized(adaptor.getSrc()); - Value dst = peelUnrealized(adaptor.getDst()); - Value scalar = peelUnrealized(adaptor.getScalar()); - - SmallVector operands{dst, src, scalar}; - rewriter.create( - loc, TypeRange{}, "TFMODS", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/operands); - - rewriter.eraseOp(op); - return success(); - } -}; - //===----------------------------------------------------------------------===// // PTOConvert.cpp (add lowering + patterns.add for TROWEXPAND DPS/memref op) //===----------------------------------------------------------------------===// @@ -9930,30 +9863,32 @@ struct PTOShrSConstToEmitC : public OpConversionPattern { }; //===----------------------------------------------------------------------===// -// PTOConvert.cpp (TSORT32 DPS/memref op: ins(src, idx[, tmp]) outs(dst)) +// PTOConvert.cpp (TSORT DPS/memref op: ins(src) outs(dst, dst_indices)) //===----------------------------------------------------------------------===// -struct PTOSORT32SToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; +struct PTOSORTToEmitC : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(pto::TSort32Op op, OpAdaptor adaptor, + LogicalResult matchAndRewrite(pto::TSortOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto loc = op.getLoc(); + auto *ctx = rewriter.getContext(); Value src = peelUnrealized(adaptor.getSrc()); Value dst = peelUnrealized(adaptor.getDst()); - Value idx = peelUnrealized(adaptor.getIdx()); - Value tmp = op.getTmp() ? peelUnrealized(adaptor.getTmp()) : Value(); + Value dstIndices = peelUnrealized(adaptor.getDstIndices()); + src = loadEmitCLValueIfNeeded(rewriter, loc, src); + dst = loadEmitCLValueIfNeeded(rewriter, loc, dst); + dstIndices = loadEmitCLValueIfNeeded(rewriter, loc, dstIndices); + auto boolTy = emitc::OpaqueType::get(ctx, "bool"); + Value descending = rewriter.create( + loc, boolTy, + emitc::OpaqueAttr::get(ctx, op.getDescending() ? "true" : "false")); - SmallVector operands; - if (tmp) - operands.assign({dst, src, idx, tmp}); - else - operands.assign({dst, src, idx}); rewriter.create( - loc, TypeRange{}, "TSORT32", + loc, TypeRange{}, "TSORT", /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/operands); + /*operands=*/ValueRange{dst, dstIndices, src, descending}); rewriter.eraseOp(op); return success(); @@ -10036,37 +9971,6 @@ struct PTOSubSToEmitC : public OpConversionPattern { } }; //===----------------------------------------------------------------------===// -// PTOConvert.cpp (add lowering + patterns.add for TSUBC DPS/memref op) -//===----------------------------------------------------------------------===// - -struct PTOSubCSToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TSubCOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - - Value src0 = peelUnrealized(adaptor.getSrc0()); - Value src1 = peelUnrealized(adaptor.getSrc1()); - Value src2 = peelUnrealized(adaptor.getSrc2()); - Value dst = peelUnrealized(adaptor.getDst()); - - // pto-isa does not provide NPU implementation for TSUBC yet. - // Decompose: dst = src0 - src1 + src2 - rewriter.create( - loc, TypeRange{}, "TSUB", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/ValueRange{dst, src0, src1}); - rewriter.create( - loc, TypeRange{}, "TADD", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/ValueRange{dst, dst, src2}); - - rewriter.eraseOp(op); - return success(); - } -}; -//===----------------------------------------------------------------------===// // PTOConvert.cpp (add lowering + patterns.add for TSUBS DPS/memref op) //===----------------------------------------------------------------------===// @@ -10091,39 +9995,6 @@ struct PTOSubSSToEmitC : public OpConversionPattern { return success(); } }; -//===----------------------------------------------------------------------===// -// PTOConvert.cpp (add lowering + patterns.add for TSUBSC DPS/memref op) -//===----------------------------------------------------------------------===// - -struct PTOSubSCToEmitC : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchAndRewrite(pto::TSubSCOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - auto loc = op.getLoc(); - - Value src0 = peelUnrealized(adaptor.getSrc0()); - Value scalar = peelUnrealized(adaptor.getScalar()); - Value src1 = peelUnrealized(adaptor.getSrc1()); - Value dst = peelUnrealized(adaptor.getDst()); - - // pto-isa does not provide NPU implementation for TSUBSC yet. - // Decompose: dst = src0 - scalar + src1 - rewriter.create( - loc, TypeRange{}, "TSUBS", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/ValueRange{dst, src0, scalar}); - rewriter.create( - loc, TypeRange{}, "TADD", - /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{}, - /*operands=*/ValueRange{dst, dst, src1}); - - rewriter.eraseOp(op); - return success(); - } -}; - - //===----------------------------------------------------------------------===// // PTOConvert.cpp (add lowering + patterns.add for TXOR DPS/memref op) //===----------------------------------------------------------------------===// @@ -11856,8 +11727,6 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, "set_flag"); patterns.add(typeConverter, ctx, "pto.wait_flag_d", "wait_flag"); - patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -11891,7 +11760,7 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); + patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -11905,14 +11774,12 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -11946,9 +11813,7 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); + patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -12019,10 +11884,11 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); + patterns.add( + typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); + patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -12030,7 +11896,6 @@ static void populatePTOToEmitCPatterns(RewritePatternSet &patterns, patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); - patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); patterns.add(typeConverter, ctx); @@ -12164,14 +12029,11 @@ struct EmitPTOManualPass } bool needsEventIdArrayHelper = false; - bool needsTRandomHelper = false; bool needsGlobalTensorDataHelper = false; bool needsCommInclude = false; mop.walk([&](Operation *op) { if (isa(op)) needsEventIdArrayHelper = true; - if (isa(op)) - needsTRandomHelper = true; if (isa(op)) needsGlobalTensorDataHelper = true; if (isa( - loc, builder.getStringAttr(R"cpp( -template -static AICORE inline void PTOAS__TRANDOM( - DstTile &dst, uint32_t key0, uint32_t key1, uint32_t counter0, - uint32_t counter1, uint32_t counter2, uint32_t counter3) { - TRandomKey key = {key0, key1}; - TRandomCounter counter = {counter0, counter1, counter2, counter3}; - TRANDOM(dst, key, counter); -} )cpp")); } builder.create( diff --git a/lib/PTO/Transforms/PTOViewToMemref.cpp b/lib/PTO/Transforms/PTOViewToMemref.cpp index 6706cd56ca..4db2aa6246 100644 --- a/lib/PTO/Transforms/PTOViewToMemref.cpp +++ b/lib/PTO/Transforms/PTOViewToMemref.cpp @@ -1911,37 +1911,6 @@ struct PTOViewToMemrefPass dst); } - SmallVector addcops; - func.walk([&](mlir::pto::TAddCOp op) { addcops.push_back(op); }); - - for (auto op : addcops) { - IRRewriter rewriter(ctx); - rewriter.setInsertionPoint(op); - - Value src0 = op.getSrc0(); - Value src1 = op.getSrc1(); - Value src2 = op.getSrc2(); - Value dst = op.getDst(); - - auto src0Ty = dyn_cast(src0.getType()); - auto src1Ty = dyn_cast(src1.getType()); - auto src2Ty = dyn_cast(src2.getType()); - auto dstTy = dyn_cast(dst.getType()); - if (!src0Ty || !src1Ty || !src2Ty ||!dstTy) { - op.emitError("ins/outs are not memref yet"); - signalPassFailure(); - return; - } - - rewriter.replaceOpWithNewOp( - op, - TypeRange{}, - src0, - src1, - src2, - dst); - } - SmallVector addsops; func.walk([&](mlir::pto::TAddSOp op) { addsops.push_back(op); }); @@ -1969,36 +1938,6 @@ struct PTOViewToMemrefPass dst); } - SmallVector addscops; - func.walk([&](mlir::pto::TAddSCOp op) { addscops.push_back(op); }); - - for (auto op : addscops) { - IRRewriter rewriter(ctx); - rewriter.setInsertionPoint(op); - - Value src0 = op.getSrc0(); - Value scalar = op.getScalar(); - Value src1 = op.getSrc1(); - Value dst = op.getDst(); - - auto src0Ty = dyn_cast(src0.getType()); - auto src1Ty = dyn_cast(src1.getType()); - auto dstTy = dyn_cast(dst.getType()); - if (!src0Ty || !src1Ty || !dstTy) { - op.emitError("ins/outs are not memref yet"); - signalPassFailure(); - return; - } - - rewriter.replaceOpWithNewOp( - op, - TypeRange{}, - src0, - scalar, - src1, - dst); - } - SmallVector andops; func.walk([&](mlir::pto::TAndOp op) { andops.push_back(op); }); @@ -2865,34 +2804,6 @@ struct PTOViewToMemrefPass dst); } - SmallVector lreluops; - func.walk([&](mlir::pto::TLReluOp op) { lreluops.push_back(op); }); - - for (auto op : lreluops) { - IRRewriter rewriter(ctx); - rewriter.setInsertionPoint(op); - - Value src = op.getSrc(); - Value slope = op.getSlope(); - Value dst = op.getDst(); - - auto srcTy = dyn_cast(src.getType()); - auto slopeTy = dyn_cast(slope.getType()); - auto dstTy = dyn_cast(dst.getType()); - if (!srcTy || !slopeTy || !dstTy) { - op.emitError("ins/outs are not correct type yet"); - signalPassFailure(); - return; - } - - rewriter.replaceOpWithNewOp( - op, - TypeRange{}, - src, - slope, - dst); - } - SmallVector maxops; func.walk([&](mlir::pto::TMaxOp op) { maxops.push_back(op); }); @@ -3331,6 +3242,60 @@ struct PTOViewToMemrefPass op.getGatherOobAttr()); } + SmallVector mgatherMaskOps; + func.walk([&](mlir::pto::MGatherMaskOp op) { + mgatherMaskOps.push_back(op); + }); + + for (auto op : mgatherMaskOps) { + IRRewriter rewriter(ctx); + rewriter.setInsertionPoint(op); + + Value mem = op.getMem(); + Value idx = op.getIdx(); + Value mask = op.getMask(); + Value dst = op.getDst(); + if (!isa(mem.getType()) || + !isa(idx.getType()) || + !isa(mask.getType()) || + !isa(dst.getType())) { + op.emitError("ins/outs are not memref yet"); + signalPassFailure(); + return; + } + + rewriter.replaceOpWithNewOp( + op, TypeRange{}, mem, idx, mask, dst); + } + + SmallVector mgatherCasOps; + func.walk([&](mlir::pto::MGatherCasOp op) { + mgatherCasOps.push_back(op); + }); + + for (auto op : mgatherCasOps) { + IRRewriter rewriter(ctx); + rewriter.setInsertionPoint(op); + + Value mem = op.getMem(); + Value idx = op.getIdx(); + Value expected = op.getExpected(); + Value replacement = op.getReplacement(); + Value dst = op.getDst(); + if (!isa(mem.getType()) || + !isa(idx.getType()) || + !isa(expected.getType()) || + !isa(replacement.getType()) || + !isa(dst.getType())) { + op.emitError("ins/outs are not memref yet"); + signalPassFailure(); + return; + } + + rewriter.replaceOpWithNewOp( + op, TypeRange{}, mem, idx, expected, replacement, dst); + } + SmallVector mascatterops; func.walk([&](mlir::pto::MScatterOp op) { mascatterops.push_back(op); }); @@ -3360,6 +3325,32 @@ struct PTOViewToMemrefPass op.getScatterAtomicOpAttr(), op.getScatterOobAttr()); } + + SmallVector mscatterMaskOps; + func.walk([&](mlir::pto::MScatterMaskOp op) { + mscatterMaskOps.push_back(op); + }); + + for (auto op : mscatterMaskOps) { + IRRewriter rewriter(ctx); + rewriter.setInsertionPoint(op); + + Value src = op.getSrc(); + Value idx = op.getIdx(); + Value mask = op.getMask(); + Value mem = op.getMem(); + if (!isa(src.getType()) || + !isa(idx.getType()) || + !isa(mask.getType()) || + !isa(mem.getType())) { + op.emitError("ins/outs are not memref yet"); + signalPassFailure(); + return; + } + + rewriter.replaceOpWithNewOp( + op, TypeRange{}, src, idx, mask, mem); + } SmallVector printops; func.walk([&](mlir::pto::TPrintOp op) { printops.push_back(op); }); diff --git a/test/lit/pto/acccvt_verify_acc_destination_rejected.pto b/test/lit/pto/acccvt_verify_acc_destination_rejected.pto new file mode 100644 index 0000000000..b2dadff3f4 --- /dev/null +++ b/test/lit/pto/acccvt_verify_acc_destination_rejected.pto @@ -0,0 +1,11 @@ +// RUN: not ptoas --pto-arch=a5 %s 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5"} { + func.func @acccvt_rejects_explicit_acc_destination() { + %dst = pto.alloc_tile : !pto.tile_buf + pto.acccvt outs(%dst : !pto.tile_buf) + return + } +} + +// CHECK: error: 'pto.acccvt' op expects explicit dst to be a publishable tile/memref, not loc=acc; ACCCVT reads ACC implicitly diff --git a/test/lit/pto/tgemv_mx_emitc.pto b/test/lit/pto/tgemv_mx_emitc.pto index 25f5a3d125..96293f6d6c 100644 --- a/test/lit/pto/tgemv_mx_emitc.pto +++ b/test/lit/pto/tgemv_mx_emitc.pto @@ -13,4 +13,6 @@ module { } } -// CHECK: error: 'pto.tgemv.mx' op lhs: dtype 'f8E4M3' is not supported by this op yet +// MX operand verification accepts FP8 function arguments. EmitC lowering still +// stops when function-argument materialization tries to allocate an FP8 tile. +// CHECK: error: 'pto.alloc_tile' op result dtype 'f8E4M3' is not supported by pto.alloc_tile yet diff --git a/test/lit/pto/tgemv_mx_variants_emitc.pto b/test/lit/pto/tgemv_mx_variants_emitc.pto index 5c5e07755f..e5b3e88295 100644 --- a/test/lit/pto/tgemv_mx_variants_emitc.pto +++ b/test/lit/pto/tgemv_mx_variants_emitc.pto @@ -17,4 +17,6 @@ module { } } -// CHECK: error: 'pto.tgemv.mx.acc' op lhs: dtype 'f8E4M3' is not supported by this op yet +// MX operand verification accepts FP8 function arguments. EmitC lowering still +// stops when function-argument materialization tries to allocate an FP8 tile. +// CHECK: error: 'pto.alloc_tile' op result dtype 'f8E4M3' is not supported by pto.alloc_tile yet diff --git a/test/lit/pto/tprefetch_emitc.pto b/test/lit/pto/tprefetch_emitc.pto index 5553eb4c17..70f307164d 100644 --- a/test/lit/pto/tprefetch_emitc.pto +++ b/test/lit/pto/tprefetch_emitc.pto @@ -1,19 +1,17 @@ // RUN: ptoas %s | FileCheck %s --check-prefix=A3 module attributes {pto.target_arch = "a3"} { - func.func @tprefetch_emitc(%arg0: !pto.ptr) { - %c0 = arith.constant 0 : index - %c64 = arith.constant 64 : index - %c16 = arith.constant 16 : index - %c1 = arith.constant 1 : index - %tv = pto.make_tensor_view %arg0, shape = [%c64, %c16], strides = [%c16, %c1] : !pto.tensor_view - %part = pto.partition_view %tv, offsets = [%c0, %c0], sizes = [%c16, %c16] - : !pto.tensor_view -> !pto.partition_tensor_view<16x16xf16> - %tile = pto.alloc_tile : !pto.tile_buf - pto.tprefetch ins(%part : !pto.partition_tensor_view<16x16xf16>) - outs(%tile : !pto.tile_buf) + func.func @tprefetch_emitc(%dynamic_bytes: index) { + %addr = arith.constant 4096 : i64 + %bytes = arith.constant 512 : index + %max_bytes = arith.constant 262144 : index + pto.tprefetch ins(%addr, %bytes : i64, index) + pto.tprefetch ins(%addr, %max_bytes : i64, index) + pto.tprefetch ins(%addr, %dynamic_bytes : i64, index) return } } // A3: TPREFETCH( +// A3: TPREFETCH( +// A3: TPREFETCH( diff --git a/test/lit/pto/tprefetch_verify_negative_byte_count.pto b/test/lit/pto/tprefetch_verify_negative_byte_count.pto new file mode 100644 index 0000000000..926ff08ae0 --- /dev/null +++ b/test/lit/pto/tprefetch_verify_negative_byte_count.pto @@ -0,0 +1,12 @@ +// RUN: not ptoas --pto-arch=a5 %s 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5"} { + func.func @tprefetch_rejects_negative_byte_count() { + %addr = arith.constant 4096 : i64 + %bytes = arith.constant -1 : index + pto.tprefetch ins(%addr, %bytes : i64, index) + return + } +} + +// CHECK: error: 'pto.tprefetch' op expects a static byte_count in the inclusive range 0..262144 diff --git a/test/lit/pto/tprefetch_verify_oversized_byte_count.pto b/test/lit/pto/tprefetch_verify_oversized_byte_count.pto new file mode 100644 index 0000000000..4026bb10ce --- /dev/null +++ b/test/lit/pto/tprefetch_verify_oversized_byte_count.pto @@ -0,0 +1,12 @@ +// RUN: not ptoas --pto-arch=a5 %s 2>&1 | FileCheck %s + +module attributes {pto.target_arch = "a5"} { + func.func @tprefetch_rejects_oversized_byte_count() { + %addr = arith.constant 4096 : i64 + %bytes = arith.constant 262145 : index + pto.tprefetch ins(%addr, %bytes : i64, index) + return + } +} + +// CHECK: error: 'pto.tprefetch' op expects a static byte_count in the inclusive range 0..262144 diff --git a/test/lit/pto/trandom_emitc.pto b/test/lit/pto/trandom_emitc.pto deleted file mode 100644 index 8e6a1838b2..0000000000 --- a/test/lit/pto/trandom_emitc.pto +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2026 Huawei Technologies Co., Ltd. -// This program is free software, you can redistribute it and/or modify it under the terms and conditions of -// CANN Open Software License Agreement Version 2.0 (the "License"). -// Please refer to the License for details. You may not use this file except in compliance with the License. -// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -// See LICENSE in the root of the software repository for the full text of the License. - -// RUN: ptoas --pto-arch=a5 %s 2>&1 | FileCheck %s --check-prefix=A5 - -module attributes {pto.target_arch = "a5"} { - func.func @trandom_rounds10() attributes {pto.kernel_kind = #pto.kernel_kind} { - %k0 = arith.constant 305419896 : i32 - %k1 = arith.constant -1698898192 : i32 - %c0 = arith.constant 0 : i32 - %c1 = arith.constant 1 : i32 - %c2 = arith.constant 2 : i32 - %c3 = arith.constant 3 : i32 - %dst = pto.alloc_tile : !pto.tile_buf - pto.trandom ins(%k0, %k1, %c0, %c1, %c2, %c3 : i32, i32, i32, i32, i32, i32) - outs(%dst : !pto.tile_buf) - return - } - - func.func @trandom_rounds7() attributes {pto.kernel_kind = #pto.kernel_kind} { - %k0 = arith.constant 324508639 : i32 - %k1 = arith.constant 610839776 : i32 - %c0 = arith.constant 16 : i32 - %c1 = arith.constant 32 : i32 - %c2 = arith.constant 48 : i32 - %c3 = arith.constant 64 : i32 - %dst = pto.alloc_tile : !pto.tile_buf - pto.trandom ins(%k0, %k1, %c0, %c1, %c2, %c3 {rounds = 7 : i32} : i32, i32, i32, i32, i32, i32) - outs(%dst : !pto.tile_buf) - return - } -} - -// A5: template -// A5-LABEL: AICORE void trandom_rounds10( -// A5: PTOAS__TRANDOM<10>( -// A5-LABEL: AICORE void trandom_rounds7( -// A5: PTOAS__TRANDOM<7>( diff --git a/test/lit/pto/tsort32_emitc.pto b/test/lit/pto/tsort32_emitc.pto deleted file mode 100644 index a49768169b..0000000000 --- a/test/lit/pto/tsort32_emitc.pto +++ /dev/null @@ -1,22 +0,0 @@ -// RUN: ptoas --pto-arch=a3 --mlir-print-ir-after=pto-view-to-memref %s 2>&1 | FileCheck %s --check-prefix=A3 - -module { - func.func @tsort32_forms() { - %src = pto.alloc_tile : !pto.tile_buf - %idx = pto.alloc_tile : !pto.tile_buf - %dst0 = pto.alloc_tile : !pto.tile_buf - pto.tsort32 ins(%src, %idx : !pto.tile_buf, !pto.tile_buf) outs(%dst0 : !pto.tile_buf) - - %tmp = pto.alloc_tile : !pto.tile_buf - %dst1 = pto.alloc_tile : !pto.tile_buf - pto.tsort32 ins(%src, %idx, %tmp : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst1 : !pto.tile_buf) - return - } -} - -// A3: pto.tsort32 ins([[SRC:%.*]], [[IDX:%.*]] : memref<1x32xf16 -// A3: outs([[DST0:%.*]] : memref<1x64xf16 -// A3: pto.tsort32 ins([[SRC]], [[IDX]], [[TMP:%.*]] : memref<1x32xf16 -// A3: outs([[DST1:%.*]] : memref<1x64xf16 -// A3: TSORT32([[VDST0:v[0-9]+]], [[VSRC:v[0-9]+]], [[VIDX:v[0-9]+]]); -// A3: TSORT32([[VDST1:v[0-9]+]], [[VSRC]], [[VIDX]], [[VTMP:v[0-9]+]]); diff --git a/test/lit/pto/tsort_emitc.pto b/test/lit/pto/tsort_emitc.pto new file mode 100644 index 0000000000..4fe7206823 --- /dev/null +++ b/test/lit/pto/tsort_emitc.pto @@ -0,0 +1,28 @@ +// RUN: ptoas --pto-arch=a3 --mlir-print-ir-after=pto-view-to-memref %s 2>&1 | FileCheck %s --check-prefix=VIEW +// RUN: ptoas --pto-arch=a3 %s | FileCheck %s --check-prefix=EMITC + +module { + func.func @tsort_forms() { + %src = pto.alloc_tile : !pto.tile_buf + %dst_asc = pto.alloc_tile : !pto.tile_buf + %idx_asc = pto.alloc_tile : !pto.tile_buf + pto.tsort ins(%src : !pto.tile_buf) outs(%dst_asc, %idx_asc : !pto.tile_buf, !pto.tile_buf) + + %dst_desc = pto.alloc_tile : !pto.tile_buf + %idx_desc = pto.alloc_tile : !pto.tile_buf + pto.tsort ins(%src : !pto.tile_buf) outs(%dst_desc, %idx_desc : !pto.tile_buf, !pto.tile_buf) {descending = true} + return + } +} + +// VIEW: pto.tsort ins([[SRC:%[0-9]+]] : memref<1x64xf16 +// VIEW-SAME: outs([[DST_ASC:%[0-9]+]], [[IDX_ASC:%[0-9]+]] : memref<1x64xf16 +// VIEW-SAME: memref<1x64xui32 +// VIEW: pto.tsort ins([[SRC]] : memref<1x64xf16 +// VIEW-SAME: outs([[DST_DESC:%[0-9]+]], [[IDX_DESC:%[0-9]+]] : memref<1x64xf16 +// VIEW-SAME: memref<1x64xui32 +// VIEW-SAME: {descending = true} +// EMITC-DAG: const bool [[ASC:v[0-9]+]] = false; +// EMITC-DAG: const bool [[DESC:v[0-9]+]] = true; +// EMITC: TSORT([[VDST_ASC:v[0-9]+]], [[VIDX_ASC:v[0-9]+]], [[VSRC:v[0-9]+]], [[ASC]]); +// EMITC: TSORT([[VDST_DESC:v[0-9]+]], [[VIDX_DESC:v[0-9]+]], [[VSRC_DESC:v[0-9]+]], [[DESC]]); diff --git a/test/lit/pto/tsort_old_form_rejected.pto b/test/lit/pto/tsort_old_form_rejected.pto new file mode 100644 index 0000000000..fa897fa01e --- /dev/null +++ b/test/lit/pto/tsort_old_form_rejected.pto @@ -0,0 +1,13 @@ +// RUN: not ptoas --pto-arch=a3 %s 2>&1 | FileCheck %s + +module { + func.func @old_tsort_form_is_not_an_alias() { + %src = pto.alloc_tile : !pto.tile_buf + %idx = pto.alloc_tile : !pto.tile_buf + %dst = pto.alloc_tile : !pto.tile_buf + pto.tsort ins(%src, %idx : !pto.tile_buf, !pto.tile_buf) outs(%dst : !pto.tile_buf) + return + } +} + +// CHECK: error: expected ':' diff --git a/test/lit/pto/tsort_verify_aliased_outputs.pto b/test/lit/pto/tsort_verify_aliased_outputs.pto new file mode 100644 index 0000000000..fed5db8ef4 --- /dev/null +++ b/test/lit/pto/tsort_verify_aliased_outputs.pto @@ -0,0 +1,12 @@ +// RUN: not ptoas --pto-arch=a3 %s 2>&1 | FileCheck %s + +module { + func.func @aliased_outputs_are_illegal() { + %src = pto.alloc_tile : !pto.tile_buf + %dst = pto.alloc_tile : !pto.tile_buf + pto.tsort ins(%src : !pto.tile_buf) outs(%dst, %dst : !pto.tile_buf, !pto.tile_buf) + return + } +} + +// CHECK: error: 'pto.tsort' op expects dst and dst_indices to be distinct destinations diff --git a/test/lit/pto/tsort_verify_invalid_group.pto b/test/lit/pto/tsort_verify_invalid_group.pto new file mode 100644 index 0000000000..89c7859d76 --- /dev/null +++ b/test/lit/pto/tsort_verify_invalid_group.pto @@ -0,0 +1,13 @@ +// RUN: not ptoas --pto-arch=a3 %s 2>&1 | FileCheck %s + +module { + func.func @partial_group_is_illegal() { + %src = pto.alloc_tile : !pto.tile_buf + %dst = pto.alloc_tile : !pto.tile_buf + %idx = pto.alloc_tile : !pto.tile_buf + pto.tsort ins(%src : !pto.tile_buf) outs(%dst, %idx : !pto.tile_buf, !pto.tile_buf) + return + } +} + +// CHECK: error: 'pto.tsort' op expects the valid element count to contain complete 32-element groups diff --git a/test/lit/pto/v0571_direct_ops_emitc.pto b/test/lit/pto/v0571_direct_ops_emitc.pto new file mode 100644 index 0000000000..2d8aeced99 --- /dev/null +++ b/test/lit/pto/v0571_direct_ops_emitc.pto @@ -0,0 +1,58 @@ +// RUN: ptoas --pto-arch=a5 %s | FileCheck %s +// RUN: ptoas --pto-arch=a5 --mlir-print-ir-after=pto-view-to-memref %s 2>&1 | FileCheck %s --check-prefix=VIEW + +module attributes {pto.target_arch = "a5"} { + func.func @v0571_direct_ops(%mem: !pto.ptr, %idx_mem: !pto.ptr) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c16 = arith.constant 16 : index + %c256 = arith.constant 256 : index + + %mem_view = pto.make_tensor_view %mem, + shape = [%c256], strides = [%c1] {layout = #pto.layout} + : !pto.tensor_view + %idx_view = pto.make_tensor_view %idx_mem, + shape = [%c16, %c16], strides = [%c16, %c1] {layout = #pto.layout} + : !pto.tensor_view + %mem_part = pto.partition_view %mem_view, offsets = [%c0], sizes = [%c256] + : !pto.tensor_view -> !pto.partition_tensor_view<256xf32> + %idx_part = pto.partition_view %idx_view, offsets = [%c0, %c0], sizes = [%c16, %c16] + : !pto.tensor_view -> !pto.partition_tensor_view<16x16xi32> + + %idx = pto.alloc_tile : !pto.tile_buf + %src = pto.alloc_tile : !pto.tile_buf + %dst = pto.alloc_tile : !pto.tile_buf + %mask = pto.alloc_tile : !pto.tile_buf + %acc_dst = pto.alloc_tile : !pto.tile_buf + + pto.tload ins(%idx_part : !pto.partition_tensor_view<16x16xi32>) + outs(%idx : !pto.tile_buf) + pto.mgather_mask ins(%mem_part, %idx, %mask : !pto.partition_tensor_view<256xf32>, !pto.tile_buf, !pto.tile_buf) + outs(%dst : !pto.tile_buf) + pto.mscatter_mask ins(%src, %idx, %mask : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) + outs(%mem_part : !pto.partition_tensor_view<256xf32>) + pto.mgather_cas ins(%mem_part, %idx, %src, %dst : !pto.partition_tensor_view<256xf32>, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) + outs(%dst : !pto.tile_buf) + pto.acccvt outs(%acc_dst : !pto.tile_buf) + return + } +} + +// CHECK: MGATHER_MASK( +// CHECK: MSCATTER_MASK( +// CHECK: MGATHER_CAS( +// CHECK: ACCCVT( +// VIEW: pto.mgather_mask ins({{.*}} : memref<256xf32 +// VIEW-SAME: memref<16x16xi32 +// VIEW-SAME: memref<16x16xi32 +// VIEW-SAME: outs({{.*}} : memref<16x16xf32 +// VIEW: pto.mscatter_mask ins({{.*}} : memref<16x16xf32 +// VIEW-SAME: memref<16x16xi32 +// VIEW-SAME: memref<16x16xi32 +// VIEW-SAME: outs({{.*}} : memref<256xf32 +// VIEW: pto.mgather_cas ins({{.*}} : memref<256xf32 +// VIEW-SAME: memref<16x16xi32 +// VIEW-SAME: memref<16x16xf32 +// VIEW-SAME: memref<16x16xf32 +// VIEW-SAME: outs({{.*}} : memref<16x16xf32 diff --git a/test/samples/Addc/addc.pto b/test/samples/Addc/addc.pto deleted file mode 100644 index e352f25389..0000000000 --- a/test/samples/Addc/addc.pto +++ /dev/null @@ -1,26 +0,0 @@ -module { - func.func @vec_add_ternary_kernel_2d(%arg0: !pto.ptr, %arg1: !pto.ptr, %arg2: !pto.ptr, %arg3: !pto.ptr) { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c32 = arith.constant 32 : index - %0 = pto.make_tensor_view %arg0, shape = [%c32, %c32], strides = [%c32, %c1] : !pto.tensor_view<32x32xf32> - %1 = pto.make_tensor_view %arg1, shape = [%c32, %c32], strides = [%c32, %c1] : !pto.tensor_view<32x32xf32> - %2 = pto.make_tensor_view %arg2, shape = [%c32, %c32], strides = [%c32, %c1] : !pto.tensor_view<32x32xf32> - %3 = pto.make_tensor_view %arg3, shape = [%c32, %c32], strides = [%c32, %c1] : !pto.tensor_view<32x32xf32> - %4 = pto.partition_view %0, offsets = [%c0, %c0], sizes = [%c32, %c32] : !pto.tensor_view<32x32xf32> -> !pto.partition_tensor_view<32x32xf32> - %5 = pto.partition_view %1, offsets = [%c0, %c0], sizes = [%c32, %c32] : !pto.tensor_view<32x32xf32> -> !pto.partition_tensor_view<32x32xf32> - %6 = pto.partition_view %2, offsets = [%c0, %c0], sizes = [%c32, %c32] : !pto.tensor_view<32x32xf32> -> !pto.partition_tensor_view<32x32xf32> - %7 = pto.alloc_tile : !pto.tile_buf - %8 = pto.alloc_tile : !pto.tile_buf - %9 = pto.alloc_tile : !pto.tile_buf - %10 = pto.alloc_tile : !pto.tile_buf - pto.tload ins(%4 : !pto.partition_tensor_view<32x32xf32>) outs(%7 : !pto.tile_buf) - pto.tload ins(%5 : !pto.partition_tensor_view<32x32xf32>) outs(%8 : !pto.tile_buf) - pto.tload ins(%6 : !pto.partition_tensor_view<32x32xf32>) outs(%9 : !pto.tile_buf) - pto.taddc ins(%7, %8, %9 : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%10 : !pto.tile_buf) - %11 = pto.partition_view %3, offsets = [%c0, %c0], sizes = [%c32, %c32] : !pto.tensor_view<32x32xf32> -> !pto.partition_tensor_view<32x32xf32> - pto.tstore ins(%10 : !pto.tile_buf) outs(%11 : !pto.partition_tensor_view<32x32xf32>) - return - } -} - diff --git a/test/samples/Addc/addc.py b/test/samples/Addc/addc.py deleted file mode 100644 index 048ffb255d..0000000000 --- a/test/samples/Addc/addc.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get([32, 32], f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32, ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("vec_add_ternary_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - # Create constants instead of immediate numbers - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - - arg0, arg1, arg2, arg3 = entry.arguments - - # Use constants in tensor views instead of literals - tv0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv1 = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - tv2 = pto.MakeTensorViewOp(tv2_f32, arg2, [c32, c32], [c32, c1]).result - tv3 = pto.MakeTensorViewOp(tv2_f32, arg3, [c32, c32], [c32, c1]).result - - # Use constants for offsets and sizes in SubviewOp - sv0 = pto.PartitionViewOp(tile_view_32, tv0, offsets=[c0, c0], sizes=[c32, c32]).result - sv1 = pto.PartitionViewOp(tile_view_32, tv1, offsets=[c0, c0], sizes=[c32, c32]).result - sv2 = pto.PartitionViewOp(tile_view_32, tv2, offsets=[c0, c0], sizes=[c32, c32]).result - - # Allocate tiles - tb0 = pto.AllocTileOp(tile_buf_32).result - tb1 = pto.AllocTileOp(tile_buf_32).result - tb2 = pto.AllocTileOp(tile_buf_32).result - tb3 = pto.AllocTileOp(tile_buf_32).result - - # Load data into tiles - pto.TLoadOp(None, sv0, tb0) # result=None - pto.TLoadOp(None, sv1, tb1) # result=None - pto.TLoadOp(None, sv2, tb2) # result=None - - # Perform operation (Add in this case, it can be other operations) - pto.TAddCOp(tb0, tb1, tb2, tb3) - - # Subview on output tensor view - sv3 = pto.PartitionViewOp(tile_view_32, tv3, offsets=[c0, c0], sizes=[c32, c32]).result - - # Store the result - pto.TStoreOp(None, tb3, sv3) - - # Return from function - func.ReturnOp([]) - - # Verify the operation - m.operation.verify() - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Addc/addc_compare.py b/test/samples/Addc/addc_compare.py deleted file mode 100755 index 2a923d5faa..0000000000 --- a/test/samples/Addc/addc_compare.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import compare_outputs - -if __name__ == '__main__': - compare_outputs(np.float32, atol=0.0001) diff --git a/test/samples/Addc/addc_golden.py b/test/samples/Addc/addc_golden.py deleted file mode 100755 index e897dc7a8e..0000000000 --- a/test/samples/Addc/addc_golden.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import default_buffers, float_values, load_case_meta, rng, single_output, write_buffers, write_golden - - -def main(): - meta = load_case_meta() - a_name, b_name, c_name = meta.inputs - generator = rng() - a = float_values(generator, meta.elem_counts[a_name], style='signed') - b = float_values(generator, meta.elem_counts[b_name], style='signed') - c = float_values(generator, meta.elem_counts[c_name], style='signed_small') - buffers = default_buffers(meta) - buffers[a_name] = a - buffers[b_name] = b - buffers[c_name] = c - write_buffers(meta, buffers) - out = a + b + c - write_golden(meta, {single_output(meta): np.asarray(out, dtype=np.float32)}) - - -if __name__ == '__main__': - main() diff --git a/test/samples/Addsc/addsc.py b/test/samples/Addsc/addsc.py deleted file mode 100644 index 65b264c294..0000000000 --- a/test/samples/Addsc/addsc.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("vec_add_scalar_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - # constants - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - scale = arith.ConstantOp(f32, 3.14).result - - arg0, arg1 = entry.arguments - - # %0/%1/%2 = pto.make_tensor_view %arg?, shape=[%c32,%c32] strides=[%c32,%c1] - tv_src0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv_src1 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv_dst = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - - # Using constants for offsets and sizes in SubviewOp - sv_src0 = pto.PartitionViewOp(tile_view_32, tv_src0, offsets=[c0, c0], sizes=[c32, c32]).result - sv_src1 = pto.PartitionViewOp(tile_view_32, tv_src1, offsets=[c0, c0], sizes=[c32, c32]).result - - # Allocate tiles - tb_src0 = pto.AllocTileOp(tile_buf_32).result - tb_src1 = pto.AllocTileOp(tile_buf_32).result - tb_dst = pto.AllocTileOp(tile_buf_32).result - - # Load data into tiles - pto.TLoadOp(None, sv_src0, tb_src0) # result=None - pto.TLoadOp(None, sv_src1, tb_src1) - - # Perform operation (Add with scale) - pto.TAddSCOp(tb_src0, scale, tb_src1, tb_dst) - - # Subview on output tensor view - sv_dst = pto.PartitionViewOp(tile_view_32, tv_dst, offsets=[c0, c0], sizes=[c32, c32]).result - - # Store the result - pto.TStoreOp(None, tb_dst, sv_dst) - - # Return from function - func.ReturnOp([]) - - # Verify the operation - m.operation.verify() - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Addsc/addsc_compare.py b/test/samples/Addsc/addsc_compare.py deleted file mode 100755 index 2a923d5faa..0000000000 --- a/test/samples/Addsc/addsc_compare.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import compare_outputs - -if __name__ == '__main__': - compare_outputs(np.float32, atol=0.0001) diff --git a/test/samples/Addsc/addsc_golden.py b/test/samples/Addsc/addsc_golden.py deleted file mode 100755 index b2fb27d4d9..0000000000 --- a/test/samples/Addsc/addsc_golden.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import default_buffers, float_values, load_case_meta, rng, single_output, write_buffers, write_golden - - -def main(): - meta = load_case_meta() - [src_name] = meta.inputs - generator = rng() - src = float_values(generator, meta.elem_counts[src_name], style='signed') - buffers = default_buffers(meta) - buffers[src_name] = src - write_buffers(meta, buffers) - out = src + np.float32(3.14) + src - write_golden(meta, {single_output(meta): np.asarray(out, dtype=np.float32)}) - - -if __name__ == '__main__': - main() diff --git a/test/samples/Fmod/fmod.py b/test/samples/Fmod/fmod.py deleted file mode 100644 index 88dbe3fa39..0000000000 --- a/test/samples/Fmod/fmod.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("tfmod_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - - arg0, arg1, arg2 = entry.arguments - - tv0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv1 = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - tv2 = pto.MakeTensorViewOp(tv2_f32, arg2, [c32, c32], [c32, c1]).result - - sv0 = pto.PartitionViewOp(tile_view_32, tv0, offsets=[c0, c0], sizes=[c32, c32]).result - sv1 = pto.PartitionViewOp(tile_view_32, tv1, offsets=[c0, c0], sizes=[c32, c32]).result - - tb0 = pto.AllocTileOp(tile_buf_32).result - tb1 = pto.AllocTileOp(tile_buf_32).result - tb2 = pto.AllocTileOp(tile_buf_32).result - - pto.TLoadOp(None, sv0, tb0) - pto.TLoadOp(None, sv1, tb1) - pto.TFModOp(tb0, tb1, tb2) - - sv2 = pto.PartitionViewOp(tile_view_32, tv2, offsets=[c0, c0], sizes=[c32, c32]).result - pto.TStoreOp(None, tb2, sv2) - - func.ReturnOp([]) - - m.operation.verify() - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Fmods/fmods.py b/test/samples/Fmods/fmods.py deleted file mode 100644 index 6572582a14..0000000000 --- a/test/samples/Fmods/fmods.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("tfmods_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - scalar = arith.ConstantOp(f32, 3.25).result - - arg0, arg1 = entry.arguments - - tv0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv1 = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - - sv0 = pto.PartitionViewOp(tile_view_32, tv0, offsets=[c0, c0], sizes=[c32, c32]).result - - tb0 = pto.AllocTileOp(tile_buf_32).result - tb1 = pto.AllocTileOp(tile_buf_32).result - - pto.TLoadOp(None, sv0, tb0) - pto.TFModSOp(tb0, scalar, tb1) - - sv1 = pto.PartitionViewOp(tile_view_32, tv1, offsets=[c0, c0], sizes=[c32, c32]).result - pto.TStoreOp(None, tb1, sv1) - - func.ReturnOp([]) - - m.operation.verify() - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Fmods/tfmods_scalar_type_invalid.py b/test/samples/Fmods/tfmods_scalar_type_invalid.py deleted file mode 100644 index 179a762c21..0000000000 --- a/test/samples/Fmods/tfmods_scalar_type_invalid.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IntegerType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - i32 = IntegerType.get_signless(32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_ty = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([], []) - with InsertionPoint(m.body): - fn = func.FuncOp("tfmods_scalar_type_invalid", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - src = pto.AllocTileOp(tile_ty).result - dst = pto.AllocTileOp(tile_ty).result - scalar = arith.ConstantOp(i32, 7).result - pto.TFModSOp(src, scalar, dst) - func.ReturnOp([]) - - ok = m.operation.verify() - if ok: - return m - raise SystemExit(1) - - -if __name__ == "__main__": - try: - print(build()) - except MLIRError as err: - print(f"EXPECTED_VERIFIER_FAILURE: {err}") diff --git a/test/samples/Lrelu/lrelu.py b/test/samples/Lrelu/lrelu.py deleted file mode 100644 index 52c7ed383e..0000000000 --- a/test/samples/Lrelu/lrelu.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("vec_add_scalar_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - # constants - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - scale = arith.ConstantOp(f32, 3.14).result - - arg0, arg1 = entry.arguments - - # %0/%1/%2 = pto.make_tensor_view %arg?, shape=[%c32,%c32] strides=[%c32,%c1] - # 这里用原生 builder：通常签名会是 (result_type, ptr, shape, strides) - tv0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv1 = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - - # %3/%4/%8 = pto.subview %tv, offsets=[%c0,%c0], sizes=[32,32] - # Modify this line to use constant values instead of immediate numbers - sv0 = pto.PartitionViewOp(tile_view_32, tv0, offsets=[c0, c0], sizes=[c32, c32]).result - - # %5/%6/%7 = pto.alloc_tile : <32x32xf32> - tb0 = pto.AllocTileOp(tile_buf_32).result - tb1 = pto.AllocTileOp(tile_buf_32).result - - pto.TLoadOp(None, sv0, tb0) # result=None - - pto.TLReluOp(tb0, scale, tb1) - - # %8 = subview on output tensor_view - # Modify this line to use constant values instead of immediate numbers - sv1 = pto.PartitionViewOp(tile_view_32, tv1, offsets=[c0, c0], sizes=[c32, c32]).result - - # pto.store_dps_tb ins(%tb1) outs(%sv1) - pto.TStoreOp(None, tb1, sv1) - - func.ReturnOp([]) - - m.operation.verify() - - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Lrelu/lrelu_compare.py b/test/samples/Lrelu/lrelu_compare.py deleted file mode 100755 index 2a923d5faa..0000000000 --- a/test/samples/Lrelu/lrelu_compare.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import compare_outputs - -if __name__ == '__main__': - compare_outputs(np.float32, atol=0.0001) diff --git a/test/samples/Lrelu/lrelu_golden.py b/test/samples/Lrelu/lrelu_golden.py deleted file mode 100755 index 119e8711f7..0000000000 --- a/test/samples/Lrelu/lrelu_golden.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import default_buffers, float_values, load_case_meta, rng, single_output, write_buffers, write_golden - - -def main(): - meta = load_case_meta() - [src_name] = meta.inputs - generator = rng() - src = float_values(generator, meta.elem_counts[src_name], style='signed') - buffers = default_buffers(meta) - buffers[src_name] = src - write_buffers(meta, buffers) - out = np.where(src > 0.0, src, src * np.float32(3.14)) - write_golden(meta, {single_output(meta): np.asarray(out, dtype=np.float32)}) - - -if __name__ == '__main__': - main() diff --git a/test/samples/Sort32/sort32.py b/test/samples/Sort/sort.py similarity index 85% rename from test/samples/Sort32/sort32.py rename to test/samples/Sort/sort.py index 060ba26f3d..292b10c4ab 100644 --- a/test/samples/Sort32/sort32.py +++ b/test/samples/Sort/sort.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint +from mlir.ir import BoolAttr, Context, Location, Module, InsertionPoint from mlir.dialects import func, arith, pto from mlir.ir import F32Type, IntegerType, IndexType @@ -41,7 +41,7 @@ def build(): fn_ty = func.FunctionType.get([ptr_f32, ptr_f32, ptr_u32], []) with InsertionPoint(m.body): - fn = func.FuncOp("sort32_kernel", fn_ty) + fn = func.FuncOp("sort_kernel", fn_ty) entry = fn.add_entry_block() with InsertionPoint(entry): @@ -60,18 +60,16 @@ def build(): sv2 = pto.PartitionViewOp(tile_view_u32, tv2, offsets=[c0, c0], sizes=[c32, c32]).result tb_src = pto.AllocTileOp(tile_buf_f32).result - tb_stage0 = pto.AllocTileOp(tile_buf_f32).result tb_dst = pto.AllocTileOp(tile_buf_f32).result tb_idx = pto.AllocTileOp(tile_buf_u32).result - tb_tmp = pto.AllocTileOp(tile_buf_f32).result pto.TLoadOp(None, sv0, tb_src) - pto.TLoadOp(None, sv2, tb_idx) - - # Exercise the no-tmp form first. - pto.TSort32Op(src=tb_src, idx=tb_idx, dst=tb_stage0) - # Then exercise the tmp-taking form using the first result as input. - pto.TSort32Op(src=tb_stage0, idx=tb_idx, dst=tb_dst, tmp=tb_tmp) + pto.TSortOp( + src=tb_src, + dst=tb_dst, + dst_indices=tb_idx, + descending=BoolAttr.get(True), + ) pto.TStoreOp(None, tb_dst, sv1) pto.TStoreOp(None, tb_idx, sv2) diff --git a/test/samples/Subc/subc.py b/test/samples/Subc/subc.py deleted file mode 100644 index 4f6ebb04d9..0000000000 --- a/test/samples/Subc/subc.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32, ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("subc_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - # constants - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - - arg0, arg1, arg2, arg3 = entry.arguments - - # %0/%1/%2 = pto.make_tensor_view %arg?, shape=[%c32,%c32] strides=[%c32,%c1] - tv0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv1 = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - tv2 = pto.MakeTensorViewOp(tv2_f32, arg2, [c32, c32], [c32, c1]).result - tv3 = pto.MakeTensorViewOp(tv2_f32, arg3, [c32, c32], [c32, c1]).result - - # Replacing immediate values with constants c0 and c32 - sv0 = pto.PartitionViewOp(tile_view_32, tv0, offsets=[c0, c0], sizes=[c32, c32]).result - sv1 = pto.PartitionViewOp(tile_view_32, tv1, offsets=[c0, c0], sizes=[c32, c32]).result - sv2 = pto.PartitionViewOp(tile_view_32, tv2, offsets=[c0, c0], sizes=[c32, c32]).result - - # %5/%6/%7 = pto.alloc_tile : <32x32xf32> - tb0 = pto.AllocTileOp(tile_buf_32).result - tb1 = pto.AllocTileOp(tile_buf_32).result - tb2 = pto.AllocTileOp(tile_buf_32).result - tb3 = pto.AllocTileOp(tile_buf_32).result - - pto.TLoadOp(None, sv0, tb0) # result=None - pto.TLoadOp(None, sv1, tb1) # result=None - pto.TLoadOp(None, sv2, tb2) # result=None - - # Performing subtraction - pto.TSubCOp(tb0, tb1, tb2, tb3) - - # %8 = subview on output tensor_view - sv3 = pto.PartitionViewOp(tile_view_32, tv3, offsets=[c0, c0], sizes=[c32, c32]).result - - # Store the result back - pto.TStoreOp(None, tb3, sv3) - - func.ReturnOp([]) - - m.operation.verify() - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Subc/subc_compare.py b/test/samples/Subc/subc_compare.py deleted file mode 100755 index 2a923d5faa..0000000000 --- a/test/samples/Subc/subc_compare.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import compare_outputs - -if __name__ == '__main__': - compare_outputs(np.float32, atol=0.0001) diff --git a/test/samples/Subc/subc_golden.py b/test/samples/Subc/subc_golden.py deleted file mode 100755 index f782b05855..0000000000 --- a/test/samples/Subc/subc_golden.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import default_buffers, float_values, load_case_meta, rng, single_output, write_buffers, write_golden - - -def main(): - meta = load_case_meta() - a_name, b_name, c_name = meta.inputs - generator = rng() - a = float_values(generator, meta.elem_counts[a_name], style='signed') - b = float_values(generator, meta.elem_counts[b_name], style='signed') - c = float_values(generator, meta.elem_counts[c_name], style='signed_small') - buffers = default_buffers(meta) - buffers[a_name] = a - buffers[b_name] = b - buffers[c_name] = c - write_buffers(meta, buffers) - out = a - b + c - write_golden(meta, {single_output(meta): np.asarray(out, dtype=np.float32)}) - - -if __name__ == '__main__': - main() diff --git a/test/samples/Subsc/subsc.py b/test/samples/Subsc/subsc.py deleted file mode 100644 index a951f7a739..0000000000 --- a/test/samples/Subsc/subsc.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -from mlir.ir import Context, Location, Module, InsertionPoint -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType - - -def build(): - with Context() as ctx: - pto.register_dialect(ctx, load=True) - - with Location.unknown(ctx): - m = Module.create() - - f32 = F32Type.get(ctx) - ptr_f32 = pto.PtrType.get(f32, ctx) - - tv2_f32 = pto.TensorViewType.get(2, f32, ctx) - tile_view_32 = pto.PartitionTensorViewType.get([32, 32], f32, ctx) - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - - fractal_ab_size = pto.TileConfig.fractalABSize - cfg = pto.TileBufConfigAttr.get(bl, sl, fractal_ab_size, pd, ctx) - tile_buf_32 = pto.TileBufType.get([32, 32], f32, vec, [32, 32], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f32, ptr_f32], []) - with InsertionPoint(m.body): - fn = func.FuncOp("subsc_kernel_2d", fn_ty) - entry = fn.add_entry_block() - - with InsertionPoint(entry): - # constants - c0 = arith.ConstantOp(IndexType.get(ctx), 0).result - c1 = arith.ConstantOp(IndexType.get(ctx), 1).result - c32 = arith.ConstantOp(IndexType.get(ctx), 32).result - scale = arith.ConstantOp(f32, 3.14).result - - arg0, arg1 = entry.arguments - - # %0/%1/%2 = pto.make_tensor_view %arg?, shape=[%c32,%c32] strides=[%c32,%c1] - tv_src0 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv_src1 = pto.MakeTensorViewOp(tv2_f32, arg0, [c32, c32], [c32, c1]).result - tv_dst = pto.MakeTensorViewOp(tv2_f32, arg1, [c32, c32], [c32, c1]).result - - # %3/%4/%8 = pto.subview %tv, offsets=[%c0,%c0], sizes=[32,32] - sv_src0 = pto.PartitionViewOp(tile_view_32, tv_src0, offsets=[c0, c0], sizes=[c32, c32]).result - sv_src1 = pto.PartitionViewOp(tile_view_32, tv_src1, offsets=[c0, c0], sizes=[c32, c32]).result - - # %5/%6/%7 = pto.alloc_tile : <32x32xf32> - tb_src0 = pto.AllocTileOp(tile_buf_32).result - tb_src1 = pto.AllocTileOp(tile_buf_32).result - tb_dst = pto.AllocTileOp(tile_buf_32).result - - pto.TLoadOp(None, sv_src0, tb_src0) # result=None - pto.TLoadOp(None, sv_src1, tb_src1) - - pto.TSubSCOp(tb_src0, scale, tb_src1, tb_dst) - - # %8 = subview on output tensor_view - sv_dst = pto.PartitionViewOp(tile_view_32, tv_dst, offsets=[c0, c0], sizes=[c32, c32]).result - - # pto.store_dps_tb ins(%tb1) outs(%sv1) - pto.TStoreOp(None, tb_dst, sv_dst) - - func.ReturnOp([]) - - m.operation.verify() - - return m - - -if __name__ == "__main__": - print(build()) diff --git a/test/samples/Subsc/subsc_compare.py b/test/samples/Subsc/subsc_compare.py deleted file mode 100755 index 2a923d5faa..0000000000 --- a/test/samples/Subsc/subsc_compare.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import compare_outputs - -if __name__ == '__main__': - compare_outputs(np.float32, atol=0.0001) diff --git a/test/samples/Subsc/subsc_golden.py b/test/samples/Subsc/subsc_golden.py deleted file mode 100755 index 7d625d7e72..0000000000 --- a/test/samples/Subsc/subsc_golden.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python3 -import numpy as np -from pathlib import Path -import sys - -for search_root in (Path(__file__).resolve().parent, Path(__file__).resolve().parents[1]): - if (search_root / 'validation_runtime.py').is_file(): - sys.path.insert(0, str(search_root)) - break - -from validation_runtime import default_buffers, float_values, load_case_meta, rng, single_output, write_buffers, write_golden - - -def main(): - meta = load_case_meta() - [src_name] = meta.inputs - generator = rng() - src = float_values(generator, meta.elem_counts[src_name], style='signed') - buffers = default_buffers(meta) - buffers[src_name] = src - write_buffers(meta, buffers) - out = src - np.float32(3.14) + src - write_golden(meta, {single_output(meta): np.asarray(out, dtype=np.float32)}) - - -if __name__ == '__main__': - main() diff --git a/test/samples/TPrefetch/tprefetch.py b/test/samples/TPrefetch/tprefetch.py index f6a5a22bac..7b132806f2 100644 --- a/test/samples/TPrefetch/tprefetch.py +++ b/test/samples/TPrefetch/tprefetch.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, IndexType, F16Type -from mlir.dialects import arith, func, pto +from mlir.ir import Context, InsertionPoint, IntegerType, Location, Module, IndexType +from mlir.dialects import func, pto def build(): @@ -16,41 +16,15 @@ def build(): with Location.unknown(ctx): module = Module.create() - f16 = F16Type.get(ctx) + i64 = IntegerType.get_signless(64, ctx) idx = IndexType.get(ctx) - ptr_f16 = pto.PtrType.get(f16, ctx) - tv2_f16 = pto.TensorViewType.get(2, f16, ctx) - ptv_16x16 = pto.PartitionTensorViewType.get([16, 16], f16, ctx) - - vec = pto.AddressSpaceAttr.get(pto.AddressSpace.VEC, ctx) - bl = pto.BLayoutAttr.get(pto.BLayout.RowMajor, ctx) - sl = pto.SLayoutAttr.get(pto.SLayout.NoneBox, ctx) - pd = pto.PadValueAttr.get(pto.PadValue.Null, ctx) - cfg = pto.TileBufConfigAttr.get(bl, sl, pto.TileConfig.fractalABSize, pd, ctx) - tb_16x16 = pto.TileBufType.get([16, 16], f16, vec, [16, 16], cfg, ctx) - - fn_ty = func.FunctionType.get([ptr_f16, ptr_f16], []) + fn_ty = func.FunctionType.get([i64, idx], []) with InsertionPoint(module.body): fn = func.FuncOp("tprefetch_kernel", fn_ty) entry = fn.add_entry_block() with InsertionPoint(entry): - c0 = arith.ConstantOp(idx, 0).result - c1 = arith.ConstantOp(idx, 1).result - c16 = arith.ConstantOp(idx, 16).result - - src_view = pto.MakeTensorViewOp(tv2_f16, entry.arguments[0], [c16, c16], [c16, c1]).result - dst_view = pto.MakeTensorViewOp(tv2_f16, entry.arguments[1], [c16, c16], [c16, c1]).result - src_part = pto.PartitionViewOp( - ptv_16x16, src_view, offsets=[c0, c0], sizes=[c16, c16] - ).result - dst_part = pto.PartitionViewOp( - ptv_16x16, dst_view, offsets=[c0, c0], sizes=[c16, c16] - ).result - - tile = pto.AllocTileOp(tb_16x16).result - pto.TPrefetchOp(src_part, tile) - pto.TStoreOp(None, tile, dst_part) + pto.TPrefetchOp(entry.arguments[0], entry.arguments[1]) func.ReturnOp([]) module.operation.verify() diff --git a/tools/check_v057_pto_manifest.py b/tools/check_v057_pto_manifest.py index b223d9d096..35debc29df 100644 --- a/tools/check_v057_pto_manifest.py +++ b/tools/check_v057_pto_manifest.py @@ -7,105 +7,370 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -"""Check PTOAS PTO-op coverage against the LinxISA v0.57 PTO manifest. +"""Check PTOAS PTO-op contracts against the LinxISA v0.57.1 PTO manifest. PTOAS is an MLIR PTO dialect-to-EmitC compiler, not a Linx scalar assembler. -This check therefore validates dialect/EmitC surface coverage for PTO tile -operations and documents the intentionally external TMA/CUBE selector encoding -families owned by the Linx ISA golden catalog. +This check validates all 120 public operations' exact Linx operand roles/arity, +the matching PTOAS ODS argument surface, and the explicitly enumerated +dialect-only boundary. """ from __future__ import annotations import argparse +import hashlib import json import re +import subprocess from pathlib import Path -CANONICAL_ALIASES = { - "TTRANSPOSE": "TTRANS", - "TSORT": "TSORT32", +EXPECTED_LOCK = { + "release": "0.57.1", + "encoding_abi": "pto-isa-0.57.1-mode-function-v1", + "encoding_projection_sha256": "9705a984e2e48e0d4e856d3fbcfa07041c8578dd326d81f1c90279e826354c32", + "content_sha256": "748c1c6ac1bd5482d219cd08b8f0a871c7eda2b8f8afb1e81d498ec742e8abe8", + "release_manifest_sha256": "d85ee1cfcd51b1b86184ba015e9fb24ee61371165cf4691a5794f79618f7e273", + "source_commit": "2f3f605e289b09d56ef5a9ba39fc80b52948a5f5", + "hardware_profile_path": "spec/hardware-conformance-profile.json", + "hardware_profile_id": "pto-hardware-numeric-0.57.1-ieee-v1", + "hardware_profile_sha256": "becfbefcc7a31408e5e5293802493f833f68a2fccebb3f9e8008d8b9f4e7658c", + "numeric_vectors_path": "spec/evidence/pto-isa-0571-hardware-numeric-vectors.json", + "numeric_vectors_sha256": "b9f908deb9cfec412388e95f4532563cf4e462032b43d15996f0f7b4b93b4ec9", + "command_forms_path": "spec/catalog/command-forms.json", + "command_forms_sha256": "e5a98441098fdd88c0f5837bb3e1367d7b272ebc2ba506275e9e2be1b4c1978a", + "command_forms_count": 99, + "tile_operations_path": "spec/catalog/tile-operations.json", + "tile_operations_sha256": "e77c762122dd776ea8b1b2113e6d095b2db0fe24b5b98d0c26012db9fb98e64e", + "tile_operation_count": 120, + "release_manifest_path": "spec/release-manifest.json", + "source_repository": "https://github.com/PTO-ISA/pto-spec.git", } -# These are selector-family entries in the Linx ISA manifest. PTOAS lowers the -# high-level dialect to PTO-ISA C++ APIs and does not encode BSTART.TMA/CUBE. -EXTERNAL_ENCODING_FAMILY = { - "TLOAD", - "TSTORE", - "TMOV", - "TPREFETCH", - "MGATHER", - "MSCATTER", - "TMATMUL", - "TMATMUL.ACC", - "TMATMUL.BIAS", - "TMATMULMX", - "TMATMULMX.ACC", - "TMATMULMX.BIAS", - "TGEMV", - "TGEMV.ACC", - "TGEMV.BIAS", - "TGEMVMX", - "TGEMVMX.ACC", - "TGEMVMX.BIAS", - "ACCCVT", +DELETED_ACTIVE_NAMES = { + "TADDC", + "TADDSC", + "TFMA", + "TFMOD", + "TFMODS", + "TLRELU", + "TRANDOM", + "TSUBC", + "TSUBSC", + "TSORT32", } - def normalize(name: str) -> str: return name.replace(".", "").replace("_", "").upper() -def load_manifest(repo_root: Path) -> list[str]: - manifest_path = repo_root / "isa/v0.57/state/pto_encoding_map.json" +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def load_lock(ptoas_root: Path) -> dict: + lock_path = ptoas_root / "tools/pto_isa_v0_57_1_lock.json" + lock = json.loads(lock_path.read_text()) + errors = [] + for key in ("release", "encoding_abi", "encoding_projection_sha256", "content_sha256"): + if lock.get(key) != EXPECTED_LOCK[key]: + errors.append(f"{key}: expected {EXPECTED_LOCK[key]}, got {lock.get(key)}") + release_manifest = lock.get("release_manifest", {}) + for key, expected_key in ( + ("path", "release_manifest_path"), + ("sha256", "release_manifest_sha256"), + ): + if release_manifest.get(key) != EXPECTED_LOCK[expected_key]: + errors.append(f"release_manifest.{key} mismatch") + source = lock.get("source", {}) + for key, expected_key in ( + ("commit", "source_commit"), + ("repository", "source_repository"), + ): + if source.get(key) != EXPECTED_LOCK[expected_key]: + errors.append(f"source.{key} mismatch") + hardware_profile = lock.get("hardware_conformance_profile", {}) + for key, expected_key in ( + ("path", "hardware_profile_path"), + ("profile_id", "hardware_profile_id"), + ("sha256", "hardware_profile_sha256"), + ): + if hardware_profile.get(key) != EXPECTED_LOCK[expected_key]: + errors.append(f"hardware_conformance_profile.{key} mismatch") + numeric_vectors = lock.get("numeric_conformance_vectors", {}) + for key, expected_key in ( + ("path", "numeric_vectors_path"), + ("sha256", "numeric_vectors_sha256"), + ): + if numeric_vectors.get(key) != EXPECTED_LOCK[expected_key]: + errors.append(f"numeric_conformance_vectors.{key} mismatch") + catalogs = lock.get("catalogs", {}) + for catalog, expected_keys in ( + ( + "command_forms", + ("command_forms_path", "command_forms_sha256", "command_forms_count"), + ), + ( + "tile_operations", + ("tile_operations_path", "tile_operations_sha256", "tile_operation_count"), + ), + ): + entry = catalogs.get(catalog, {}) + for key, expected_key in zip(("path", "sha256", "count"), expected_keys): + if entry.get(key) != EXPECTED_LOCK[expected_key]: + errors.append(f"catalogs.{catalog}.{key} mismatch") + if errors: + raise SystemExit("unexpected PTO ISA 0.57.1 lock:\n " + "\n ".join(errors)) + return lock + + +def validate_source_tree(source_root: Path, lock: dict) -> None: + try: + head = subprocess.run( + ["git", "-C", str(source_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as error: + raise SystemExit(f"cannot resolve PTO ISA source commit in {source_root}: {error}") + if head != lock["source"]["commit"]: + raise SystemExit( + f"PTO ISA source commit mismatch: expected {lock['source']['commit']}, got {head}" + ) + + pinned_files = ( + lock["catalogs"]["command_forms"], + lock["catalogs"]["tile_operations"], + lock["release_manifest"], + lock["hardware_conformance_profile"], + lock["numeric_conformance_vectors"], + ) + for entry in pinned_files: + path = source_root / entry["path"] + if not path.is_file(): + raise SystemExit(f"missing pinned PTO ISA source file: {path}") + actual = sha256(path) + if actual != entry["sha256"]: + raise SystemExit( + f"PTO ISA source hash mismatch for {entry['path']}: " + f"expected {entry['sha256']}, got {actual}" + ) + + manifest = json.loads((source_root / lock["release_manifest"]["path"]).read_text()) + expected_manifest_fields = { + "release": lock["release"], + "encoding_abi": lock["encoding_abi"], + "encoding_projection_sha256": lock["encoding_projection_sha256"], + "content_sha256": lock["content_sha256"], + } + for key, expected in expected_manifest_fields.items(): + if manifest.get(key) != expected: + raise SystemExit( + f"PTO ISA release manifest {key} mismatch: " + f"expected {expected}, got {manifest.get(key)}" + ) + hardware = manifest.get("hardware_conformance_profile", {}) + for key in ("path", "profile_id", "sha256"): + if hardware.get(key) != lock["hardware_conformance_profile"][key]: + raise SystemExit(f"PTO ISA release manifest hardware profile {key} mismatch") + if hardware.get("evidence") != lock["numeric_conformance_vectors"]["path"]: + raise SystemExit("PTO ISA release manifest hardware numeric evidence path mismatch") + owned_hashes = { + entry.get("path"): entry.get("sha256") + for entry in manifest.get("owned_artifacts", []) + } + numeric_vectors = lock["numeric_conformance_vectors"] + if owned_hashes.get(numeric_vectors["path"]) != numeric_vectors["sha256"]: + raise SystemExit("PTO ISA release manifest hardware numeric evidence hash mismatch") + if ( + manifest.get("catalog_counts", {}).get("tile_operations_total") + != lock["catalogs"]["tile_operations"]["count"] + ): + raise SystemExit("PTO ISA release manifest tile operation count mismatch") + if ( + manifest.get("catalog_counts", {}).get("command_forms") + != lock["catalogs"]["command_forms"]["count"] + ): + raise SystemExit("PTO ISA release manifest command form count mismatch") + + +def load_manifest(linx_root: Path) -> dict[str, dict]: + manifest_path = linx_root / "isa/v0.57/state/pto_ops.json" manifest = json.loads(manifest_path.read_text()) - if manifest["profile"] != "v0.57" or manifest["entry_count"] != 111: - raise SystemExit(f"unexpected v0.57 manifest header in {manifest_path}") - names = [] - for entry in manifest["entries"]: - names.append(entry["canonical_name"]) - return names + if manifest["profile"] != "v0.57" or manifest["operation_count"] != 120: + raise SystemExit(f"unexpected v0.57.1 manifest header in {manifest_path}") + source_lock = manifest.get("source_lock") + if source_lock != "isa/v0.57/pto-spec.lock.json": + raise SystemExit(f"unexpected v0.57.1 source_lock in {manifest_path}: {source_lock}") + operations = manifest["operations"] + names = [entry["name"] for entry in operations] + if len(names) != len(set(names)): + raise SystemExit(f"duplicate operation names in {manifest_path}") + return {entry["name"]: entry for entry in operations} -def load_ptoas_ops(repo_root: Path) -> set[str]: - ods_path = repo_root / "compiler/ptoas/include/PTO/IR/PTOOps.td" +def load_expected_contracts(ptoas_root: Path) -> tuple[dict[str, dict], dict[str, dict]]: + contract_path = ptoas_root / "tools/pto_isa_v0_57_1_operation_contracts.json" + contracts = json.loads(contract_path.read_text()) + if contracts.get("release") != EXPECTED_LOCK["release"]: + raise SystemExit(f"unexpected release in {contract_path}") + operations = contracts.get("operations", {}) + if (contracts.get("public_operation_count") != EXPECTED_LOCK["tile_operation_count"] or + len(operations) != EXPECTED_LOCK["tile_operation_count"]): + raise SystemExit(f"unexpected public operation count in {contract_path}") + public = operations + dialect_only_entries = contracts.get("dialect_only_operations", {}) + dialect_only = { + normalize(mnemonic): { + "ptoas_mnemonic": mnemonic, + "ptoas_arguments": arguments, + } + for mnemonic, arguments in dialect_only_entries.items() + } + if len(dialect_only) != len(dialect_only_entries): + raise SystemExit(f"duplicate dialect-only operation names in {contract_path}") + public_mnemonics = { + normalize(entry["ptoas_mnemonic"]) for entry in operations.values() + } + overlap = sorted(public_mnemonics & set(dialect_only)) + if overlap: + raise SystemExit( + f"public/dialect-only operation overlap in {contract_path}: {overlap}" + ) + return public, dialect_only + + +def load_ptoas_ops(ptoas_root: Path) -> dict[str, dict]: + ods_path = ptoas_root / "include/PTO/IR/PTOOps.td" + if not ods_path.exists(): + ods_path = ptoas_root / "compiler/ptoas/include/PTO/IR/PTOOps.td" text = ods_path.read_text() - mnemonics = re.findall(r'PTO_TOp<"([^"]+)"', text) - return {normalize(mnemonic) for mnemonic in mnemonics} + definitions = list( + re.finditer(r'\bdef\s+\w+\s*:\s*PTO_TOp<"([^"]+)"', text) + ) + operations = {} + for index, definition in enumerate(definitions): + mnemonic = definition.group(1) + start = definition.start() + end = definitions[index + 1].start() if index + 1 < len(definitions) else len(text) + body = text[start:end] + match = re.search(r"let\s+arguments\s*=\s*\(ins(.*?)\);", body, re.DOTALL) + arguments = tuple(re.findall(r"\$([A-Za-z0-9_]+)", match.group(1))) if match else () + key = normalize(mnemonic) + if key in operations: + raise SystemExit(f"duplicate normalized PTOAS mnemonic in {ods_path}: {mnemonic}") + operations[key] = {"mnemonic": mnemonic, "arguments": arguments} + return operations def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[3]) + parser.add_argument("--ptoas-root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--linx-root", type=Path) + parser.add_argument( + "--pto-spec-root", + type=Path, + help="optionally verify every pinned identity against a PTO-ISA/pto-spec checkout", + ) args = parser.parse_args() - repo_root = args.repo_root.resolve() - manifest_names = load_manifest(repo_root) - ptoas_ops = load_ptoas_ops(repo_root) - - missing = [] - external = [] - covered = [] - for canonical_name in manifest_names: - alias = CANONICAL_ALIASES.get(canonical_name, canonical_name) - if normalize(alias) in ptoas_ops: - covered.append(canonical_name) - elif canonical_name in EXTERNAL_ENCODING_FAMILY: - external.append(canonical_name) - else: - missing.append(canonical_name) - - if missing: - print("PTOAS missing v0.57 PTO dialect coverage:") - for name in missing: + ptoas_root = args.ptoas_root.resolve() + lock = load_lock(ptoas_root) + if args.pto_spec_root is not None: + validate_source_tree(args.pto_spec_root.resolve(), lock) + expected_public, expected_dialect_only = load_expected_contracts(ptoas_root) + ptoas_ops = load_ptoas_ops(ptoas_root) + + deleted_present = sorted(name for name in DELETED_ACTIVE_NAMES if normalize(name) in ptoas_ops) + if deleted_present: + print("PTOAS has deleted PTO ISA 0.57.1 names active in the dialect:") + for name in deleted_present: print(f" - {name}") return 1 + expected_all = { + normalize(contract["ptoas_mnemonic"]) + for contract in expected_public.values() + } | set(expected_dialect_only) + missing_ops = sorted(expected_all - set(ptoas_ops)) + undeclared_ops = sorted(set(ptoas_ops) - expected_all) + if missing_ops or undeclared_ops: + print("PTOAS public/dialect-only operation boundary mismatch:") + for name in missing_ops: + print(f" - missing declared operation: {name}") + for name in undeclared_ops: + print(f" - undeclared dialect-only operation: {name}") + return 1 + + contract_errors = [] + for name, contract in expected_public.items(): + actual = ptoas_ops[normalize(contract["ptoas_mnemonic"])]["arguments"] + expected = tuple(contract["ptoas_arguments"]) + if actual != expected: + contract_errors.append(f"{name}: expected PTOAS arguments {expected}, got {actual}") + for key, contract in expected_dialect_only.items(): + actual = ptoas_ops[key]["arguments"] + expected = tuple(contract["ptoas_arguments"]) + if actual != expected: + contract_errors.append( + f"dialect-only {contract['ptoas_mnemonic']}: " + f"expected PTOAS arguments {expected}, got {actual}" + ) + if contract_errors: + print("PTOAS has incorrect PTO ISA 0.57.1 operation roles/arity:") + for error in contract_errors: + print(f" - {error}") + return 1 + + if args.linx_root is None: + print( + "PTOAS v0.57.1 PTO lock/dialect check OK: all 120 public " + f"operation argument contracts and {len(expected_dialect_only)} explicit " + "dialect-only contracts match; hardware numeric profile/vectors are " + "identity metadata only (execution conformance not evaluated)" + ) + return 0 + + manifest = load_manifest(args.linx_root.resolve()) + + expected_names = set(expected_public) + actual_names = set(manifest) + if actual_names != expected_names: + print("LinxISA v0.57.1 public operation boundary mismatch:") + for name in sorted(expected_names - actual_names): + print(f" - missing manifest operation: {name}") + for name in sorted(actual_names - expected_names): + print(f" - unexpected manifest operation: {name}") + return 1 + + role_errors = [] + for name, contract in expected_public.items(): + entry = manifest[name] + actual = tuple( + (operand.get("field"), operand.get("role")) + for operand in entry.get("operands", []) + ) + expected = tuple( + (operand[0], operand[1]) + for operand in contract["isa_operands"] + ) + if actual != expected: + role_errors.append( + f"{name}: expected ISA operands {expected}, got {actual}" + ) + if role_errors: + print("LinxISA v0.57.1 manifest role/arity mismatch:") + for error in role_errors: + print(f" - {error}") + return 1 + print( - "PTOAS v0.57 PTO manifest coverage OK: " - f"{len(covered)} dialect ops, {len(external)} external encoding-family ops, " - f"{len(manifest_names)} manifest entries" + "PTOAS v0.57.1 PTO manifest contract check OK: " + f"all {len(manifest)} public operations match exact Linx roles/arity; " + f"{len(expected_dialect_only)} dialect-only operations are explicitly bounded; " + "hardware numeric profile/vectors are identity metadata only " + "(execution conformance not evaluated)" ) return 0 diff --git a/tools/pto_isa_v0_57_1_lock.json b/tools/pto_isa_v0_57_1_lock.json new file mode 100644 index 0000000000..319547a370 --- /dev/null +++ b/tools/pto_isa_v0_57_1_lock.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://docs.openclaw.ai/schemas/linxisa/pto_spec_lock.v1.json", + "catalogs": { + "command_forms": { + "path": "spec/catalog/command-forms.json", + "sha256": "e5a98441098fdd88c0f5837bb3e1367d7b272ebc2ba506275e9e2be1b4c1978a", + "count": 99 + }, + "tile_operations": { + "path": "spec/catalog/tile-operations.json", + "sha256": "e77c762122dd776ea8b1b2113e6d095b2db0fe24b5b98d0c26012db9fb98e64e", + "count": 120 + } + }, + "content_sha256": "748c1c6ac1bd5482d219cd08b8f0a871c7eda2b8f8afb1e81d498ec742e8abe8", + "encoding_abi": "pto-isa-0.57.1-mode-function-v1", + "encoding_projection_sha256": "9705a984e2e48e0d4e856d3fbcfa07041c8578dd326d81f1c90279e826354c32", + "hardware_conformance_profile": { + "path": "spec/hardware-conformance-profile.json", + "profile_id": "pto-hardware-numeric-0.57.1-ieee-v1", + "sha256": "becfbefcc7a31408e5e5293802493f833f68a2fccebb3f9e8008d8b9f4e7658c" + }, + "numeric_conformance_vectors": { + "path": "spec/evidence/pto-isa-0571-hardware-numeric-vectors.json", + "sha256": "b9f908deb9cfec412388e95f4532563cf4e462032b43d15996f0f7b4b93b4ec9" + }, + "release": "0.57.1", + "release_manifest": { + "path": "spec/release-manifest.json", + "sha256": "d85ee1cfcd51b1b86184ba015e9fb24ee61371165cf4691a5794f79618f7e273" + }, + "source": { + "commit": "2f3f605e289b09d56ef5a9ba39fc80b52948a5f5", + "repository": "https://github.com/PTO-ISA/pto-spec.git" + } +} diff --git a/tools/pto_isa_v0_57_1_operation_contracts.json b/tools/pto_isa_v0_57_1_operation_contracts.json new file mode 100644 index 0000000000..f39ce584e6 --- /dev/null +++ b/tools/pto_isa_v0_57_1_operation_contracts.json @@ -0,0 +1,150 @@ +{ + "release": "0.57.1", + "source": "PTO-ISA/pto-spec via LinxISA isa/v0.57/state/pto_ops.json", + "public_operation_count": 120, + "operations": { + "TADD": {"ptoas_mnemonic":"tadd","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TSUB": {"ptoas_mnemonic":"tsub","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TMUL": {"ptoas_mnemonic":"tmul","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TMAX": {"ptoas_mnemonic":"tmax","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TMIN": {"ptoas_mnemonic":"tmin","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TAND": {"ptoas_mnemonic":"tand","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TOR": {"ptoas_mnemonic":"tor","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TXOR": {"ptoas_mnemonic":"txor","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TSHL": {"ptoas_mnemonic":"tshl","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TSHR": {"ptoas_mnemonic":"tshr","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TCMP": {"ptoas_mnemonic":"tcmp","ptoas_arguments":["src0","src1","dst","cmpMode"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"],["comparison","comparison"]]}, + "TSEL": {"ptoas_mnemonic":"tsel","ptoas_arguments":["mask","src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","mask"],["source1","source-true"],["source2","source-false"]]}, + "TABS": {"ptoas_mnemonic":"tabs","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TNOT": {"ptoas_mnemonic":"tnot","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TNEG": {"ptoas_mnemonic":"tneg","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TRELU": {"ptoas_mnemonic":"trelu","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TPRELU": {"ptoas_mnemonic":"tprelu","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","negative-slope"]]}, + "TDIV": {"ptoas_mnemonic":"tdiv","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TREM": {"ptoas_mnemonic":"trem","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TSQRT": {"ptoas_mnemonic":"tsqrt","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TLOG": {"ptoas_mnemonic":"tlog","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TRECIP": {"ptoas_mnemonic":"trecip","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TEXP": {"ptoas_mnemonic":"texp","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TRSQRT": {"ptoas_mnemonic":"trsqrt","ptoas_arguments":["src","dst","tmp"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TADDS": {"ptoas_mnemonic":"tadds","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TAXPY": {"ptoas_mnemonic":"taxpy","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TSUBS": {"ptoas_mnemonic":"tsubs","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TMULS": {"ptoas_mnemonic":"tmuls","ptoas_arguments":["src0","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TDIVS": {"ptoas_mnemonic":"tdivs","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TMINS": {"ptoas_mnemonic":"tmins","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TMAXS": {"ptoas_mnemonic":"tmaxs","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TREMS": {"ptoas_mnemonic":"trems","ptoas_arguments":["src","scalar","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TANDS": {"ptoas_mnemonic":"tands","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TORS": {"ptoas_mnemonic":"tors","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TXORS": {"ptoas_mnemonic":"txors","ptoas_arguments":["src","scalar","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TCMPS": {"ptoas_mnemonic":"tcmps","ptoas_arguments":["src","scalar","cmpMode","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"],["comparison","comparison"]]}, + "TSELS": {"ptoas_mnemonic":"tsels","ptoas_arguments":["mask","src","tmp","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","mask"],["source1","source-true"],["scalar0","scalar-false"]]}, + "TSHLS": {"ptoas_mnemonic":"tshls","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TSHRS": {"ptoas_mnemonic":"tshrs","ptoas_arguments":["src","scalar","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scalar"]]}, + "TROWSUM": {"ptoas_mnemonic":"trowsum","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWPROD": {"ptoas_mnemonic":"trowprod","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWMAX": {"ptoas_mnemonic":"trowmax","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWMIN": {"ptoas_mnemonic":"trowmin","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWARGMAX": {"ptoas_mnemonic":"trowargmax","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWARGMIN": {"ptoas_mnemonic":"trowargmin","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLSUM": {"ptoas_mnemonic":"tcolsum","ptoas_arguments":["src","tmp","dst","isBinary"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLPROD": {"ptoas_mnemonic":"tcolprod","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLMAX": {"ptoas_mnemonic":"tcolmax","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLMIN": {"ptoas_mnemonic":"tcolmin","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLARGMAX": {"ptoas_mnemonic":"tcolargmax","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TCOLARGMIN": {"ptoas_mnemonic":"tcolargmin","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TROWEXPAND": {"ptoas_mnemonic":"trowexpand","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDADD": {"ptoas_mnemonic":"trowexpandadd","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDSUB": {"ptoas_mnemonic":"trowexpandsub","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDMUL": {"ptoas_mnemonic":"trowexpandmul","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDDIV": {"ptoas_mnemonic":"trowexpanddiv","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDMAX": {"ptoas_mnemonic":"trowexpandmax","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDMIN": {"ptoas_mnemonic":"trowexpandmin","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TROWEXPANDEXPDIF": {"ptoas_mnemonic":"trowexpandexpdif","ptoas_arguments":["src0","src1","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPAND": {"ptoas_mnemonic":"tcolexpand","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDADD": {"ptoas_mnemonic":"tcolexpandadd","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDSUB": {"ptoas_mnemonic":"tcolexpandsub","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDMUL": {"ptoas_mnemonic":"tcolexpandmul","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDDIV": {"ptoas_mnemonic":"tcolexpanddiv","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDMAX": {"ptoas_mnemonic":"tcolexpandmax","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDMIN": {"ptoas_mnemonic":"tcolexpandmin","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TCOLEXPANDEXPDIF": {"ptoas_mnemonic":"tcolexpandexpdif","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","broadcast-source"]]}, + "TMATMUL": {"ptoas_mnemonic":"tmatmul","ptoas_arguments":["lhs","rhs","dst","accPhase"],"isa_operands":[["source0","left"],["source1","right"]]}, + "TMATMUL_BIAS": {"ptoas_mnemonic":"tmatmul.bias","ptoas_arguments":["a","b","bias","dst"],"isa_operands":[["source0","left"],["source1","right"],["source2","bias"]]}, + "TMATMUL_ACC": {"ptoas_mnemonic":"tmatmul.acc","ptoas_arguments":["acc_in","lhs","rhs","dst","accPhase"],"isa_operands":[["source0","left"],["source1","right"]]}, + "TMATMUL_MX": {"ptoas_mnemonic":"tmatmul.mx","ptoas_arguments":["a","a_scale","b","b_scale","dst"],"isa_operands":[["source0","left"],["source1","right"],["source2","row-scale"],["source3","column-scale"]]}, + "TMATMUL_MX_BIAS": {"ptoas_mnemonic":"tmatmul.mx.bias","ptoas_arguments":["a","a_scale","b","b_scale","bias","dst"],"isa_operands":[["source0","left"],["source1","right"],["source2","row-scale"],["source3","column-scale"],["source4","bias"]]}, + "TMATMUL_MX_ACC": {"ptoas_mnemonic":"tmatmul.mx.acc","ptoas_arguments":["c_in","a","a_scale","b","b_scale","dst"],"isa_operands":[["source0","left"],["source1","right"],["source2","row-scale"],["source3","column-scale"]]}, + "ACCCVT": {"ptoas_mnemonic":"acccvt","ptoas_arguments":["dst"],"isa_operands":[["destination0","destination"]]}, + "TGEMV": {"ptoas_mnemonic":"tgemv","ptoas_arguments":["lhs","rhs","dst"],"isa_operands":[["source0","matrix"],["source1","vector"]]}, + "TGEMV_BIAS": {"ptoas_mnemonic":"tgemv.bias","ptoas_arguments":["a","b","bias","dst"],"isa_operands":[["source0","matrix"],["source1","vector"],["source2","bias"]]}, + "TGEMV_ACC": {"ptoas_mnemonic":"tgemv.acc","ptoas_arguments":["acc_in","lhs","rhs","dst"],"isa_operands":[["source0","matrix"],["source1","vector"]]}, + "TGEMV_MX": {"ptoas_mnemonic":"tgemv.mx","ptoas_arguments":["a","a_scale","b","b_scale","dst"],"isa_operands":[["source0","matrix"],["source1","vector"],["source2","row-scale"],["source3","column-scale"]]}, + "TGEMV_MX_BIAS": {"ptoas_mnemonic":"tgemv.mx.bias","ptoas_arguments":["a","a_scale","b","b_scale","bias","dst"],"isa_operands":[["source0","matrix"],["source1","vector"],["source2","row-scale"],["source3","column-scale"],["source4","bias"]]}, + "TGEMV_MX_ACC": {"ptoas_mnemonic":"tgemv.mx.acc","ptoas_arguments":["c_in","a","a_scale","b","b_scale","dst"],"isa_operands":[["source0","matrix"],["source1","vector"],["source2","row-scale"],["source3","column-scale"]]}, + "TLOAD": {"ptoas_mnemonic":"tload","ptoas_arguments":["src","dst","pad_mode","pad_value","left_padding_num","right_padding_num","init_out_buffer","init_condition"],"isa_operands":[["destination0","destination"],["address","base-address"]]}, + "TSTORE": {"ptoas_mnemonic":"tstore","ptoas_arguments":["src","dst","preQuantScalar","stPhase","atomicType","reluPreMode"],"isa_operands":[["address","base-address"],["source0","source"]]}, + "TPREFETCH": {"ptoas_mnemonic":"tprefetch","ptoas_arguments":["address","byte_count"],"isa_operands":[["address","base-address"],["byte_count","byte-count"]]}, + "MGATHER": {"ptoas_mnemonic":"mgather","ptoas_arguments":["mem","idx","dst","gatherOob"],"isa_operands":[["destination0","destination"],["address","base-address"],["source0","indices"]]}, + "MSCATTER": {"ptoas_mnemonic":"mscatter","ptoas_arguments":["src","idx","mem","scatterAtomicOp","scatterOob"],"isa_operands":[["address","base-address"],["source0","source"],["source1","indices"]]}, + "MGATHER_MASK": {"ptoas_mnemonic":"mgather_mask","ptoas_arguments":["mem","idx","mask","dst"],"isa_operands":[["destination0","destination"],["address","base-address"],["source0","indices"],["source1","mask"]]}, + "MSCATTER_MASK": {"ptoas_mnemonic":"mscatter_mask","ptoas_arguments":["src","idx","mask","mem"],"isa_operands":[["address","base-address"],["source0","source"],["source1","indices"],["source2","mask"]]}, + "MGATHER_CAS": {"ptoas_mnemonic":"mgather_cas","ptoas_arguments":["mem","idx","expected","replacement","dst"],"isa_operands":[["destination0","destination"],["address","base-address"],["source0","indices"],["source1","expected"],["source2","replacement"]]}, + "TEXPANDS": {"ptoas_mnemonic":"texpands","ptoas_arguments":["scalar","dst"],"isa_operands":[["destination0","destination"],["scalar0","scalar"]]}, + "TCI": {"ptoas_mnemonic":"tci","ptoas_arguments":["S","dst","descending"],"isa_operands":[["destination0","destination"],["scalar0","start"],["flag0","descending"]]}, + "TTRI": {"ptoas_mnemonic":"ttri","ptoas_arguments":["diagonal","dst","upperOrLower"],"isa_operands":[["destination0","destination"],["flag0","upper"],["diagonal","diagonal"]]}, + "TFILLPAD": {"ptoas_mnemonic":"tfillpad","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","padding"]]}, + "TCVT": {"ptoas_mnemonic":"tcvt","ptoas_arguments":["src","dst","rmode","sat_mode"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TQUANT": {"ptoas_mnemonic":"tquant","ptoas_arguments":["src","fp","offset","dst","quant_type"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scale"],["scalar1","zero-point"]]}, + "TDEQUANT": {"ptoas_mnemonic":"tdequant","ptoas_arguments":["src","scale","offset","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["scalar0","scale"],["scalar1","zero-point"]]}, + "TEXTRACT": {"ptoas_mnemonic":"textract","ptoas_arguments":["src","indexRow","indexCol","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["natural0","row-offset"],["natural1","column-offset"]]}, + "TINSERT": {"ptoas_mnemonic":"tinsert","ptoas_arguments":["src","indexRow","indexCol","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["natural0","row-offset"],["natural1","column-offset"]]}, + "TGATHER": {"ptoas_mnemonic":"tgather","ptoas_arguments":["src","dst","cdst","indices","tmp","kValue","maskPattern","cmpMode","offset"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","indices"]]}, + "TSCATTER": {"ptoas_mnemonic":"tscatter","ptoas_arguments":["src","indexes","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","indices"]]}, + "TCONCAT": {"ptoas_mnemonic":"tconcat","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"],["axis","axis"]]}, + "TTRANS": {"ptoas_mnemonic":"ttrans","ptoas_arguments":["src","tmp","dst"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TIMG2COL": {"ptoas_mnemonic":"timg2col","ptoas_arguments":["src","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["positive0","kernel-rows"],["positive1","kernel-columns"],["positive2","stride-rows"],["positive3","stride-columns"],["natural0","pad-rows"],["natural1","pad-columns"],["scalar0","padding"]]}, + "TMOV": {"ptoas_mnemonic":"tmov","ptoas_arguments":["src","dst","fp","preQuantScalar","accToVecMode","reluPreMode"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TGATHERB": {"ptoas_mnemonic":"tgatherb","ptoas_arguments":["src","offsets","dst"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","byte-offsets"]]}, + "TDEINTERLEAVE": {"ptoas_mnemonic":"tdeinterleave","ptoas_arguments":["src","dst0","dst1"],"isa_operands":[["destination0","destination-even"],["destination1","destination-odd"],["source0","source"]]}, + "TINTERLEAVE": {"ptoas_mnemonic":"tinterleave","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TRESHAPE": {"ptoas_mnemonic":"treshape","ptoas_arguments":["src"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TSORT": {"ptoas_mnemonic":"tsort","ptoas_arguments":["src","dst","dst_indices","descending"],"isa_operands":[["destination0","destination"],["destination1","original-indices-u32"],["source0","source"],["flag0","descending"]]}, + "TMRGSORT": {"ptoas_mnemonic":"tmrgsort","ptoas_arguments":["srcs","blockLen","dsts","tmp","excuted","exhausted"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"],["flag0","descending"]]}, + "THISTOGRAM": {"ptoas_mnemonic":"thistogram","ptoas_arguments":["src","idx","dst","isMSB"],"isa_operands":[["destination0","destination"],["source0","source"],["source1","indices"],["selected_byte","selected-byte"]]}, + "TPUSH": {"ptoas_mnemonic":"tpush","ptoas_arguments":["tile","pipe_handle","split"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TPOP": {"ptoas_mnemonic":"tpop","ptoas_arguments":["tile","pipe_handle","split"],"isa_operands":[["destination0","destination"],["source0","source"]]}, + "TALLOC": {"ptoas_mnemonic":"talloc","ptoas_arguments":["entry","pipe_handle","split"],"isa_operands":[["destination0","destination"],["byte_count","capacity-bytes"],["positive0","rows"],["positive1","columns"],["natural0","valid-rows"],["natural1","valid-columns"],["scalar0","data-type-code"],["flag0","implementation-defined-layout"]]}, + "TFREE": {"ptoas_mnemonic":"tfree","ptoas_arguments":["entry","pipe_handle","split"],"isa_operands":[["destination0","destination"]]}, + "TPARTADD": {"ptoas_mnemonic":"tpartadd","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TPARTMUL": {"ptoas_mnemonic":"tpartmul","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TPARTMAX": {"ptoas_mnemonic":"tpartmax","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TPARTMIN": {"ptoas_mnemonic":"tpartmin","ptoas_arguments":["src0","src1","dst"],"isa_operands":[["destination0","destination"],["source0","source-left"],["source1","source-right"]]}, + "TPARTARGMAX": {"ptoas_mnemonic":"tpartargmax","ptoas_arguments":["src0","src1","src0Idx","src1Idx","dst","dstIdx"],"isa_operands":[["destination0","destination"],["destination1","destination-indices"],["source0","source-left"],["source1","source-right"],["source2","left-indices"],["source3","right-indices"]]}, + "TPARTARGMIN": {"ptoas_mnemonic":"tpartargmin","ptoas_arguments":["src0","src1","src0Idx","src1Idx","dst","dstIdx"],"isa_operands":[["destination0","destination"],["destination1","destination-indices"],["source0","source-left"],["source1","source-right"],["source2","left-indices"],["source3","right-indices"]]} + }, + "dialect_only_operations": { + "talloc_to_aic": ["id","split"], + "talloc_to_aiv": ["id","split"], + "tconcatidx": ["src0","src1","src0Idx","src1Idx","dst"], + "textract_fp": ["src","fp","indexRow","indexCol","dst"], + "tfillpad_expand": ["src","dst"], + "tfillpad_inplace": ["src","dst"], + "tfree_from_aic": ["entry","id","split"], + "tfree_from_aiv": ["entry","id","split"], + "tget_scale_addr": ["src","dst"], + "tgetval": ["src","offset"], + "tinsert_fp": ["src","fp","indexRow","indexCol","dst"], + "tmov.fp": ["src","fp","dst"], + "tpop_from_aic": ["valid_row","valid_col","id","split"], + "tpop_from_aiv": ["valid_row","valid_col","id","split"], + "tprint": ["src"], + "tpush_to_aic": ["tile","id","split"], + "tpush_to_aiv": ["tile","id","split"], + "tsetval": ["dst","offset","val"], + "tstore_fp": ["src","fp","dst"], + "tsync": ["events"] + } +} + diff --git a/tools/ptobc/generated/ptobc_opcodes_v0.h b/tools/ptobc/generated/ptobc_opcodes_v0.h index 96d3431637..2575b78b51 100644 --- a/tools/ptobc/generated/ptobc_opcodes_v0.h +++ b/tools/ptobc/generated/ptobc_opcodes_v0.h @@ -27,6 +27,26 @@ inline constexpr uint8_t kSectionCubeVariant = 0; inline constexpr uint8_t kSectionVectorVariant = 1; inline constexpr uint8_t kHasVariant = 1; +// Opcode values assigned to operations removed from PTO ISA 0.57.1 remain +// permanently reserved. They must decode as unknown and must never be reused. +inline constexpr uint16_t kReservedPtoOpcodes[] = { + 0x1008, // pto.taddc + 0x100A, // pto.taddsc + 0x1026, // pto.tfmod + 0x1027, // pto.tfmods + 0x1031, // pto.tlrelu + 0x1063, // pto.tsort32 + 0x1068, // pto.tsubc + 0x106A, // pto.tsubsc +}; + +inline constexpr bool isReservedPtoOpcode(uint16_t opcode) { + for (uint16_t reserved : kReservedPtoOpcodes) + if (opcode == reserved) + return true; + return false; +} + inline constexpr int kTgemvOperandCount = 3; inline constexpr int kTgemvAccOperandCount = 4; inline constexpr int kTgemvBiasOperandCount = 4; @@ -68,9 +88,7 @@ inline constexpr OpInfo kOpTable[] = { {0x1005, "pto.record_event", 0, 0x00, 0x00, 0, 0, 0, 0x02}, {0x1006, "pto.tabs", 0, 0x00, 0x00, 2, 0, 0, 0x00}, {0x1007, "pto.tadd", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x1008, "pto.taddc", 0, 0x00, 0x00, 4, 0, 0, 0x00}, {0x1009, "pto.tadds", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x100A, "pto.taddsc", 0, 0x00, 0x00, 4, 0, 0, 0x00}, {0x100B, "pto.tand", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x100C, "pto.tands", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x100D, "pto.tci", 0, 0x00, 0x00, 2, 0, 0, 0x00}, @@ -98,8 +116,6 @@ inline constexpr OpInfo kOpTable[] = { {0x1023, "pto.tfillpad", 0, 0x00, 0x00, 2, 0, 0, 0x00}, {0x1024, "pto.tfillpad_expand", 0, 0x00, 0x00, 2, 0, 0, 0x00}, {0x1025, "pto.tfillpad_inplace", 0, 0x00, 0x00, 2, 0, 0, 0x00}, - {0x1026, "pto.tfmod", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x1027, "pto.tfmods", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x1028, "pto.tgather", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x1029, "pto.tgatherb", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x102A, "pto.tgemv", 1, 0x00, 0x01, 0, 0, 0, 0x00}, @@ -109,7 +125,6 @@ inline constexpr OpInfo kOpTable[] = { {0x102E, "pto.tinsert_fp", 0, 0x00, 0x00, 5, 0, 0, 0x00}, {0x102F, "pto.tload", 0, 0x00, 0x00, 2, 0, 0, 0x00}, {0x1030, "pto.tlog", 0, 0x00, 0x00, 2, 0, 0, 0x00}, - {0x1031, "pto.tlrelu", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x1032, "pto.tmatmul", 1, 0x00, 0x01, 0, 0, 0, 0x00}, {0x1033, "pto.tmatmul.mx", 1, 0x00, 0x01, 0, 0, 0, 0x00}, {0x1034, "pto.tmax", 0, 0x00, 0x00, 3, 0, 0, 0x00}, @@ -159,14 +174,11 @@ inline constexpr OpInfo kOpTable[] = { {0x1060, "pto.tshls", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x1061, "pto.tshr", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x1062, "pto.tshrs", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x1063, "pto.tsort32", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x1064, "pto.tsqrt", 0, 0x00, 0x00, 2, 0, 0, 0x00}, {0x1065, "pto.tstore", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x1066, "pto.tstore_fp", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x1067, "pto.tsub", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x1068, "pto.tsubc", 0, 0x00, 0x00, 4, 0, 0, 0x00}, {0x1069, "pto.tsubs", 0, 0x00, 0x00, 3, 0, 0, 0x00}, - {0x106A, "pto.tsubsc", 0, 0x00, 0x00, 4, 0, 0, 0x00}, {0x106B, "pto.trowexpandsub", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x106C, "pto.ttrans", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x106D, "pto.ttri", 0, 0x00, 0x00, 2, 0, 0, 0x00}, @@ -216,6 +228,11 @@ inline constexpr OpInfo kOpTable[] = { {0x1099, "pto.comm.treduce", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x109A, "pto.tpartargmax", 0, 0x00, 0x00, 6, 0, 0, 0x00}, {0x109B, "pto.tpartargmin", 0, 0x00, 0x00, 6, 0, 0, 0x00}, + {0x109C, "pto.acccvt", 0, 0x00, 0x00, 1, 0, 0, 0x00}, + {0x109D, "pto.mgather_mask", 0, 0x00, 0x00, 4, 0, 0, 0x00}, + {0x109E, "pto.mgather_cas", 0, 0x00, 0x00, 5, 0, 0, 0x00}, + {0x109F, "pto.mscatter_mask", 0, 0x00, 0x00, 4, 0, 0, 0x00}, + {0x10A0, "pto.tsort", 0, 0x00, 0x00, 3, 0, 0, 0x00}, {0x2000, "arith.addi", 0, 0x01, 0x00, 2, 1, 0, 0x00}, {0x2001, "arith.ceildivsi", 0, 0x01, 0x00, 2, 1, 0, 0x00}, {0x2002, "arith.cmpi", 0, 0x01, 0x00, 2, 1, 0, 0x01}, @@ -225,8 +242,8 @@ inline constexpr OpInfo kOpTable[] = { {0x2006, "arith.muli", 0, 0x01, 0x00, 2, 1, 0, 0x00}, {0x2007, "arith.select", 0, 0x01, 0x00, 3, 1, 0, 0x00}, {0x2008, "arith.subi", 0, 0x01, 0x00, 2, 1, 0, 0x00}, - {0x4000, "scf.for", 0, 0x00, 0x00, 3, 0, 1, 0x00}, - {0x4001, "scf.if", 0, 0x00, 0x00, 1, 0, 2, 0x00}, + {0x4000, "scf.for", 0, 0x02, 0x02, 0, 0, 1, 0x00}, + {0x4001, "scf.if", 0, 0x02, 0x00, 1, 0, 2, 0x00}, {0x4002, "scf.yield", 0, 0x00, 0x02, 0, 0, 0, 0x00}, {0x6000, "func.func", 0, 0x00, 0x00, 0, 0, 0, 0x00}, {0x6001, "func.return", 0, 0x00, 0x02, 0, 0, 0, 0x00}, @@ -234,6 +251,7 @@ inline constexpr OpInfo kOpTable[] = { }; inline const OpInfo *lookupByOpcode(uint16_t opcode) { + if (isReservedPtoOpcode(opcode)) return nullptr; // Binary search on kOpTable (sorted by opcode). size_t lo = 0, hi = sizeof(kOpTable) / sizeof(kOpTable[0]); while (lo < hi) { @@ -274,9 +292,9 @@ inline std::optional lookupOpcodeByName(llvm::StringRef name) { .Case("pto.section", 0x0006) .Case("pto.tabs", 0x1006) .Case("pto.tadd", 0x1007) - .Case("pto.taddc", 0x1008) + .Case("pto.acccvt", 0x109C) .Case("pto.tadds", 0x1009) - .Case("pto.taddsc", 0x100A) + .Case("pto.mgather_mask", 0x109D) .Case("pto.tand", 0x100B) .Case("pto.tands", 0x100C) .Case("pto.tci", 0x100D) @@ -304,8 +322,8 @@ inline std::optional lookupOpcodeByName(llvm::StringRef name) { .Case("pto.tfillpad", 0x1023) .Case("pto.tfillpad_expand", 0x1024) .Case("pto.tfillpad_inplace", 0x1025) - .Case("pto.tfmod", 0x1026) - .Case("pto.tfmods", 0x1027) + .Case("pto.mscatter_mask", 0x109F) + .Case("pto.mgather_cas", 0x109E) .Case("pto.tgather", 0x1028) .Case("pto.tgatherb", 0x1029) .Case("pto.tgemv", 0x102A) @@ -315,7 +333,6 @@ inline std::optional lookupOpcodeByName(llvm::StringRef name) { .Case("pto.tinsert_fp", 0x102E) .Case("pto.tload", 0x102F) .Case("pto.tlog", 0x1030) - .Case("pto.tlrelu", 0x1031) .Case("pto.tmatmul", 0x1032) .Case("pto.tmatmul.mx", 0x1033) .Case("pto.tmax", 0x1034) @@ -365,14 +382,12 @@ inline std::optional lookupOpcodeByName(llvm::StringRef name) { .Case("pto.tshls", 0x1060) .Case("pto.tshr", 0x1061) .Case("pto.tshrs", 0x1062) - .Case("pto.tsort32", 0x1063) + .Case("pto.tsort", 0x10A0) .Case("pto.tsqrt", 0x1064) .Case("pto.tstore", 0x1065) .Case("pto.tstore_fp", 0x1066) .Case("pto.tsub", 0x1067) - .Case("pto.tsubc", 0x1068) .Case("pto.tsubs", 0x1069) - .Case("pto.tsubsc", 0x106A) .Case("pto.trowexpandsub", 0x106B) .Case("pto.ttrans", 0x106C) .Case("pto.ttri", 0x106D) @@ -468,9 +483,9 @@ inline std::optional lookupOpcodeAndVariantByFullName(llvm::St .Case("pto.record_event", OpcodeAndVariant{0x1005, 0, 0}) .Case("pto.tabs", OpcodeAndVariant{0x1006, 0, 0}) .Case("pto.tadd", OpcodeAndVariant{0x1007, 0, 0}) - .Case("pto.taddc", OpcodeAndVariant{0x1008, 0, 0}) + .Case("pto.acccvt", OpcodeAndVariant{0x109C, 0, 0}) .Case("pto.tadds", OpcodeAndVariant{0x1009, 0, 0}) - .Case("pto.taddsc", OpcodeAndVariant{0x100A, 0, 0}) + .Case("pto.mgather_mask", OpcodeAndVariant{0x109D, 0, 0}) .Case("pto.tand", OpcodeAndVariant{0x100B, 0, 0}) .Case("pto.tands", OpcodeAndVariant{0x100C, 0, 0}) .Case("pto.tci", OpcodeAndVariant{0x100D, 0, 0}) @@ -498,8 +513,8 @@ inline std::optional lookupOpcodeAndVariantByFullName(llvm::St .Case("pto.tfillpad", OpcodeAndVariant{0x1023, 0, 0}) .Case("pto.tfillpad_expand", OpcodeAndVariant{0x1024, 0, 0}) .Case("pto.tfillpad_inplace", OpcodeAndVariant{0x1025, 0, 0}) - .Case("pto.tfmod", OpcodeAndVariant{0x1026, 0, 0}) - .Case("pto.tfmods", OpcodeAndVariant{0x1027, 0, 0}) + .Case("pto.mscatter_mask", OpcodeAndVariant{0x109F, 0, 0}) + .Case("pto.mgather_cas", OpcodeAndVariant{0x109E, 0, 0}) .Case("pto.tgather", OpcodeAndVariant{0x1028, 0, 0}) .Case("pto.tgatherb", OpcodeAndVariant{0x1029, 0, 0}) .Case("pto.tgetval", OpcodeAndVariant{0x102B, 0, 0}) @@ -508,7 +523,6 @@ inline std::optional lookupOpcodeAndVariantByFullName(llvm::St .Case("pto.tinsert_fp", OpcodeAndVariant{0x102E, 0, 0}) .Case("pto.tload", OpcodeAndVariant{0x102F, 0, 0}) .Case("pto.tlog", OpcodeAndVariant{0x1030, 0, 0}) - .Case("pto.tlrelu", OpcodeAndVariant{0x1031, 0, 0}) .Case("pto.tmax", OpcodeAndVariant{0x1034, 0, 0}) .Case("pto.tmaxs", OpcodeAndVariant{0x1035, 0, 0}) .Case("pto.tmin", OpcodeAndVariant{0x1036, 0, 0}) @@ -556,14 +570,12 @@ inline std::optional lookupOpcodeAndVariantByFullName(llvm::St .Case("pto.tshls", OpcodeAndVariant{0x1060, 0, 0}) .Case("pto.tshr", OpcodeAndVariant{0x1061, 0, 0}) .Case("pto.tshrs", OpcodeAndVariant{0x1062, 0, 0}) - .Case("pto.tsort32", OpcodeAndVariant{0x1063, 0, 0}) + .Case("pto.tsort", OpcodeAndVariant{0x10A0, 0, 0}) .Case("pto.tsqrt", OpcodeAndVariant{0x1064, 0, 0}) .Case("pto.tstore", OpcodeAndVariant{0x1065, 0, 0}) .Case("pto.tstore_fp", OpcodeAndVariant{0x1066, 0, 0}) .Case("pto.tsub", OpcodeAndVariant{0x1067, 0, 0}) - .Case("pto.tsubc", OpcodeAndVariant{0x1068, 0, 0}) .Case("pto.tsubs", OpcodeAndVariant{0x1069, 0, 0}) - .Case("pto.tsubsc", OpcodeAndVariant{0x106A, 0, 0}) .Case("pto.trowexpandsub", OpcodeAndVariant{0x106B, 0, 0}) .Case("pto.ttrans", OpcodeAndVariant{0x106C, 0, 0}) .Case("pto.ttri", OpcodeAndVariant{0x106D, 0, 0}) diff --git a/tools/ptobc/src/mlir_encode.cpp b/tools/ptobc/src/mlir_encode.cpp index 7445f0970a..28d018f001 100644 --- a/tools/ptobc/src/mlir_encode.cpp +++ b/tools/ptobc/src/mlir_encode.cpp @@ -404,10 +404,12 @@ void Encoder::encodeKnownOpImmediates( throw std::runtime_error( "imm_kind=alloc_tile but op is not pto.alloc_tile"); uint8_t mask = 0; - if (at.getValidRow()) + if (at.getAddr()) mask |= 0x1; - if (at.getValidCol()) + if (at.getValidRow()) mask |= 0x2; + if (at.getValidCol()) + mask |= 0x4; out.appendU8(mask); imms.push_back(mask); return; @@ -461,7 +463,8 @@ void Encoder::encodeKnownOpOperands( if (imms.empty()) throw std::runtime_error("optmask operands missing immediate"); uint8_t mask = uint8_t(imms.front()); - emitOperands(((mask & 0x1) ? 1 : 0) + ((mask & 0x2) ? 1 : 0)); + emitOperands(((mask & 0x1) ? 1 : 0) + ((mask & 0x2) ? 1 : 0) + + ((mask & 0x4) ? 1 : 0)); return; } default: diff --git a/tools/ptobc/src/ptobc_decode_print.cpp b/tools/ptobc/src/ptobc_decode_print.cpp index 1c53eb03ba..fd08f36c31 100644 --- a/tools/ptobc/src/ptobc_decode_print.cpp +++ b/tools/ptobc/src/ptobc_decode_print.cpp @@ -462,7 +462,8 @@ readKnownOperandIds(BuildCtx &bc, Reader &r, uint16_t opcode, uint8_t variant, size_t(imms.n2)); case 0x04: return readValueIds(r, ((imms.optMask & 0x1) ? 1 : 0) + - ((imms.optMask & 0x2) ? 1 : 0)); + ((imms.optMask & 0x2) ? 1 : 0) + + ((imms.optMask & 0x4) ? 1 : 0)); default: (void)bc; throw std::runtime_error("unknown operand_mode"); @@ -475,7 +476,9 @@ materializeOperands(BuildCtx &bc, llvm::ArrayRef operandIds) { operands.reserve(operandIds.size()); for (uint64_t valueId : operandIds) { if (valueId >= bc.values.size()) - throw std::runtime_error("operand value_id out of range"); + throw std::runtime_error("operand value_id " + std::to_string(valueId) + + " out of range for " + + std::to_string(bc.values.size()) + " values"); operands.push_back(bc.values[valueId]); } return operands; @@ -489,6 +492,8 @@ static mlir::Operation *buildGenericOpFromReader(BuildCtx &bc, Reader &r, if (nameSid >= bc.strings->size()) throw std::runtime_error("bad op_name sid"); std::string opName = (*bc.strings)[nameSid]; + if (debugEnabled()) + llvm::errs() << "[ptobc] generic " << opName << "\n"; uint64_t nres = r.readULEB(); llvm::SmallVector resultTypes; @@ -553,6 +558,8 @@ static mlir::Operation *buildKnownOpFromReader(BuildCtx &bc, Reader &r, const auto *info = ptobc::v0::lookupByOpcode(opcode); if (!info) throw std::runtime_error("missing opcode schema"); + if (debugEnabled()) + llvm::errs() << "[ptobc] known " << info->name << "\n"; uint8_t variant = info->has_variant_u8 ? r.readU8() : 0; KnownOpImmediates imms = readKnownOpImmediates(r, *info); diff --git a/tools/ptobc/testdata/recent_ops_v0_roundtrip.pto b/tools/ptobc/testdata/recent_ops_v0_roundtrip.pto index ecbf78fa89..b1324400bd 100644 --- a/tools/ptobc/testdata/recent_ops_v0_roundtrip.pto +++ b/tools/ptobc/testdata/recent_ops_v0_roundtrip.pto @@ -6,8 +6,10 @@ // INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. // See LICENSE in the root of the software repository for the full text of the License. -module { - func.func @recent_ops_v0() { +module attributes {pto.target_arch = "a5"} { + func.func @recent_ops_v0( + %a: !pto.tile_buf, + %b: !pto.tile_buf) { %c0 = arith.constant 0 : index %src = pto.alloc_tile : !pto.tile_buf %sub = pto.subview %src[%c0, %c0] sizes [16, 16] : !pto.tile_buf -> !pto.tile_buf @@ -15,23 +17,16 @@ module { %src0 = pto.alloc_tile : !pto.tile_buf %src1 = pto.alloc_tile : !pto.tile_buf - %tmp = pto.alloc_tile : !pto.tile_buf %dst0 = pto.alloc_tile : !pto.tile_buf %dst1 = pto.alloc_tile : !pto.tile_buf %dst2 = pto.alloc_tile : !pto.tile_buf - %dst3 = pto.alloc_tile : !pto.tile_buf %dst4 = pto.alloc_tile : !pto.tile_buf - %dst5 = pto.alloc_tile : !pto.tile_buf %dst6 = pto.alloc_tile : !pto.tile_buf - %dst7 = pto.alloc_tile : !pto.tile_buf pto.trowexpanddiv ins(%src0, %src1 : !pto.tile_buf, !pto.tile_buf) outs(%dst0 : !pto.tile_buf) - pto.trowexpandmul ins(%src0, %src1, %tmp : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst1 : !pto.tile_buf) + pto.trowexpandmul ins(%src0, %src1 : !pto.tile_buf, !pto.tile_buf) outs(%dst1 : !pto.tile_buf) pto.trowexpandexpdif ins(%src0, %src1 : !pto.tile_buf, !pto.tile_buf) outs(%dst2 : !pto.tile_buf) - pto.trowexpandexpdif ins(%src0, %src1, %tmp : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst3 : !pto.tile_buf) pto.trowexpandmax ins(%src0, %src1 : !pto.tile_buf, !pto.tile_buf) outs(%dst4 : !pto.tile_buf) - pto.trowexpandmax ins(%src0, %src1, %tmp : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst5 : !pto.tile_buf) pto.trowexpandmin ins(%src0, %src1 : !pto.tile_buf, !pto.tile_buf) outs(%dst6 : !pto.tile_buf) - pto.trowexpandmin ins(%src0, %src1, %tmp : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst7 : !pto.tile_buf) %part0 = pto.alloc_tile : !pto.tile_buf %part1 = pto.alloc_tile : !pto.tile_buf %partdst = pto.alloc_tile : !pto.tile_buf @@ -42,18 +37,16 @@ module { pto.trsqrt ins(%src : !pto.tile_buf) outs(%rs_dst0 : !pto.tile_buf) pto.trsqrt ins(%src, %rs_tmp : !pto.tile_buf, !pto.tile_buf) outs(%rs_dst1 : !pto.tile_buf) pto.tpartmul ins(%part0, %part1 : !pto.tile_buf, !pto.tile_buf) outs(%partdst : !pto.tile_buf) - %a = pto.alloc_tile : !pto.tile_buf %a_scale = pto.alloc_tile : !pto.tile_buf - %b = pto.alloc_tile : !pto.tile_buf - %b_scale = pto.alloc_tile : !pto.tile_buf - %c_in = pto.alloc_tile : !pto.tile_buf - %bias_mx = pto.alloc_tile : !pto.tile_buf - %dst_mx = pto.alloc_tile : !pto.tile_buf - %dst_mx_acc = pto.alloc_tile : !pto.tile_buf - %dst_mx_bias = pto.alloc_tile : !pto.tile_buf - pto.tgemv.mx ins(%a, %a_scale, %b, %b_scale : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx : !pto.tile_buf) - pto.tgemv.mx.acc ins(%c_in, %a, %a_scale, %b, %b_scale : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx_acc : !pto.tile_buf) - pto.tgemv.mx.bias ins(%a, %a_scale, %b, %b_scale, %bias_mx : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx_bias : !pto.tile_buf) + %b_scale = pto.alloc_tile : !pto.tile_buf + %c_in = pto.alloc_tile : !pto.tile_buf + %bias_mx = pto.alloc_tile : !pto.tile_buf + %dst_mx = pto.alloc_tile : !pto.tile_buf + %dst_mx_acc = pto.alloc_tile : !pto.tile_buf + %dst_mx_bias = pto.alloc_tile : !pto.tile_buf + pto.tgemv.mx ins(%a, %a_scale, %b, %b_scale : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx : !pto.tile_buf) + pto.tgemv.mx.acc ins(%c_in, %a, %a_scale, %b, %b_scale : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx_acc : !pto.tile_buf) + pto.tgemv.mx.bias ins(%a, %a_scale, %b, %b_scale, %bias_mx : !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf, !pto.tile_buf) outs(%dst_mx_bias : !pto.tile_buf) return } } diff --git a/tools/ptobc/tests/CMakeLists.txt b/tools/ptobc/tests/CMakeLists.txt index df3ae8b48a..a733d5087d 100644 --- a/tools/ptobc/tests/CMakeLists.txt +++ b/tools/ptobc/tests/CMakeLists.txt @@ -36,6 +36,14 @@ add_test(NAME ptobc_opcode_coverage_check ${CMAKE_SOURCE_DIR}/tools/ptobc/generated/ptobc_opcodes_v0.h ) +add_test(NAME ptobc_v0571_ops_v0_encode + COMMAND ${CMAKE_COMMAND} -E env + PTOBC_BIN=$ + TEST_INPUT_DIR=${CMAKE_SOURCE_DIR}/test/lit/pto + OUT_DIR=${CMAKE_CURRENT_BINARY_DIR}/ptobc_v0571_ops_out + ${CMAKE_CURRENT_LIST_DIR}/v0571_ops_v0_encode.sh +) + add_test(NAME ptobc_trowexpandsub_v0_encode COMMAND ${CMAKE_COMMAND} -E env PTOBC_BIN=$ diff --git a/tools/ptobc/tests/opcode_coverage_check.py b/tools/ptobc/tests/opcode_coverage_check.py index 757c7ea405..ac93530f0e 100755 --- a/tools/ptobc/tests/opcode_coverage_check.py +++ b/tools/ptobc/tests/opcode_coverage_check.py @@ -10,18 +10,186 @@ import logging import re import sys +from collections import Counter from pathlib import Path +KNOWN_NON_V0_OPCODE_OPS = { + "pto.talloc", + "pto.talloc_to_aic", + "pto.talloc_to_aiv", + "pto.tconcatidx", + "pto.tdeinterleave", + "pto.tgemv.acc", + "pto.tgemv.bias", + "pto.tgemv.mx", + "pto.tgemv.mx.acc", + "pto.tgemv.mx.bias", + "pto.tinterleave", + "pto.tmatmul.acc", + "pto.tmatmul.bias", + "pto.tmatmul.mx.acc", + "pto.tmatmul.mx.bias", +} + +DELETED_OPCODE_ASSIGNMENTS = { + "pto.taddc": 0x1008, + "pto.taddsc": 0x100A, + "pto.tfmod": 0x1026, + "pto.tfmods": 0x1027, + "pto.tlrelu": 0x1031, + "pto.tsort32": 0x1063, + "pto.tsubc": 0x1068, + "pto.tsubsc": 0x106A, +} + +NEW_V0571_OPCODE_ASSIGNMENTS = { + "pto.acccvt": (0x109C, 1), + "pto.mgather_mask": (0x109D, 4), + "pto.mgather_cas": (0x109E, 5), + "pto.mscatter_mask": (0x109F, 4), + # `descending` is a BoolAttr serialized in the attribute dictionary, so + # the PTO-BC SSA operand count is three even though the ISA has four roles. + "pto.tsort": (0x10A0, 3), +} + +EXACT_ARITY = { + "pto.tprefetch": 2, + **{name: arity for name, (_, arity) in NEW_V0571_OPCODE_ASSIGNMENTS.items()}, +} + +PRE_V0571_MAX_PTO_OPCODE = 0x109B + + def parse_td_mnemonics(td_path: Path): td = td_path.read_text(encoding="utf-8", errors="ignore") mns = set(re.findall(r"\bmnemonic\s*=\s*\"([^\"]+)\"", td)) + mns.update(re.findall(r'PTO_TOp<"([^"]+)"', td)) return {f"pto.{m}" for m in mns} -def parse_header_names(h_path: Path): - txt = h_path.read_text(encoding="utf-8", errors="ignore") - return set(re.findall(r"\{0x[0-9A-Fa-f]+,\s*\"([^\"]+)\"", txt)) +def find_duplicates(values): + return sorted(value for value, count in Counter(values).items() if count > 1) + + +def parse_u8_token(token): + constants = { + "kHasVariant": 1, + "kVariantDefault": 0, + "kVariantAcc": 1, + "kVariantBias": 2, + "kVariantMx": 3, + "kVariantMxAcc": 4, + "kVariantMxBias": 5, + "kSectionCubeVariant": 0, + "kSectionVectorVariant": 1, + } + token = token.strip() + return constants[token] if token in constants else int(token, 0) + + +def parse_header(h_path: Path): + text = h_path.read_text(encoding="utf-8", errors="ignore") + table_body = text.split("inline constexpr OpInfo kOpTable[] = {", 1)[1].split( + "};", 1 + )[0] + rows = [ + (int(opcode, 16), name, int(has_variant), int(num_operands)) + for opcode, name, has_variant, num_operands in re.findall( + r'\{(0x[0-9A-Fa-f]+),\s*"([^"]+)",\s*(\d+),' + r"\s*0x[0-9A-Fa-f]+,\s*0x[0-9A-Fa-f]+,\s*(\d+),", + table_body, + ) + ] + + reserved_body = text.split( + "inline constexpr uint16_t kReservedPtoOpcodes[] = {", 1 + )[1].split("};", 1)[0] + reserved = [int(value, 16) for value in re.findall(r"0x[0-9A-Fa-f]+", reserved_body)] + + simple_body = text.split( + "inline std::optional lookupOpcodeByName", 1 + )[1].split("inline const OpInfo *lookupByName", 1)[0] + simple_cases = [ + (name, int(opcode, 16)) + for name, opcode in re.findall( + r'\.Case\("([^"]+)",\s*(0x[0-9A-Fa-f]+)\)', simple_body + ) + ] + + full_body = text.split( + "inline std::optional lookupOpcodeAndVariantByFullName", 1 + )[1].split("inline std::optional lookupOperandCountByFullName", 1)[0] + full_cases = [ + (name, int(opcode, 16), int(has_variant), int(variant)) + for name, opcode, has_variant, variant in re.findall( + r'\.Case\("([^"]+)",\s*OpcodeAndVariant\{' + r"(0x[0-9A-Fa-f]+),\s*([^,}]+),\s*([^}]+)\}\)", + full_body, + ) + for has_variant, variant in [ + (parse_u8_token(has_variant), parse_u8_token(variant)) + ] + ] + return rows, reserved, simple_cases, full_cases + + +def check_header_contract(h_path: Path): + rows, reserved, simple_cases, full_cases = parse_header(h_path) + errors = [] + + for label, values in ( + ("kOpTable opcode", [row[0] for row in rows]), + ("kOpTable name", [row[1] for row in rows]), + ("reserved opcode", reserved), + ("lookupOpcodeByName name", [case[0] for case in simple_cases]), + ("lookupOpcodeAndVariantByFullName name", [case[0] for case in full_cases]), + ( + "lookupOpcodeAndVariantByFullName opcode/variant", + [(case[1], case[2], case[3]) for case in full_cases], + ), + ): + duplicates = find_duplicates(values) + if duplicates: + errors.append(f"duplicate {label}: {duplicates}") + + table_by_name = {name: (opcode, arity) for opcode, name, _, arity in rows} + simple_by_name = dict(simple_cases) + full_by_name = {name: (opcode, has_variant, variant) for name, opcode, has_variant, variant in full_cases} + reserved_set = set(reserved) + + if reserved_set & {row[0] for row in rows}: + errors.append("reserved PTO opcode appears in kOpTable") + + for name, opcode in DELETED_OPCODE_ASSIGNMENTS.items(): + if opcode not in reserved_set: + errors.append(f"deleted opcode 0x{opcode:04X} ({name}) is not reserved") + if name in table_by_name or name in simple_by_name or name in full_by_name: + errors.append(f"deleted operation remains encodable: {name}") + + for name, (opcode, arity) in NEW_V0571_OPCODE_ASSIGNMENTS.items(): + if opcode <= PRE_V0571_MAX_PTO_OPCODE: + errors.append(f"new operation {name} does not use a fresh opcode") + if table_by_name.get(name) != (opcode, arity): + errors.append( + f"{name} table contract must be opcode=0x{opcode:04X}, arity={arity}" + ) + if simple_by_name.get(name) != opcode: + errors.append(f"{name} lookupOpcodeByName mapping mismatch") + if full_by_name.get(name) != (opcode, 0, 0): + errors.append(f"{name} full-name opcode/variant mapping mismatch") + + for name, arity in EXACT_ARITY.items(): + if name not in table_by_name or table_by_name[name][1] != arity: + errors.append(f"{name} must have exactly {arity} bytecode operands") + + for opcode, name, has_variant, _ in rows: + if simple_by_name.get(name) != opcode: + errors.append(f"{name} is inconsistent between kOpTable and name lookup") + if not has_variant and full_by_name.get(name) != (opcode, 0, 0): + errors.append(f"{name} is inconsistent in full-name lookup") + + return {name for _, name, _, _ in rows}, errors def main(): @@ -33,9 +201,17 @@ def main(): h_path = Path(sys.argv[2]) td_ops = parse_td_mnemonics(td_path) - hdr_ops = parse_header_names(h_path) + hdr_ops, header_errors = check_header_contract(h_path) - missing = sorted(op for op in td_ops if op not in hdr_ops) + if header_errors: + logging.error("ptobc v0 opcode table contract violations:") + for error in header_errors: + logging.error(" - %s", error) + return 1 + + missing = sorted( + op for op in td_ops if op not in hdr_ops and op not in KNOWN_NON_V0_OPCODE_OPS + ) if missing: logging.error("ptobc v0 opcode table is missing ops present in PTOOps.td:") for op in missing: diff --git a/tools/ptobc/tests/recent_ops_v0_encode.sh b/tools/ptobc/tests/recent_ops_v0_encode.sh index b8c302bf53..e573060268 100755 --- a/tools/ptobc/tests/recent_ops_v0_encode.sh +++ b/tools/ptobc/tests/recent_ops_v0_encode.sh @@ -35,9 +35,9 @@ grep -F "pto.subview " "${ROUNDTRIP}" >/dev/null grep -F "pto.tprint ins(" "${ROUNDTRIP}" >/dev/null grep -F "pto.trowexpanddiv ins(" "${ROUNDTRIP}" >/dev/null grep -F "pto.trowexpandmul ins(" "${ROUNDTRIP}" >/dev/null -[[ $(grep -Fc "pto.trowexpandexpdif ins(" "${ROUNDTRIP}") -eq 2 ]] -[[ $(grep -Fc "pto.trowexpandmax ins(" "${ROUNDTRIP}") -eq 2 ]] -[[ $(grep -Fc "pto.trowexpandmin ins(" "${ROUNDTRIP}") -eq 2 ]] +[[ $(grep -Fc "pto.trowexpandexpdif ins(" "${ROUNDTRIP}") -eq 1 ]] +[[ $(grep -Fc "pto.trowexpandmax ins(" "${ROUNDTRIP}") -eq 1 ]] +[[ $(grep -Fc "pto.trowexpandmin ins(" "${ROUNDTRIP}") -eq 1 ]] grep -F "pto.trsqrt ins(" "${ROUNDTRIP}" >/dev/null grep -E "pto\\.trsqrt ins\\(%[^,]+, %[^:]+ :" "${ROUNDTRIP}" >/dev/null grep -F "pto.tpartmul ins(" "${ROUNDTRIP}" >/dev/null diff --git a/tools/ptobc/tests/stage9_e2e.sh b/tools/ptobc/tests/stage9_e2e.sh index b4f2b6bcf6..55cbee2dd1 100755 --- a/tools/ptobc/tests/stage9_e2e.sh +++ b/tools/ptobc/tests/stage9_e2e.sh @@ -64,7 +64,7 @@ PY bc2="${OUT_DIR}/${base}.roundtrip.ptobc" "${PTOBC_BIN}" encode "$f" -o "$bc1" || { echo "encode failed: $f"; failed=1; continue; } - "${PTOBC_BIN}" decode "$bc1" -o "$pto2" || { echo "decode failed: $f"; failed=1; continue; } + PTOBC_PRINT_GENERIC=1 "${PTOBC_BIN}" decode "$bc1" -o "$pto2" || { echo "decode failed: $f"; failed=1; continue; } "${PTOBC_BIN}" encode "$pto2" -o "$bc2" || { echo "re-encode failed: $f"; failed=1; continue; } if command -v cmp >/dev/null 2>&1; then diff --git a/tools/ptobc/tests/v0571_ops_v0_encode.sh b/tools/ptobc/tests/v0571_ops_v0_encode.sh new file mode 100755 index 0000000000..e14c1ccbf5 --- /dev/null +++ b/tools/ptobc/tests/v0571_ops_v0_encode.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +set -euo pipefail + +: "${PTOBC_BIN:?PTOBC_BIN not set}" +: "${TEST_INPUT_DIR:?TEST_INPUT_DIR not set}" + +OUT_DIR=${OUT_DIR:-"${PWD}/ptobc_v0571_ops_out"} +mkdir -p "${OUT_DIR}" + +roundtrip() { + local stem=$1 + local input="${TEST_INPUT_DIR}/${stem}.pto" + local bytecode="${OUT_DIR}/${stem}.ptobc" + local decoded="${OUT_DIR}/${stem}.roundtrip.pto" + "${PTOBC_BIN}" encode "${input}" -o "${bytecode}" + "${PTOBC_BIN}" decode "${bytecode}" -o "${decoded}" +} + +roundtrip tsort_emitc +roundtrip v0571_direct_ops_emitc + +grep -F "pto.tsort ins(" "${OUT_DIR}/tsort_emitc.roundtrip.pto" >/dev/null +grep -F "descending = true" "${OUT_DIR}/tsort_emitc.roundtrip.pto" >/dev/null +grep -F "pto.mgather_mask ins(" "${OUT_DIR}/v0571_direct_ops_emitc.roundtrip.pto" >/dev/null +grep -F "pto.mgather_cas ins(" "${OUT_DIR}/v0571_direct_ops_emitc.roundtrip.pto" >/dev/null +grep -F "pto.mscatter_mask ins(" "${OUT_DIR}/v0571_direct_ops_emitc.roundtrip.pto" >/dev/null +grep -F "pto.acccvt outs(" "${OUT_DIR}/v0571_direct_ops_emitc.roundtrip.pto" >/dev/null