From 3d463b3d7948520cca82afe67df7d67c4163b897 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Mon, 26 Jan 2026 20:37:28 +0000 Subject: [PATCH 01/40] weights for dense --- hls4ml/backends/oneapi/passes/core_templates.py | 15 +++++++++++---- hls4ml/templates/oneapi/firmware/myproject.cpp | 5 ++++- hls4ml/templates/oneapi/firmware/myproject.h | 3 +++ .../oneapi/firmware/nnet_utils/nnet_dense.h | 7 +++---- hls4ml/writer/oneapi_writer.py | 8 ++++++++ 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 9602b2d0fc..64a4c7097a 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -6,6 +6,7 @@ # Dense templates dense_config_template = """struct config{index} : nnet::dense_config {{ + static constexpr unsigned n_in = {n_in}; static constexpr unsigned n_out = {n_out}; static constexpr unsigned io_type = nnet::{iotype}; @@ -30,13 +31,16 @@ typedef {weight_t.name} weight_t; typedef {index_t.name} index_t; + static constexpr weight_t weights = {weights}; + static constexpr bias_t biases = {biases}; + template using product = nnet::product::{product_type}; }};\n""" -dense_function_template = 'nnet::dense_{strategy}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});' +dense_function_template = 'nnet::dense_{strategy}<{input_t}, {output_t}, {config}>({input}, {output});' dense_task_sequence_template = 'task_sequence> {name};' -dense_stream_function_template = '{name}.async({w}, {b});' +dense_stream_function_template = '{name}.async();' dense_include_list = ['nnet_utils/nnet_dense.h', 'nnet_utils/nnet_dense_stream.h'] @@ -53,6 +57,9 @@ def format(self, node): node.get_input_variable().type.precision, node.get_weights('weight').type.precision ) + params['weights'] = node.get_weights('weight').name + params['biases'] = node.get_weights('bias').name + return self.template.format(**params) @@ -63,8 +70,8 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) - params['w'] = node.get_weights('weight').name - params['b'] = node.get_weights('bias').name + #params['w'] = node.get_weights('weight').name + #params['b'] = node.get_weights('bias').name return self.template.format(**params) diff --git a/hls4ml/templates/oneapi/firmware/myproject.cpp b/hls4ml/templates/oneapi/firmware/myproject.cpp index 06e7d3fe37..da9439f74a 100644 --- a/hls4ml/templates/oneapi/firmware/myproject.cpp +++ b/hls4ml/templates/oneapi/firmware/myproject.cpp @@ -1,9 +1,12 @@ #include "myproject.h" -#include "parameters.h" #include // hls-fpga-machine-learning insert weights + +#include "parameters.h" + + // The inter-task pipes need to be declared in the global scope // hls-fpga-machine-learning insert inter-task pipes diff --git a/hls4ml/templates/oneapi/firmware/myproject.h b/hls4ml/templates/oneapi/firmware/myproject.h index 082ae5dc8c..8f313ea30f 100644 --- a/hls4ml/templates/oneapi/firmware/myproject.h +++ b/hls4ml/templates/oneapi/firmware/myproject.h @@ -3,6 +3,9 @@ #include "defines.h" +// hls-fpga-machine-learning insert weights + + // This file defines the interface to the kernel // currently this is fixed diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h index dc76189083..2b65eef42b 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h @@ -152,12 +152,11 @@ void dense_rf_lt(const data_T &data, res_T &res, const typename CONFIG_T::weight } } template -void dense_resource(const data_T &data, res_T &res, const typename CONFIG_T::weight_t &weights, - const typename CONFIG_T::bias_t &biases) { +void dense_resource(const data_T &data, res_T &res) { if (CONFIG_T::reuse_factor <= CONFIG_T::n_in) { - dense_rf_lt(data, res, weights, biases); + dense_rf_lt(data, res, CONFIG_T::weights, CONFIG_T::biases); } else { - dense_rf_gt(data, res, weights, biases); + dense_rf_gt(data, res, CONFIG_T::weights, CONFIG_T::biases); } } } // namespace nnet diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 3c0a778c50..b42ff2990f 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -242,6 +242,14 @@ def write_project_header(self, model): for out in model_outputs: newline += out.declare_cpp() + # Insert weights + elif '// hls-fpga-machine-learning insert weights' in line: + newline = line + for layer in model.get_layers(): + for w in layer.get_weights(): + #if w not in model_brams: + newline += f'#include "weights/{w.name}.h"\n' + # Simply copy line, if no inserts are required else: newline = line From d67857369385d066b7cdaad49077069b3bf9473c Mon Sep 17 00:00:00 2001 From: Chang Sun Date: Tue, 27 Jan 2026 18:58:42 +0000 Subject: [PATCH 02/40] hgq2 homogeneous quant fix --- hls4ml/converters/keras_v3/hgq2/_base.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/hls4ml/converters/keras_v3/hgq2/_base.py b/hls4ml/converters/keras_v3/hgq2/_base.py index 4a6d0a22c2..f7b4c9ddd3 100644 --- a/hls4ml/converters/keras_v3/hgq2/_base.py +++ b/hls4ml/converters/keras_v3/hgq2/_base.py @@ -30,15 +30,19 @@ def extract_fixed_quantizer_config(q, tensor: 'KerasTensor', is_input: bool) -> k, B, I = ops.convert_to_numpy(k), ops.convert_to_numpy(B), ops.convert_to_numpy(I) # noqa: E741 I = np.where(B > 0, I, 0) # noqa: E741 # type: ignore - k = np.broadcast_to(k.astype(np.int16), (1,) + shape) # type: ignore - B = np.broadcast_to(B.astype(np.int16), (1,) + shape) # type: ignore - I = np.broadcast_to(I.astype(np.int16), (1,) + shape) # noqa: E741 + if np.size(k) != 1: + k = np.broadcast_to(k.astype(np.int16), (1,) + shape) # type: ignore + B = np.broadcast_to(B.astype(np.int16), (1,) + shape) # type: ignore + I = np.broadcast_to(I.astype(np.int16), (1,) + shape) # noqa: E741 + else: + k = np.ravel(k).astype(np.int16) + B = np.ravel(B).astype(np.int16) + I = np.ravel(I).astype(np.int16) # noqa: E741 overflow_mode: str = internal_q.overflow_mode round_mode: str = internal_q.round_mode if round_mode.startswith('S_'): round_mode = round_mode[2:] - fusible = np.unique(k).size == 1 and np.unique(B).size == 1 and np.unique(I).size == 1 input_keras_tensor_names = tensor.name if is_input else f'{tensor.name}_q' output_keras_tensor_names = f'{tensor.name}_q' if is_input else tensor.name @@ -48,7 +52,7 @@ def extract_fixed_quantizer_config(q, tensor: 'KerasTensor', is_input: bool) -> 'mask_kbi': (k, B, I), 'SAT': overflow_mode, 'RND': round_mode, - 'fusible': fusible, + 'fusible': None, 'input_keras_tensor_names': [input_keras_tensor_names], 'output_keras_tensor_names': [output_keras_tensor_names], 'overrides': {}, From 59bd96f0c5e9c8e95538a9e96e0233c2d70695ba Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Mon, 9 Feb 2026 16:31:00 +0000 Subject: [PATCH 03/40] Changes required for oneAPI MHA --- hls4ml/backends/oneapi/oneapi_backend.py | 8 - .../backends/oneapi/passes/core_templates.py | 88 ++++++++++- .../keras_v3/hgq2/multi_head_attention.py | 4 +- .../firmware/nnet_utils/nnet_activation.h | 82 +++++++--- .../oneapi/firmware/nnet_utils/nnet_dense.h | 7 +- hls4ml/writer/oneapi_writer.py | 149 ++++++++++-------- 6 files changed, 233 insertions(+), 105 deletions(-) diff --git a/hls4ml/backends/oneapi/oneapi_backend.py b/hls4ml/backends/oneapi/oneapi_backend.py index 0c11c16d09..94f26c9f1c 100644 --- a/hls4ml/backends/oneapi/oneapi_backend.py +++ b/hls4ml/backends/oneapi/oneapi_backend.py @@ -19,7 +19,6 @@ Embedding, Layer, SimpleRNN, - Softmax, ) from hls4ml.model.optimizer import get_backend_passes, layer_optimizer from hls4ml.model.types import FixedPrecisionType, IntegerPrecisionType, NamedType @@ -257,13 +256,6 @@ def init_activation(self, layer): if layer.get_attr('recurrent_activation') == 'tanh': layer.set_attr('recurrent_activation', 'dense_tanh') - @layer_optimizer(Softmax) - def init_softmax(self, layer): - if layer.model.config.get_config_value('IOType') == 'io_parallel': - assert len(layer.get_input_variable().shape) == 1, ( - 'Softmax with io_parallel strategy cannot be used on multidimensional tensors.' - ) - @layer_optimizer(Embedding) def init_embed(self, layer): if layer.attributes['n_in'] is None: diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 64a4c7097a..5a2d765e8f 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -38,7 +38,7 @@ using product = nnet::product::{product_type}; }};\n""" -dense_function_template = 'nnet::dense_{strategy}<{input_t}, {output_t}, {config}>({input}, {output});' +dense_function_template = 'nnet::dense_{strategy}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});' dense_task_sequence_template = 'task_sequence> {name};' dense_stream_function_template = '{name}.async();' dense_include_list = ['nnet_utils/nnet_dense.h', 'nnet_utils/nnet_dense_stream.h'] @@ -70,8 +70,8 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) - #params['w'] = node.get_weights('weight').name - #params['b'] = node.get_weights('bias').name + params['w'] = node.get_weights('weight').name + params['b'] = node.get_weights('bias').name return self.template.format(**params) @@ -199,7 +199,7 @@ def format(self, node): static constexpr unsigned reuse_factor = {reuse}; }};\n""" -softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ +softmax_config_template_qkeras = """struct {type}_config{index} : nnet::activ_config {{ static constexpr unsigned n_in = {n_in}; static constexpr unsigned table_size = {table_size}; static constexpr unsigned io_type = nnet::{iotype}; @@ -209,6 +209,26 @@ def format(self, node): typedef {inv_table_t.name} inv_table_t; }};\n""" +softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ + static const unsigned n_in = {n_in}; + static const unsigned n_slice = {n_slice}; + static const unsigned n_outer = {n_outer}; + static const unsigned n_inner = {n_inner}; + static const unsigned parallelization_factor = {parallelization_factor}; + static const unsigned exp_table_size = {exp_table_size}; + static const unsigned inv_table_size = {inv_table_size}; + static const unsigned io_type = nnet::{iotype}; + static const unsigned reuse_factor = {reuse}; + static const unsigned axis = {axis}; + static const nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; + static constexpr float exp_scale = {exp_scale}; + typedef {exp_table_t.name} exp_table_t; + typedef {inv_table_t.name} inv_table_t; + typedef {accum_t.name} accum_t; + typedef {inv_inp_t.name} inv_inp_t; + typedef {inp_norm_t_str} inp_norm_t; +}};\n""" + activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' param_activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {param}, {output});' @@ -260,10 +280,68 @@ def __init__(self): super(ActivationConfigTemplate, self).__init__(Softmax) # Skip ActivationConfigTemplate's __init__ self.template = softmax_config_template + def format(self, node): + from math import ceil, log2 + + params = self._default_config_params(node) + params['type'] = node.get_attr('activation') + params.setdefault('exp_table_size', params['table_size']) + params.setdefault('inv_table_size', params['table_size']) + params.setdefault('n_inner', 1) + params.setdefault('n_outer', 1) + params.setdefault('exp_scale', 1.0) + params.setdefault('parallelization_factor', -1) + + n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] # type: ignore + params['n_slice'] = n_slice + + if params['accum_t'].name == 'model_default_t': # type: ignore + scale = ceil(log2(n_slice)) + exp_table_t = node.attributes['exp_table_t'].precision + signed, width, integers = exp_table_t.signed, exp_table_t.width, exp_table_t.integer + params['accum_t_str'] = f'ac_{"" if signed else "u"}fixed<{width + scale}, {integers + scale}>' + else: + params['accum_t_str'] = params['accum_t'].name # type: ignore + if params['inv_inp_t'].name == 'model_default_t': # type: ignore + params['inv_inp_t'] = params['exp_table_t'] + + if params['implementation'] == 'stable': + if 'inp_norm_t' not in params: + # Only used in stable (max-normalized) implementation + input_t = node.get_input_variable().type.precision + width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 + width, iwidth = width - signed, iwidth - signed + if signed: + # Fix table size if too large + exp_table_size = params['inv_table_size'] + params['exp_table_size'] = str(min(int(exp_table_size), 2**width)) + params['inp_norm_t_str'] = f'ac_ufixed<{width}, {iwidth}>' + else: + params['inp_norm_t_str'] = params['inp_norm_t'].name # type: ignore + else: + params['inp_norm_t_str'] = 'ac_fixed<1,0>' + + return self.template.format(**params) + + +class SoftmaxFunctionTemplate(FunctionCallTemplate): + def __init__(self): + super().__init__(Softmax, include_header=activ_include_list) + self.template = activ_function_template + + def format(self, node): + params = self._default_function_params(node) + use_multidim = node.get_attr('n_inner', 1) > 1 or node.get_attr('n_outer', 1) > 1 + use_multidim = use_multidim and node.model.config.get_config_value('IOType') == 'io_parallel' + params['activation'] = 'softmax' if not use_multidim else 'softmax_multidim' + params['config'] = f'softmax_config{node.index}' + + return self.template.format(**params) + class ActivationFunctionTemplate(FunctionCallTemplate): def __init__(self): - super().__init__((Activation, HardActivation, Softmax), include_header=activ_include_list) + super().__init__((Activation, HardActivation), include_header=activ_include_list) self.template = activ_function_template def format(self, node): diff --git a/hls4ml/converters/keras_v3/hgq2/multi_head_attention.py b/hls4ml/converters/keras_v3/hgq2/multi_head_attention.py index 24bd87d3e9..d5c1eda7b9 100644 --- a/hls4ml/converters/keras_v3/hgq2/multi_head_attention.py +++ b/hls4ml/converters/keras_v3/hgq2/multi_head_attention.py @@ -15,7 +15,7 @@ @register class QMultiHeadAttentionHandler(QLayerHandler): - handles = ('hgq.layers.multi_head_attention.QMultiHeadAttention',) + handles = ('hgq.layers.attn.mha.QMultiHeadAttention',) def handle( self, @@ -129,7 +129,7 @@ def _handle(self, layer, tensor_q, tensor_O, node_index, tensor_k, tensor_v): @register class QLinformerAttentionHandler(QMultiHeadAttentionHandler): - handles = ('hgq.layers.linformer_attention.QLinformerAttention',) + handles = ('hgq.layers.attn.linformer.QLinformerAttention',) def handle( self, diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index f118ecb05c..c2353c34a8 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -100,15 +100,8 @@ template void sigmoid(const data_ enum class softmax_implementation { latency = 0, legacy = 1, stable = 2, argmax = 3 }; template inline unsigned softmax_stable_idx_from_real_val(const data_T x) { - // Number of address bits for table - static constexpr int N = ceillog2::val; - - // Slice the top N bits of the input - [[intel::fpga_register]] ac_int y = x.template slc(x.width - N - 1); - // If x is the most negative value, the slice will be 0, so we need to set the 0-th bit to ensure correctness - if (x != 0 && y == 0) - y[0] = 1; - return y.to_uint(); + // Extract the lower 'width' bits of x + return x.template slc(0).to_uint(); } template inline unsigned softmax_latency_idx_from_real_val(const data_T x) { @@ -121,7 +114,6 @@ template inline unsigned softmax_latency_idx_f } template void softmax_stable(const data_T &data, res_T &res) { -// Look-up tables #include "activation_tables/exp_table.tb" #include "activation_tables/invert_table.tb" @@ -130,29 +122,34 @@ template void softmax_stable(cons [[intel::fpga_register]] auto x_max = reduce>(data.data(), op_max); - // For the diffs, use the same type as the input but force rounding and saturation - [[intel::fpga_register]] ac_fixed - d_xi_xmax[CONFIG_T::n_in]; + // Normalize inputs: d = x_max - x + [[intel::fpga_register]] typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - d_xi_xmax[i] = data[i] - x_max; + // HGQ stable: d = x_max - data + d_xi_xmax[i] = x_max - data[i]; } - // Calculate all the e^x's + // Exponentials [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - exp_res[i] = exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[i])]; + unsigned idx = softmax_stable_idx_from_real_val(d_xi_xmax[i]); + exp_res[i] = exp_table[idx]; } - // Explicitly sum previously calculated exponentials with an adder tree - Op_add op_add; - [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_sum = - reduce>(exp_res, op_add); + // Sum of Exponentials + Op_add op_add; + [[intel::fpga_register]] typename CONFIG_T::accum_t exp_sum = + reduce>(exp_res, op_add); - // Multiply previously calculated exponetials with the reciprocal of the sum - [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table[softmax_stable_idx_from_real_val(exp_sum)]; + // Reciprocal of Sum + typename CONFIG_T::inv_inp_t exp_sum_cast = exp_sum; + unsigned inv_idx = softmax_stable_idx_from_real_val(exp_sum_cast); + + [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = invert_table[inv_idx]; + + // Final Multiplication #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { res[i] = exp_res[i] * inv_exp_sum; @@ -265,6 +262,45 @@ template inline void softmax(cons } } +// ************************************************* +// Multidimensional Softmax +// ************************************************* + +// Helper to remap the config for the core softmax function +template struct softmax_multidim_slice_config : CONFIG_T { + static constexpr unsigned n_in = CONFIG_T::n_slice; +}; + +template inline void softmax_multidim(const data_T &data, res_T &res) { + using buffer_data_t = std::array; + using buffer_res_t = std::array; + using slice_config = softmax_multidim_slice_config; + + #pragma unroll + for (unsigned i = 0; i < CONFIG_T::n_outer; i++) { + #pragma unroll + for (unsigned k = 0; k < CONFIG_T::n_inner; k++) { + + [[intel::fpga_register]] buffer_data_t buffer_in; + [[intel::fpga_register]] buffer_res_t buffer_out; + + // Gather Phase + #pragma unroll + for (unsigned j = 0; j < CONFIG_T::n_slice; j++) { + unsigned idx = (i * CONFIG_T::n_slice * CONFIG_T::n_inner) + (j * CONFIG_T::n_inner) + k; + buffer_in[j] = data[idx]; + } + + nnet::softmax(buffer_in, buffer_out); + + #pragma unroll + for (unsigned j = 0; j < CONFIG_T::n_slice; j++) { + unsigned idx = (i * CONFIG_T::n_slice * CONFIG_T::n_inner) + (j * CONFIG_T::n_inner) + k; + res[idx] = buffer_out[j]; + } + } + } +} // ************************************************* // TanH Activation // ************************************************* diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h index 2b65eef42b..dc76189083 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_dense.h @@ -152,11 +152,12 @@ void dense_rf_lt(const data_T &data, res_T &res, const typename CONFIG_T::weight } } template -void dense_resource(const data_T &data, res_T &res) { +void dense_resource(const data_T &data, res_T &res, const typename CONFIG_T::weight_t &weights, + const typename CONFIG_T::bias_t &biases) { if (CONFIG_T::reuse_factor <= CONFIG_T::n_in) { - dense_rf_lt(data, res, CONFIG_T::weights, CONFIG_T::biases); + dense_rf_lt(data, res, weights, biases); } else { - dense_rf_gt(data, res, CONFIG_T::weights, CONFIG_T::biases); + dense_rf_gt(data, res, weights, biases); } } } // namespace nnet diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index b42ff2990f..007b645cb0 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -242,13 +242,13 @@ def write_project_header(self, model): for out in model_outputs: newline += out.declare_cpp() - # Insert weights + # Insert weights elif '// hls-fpga-machine-learning insert weights' in line: newline = line for layer in model.get_layers(): for w in layer.get_weights(): - #if w not in model_brams: - newline += f'#include "weights/{w.name}.h"\n' + # if w not in model_brams: + newline += f'#include "weights/{w.name}.h"\n' # Simply copy line, if no inserts are required else: @@ -557,16 +557,16 @@ def write_nnet_utils(self, model): dstpath = f'{model.config.get_output_dir()}/src/firmware/{dst}' copyfile(srcpath, dstpath) - def __get_table_size(self, model, activation): + def __get_table_size(self, model, activation, table_name='table_size'): for layer in model.get_layers(): if ( layer.get_attr('activation') == activation or layer.get_attr('recurrent_activation') == activation - ) and layer.get_attr('table_size') is not None: - return int(layer.get_attr('table_size')) + ) and layer.get_attr(table_name) is not None: + return int(layer.get_attr(table_name)) return 1024 - def __get_table_header(self, table_name, table_size): - table_header = f'static const typename CONFIG_T::table_t {table_name}[{table_size}] = {{' + def __get_table_header(self, table_name, table_size, table_type='table_t'): + table_header = f'static const typename CONFIG_T::{table_type} {table_name}[{table_size}] = {{' return table_header def __write_elu_table(self, model, path): @@ -695,46 +695,58 @@ def __write_selu_table(self, model, path): h_file.write('};\n') h_file.close() + def __get_table_precision(self, model, activation, table_name='table_precision'): + for layer in model.get_layers(): + if layer.get_attr('activation') == activation and layer.get_attr(table_name) is not None: + precision = layer.get_attr(table_name) + return precision.precision + + return None # fp_bits, fp_integer, fp_signed + def __write_exp_table(self, model, path): table_name = 'exp_table' - table_size = self.__get_table_size(model, 'softmax') + table_size = self.__get_table_size(model, 'softmax', table_name='exp_table_size') h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) + h_file.write(self.__get_table_header(table_name, table_size, table_type='exp_table_t')) # Default fixed point precision # 6 bits for integer part, 10 bits for decimal - total, 16 - fp_bits = 16 - fp_integer = 6 - fp_signed = True + precision = self.__get_table_precision(model, 'softmax', table_name='inp_norm_t') + + if precision is None: + fp_bits = 16 + fp_integer = 6 + fp_signed = True + + for layer in model.get_layers(): + if layer.name == 'softmax': + ac_type = layer.get_input_variable().type + if ac_type is not None: + try: + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed + except Exception: + # FixedPrecisionType wasn't correctly stored in layer attributes, use default values + pass + if fp_signed is False: + raise Exception('Softmax types need to be signed') - # Exp table should use the same precision as exp_table, as seen in Vivado code - # init_exp_table(exp_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_input_variable().type - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') + else: + fp_bits = precision.width + fp_integer = precision.integer + fp_signed = precision.signed + f_bits = fp_bits - fp_integer sep = '' - N = ceil_log2(table_size) for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) - b = uint_to_binary(i, N) - if i == 0: - b.insert(0, 0) - else: - b.insert(0, 1) - f.set_msb_bits(b) - real_val = f.exp_float() + # Index represents the raw bit pattern of the input + real_val_in = i * (2.0 ** (-f_bits)) + + # Calculate exp(-x) for the stable implementation + real_val = np.exp(-real_val_in) + h_file.write(sep + str(real_val)) sep = ', ' @@ -743,41 +755,50 @@ def __write_exp_table(self, model, path): def __write_invert_table(self, model, path): table_name = 'invert_table' - table_size = self.__get_table_size(model, 'softmax') + table_size = self.__get_table_size(model, 'softmax', table_name='inv_table_size') h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) - + h_file.write(self.__get_table_header(table_name, table_size, table_type='inv_table_t')) # Default fixed point precision, in case values from layer attributes cannot be extracted # 8 bits for integer part, 10 bits for decimal - total, 18 - fp_bits = 18 - fp_integer = 8 - fp_signed = True - # Invert table should use the same precision as exp_table, as seen in Vivado code - # init_invert_table(invert_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_attr('exp_table_t') - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') + precision = self.__get_table_precision(model, 'softmax', table_name='inv_inp_t') + + if precision is None: + fp_bits = 18 + fp_integer = 8 + fp_signed = True + + for layer in model.get_layers(): + if layer.name == 'softmax': + ac_type = layer.get_attr('exp_table_t') + if ac_type is not None: + try: + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed + except Exception: + # FixedPrecisionType wasn't correctly stored in layer attributes, use default values + pass + if fp_signed is False: + raise Exception('Softmax types need to be signed') + + else: + fp_bits = precision.width + fp_integer = precision.integer + fp_signed = precision.signed + f_bits = fp_bits - fp_integer sep = '' - N = ceil_log2(table_size) for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) - b = uint_to_binary(i, N) - b.insert(0, 0) - f.set_msb_bits(b) - real_val = f.inv_float() + # Index represents the raw bit pattern of the input + real_val_in = i * (2.0 ** (-f_bits)) + + if real_val_in == 0: + real_val = 999.0 + else: + real_val = 1.0 / real_val_in + h_file.write(sep + str(real_val)) sep = ', ' From dbb207b7a5c1f343d8100bba9645340a2098730c Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Mon, 9 Feb 2026 16:33:38 +0000 Subject: [PATCH 04/40] Original weight implementation --- .../backends/oneapi/passes/core_templates.py | 91 +------------------ 1 file changed, 3 insertions(+), 88 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 5a2d765e8f..9602b2d0fc 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -6,7 +6,6 @@ # Dense templates dense_config_template = """struct config{index} : nnet::dense_config {{ - static constexpr unsigned n_in = {n_in}; static constexpr unsigned n_out = {n_out}; static constexpr unsigned io_type = nnet::{iotype}; @@ -31,16 +30,13 @@ typedef {weight_t.name} weight_t; typedef {index_t.name} index_t; - static constexpr weight_t weights = {weights}; - static constexpr bias_t biases = {biases}; - template using product = nnet::product::{product_type}; }};\n""" dense_function_template = 'nnet::dense_{strategy}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});' dense_task_sequence_template = 'task_sequence> {name};' -dense_stream_function_template = '{name}.async();' +dense_stream_function_template = '{name}.async({w}, {b});' dense_include_list = ['nnet_utils/nnet_dense.h', 'nnet_utils/nnet_dense_stream.h'] @@ -57,9 +53,6 @@ def format(self, node): node.get_input_variable().type.precision, node.get_weights('weight').type.precision ) - params['weights'] = node.get_weights('weight').name - params['biases'] = node.get_weights('bias').name - return self.template.format(**params) @@ -199,7 +192,7 @@ def format(self, node): static constexpr unsigned reuse_factor = {reuse}; }};\n""" -softmax_config_template_qkeras = """struct {type}_config{index} : nnet::activ_config {{ +softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ static constexpr unsigned n_in = {n_in}; static constexpr unsigned table_size = {table_size}; static constexpr unsigned io_type = nnet::{iotype}; @@ -209,26 +202,6 @@ def format(self, node): typedef {inv_table_t.name} inv_table_t; }};\n""" -softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ - static const unsigned n_in = {n_in}; - static const unsigned n_slice = {n_slice}; - static const unsigned n_outer = {n_outer}; - static const unsigned n_inner = {n_inner}; - static const unsigned parallelization_factor = {parallelization_factor}; - static const unsigned exp_table_size = {exp_table_size}; - static const unsigned inv_table_size = {inv_table_size}; - static const unsigned io_type = nnet::{iotype}; - static const unsigned reuse_factor = {reuse}; - static const unsigned axis = {axis}; - static const nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; - static constexpr float exp_scale = {exp_scale}; - typedef {exp_table_t.name} exp_table_t; - typedef {inv_table_t.name} inv_table_t; - typedef {accum_t.name} accum_t; - typedef {inv_inp_t.name} inv_inp_t; - typedef {inp_norm_t_str} inp_norm_t; -}};\n""" - activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' param_activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {param}, {output});' @@ -280,68 +253,10 @@ def __init__(self): super(ActivationConfigTemplate, self).__init__(Softmax) # Skip ActivationConfigTemplate's __init__ self.template = softmax_config_template - def format(self, node): - from math import ceil, log2 - - params = self._default_config_params(node) - params['type'] = node.get_attr('activation') - params.setdefault('exp_table_size', params['table_size']) - params.setdefault('inv_table_size', params['table_size']) - params.setdefault('n_inner', 1) - params.setdefault('n_outer', 1) - params.setdefault('exp_scale', 1.0) - params.setdefault('parallelization_factor', -1) - - n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] # type: ignore - params['n_slice'] = n_slice - - if params['accum_t'].name == 'model_default_t': # type: ignore - scale = ceil(log2(n_slice)) - exp_table_t = node.attributes['exp_table_t'].precision - signed, width, integers = exp_table_t.signed, exp_table_t.width, exp_table_t.integer - params['accum_t_str'] = f'ac_{"" if signed else "u"}fixed<{width + scale}, {integers + scale}>' - else: - params['accum_t_str'] = params['accum_t'].name # type: ignore - if params['inv_inp_t'].name == 'model_default_t': # type: ignore - params['inv_inp_t'] = params['exp_table_t'] - - if params['implementation'] == 'stable': - if 'inp_norm_t' not in params: - # Only used in stable (max-normalized) implementation - input_t = node.get_input_variable().type.precision - width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 - width, iwidth = width - signed, iwidth - signed - if signed: - # Fix table size if too large - exp_table_size = params['inv_table_size'] - params['exp_table_size'] = str(min(int(exp_table_size), 2**width)) - params['inp_norm_t_str'] = f'ac_ufixed<{width}, {iwidth}>' - else: - params['inp_norm_t_str'] = params['inp_norm_t'].name # type: ignore - else: - params['inp_norm_t_str'] = 'ac_fixed<1,0>' - - return self.template.format(**params) - - -class SoftmaxFunctionTemplate(FunctionCallTemplate): - def __init__(self): - super().__init__(Softmax, include_header=activ_include_list) - self.template = activ_function_template - - def format(self, node): - params = self._default_function_params(node) - use_multidim = node.get_attr('n_inner', 1) > 1 or node.get_attr('n_outer', 1) > 1 - use_multidim = use_multidim and node.model.config.get_config_value('IOType') == 'io_parallel' - params['activation'] = 'softmax' if not use_multidim else 'softmax_multidim' - params['config'] = f'softmax_config{node.index}' - - return self.template.format(**params) - class ActivationFunctionTemplate(FunctionCallTemplate): def __init__(self): - super().__init__((Activation, HardActivation), include_header=activ_include_list) + super().__init__((Activation, HardActivation, Softmax), include_header=activ_include_list) self.template = activ_function_template def format(self, node): From 51efff0c34744ab2fa70d7e3a52fdbf196ffcf0a Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Mon, 9 Feb 2026 16:51:19 +0000 Subject: [PATCH 05/40] Restore oneAPI weight placement --- hls4ml/templates/oneapi/firmware/myproject.cpp | 5 +---- hls4ml/templates/oneapi/firmware/myproject.h | 3 --- hls4ml/writer/oneapi_writer.py | 7 ------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/hls4ml/templates/oneapi/firmware/myproject.cpp b/hls4ml/templates/oneapi/firmware/myproject.cpp index da9439f74a..06e7d3fe37 100644 --- a/hls4ml/templates/oneapi/firmware/myproject.cpp +++ b/hls4ml/templates/oneapi/firmware/myproject.cpp @@ -1,12 +1,9 @@ #include "myproject.h" +#include "parameters.h" #include // hls-fpga-machine-learning insert weights - -#include "parameters.h" - - // The inter-task pipes need to be declared in the global scope // hls-fpga-machine-learning insert inter-task pipes diff --git a/hls4ml/templates/oneapi/firmware/myproject.h b/hls4ml/templates/oneapi/firmware/myproject.h index 8f313ea30f..082ae5dc8c 100644 --- a/hls4ml/templates/oneapi/firmware/myproject.h +++ b/hls4ml/templates/oneapi/firmware/myproject.h @@ -3,9 +3,6 @@ #include "defines.h" -// hls-fpga-machine-learning insert weights - - // This file defines the interface to the kernel // currently this is fixed diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 007b645cb0..8ef2b0b0a1 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -242,13 +242,6 @@ def write_project_header(self, model): for out in model_outputs: newline += out.declare_cpp() - # Insert weights - elif '// hls-fpga-machine-learning insert weights' in line: - newline = line - for layer in model.get_layers(): - for w in layer.get_weights(): - # if w not in model_brams: - newline += f'#include "weights/{w.name}.h"\n' # Simply copy line, if no inserts are required else: From 6067bea99e35fd0bb3b2d89323e721e3916b0960 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Mon, 9 Feb 2026 16:52:42 +0000 Subject: [PATCH 06/40] pre-commit --- hls4ml/writer/oneapi_writer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 8ef2b0b0a1..b945f3faf9 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -242,7 +242,6 @@ def write_project_header(self, model): for out in model_outputs: newline += out.declare_cpp() - # Simply copy line, if no inserts are required else: newline = line From 16ca197d57e0d72485265cf25e170ee7fc576280 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Tue, 24 Feb 2026 16:40:32 +0000 Subject: [PATCH 07/40] softmax multidim templates --- .../backends/oneapi/passes/core_templates.py | 74 ++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 9602b2d0fc..8205fb1e17 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -1,3 +1,5 @@ +from math import ceil, log2 + from hls4ml.backends.backend import get_backend from hls4ml.backends.oneapi.oneapi_template import StreamFunctionCallTemplate, TaskSequenceTemplate from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate @@ -192,14 +194,24 @@ def format(self, node): static constexpr unsigned reuse_factor = {reuse}; }};\n""" + softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ - static constexpr unsigned n_in = {n_in}; - static constexpr unsigned table_size = {table_size}; - static constexpr unsigned io_type = nnet::{iotype}; - static constexpr unsigned reuse_factor = {reuse}; - static constexpr nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; + static const unsigned n_in = {n_in}; + static const unsigned n_slice = {n_slice}; + static const unsigned n_outer = {n_outer}; + static const unsigned n_inner = {n_inner}; + static const unsigned parallelization_factor = {parallelization_factor}; + static const unsigned exp_table_size = {exp_table_size}; + static const unsigned inv_table_size = {inv_table_size}; + static const unsigned io_type = nnet::{iotype}; + static const unsigned reuse_factor = {reuse}; + static const unsigned axis = {axis}; + static const nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; + static constexpr float exp_scale = {exp_scale}; typedef {exp_table_t.name} exp_table_t; typedef {inv_table_t.name} inv_table_t; + //typedef {accum_t.name} accum_t; + //typedef {inp_norm_t_str} inp_norm_t; }};\n""" activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' @@ -253,6 +265,48 @@ def __init__(self): super(ActivationConfigTemplate, self).__init__(Softmax) # Skip ActivationConfigTemplate's __init__ self.template = softmax_config_template + def format(self, node): + params = self._default_config_params(node) + params['type'] = node.get_attr('activation') + params.setdefault('exp_table_size', params['table_size']) + params.setdefault('inv_table_size', params['table_size']) + params.setdefault('n_inner', 1) + params.setdefault('n_outer', 1) + params.setdefault('exp_scale', 1.0) + params.setdefault('parallelization_factor', -1) + + n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] # type: ignore + params['n_slice'] = n_slice + + if params['accum_t'].name == 'model_default_t': # type: ignore + scale = ceil(log2(n_slice)) + exp_table_t = node.attributes['exp_table_t'].precision + signed, width, integers = exp_table_t.signed, exp_table_t.width, exp_table_t.integer + params['accum_t_str'] = f'ac_fixed<{width + scale}, {integers + scale}, {"true" if signed else "false"}>' + else: + params['accum_t_str'] = params['accum_t'].name # type: ignore + if params['inv_inp_t'].name == 'model_default_t': # type: ignore + params['inv_inp_t'] = params['exp_table_t'] + + if params['implementation'] == 'stable': + if 'inp_norm_t' not in params: + # Only used in stable (max-normalized) implementation + input_t = node.get_input_variable().type.precision + width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 + width, iwidth = width - signed, iwidth - signed + if signed: + # Fix table size if too large + exp_table_size = params['inv_table_size'] + params['exp_table_size'] = str(min(int(exp_table_size), 2**width)) + params['inp_norm_t_str'] = f'ac_fixed<{width}, {iwidth}, false>' + else: + params['inp_norm_t_str'] = params['inp_norm_t'].name # type: ignore + else: + params['inp_norm_t_str'] = 'ac_fixed<2,0>' + + return self.template.format(**params) + + class ActivationFunctionTemplate(FunctionCallTemplate): def __init__(self): @@ -262,7 +316,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() - params['config'] = f'{node.get_attr("activation")}_config{node.index}' + params['config'] = f"{node.get_attr('activation')}_config{node.index}" return self.template.format(**params) @@ -276,7 +330,7 @@ def format(self, node): params = self._default_function_params(node) params['activation'] = node._get_act_function_name() params['param'] = node.get_attr('activ_param', 1.0) - params['config'] = f'{node.get_attr("activation")}_config{node.index}' + params['config'] = f"{node.get_attr('activation')}_config{node.index}" return self.template.format(**params) @@ -290,7 +344,7 @@ def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() params['param'] = node.get_weights('param').name - params['config'] = f'{node.get_attr("activation")}_config{node.index}' + params['config'] = f"{node.get_attr('activation')}_config{node.index}" return self.template.format(**params) @@ -303,7 +357,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() - params['config'] = f'{node.get_attr("activation")}_config{node.index}' + params['config'] = f"{node.get_attr('activation')}_config{node.index}" return self.template.format(**params) @@ -315,7 +369,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node._get_act_function_name() - params['config'] = f'{node.get_attr("activation")}_config{node.index}' + params['config'] = f"{node.get_attr('activation')}_config{node.index}" return self.template.format(**params) From 974e75a3962de9afa9b832d9c6d97edf85659a4f Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Tue, 24 Feb 2026 16:43:13 +0000 Subject: [PATCH 08/40] pre-commit --- hls4ml/backends/oneapi/passes/core_templates.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 8205fb1e17..4fae515efc 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -307,7 +307,6 @@ def format(self, node): return self.template.format(**params) - class ActivationFunctionTemplate(FunctionCallTemplate): def __init__(self): super().__init__((Activation, HardActivation, Softmax), include_header=activ_include_list) @@ -316,7 +315,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() - params['config'] = f"{node.get_attr('activation')}_config{node.index}" + params['config'] = f'{node.get_attr("activation")}_config{node.index}' return self.template.format(**params) @@ -330,7 +329,7 @@ def format(self, node): params = self._default_function_params(node) params['activation'] = node._get_act_function_name() params['param'] = node.get_attr('activ_param', 1.0) - params['config'] = f"{node.get_attr('activation')}_config{node.index}" + params['config'] = f'{node.get_attr("activation")}_config{node.index}' return self.template.format(**params) @@ -344,7 +343,7 @@ def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() params['param'] = node.get_weights('param').name - params['config'] = f"{node.get_attr('activation')}_config{node.index}" + params['config'] = f'{node.get_attr("activation")}_config{node.index}' return self.template.format(**params) @@ -357,7 +356,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() - params['config'] = f"{node.get_attr('activation')}_config{node.index}" + params['config'] = f'{node.get_attr("activation")}_config{node.index}' return self.template.format(**params) @@ -369,7 +368,7 @@ def __init__(self): def format(self, node): params = self._default_function_params(node) params['activation'] = node._get_act_function_name() - params['config'] = f"{node.get_attr('activation')}_config{node.index}" + params['config'] = f'{node.get_attr("activation")}_config{node.index}' return self.template.format(**params) From 060c398933705518e10007b80a1d5dd4bc1a5b10 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Tue, 24 Feb 2026 17:07:48 +0000 Subject: [PATCH 09/40] uncomment --- hls4ml/backends/oneapi/passes/core_templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 4fae515efc..21dcc69490 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -210,8 +210,8 @@ def format(self, node): static constexpr float exp_scale = {exp_scale}; typedef {exp_table_t.name} exp_table_t; typedef {inv_table_t.name} inv_table_t; - //typedef {accum_t.name} accum_t; - //typedef {inp_norm_t_str} inp_norm_t; + typedef {accum_t.name} accum_t; + typedef {inp_norm_t_str} inp_norm_t; }};\n""" activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' From 772b93ad31d680e2df79ee756dbef06a5e783284 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Wed, 25 Feb 2026 17:16:51 +0000 Subject: [PATCH 10/40] int_inp_t to config --- hls4ml/backends/oneapi/passes/core_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 21dcc69490..f8d1b573d5 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -211,6 +211,7 @@ def format(self, node): typedef {exp_table_t.name} exp_table_t; typedef {inv_table_t.name} inv_table_t; typedef {accum_t.name} accum_t; + typedef {inv_inp_t.name} inv_inp_t; typedef {inp_norm_t_str} inp_norm_t; }};\n""" From c3a45848a04dd98c0124b7ec8ae33545ab28e680 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Mon, 13 Apr 2026 02:26:17 +0100 Subject: [PATCH 11/40] softmax fixed --- .../backends/oneapi/passes/core_templates.py | 132 ++++++++------- .../firmware/nnet_utils/nnet_activation.h | 50 +++--- .../nnet_utils/nnet_activation_stream.h | 57 ++++--- hls4ml/templates/oneapi/firmware/parameters.h | 2 + hls4ml/writer/oneapi_writer.py | 156 ++++++++++-------- 5 files changed, 220 insertions(+), 177 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index f8d1b573d5..c6050dfb57 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -1,9 +1,10 @@ -from math import ceil, log2 - from hls4ml.backends.backend import get_backend from hls4ml.backends.oneapi.oneapi_template import StreamFunctionCallTemplate, TaskSequenceTemplate from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate +from hls4ml.model.types import FixedPrecisionType, RoundingMode, SaturationMode from hls4ml.model.layers import Activation, BatchNormalization, Dense, HardActivation, ParametrizedActivation, PReLU, Softmax +from hls4ml.utils.fixed_point_utils import FixedPointEmulator, ceil_log2, uint_to_binary +import numpy as np # Dense templates @@ -194,25 +195,28 @@ def format(self, node): static constexpr unsigned reuse_factor = {reuse}; }};\n""" - softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ - static const unsigned n_in = {n_in}; - static const unsigned n_slice = {n_slice}; - static const unsigned n_outer = {n_outer}; - static const unsigned n_inner = {n_inner}; - static const unsigned parallelization_factor = {parallelization_factor}; - static const unsigned exp_table_size = {exp_table_size}; - static const unsigned inv_table_size = {inv_table_size}; - static const unsigned io_type = nnet::{iotype}; - static const unsigned reuse_factor = {reuse}; - static const unsigned axis = {axis}; - static const nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; - static constexpr float exp_scale = {exp_scale}; + static constexpr unsigned n_in = {n_in}; + static constexpr unsigned exp_table_size = {exp_table_size}; + static constexpr unsigned inv_table_size = {inv_table_size}; + static constexpr unsigned io_type = nnet::{iotype}; + static constexpr unsigned reuse_factor = {reuse}; + static constexpr nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; typedef {exp_table_t.name} exp_table_t; - typedef {inv_table_t.name} inv_table_t; - typedef {accum_t.name} accum_t; + typedef {inv_table_t.name} inv_table_t;""" + +softmax_config_table_template = """ + + static constexpr const exp_table_t *exp_table = &{exp_table_name}[0]; + static constexpr const inv_table_t *invert_table = &{inv_table_name}[0]; +}};\n""" + +softmax_config_table_template_stable = """ typedef {inv_inp_t.name} inv_inp_t; - typedef {inp_norm_t_str} inp_norm_t; + typedef {inp_norm_t.name} inp_norm_t; + + static constexpr const exp_table_t *exp_table = &{exp_table_name}[0]; + static constexpr const inv_table_t *invert_table = &{inv_table_name}[0]; }};\n""" activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' @@ -233,7 +237,58 @@ def __init__(self): def format(self, node): params = self._default_config_params(node) params['type'] = node.get_attr('activation') + + if params['type'] == 'softmax': + + if 'exp_table_size' in params: + params['exp_table_size'] //= 2 + else: + params['exp_table_size'] = 1024 + params['exp_table_t'].precision.width = ceil_log2(params['exp_table_size']) + params['exp_table_t'].precision.integer = 3 + params['exp_table_t'].precision.signed = False + + if 'inp_norm_t' not in params: + input_t = node.get_input_variable().type.precision + width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 + width, iwidth = width - signed, iwidth - signed + import copy + params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) #assign type,later override + + #this checks if table sizes will be default, if it is just use the table size to derive precision + if 'inv_table_size' not in params: + params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 + params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 + params['inp_norm_t'].precision.signed = True + params['inp_norm_t'].name = f'{node.name}_inp_norm_t' + else: + params['inp_norm_t'].name = f'ac_fixed<{width},{iwidth},{'true' if signed else 'false'},AC_RND,AC_SAT_SYM>' + + node.set_attr('inp_norm_t', params['inp_norm_t']) + + if 'inv_table_size' in params: + params['inv_table_size'] //= 2 + else: + params['inv_table_size'] = 1024 + + params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) + params['inv_table_t'].precision.integer = 3 + params['inv_table_t'].precision.signed = False + + params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 + params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 + params['inv_inp_t'].precision.signed = True + + + if params['implementation'] == 'stable': + self.template += softmax_config_table_template_stable + else: + self.template += softmax_config_table_template + + params['exp_table_name'] = node.name + '_exp_table' + params['inv_table_name'] = node.name + '_inv_table' + return self.template.format(**params) @@ -266,47 +321,6 @@ def __init__(self): super(ActivationConfigTemplate, self).__init__(Softmax) # Skip ActivationConfigTemplate's __init__ self.template = softmax_config_template - def format(self, node): - params = self._default_config_params(node) - params['type'] = node.get_attr('activation') - params.setdefault('exp_table_size', params['table_size']) - params.setdefault('inv_table_size', params['table_size']) - params.setdefault('n_inner', 1) - params.setdefault('n_outer', 1) - params.setdefault('exp_scale', 1.0) - params.setdefault('parallelization_factor', -1) - - n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] # type: ignore - params['n_slice'] = n_slice - - if params['accum_t'].name == 'model_default_t': # type: ignore - scale = ceil(log2(n_slice)) - exp_table_t = node.attributes['exp_table_t'].precision - signed, width, integers = exp_table_t.signed, exp_table_t.width, exp_table_t.integer - params['accum_t_str'] = f'ac_fixed<{width + scale}, {integers + scale}, {"true" if signed else "false"}>' - else: - params['accum_t_str'] = params['accum_t'].name # type: ignore - if params['inv_inp_t'].name == 'model_default_t': # type: ignore - params['inv_inp_t'] = params['exp_table_t'] - - if params['implementation'] == 'stable': - if 'inp_norm_t' not in params: - # Only used in stable (max-normalized) implementation - input_t = node.get_input_variable().type.precision - width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 - width, iwidth = width - signed, iwidth - signed - if signed: - # Fix table size if too large - exp_table_size = params['inv_table_size'] - params['exp_table_size'] = str(min(int(exp_table_size), 2**width)) - params['inp_norm_t_str'] = f'ac_fixed<{width}, {iwidth}, false>' - else: - params['inp_norm_t_str'] = params['inp_norm_t'].name # type: ignore - else: - params['inp_norm_t_str'] = 'ac_fixed<2,0>' - - return self.template.format(**params) - class ActivationFunctionTemplate(FunctionCallTemplate): def __init__(self): diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index c2353c34a8..385457204d 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -99,11 +99,21 @@ template void sigmoid(const data_ enum class softmax_implementation { latency = 0, legacy = 1, stable = 2, argmax = 3 }; -template inline unsigned softmax_stable_idx_from_real_val(const data_T x) { - // Extract the lower 'width' bits of x - return x.template slc(0).to_uint(); + +template inline unsigned softmax_stable_idx_from_real_val(const data_T x) { + // Number of address bits for table + static constexpr int N = ceillog2::val; + + // Slice the top N bits of the input + [[intel::fpga_register]] ac_int y = x.template slc(x.width - N - 1); + + // If x is the most negative value, the slice will be 0, so we need to set the 0-th bit to ensure correctness + if (x != 0 && y == 0) + y[0] = 1; + return y.to_uint(); } + template inline unsigned softmax_latency_idx_from_real_val(const data_T x) { // Number of address bits for table static constexpr int N = ceillog2::val; @@ -113,49 +123,45 @@ template inline unsigned softmax_latency_idx_f return y.to_uint(); } + template void softmax_stable(const data_T &data, res_T &res) { -#include "activation_tables/exp_table.tb" -#include "activation_tables/invert_table.tb" // Find maximum Op_max op_max; [[intel::fpga_register]] auto x_max = reduce>(data.data(), op_max); - // Normalize inputs: d = x_max - x - [[intel::fpga_register]] typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; + // For the diffs, use the same type as the input but force rounding and saturation + [[intel::fpga_register]] + typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - // HGQ stable: d = x_max - data - d_xi_xmax[i] = x_max - data[i]; + d_xi_xmax[i] = data[i] - x_max; } - // Exponentials + // Calculate all the e^x's [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - unsigned idx = softmax_stable_idx_from_real_val(d_xi_xmax[i]); - exp_res[i] = exp_table[idx]; + exp_res[i] = CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[i])]; //input_t, CONFIG_T } - // Sum of Exponentials - Op_add op_add; - [[intel::fpga_register]] typename CONFIG_T::accum_t exp_sum = - reduce>(exp_res, op_add); - - // Reciprocal of Sum - typename CONFIG_T::inv_inp_t exp_sum_cast = exp_sum; - unsigned inv_idx = softmax_stable_idx_from_real_val(exp_sum_cast); + // Explicitly sum previously calculated exponentials with an adder tree + Op_add op_add; + [[intel::fpga_register]] typename CONFIG_T::inv_inp_t exp_sum = + reduce>(exp_res, op_add); - [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = invert_table[inv_idx]; + // Multiply previously calculated exponetials with the reciprocal of the sum + [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = + CONFIG_T::invert_table[softmax_stable_idx_from_real_val(exp_sum)]; - // Final Multiplication #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { res[i] = exp_res[i] * inv_exp_sum; } } + // TODO - Improve accuracy template void softmax_latency(const data_T &data, res_T &res) { #include "activation_tables/exp_table_latency.tb" diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index e860c38988..d640f89f7e 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -271,64 +271,63 @@ template void softsign_stre // ************************************************* template void softmax_stable_stream() { -#include "activation_tables/exp_table.tb" -#include "activation_tables/invert_table.tb" + + using input_arr_t = typename ExtractPipeType::value_type; + using input_t = typename ExtractPipeType::value_type::value_type; + constexpr unsigned input_arr_size = std::tuple_size{}; + constexpr unsigned multiplier_limit = - DIV_ROUNDUP(std::tuple_size::value_type>{}, CONFIG_T::reuse_factor); - constexpr unsigned pipeline = std::tuple_size::value_type>{} / multiplier_limit; + DIV_ROUNDUP(input_arr_size, CONFIG_T::reuse_factor); + constexpr unsigned pipeline = input_arr_size / multiplier_limit; - [[intel::fpga_register]] typename ExtractPipeType::value_type::value_type - data_array[std::tuple_size::value_type>{}]; + + [[intel::fpga_register]] input_t data_array[input_arr_size]; SoftmaxArrayLoop: - [[intel::initiation_interval(pipeline)]] for (unsigned i = 0; - i < CONFIG_T::n_in / - std::tuple_size::value_type>{}; - i++) { + [[intel::initiation_interval(pipeline)]] + for (unsigned i = 0; i < CONFIG_T::n_in / input_arr_size; i++) { auto in_pack = data_pipe::read(); SoftmaxArrayPackLoop: #pragma unroll - for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { + for (unsigned j = 0; j < input_arr_size; j++) { data_array[j] = in_pack[j]; } // Find the max and compute all delta(x_i, x_max) - Op_max::value_type::value_type> op_max; - [[intel::fpga_register]] typename ExtractPipeType::value_type::value_type x_max = - reduce::value_type::value_type, - std::tuple_size::value_type>{}, - Op_max::value_type::value_type>>(data_array, op_max); - - // For the diffs, use the same type as the input but force rounding and saturation - [[intel::fpga_register]] ac_fixed::value_type::value_type::width, - ExtractPipeType::value_type::value_type::i_width, true, AC_RND, AC_SAT> - d_xi_xmax[std::tuple_size::value_type>{}]; + Op_max op_max; + [[intel::fpga_register]] + input_t x_max = reduce>(data_array, op_max); + + [[intel::fpga_register]] + typename CONFIG_T::inp_norm_t d_xi_xmax[input_arr_size]; + #pragma unroll - for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { + for (unsigned j = 0; j < input_arr_size; j++) { d_xi_xmax[j] = data_array[j] - x_max; } // Calculate all the e^x's [[intel::fpga_register]] - typename CONFIG_T::exp_table_t exp_res[std::tuple_size::value_type>{}]; + typename CONFIG_T::exp_table_t exp_res[input_arr_size]; + #pragma unroll - for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { + for (unsigned j = 0; j < input_arr_size; j++) { exp_res[j] = - exp_table[softmax_stable_idx_from_real_val::value_type::value_type, - CONFIG_T>(d_xi_xmax[j])]; + CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[j])]; } // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing Op_add op_add; - [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_sum = - reduce::value_type>{}, + [[intel::fpga_register]] typename CONFIG_T::inv_inp_t exp_sum = + reduce>(exp_res, op_add); [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table[softmax_stable_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_stable_idx_from_real_val(exp_sum)]; + typename ExtractPipeType::value_type out_pack; SoftmaxInvPackLoop: diff --git a/hls4ml/templates/oneapi/firmware/parameters.h b/hls4ml/templates/oneapi/firmware/parameters.h index 717059f1e8..ef4e5d26b9 100644 --- a/hls4ml/templates/oneapi/firmware/parameters.h +++ b/hls4ml/templates/oneapi/firmware/parameters.h @@ -6,6 +6,8 @@ #include "nnet_utils/nnet_code_gen.h" #include "nnet_utils/nnet_helpers.h" +// hls-fpga-machine-learning insert softmax tables + // hls-fpga-machine-learning insert includes // hls-fpga-machine-learning insert layer-config diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index b945f3faf9..7f95830c21 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -302,6 +302,14 @@ def write_parameters(self, model): config = layer.get_attr('config_cpp', None) if config: newline += config + '\n' + + elif '// hls-fpga-machine-learning insert softmax tables' in line: + newline = line + for layer in model.get_layers(): + if 'softmax' in layer.name: + newline += f'#include "nnet_utils/activation_tables/{layer.name}_exp_table.h"\n' + newline += f'#include "nnet_utils/activation_tables/{layer.name}_inv_table.h"\n' + else: newline = line fout.write(newline) @@ -695,25 +703,29 @@ def __get_table_precision(self, model, activation, table_name='table_precision') return None # fp_bits, fp_integer, fp_signed + def __write_exp_table(self, model, path): - table_name = 'exp_table' - table_size = self.__get_table_size(model, 'softmax', table_name='exp_table_size') - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size, table_type='exp_table_t')) + for layer in model.get_layers(): + + if 'softmax' in layer.name: + + table_name = layer.name + '_exp_table' + table_size = int(layer.get_attr('exp_table_size'))//2 if ( + layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' + ) and layer.get_attr('exp_table_size') is not None else 1024 - # Default fixed point precision - # 6 bits for integer part, 10 bits for decimal - total, 16 - precision = self.__get_table_precision(model, 'softmax', table_name='inp_norm_t') + with open(f'{path}/{table_name}.h', 'w') as h_file: + + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') - if precision is None: - fp_bits = 16 - fp_integer = 6 - fp_signed = True + h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_input_variable().type + #ac_type = layer.get_input_variable().type + ac_type = layer.get_attr('inp_norm_t') + if ac_type is not None: try: fp_bits = ac_type.precision.integer + ac_type.precision.fractional @@ -721,49 +733,55 @@ def __write_exp_table(self, model, path): fp_signed = ac_type.precision.signed except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + fp_bits = 16 + fp_integer = 6 + fp_signed = True + if fp_signed is False: raise Exception('Softmax types need to be signed') + + else: + fp_bits = 16 + fp_integer = 6 + fp_signed = True + + sep = '' + N = ceil_log2(table_size) + for i in range(table_size): + f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + b = uint_to_binary(i, N) + if i == 0: + b.insert(0, 0) + else: + b.insert(0, 1) + f.set_msb_bits(b) + real_val = f.exp_float() + h_file.write(sep + str(real_val)) + sep = ', ' + + h_file.write('};\n\n') + h_file.write('#endif') - else: - fp_bits = precision.width - fp_integer = precision.integer - fp_signed = precision.signed - - f_bits = fp_bits - fp_integer - sep = '' - for i in range(table_size): - # Index represents the raw bit pattern of the input - real_val_in = i * (2.0 ** (-f_bits)) - - # Calculate exp(-x) for the stable implementation - real_val = np.exp(-real_val_in) - - h_file.write(sep + str(real_val)) - sep = ', ' - - h_file.write('};\n') - h_file.close() def __write_invert_table(self, model, path): - table_name = 'invert_table' - table_size = self.__get_table_size(model, 'softmax', table_name='inv_table_size') + for layer in model.get_layers(): + if 'softmax' in layer.name: - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size, table_type='inv_table_t')) - # Default fixed point precision, in case values from layer attributes cannot be extracted - # 8 bits for integer part, 10 bits for decimal - total, 18 + table_name = layer.name + '_inv_table' + table_size = int(layer.get_attr('inv_table_size')) //2 if ( + layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' + ) and layer.get_attr('inv_table_size') is not None else 1024 + + with open(f'{path}/{table_name}.h', 'w') as h_file: + + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') - precision = self.__get_table_precision(model, 'softmax', table_name='inv_inp_t') + h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') - if precision is None: - fp_bits = 18 - fp_integer = 8 - fp_signed = True + ac_type = layer.get_attr('inv_inp_t') - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_attr('exp_table_t') if ac_type is not None: try: fp_bits = ac_type.precision.integer + ac_type.precision.fractional @@ -771,31 +789,32 @@ def __write_invert_table(self, model, path): fp_signed = ac_type.precision.signed except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + fp_bits = 18 + fp_integer = 8 + fp_signed = True + if fp_signed is False: raise Exception('Softmax types need to be signed') - else: - fp_bits = precision.width - fp_integer = precision.integer - fp_signed = precision.signed - - f_bits = fp_bits - fp_integer - sep = '' - for i in range(table_size): - # Index represents the raw bit pattern of the input - real_val_in = i * (2.0 ** (-f_bits)) + else: + fp_bits = 18 + fp_integer = 8 + fp_signed = True - if real_val_in == 0: - real_val = 999.0 - else: - real_val = 1.0 / real_val_in + sep = '' + N = ceil_log2(table_size) + for i in range(table_size): + f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + b = uint_to_binary(i, N) + b.insert(0, 0) + f.set_msb_bits(b) + real_val = f.inv_float() + h_file.write(sep + str(real_val)) + sep = ', ' - h_file.write(sep + str(real_val)) - sep = ', ' + h_file.write('};\n\n') + h_file.write('#endif') - h_file.write('};\n') - h_file.close() def __write_exp_table_latency(self, model, path): table_name = 'exp_table_latency' @@ -1015,3 +1034,6 @@ def write_hls(self, model): self.write_generated_code(model) self.write_yml(model) self.write_tar(model) + + + From 31b7ad65eca392429b2da852097261acebf883be Mon Sep 17 00:00:00 2001 From: bugracyln Date: Tue, 14 Apr 2026 21:39:58 +0100 Subject: [PATCH 12/40] table generation cleanup --- hls4ml/writer/oneapi_writer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 7f95830c21..320afa74db 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -722,8 +722,7 @@ def __write_exp_table(self, model, path): h_file.write(f'#define {header_name.upper()}_H_\n\n') h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') - - #ac_type = layer.get_input_variable().type + ac_type = layer.get_attr('inp_norm_t') if ac_type is not None: From 091cd8ec65d261a6c436b0ed28e7cbb25b119868 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 20 May 2026 12:08:26 -0500 Subject: [PATCH 13/40] fix overflow for legacy softmax, start looking at others --- hls4ml/backends/fpga/fpga_backend.py | 21 +++++++++--- .../model/optimizer/passes/infer_precision.py | 23 +++++++++++++ .../vivado/nnet_utils/nnet_activation.h | 33 ++++++++++--------- .../nnet_utils/nnet_activation_stream.h | 12 +++---- 4 files changed, 62 insertions(+), 27 deletions(-) diff --git a/hls4ml/backends/fpga/fpga_backend.py b/hls4ml/backends/fpga/fpga_backend.py index db0f256172..1518f1a04b 100644 --- a/hls4ml/backends/fpga/fpga_backend.py +++ b/hls4ml/backends/fpga/fpga_backend.py @@ -129,22 +129,33 @@ def __init__(self, name): ConfigurableAttribute('skip', value_type=bool, default=False, description=descriptions.softmax_skip), TypeAttribute( 'exp_table', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), description=descriptions.table_type, ), TypeAttribute( 'inv_table', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), description=descriptions.table_type, ), TypeAttribute( 'inv_inp', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), + description='What the accumulated value is cast to before accessing the inversion table (only in stable)', ), TypeAttribute( - 'accum', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + 'inp_norm', + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), + description='The internal width used for the exp table lookup (only in stable)', ), + TypeAttribute('accum', description=descriptions.accum_type), ] self.attribute_map[Softmax] = softmax_attrs diff --git a/hls4ml/model/optimizer/passes/infer_precision.py b/hls4ml/model/optimizer/passes/infer_precision.py index 0bc29a2955..970a8f9ead 100644 --- a/hls4ml/model/optimizer/passes/infer_precision.py +++ b/hls4ml/model/optimizer/passes/infer_precision.py @@ -90,6 +90,9 @@ def _infer_precision(self, node, types_to_infer): if node_class in ['PReLU']: return self._infer_prelu_act_precision(node, types_to_infer) + + if node_class in ['Softmax']: + return self._infer_softmax_precision(node, types_to_infer) # What about quantized activation layer? Setting it to 'auto' manually will break it here. We should prevent # this in config_from_* functions @@ -605,6 +608,26 @@ def _infer_prelu_act_precision(self, node, types_to_infer): return inferred_types + def _infer_softmax_precision(self, node, types_to_infer): + inferred_types = [] + + # for softmax, the table parameters have a default seting, so they don't need to be inferred + # here. We never expect them to be of type auto. + + # For result, we leave it to be set externally (model default if not set). We expect it to + # likely be the output value, in which case the output format would determine it's precision. + # Therefore, only the accum is configured here + + if 'accum_t' in types_to_infer: + exp_w = node.types['exp_table_t'].precision.width + exp_i = node.types['exp_table_t'].precision.integer + exp_s = node.types['exp_table_t'].precision.signed + ceillog = math.ceil(np.log2(node.get_attr('n_in'))) + node.types['accum_t'].precision = FixedPrecisionType(exp_w + ceillog, exp_i + ceillog, signed=exp_s) + inferred_types.append('accum_t') + + return inferred_types + def _get_precision_from_constant(value: int | float, max_width=8): """A utility function to find a fixed type to store the constant diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h index ac85e0b2cc..db771a581a 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h @@ -165,7 +165,7 @@ template void init_invert_table(typename CONFIG_T::inv_table_t table_out[CONFIG_T::inv_table_size]) { // The template data_T is the data type used to address the table for (unsigned i = 0; i < CONFIG_T::inv_table_size; i++) { - float x = softmax_real_val_from_idx(i); + float x = softmax_real_val_from_idx(i); typename CONFIG_T::inv_table_t inv_x = 1 / x; table_out[i] = inv_x; } @@ -196,7 +196,6 @@ void softmax_latency(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice // Calculate all the e^x's typename CONFIG_T::accum_t exp_res[CONFIG_T::n_slice]; #pragma HLS array_partition variable=exp_res complete - typename CONFIG_T::inv_inp_t exp_sum(0); for (unsigned i = 0; i < CONFIG_T::n_slice; i++) { #pragma HLS unroll unsigned x = softmax_idx_from_real_val(data[i]); @@ -205,11 +204,12 @@ void softmax_latency(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing + typename CONFIG_T::accum_t exp_sum(0); Op_add op_add; exp_sum = reduce>(exp_res, op_add); typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table[softmax_idx_from_real_val(exp_sum)]; + invert_table[softmax_idx_from_real_val(exp_sum)]; for (unsigned i = 0; i < CONFIG_T::n_slice; i++) { #pragma HLS unroll res[i] = exp_res[i] * inv_exp_sum; @@ -251,7 +251,6 @@ void softmax_stable(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] // Calculate all the e^x's typename CONFIG_T::accum_t exp_res[CONFIG_T::n_slice]; #pragma HLS array_partition variable=exp_res complete - typename CONFIG_T::inv_inp_t exp_sum(0); for (unsigned i = 0; i < CONFIG_T::n_slice; i++) { #pragma HLS unroll unsigned x = softmax_idx_from_real_val(d_xi_xmax[i]); @@ -260,6 +259,7 @@ void softmax_stable(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing + typename CONFIG_T::inv_inp_t exp_sum(0); Op_add op_add; exp_sum = reduce>(exp_res, op_add); @@ -271,18 +271,18 @@ void softmax_stable(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] } } -template void init_exp_table_legacy(typename CONFIG_T::table_t table_out[N_TABLE]) { +template void init_exp_table_legacy(typename CONFIG_T::exp_table_t table_out[N_TABLE]) { for (int ii = 0; ii < N_TABLE; ii++) { // First, convert from table index to X-value (signed 8-bit, range -8 to +8) float in_val = 2 * 8.0 * (ii - float(N_TABLE) / 2.0) / float(N_TABLE); // Next, compute lookup table function - typename CONFIG_T::table_t real_val = exp_fcn_float(in_val); + typename CONFIG_T::exp_table_t real_val = exp_fcn_float(in_val); // std::cout << "Lookup table In Value: " << in_val << " Result: " << real_val << std::endl; table_out[ii] = real_val; } } -template void init_invert_table_legacy(typename CONFIG_T::table_t table_out[N_TABLE]) { +template void init_invert_table_legacy(typename CONFIG_T::inv_table_t table_out[N_TABLE]) { // Inversion function: // result = 1/x for (int ii = 0; ii < N_TABLE; ii++) { @@ -301,12 +301,12 @@ void softmax_legacy(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] // Initialize the lookup table #ifdef __HLS_SYN__ bool initialized = false; - typename CONFIG_T::table_t exp_table[CONFIG_T::exp_table_size]; - typename CONFIG_T::table_t invert_table[CONFIG_T::inv_table_size]; + typename CONFIG_T::exp_table_t exp_table[CONFIG_T::exp_table_size]; + typename CONFIG_T::inv_table_t invert_table[CONFIG_T::inv_table_size]; #else static bool initialized = false; - static typename CONFIG_T::table_t exp_table[CONFIG_T::exp_table_size]; - static typename CONFIG_T::table_t invert_table[CONFIG_T::inv_table_size]; + static typename CONFIG_T::exp_table_t exp_table[CONFIG_T::exp_table_size]; + static typename CONFIG_T::inv_table_t invert_table[CONFIG_T::inv_table_size]; #endif if (!initialized) { init_exp_table_legacy(exp_table); @@ -317,22 +317,23 @@ void softmax_legacy(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] #pragma HLS PIPELINE // Index into the lookup table based on data for exponentials - typename CONFIG_T::table_t exp_res[CONFIG_T::n_slice]; // different, independent, fixed point precision - typename CONFIG_T::table_t exp_diff_res; // different, independent, fixed point precision + typename CONFIG_T::accum_t exp_res[CONFIG_T::n_slice]; // different, independent, fixed point precision + typename CONFIG_T::exp_table_t exp_diff_res; // different, independent, fixed point precision data_T data_cache[CONFIG_T::n_slice]; - int data_round; int index; + for (int ii = 0; ii < CONFIG_T::n_slice; ii++) { data_cache[ii] = data[ii]; exp_res[ii] = 0; } + // first calculate 1/softmax as a sum over fractions. for (int ii = 0; ii < CONFIG_T::n_slice; ii++) { for (int jj = 0; jj < CONFIG_T::n_slice; jj++) { if (ii == jj) exp_diff_res = 1; else { - data_round = (data_cache[jj] - data_cache[ii]) * CONFIG_T::exp_table_size / 16; + auto data_round = (data_cache[jj] - data_cache[ii]) * CONFIG_T::exp_table_size / 16; index = data_round + 8 * CONFIG_T::exp_table_size / 16; if (index < 0) index = 0; @@ -352,7 +353,7 @@ void softmax_legacy(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] if (exp_res_index > CONFIG_T::inv_table_size - 1) exp_res_index = CONFIG_T::inv_table_size - 1; // typename CONFIG_T::table_t exp_res_invert = invert_table[exp_res_index]; - res[ii] = (res_T)invert_table[exp_res_index]; + res[ii] = static_cast(invert_table[exp_res_index]); } } diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h index 50c6c4068c..e03adc5d58 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h @@ -249,12 +249,12 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { // Initialize the lookup table #ifdef __HLS_SYN__ bool initialized = false; - typename CONFIG_T::table_t exp_table[CONFIG_T::table_size]; - typename CONFIG_T::table_t invert_table[CONFIG_T::table_size]; + typename CONFIG_T::exp_table_t exp_table[CONFIG_T::table_size]; + typename CONFIG_T::inv_table_t invert_table[CONFIG_T::table_size]; #else static bool initialized = false; - static typename CONFIG_T::table_t exp_table[CONFIG_T::table_size]; - static typename CONFIG_T::table_t invert_table[CONFIG_T::table_size]; + static typename CONFIG_T::exp_table_t exp_table[CONFIG_T::table_size]; + static typename CONFIG_T::inv_table_t invert_table[CONFIG_T::table_size]; #endif if (!initialized) { init_exp_table_legacy(exp_table); @@ -263,8 +263,8 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { } // Index into the lookup table based on data for exponentials - typename CONFIG_T::table_t exp_res[data_T::size]; - typename CONFIG_T::table_t exp_diff_res; + typename CONFIG_T::accum_t exp_res[data_T::size]; + typename CONFIG_T::exp_table_t exp_diff_res; typename data_T::value_type data_cache[data_T::size]; SoftmaxInitLoop: From 6c9f9a7330eb44600c995d0df01448bf45d6265c Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 20 May 2026 15:54:33 -0500 Subject: [PATCH 14/40] fix stable and streaming legacy softmax --- .../vivado/nnet_utils/nnet_activation.h | 6 +++--- .../nnet_utils/nnet_activation_stream.h | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h index db771a581a..16e78a1d1e 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h @@ -165,7 +165,7 @@ template void init_invert_table(typename CONFIG_T::inv_table_t table_out[CONFIG_T::inv_table_size]) { // The template data_T is the data type used to address the table for (unsigned i = 0; i < CONFIG_T::inv_table_size; i++) { - float x = softmax_real_val_from_idx(i); + float x = softmax_real_val_from_idx(i); typename CONFIG_T::inv_table_t inv_x = 1 / x; table_out[i] = inv_x; } @@ -259,9 +259,9 @@ void softmax_stable(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice] // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing - typename CONFIG_T::inv_inp_t exp_sum(0); Op_add op_add; - exp_sum = reduce>(exp_res, op_add); + typename CONFIG_T::inv_inp_t exp_sum = + reduce>(exp_res, op_add); typename CONFIG_T::inv_table_t inv_exp_sum = invert_table[softmax_idx_from_real_val(exp_sum)]; diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h index e03adc5d58..af1e444f13 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h @@ -216,7 +216,6 @@ void softmax_stable(hls::stream &data, hls::stream &res) { // Calculate all the e^x's typename CONFIG_T::accum_t exp_res[data_T::size]; #pragma HLS ARRAY_PARTITION variable=exp_res complete - typename CONFIG_T::inv_inp_t exp_sum(0); for (unsigned j = 0; j < data_T::size; j++) { #pragma HLS UNROLL unsigned x = softmax_idx_from_real_val(d_xi_xmax[j]); @@ -226,7 +225,8 @@ void softmax_stable(hls::stream &data, hls::stream &res) { // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing Op_add op_add; - exp_sum = reduce>(exp_res, op_add); + typename CONFIG_T::inv_inp_t exp_sum = + reduce>(exp_res, op_add); typename CONFIG_T::inv_table_t inv_exp_sum = invert_table[softmax_idx_from_real_val(exp_sum)]; @@ -249,16 +249,16 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { // Initialize the lookup table #ifdef __HLS_SYN__ bool initialized = false; - typename CONFIG_T::exp_table_t exp_table[CONFIG_T::table_size]; - typename CONFIG_T::inv_table_t invert_table[CONFIG_T::table_size]; + typename CONFIG_T::exp_table_t exp_table[CONFIG_T::exp_table_size]; + typename CONFIG_T::inv_table_t invert_table[CONFIG_T::inv_table_size]; #else static bool initialized = false; - static typename CONFIG_T::exp_table_t exp_table[CONFIG_T::table_size]; - static typename CONFIG_T::inv_table_t invert_table[CONFIG_T::table_size]; + static typename CONFIG_T::exp_table_t exp_table[CONFIG_T::exp_table_size]; + static typename CONFIG_T::inv_table_t invert_table[CONFIG_T::inv_table_size]; #endif if (!initialized) { - init_exp_table_legacy(exp_table); - init_invert_table_legacy(invert_table); + init_exp_table_legacy(exp_table); + init_invert_table_legacy(invert_table); initialized = true; } @@ -288,7 +288,7 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { if (i == j) { exp_diff_res = 1; } else { - int data_round = (data_cache[j] - data_cache[i]) * CONFIG_T::table_size / 16; + auto data_round = (data_cache[j] - data_cache[i]) * CONFIG_T::table_size / 16; int index = data_round + 8 * CONFIG_T::table_size / 16; if (index < 0) index = 0; @@ -314,7 +314,7 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { if (exp_res_index > CONFIG_T::table_size - 1) exp_res_index = CONFIG_T::table_size - 1; - out_pack[j] = (typename res_T::value_type)invert_table[exp_res_index]; + out_pack[j] = static_cast(invert_table[exp_res_index]); } res.write(out_pack); } From 7a004b2752afe54b516e1236bf9a4415cd5ae1c4 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 20 May 2026 16:47:54 -0500 Subject: [PATCH 15/40] minor softmax latency fixes --- hls4ml/templates/vivado/nnet_utils/nnet_activation.h | 6 +++--- hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h index 16e78a1d1e..b262c86859 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation.h @@ -189,7 +189,7 @@ void softmax_latency(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice // Note we are exponentiating the inputs, which have type data_T init_exp_table(exp_table); // Note we are inverting the exponentials, which have type exp_table_t - init_invert_table(invert_table); + init_invert_table(invert_table); initialized = true; } @@ -204,9 +204,9 @@ void softmax_latency(data_T data[CONFIG_T::n_slice], res_T res[CONFIG_T::n_slice // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing - typename CONFIG_T::accum_t exp_sum(0); Op_add op_add; - exp_sum = reduce>(exp_res, op_add); + typename CONFIG_T::accum_t exp_sum = + reduce>(exp_res, op_add); typename CONFIG_T::inv_table_t inv_exp_sum = invert_table[softmax_idx_from_real_val(exp_sum)]; diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h index af1e444f13..ad1aa77ec3 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h @@ -150,7 +150,7 @@ void softmax_latency(hls::stream &data, hls::stream &res) { exp_sum = reduce>(exp_res, op_add); typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table[softmax_idx_from_real_val(exp_sum)]; + invert_table[softmax_idx_from_real_val(exp_sum)]; res_T out_pack; PRAGMA_DATA_PACK(out_pack) From d5e3493c124a7c3314dd9e5c66f9f81d02896791 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Mon, 8 Jun 2026 18:29:28 -0500 Subject: [PATCH 16/40] fix copilot review issues (other than adding a test) --- hls4ml/model/optimizer/passes/infer_precision.py | 2 +- .../vivado/nnet_utils/nnet_activation_stream.h | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hls4ml/model/optimizer/passes/infer_precision.py b/hls4ml/model/optimizer/passes/infer_precision.py index 970a8f9ead..189ffb7dda 100644 --- a/hls4ml/model/optimizer/passes/infer_precision.py +++ b/hls4ml/model/optimizer/passes/infer_precision.py @@ -611,7 +611,7 @@ def _infer_prelu_act_precision(self, node, types_to_infer): def _infer_softmax_precision(self, node, types_to_infer): inferred_types = [] - # for softmax, the table parameters have a default seting, so they don't need to be inferred + # for softmax, the table parameters have a default setting, so they don't need to be inferred # here. We never expect them to be of type auto. # For result, we leave it to be set externally (model default if not set). We expect it to diff --git a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h index ad1aa77ec3..814ed2f50a 100644 --- a/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/vivado/nnet_utils/nnet_activation_stream.h @@ -120,8 +120,8 @@ void softmax_latency(hls::stream &data, hls::stream &res) { if (!initialized) { // Note we are exponentiating the inputs, which have type data_T init_exp_table(exp_table); - // Note we are inverting the exponentials, which have type exp_table_t - init_invert_table(invert_table); + // Note we are inverting the summed exponentials, which have type accum_t + init_invert_table(invert_table); initialized = true; } @@ -292,8 +292,8 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { int index = data_round + 8 * CONFIG_T::table_size / 16; if (index < 0) index = 0; - if (index > CONFIG_T::table_size - 1) - index = CONFIG_T::table_size - 1; + if (index > CONFIG_T::exp_table_size - 1) + index = CONFIG_T::exp_table_size - 1; exp_diff_res = exp_table[index]; } @@ -311,8 +311,8 @@ void softmax_legacy(hls::stream &data, hls::stream &res) { int exp_res_index = exp_res[j] * CONFIG_T::table_size / 64; if (exp_res_index < 0) exp_res_index = 0; - if (exp_res_index > CONFIG_T::table_size - 1) - exp_res_index = CONFIG_T::table_size - 1; + if (exp_res_index > CONFIG_T::inv_table_size - 1) + exp_res_index = CONFIG_T::inv_table_size - 1; out_pack[j] = static_cast(invert_table[exp_res_index]); } From a56a8d66dce45c40bd951d6b4c0c9f36acc53e18 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Mon, 8 Jun 2026 23:02:35 -0500 Subject: [PATCH 17/40] add test for softmax auto inferrence --- test/pytest/test_auto_precision.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/pytest/test_auto_precision.py b/test/pytest/test_auto_precision.py index d3738c8461..57ea268340 100644 --- a/test/pytest/test_auto_precision.py +++ b/test/pytest/test_auto_precision.py @@ -1,3 +1,4 @@ +import math from pathlib import Path import numpy as np @@ -13,10 +14,12 @@ ReLU, SeparableConv1D, SeparableConv2D, + Softmax, ) from tensorflow.keras.models import Sequential import hls4ml +import hls4ml.model.layers from hls4ml.model.optimizer.passes.infer_precision import _get_precision_from_constant test_root_path = Path(__file__).parent @@ -285,3 +288,37 @@ def test_precision_from_constant_unit(val, expected_width): quantum = 2.0**-fp.fractional if expected_width < max_width: assert val % quantum == 0 + + +@pytest.mark.parametrize('n_in', [4, 8, 16]) +@pytest.mark.parametrize('backend', ['Vitis', 'oneAPI']) +def test_auto_precision_softmax(test_case_id, n_in, backend): + """Test that auto accumulator precision is correctly inferred for softmax layers.""" + model = Sequential() + model.add(Softmax(input_shape=(n_in,))) + model.compile() + + config = hls4ml.utils.config_from_keras_model(model, backend=backend, granularity='name') + + odir = str(test_root_path / test_case_id) + hls_model = hls4ml.converters.convert_from_keras_model(model, hls_config=config, output_dir=odir, backend=backend) + + # Find the Softmax layer and verify accum_t precision + softmax_layer = next((layer for layer in hls_model.get_layers() if isinstance(layer, hls4ml.model.layers.Softmax)), None) + assert softmax_layer is not None, 'No Softmax layer found in converted model' + + accum_t = softmax_layer.types['accum_t'].precision + exp_table_t = softmax_layer.types['exp_table_t'].precision + + ceillog = math.ceil(math.log2(n_in)) + expected_width = exp_table_t.width + ceillog + expected_integer = exp_table_t.integer + ceillog + expected_signed = exp_table_t.signed + + assert accum_t.width == expected_width, f'Expected accum_t width {expected_width}, got {accum_t.width} (n_in={n_in})' + assert accum_t.integer == expected_integer, ( + f'Expected accum_t integer {expected_integer}, got {accum_t.integer} (n_in={n_in})' + ) + assert accum_t.signed == expected_signed, ( + f'Expected accum_t signed={expected_signed}, got {accum_t.signed} (n_in={n_in})' + ) From cab4cbcd3ac01e4b33acd2fba947140f910f3bca Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Wed, 10 Jun 2026 07:59:34 +0100 Subject: [PATCH 18/40] Fix formatting of inp_norm_t name string --- hls4ml/backends/oneapi/passes/core_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index c6050dfb57..21752526ec 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -263,7 +263,7 @@ def format(self, node): params['inp_norm_t'].precision.signed = True params['inp_norm_t'].name = f'{node.name}_inp_norm_t' else: - params['inp_norm_t'].name = f'ac_fixed<{width},{iwidth},{'true' if signed else 'false'},AC_RND,AC_SAT_SYM>' + params['inp_norm_t'].name = f'ac_fixed<{width},{iwidth},{str(signed).lower()},AC_RND,AC_SAT_SYM>' node.set_attr('inp_norm_t', params['inp_norm_t']) From 42ece34396a6ea0184a4e9357c8a6d65315ed1cb Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Wed, 10 Jun 2026 09:52:19 +0100 Subject: [PATCH 19/40] pre-commit for core templates --- .../backends/oneapi/passes/core_templates.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 21752526ec..87d7e1519e 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -1,10 +1,8 @@ from hls4ml.backends.backend import get_backend from hls4ml.backends.oneapi.oneapi_template import StreamFunctionCallTemplate, TaskSequenceTemplate from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate -from hls4ml.model.types import FixedPrecisionType, RoundingMode, SaturationMode from hls4ml.model.layers import Activation, BatchNormalization, Dense, HardActivation, ParametrizedActivation, PReLU, Softmax -from hls4ml.utils.fixed_point_utils import FixedPointEmulator, ceil_log2, uint_to_binary -import numpy as np +from hls4ml.utils.fixed_point_utils import ceil_log2 # Dense templates @@ -211,7 +209,7 @@ def format(self, node): static constexpr const inv_table_t *invert_table = &{inv_table_name}[0]; }};\n""" -softmax_config_table_template_stable = """ +softmax_config_table_template_stable = """ typedef {inv_inp_t.name} inv_inp_t; typedef {inp_norm_t.name} inp_norm_t; @@ -237,9 +235,8 @@ def __init__(self): def format(self, node): params = self._default_config_params(node) params['type'] = node.get_attr('activation') - - if params['type'] == 'softmax': + if params['type'] == 'softmax': if 'exp_table_size' in params: params['exp_table_size'] //= 2 else: @@ -248,23 +245,24 @@ def format(self, node): params['exp_table_t'].precision.width = ceil_log2(params['exp_table_size']) params['exp_table_t'].precision.integer = 3 params['exp_table_t'].precision.signed = False - + if 'inp_norm_t' not in params: input_t = node.get_input_variable().type.precision width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 width, iwidth = width - signed, iwidth - signed import copy - params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) #assign type,later override - #this checks if table sizes will be default, if it is just use the table size to derive precision - if 'inv_table_size' not in params: + params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override + + # this checks if table sizes will be default, if it is just use the table size to derive precision + if 'inv_table_size' not in params: params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 params['inp_norm_t'].precision.signed = True params['inp_norm_t'].name = f'{node.name}_inp_norm_t' else: params['inp_norm_t'].name = f'ac_fixed<{width},{iwidth},{str(signed).lower()},AC_RND,AC_SAT_SYM>' - + node.set_attr('inp_norm_t', params['inp_norm_t']) if 'inv_table_size' in params: @@ -275,12 +273,11 @@ def format(self, node): params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) params['inv_table_t'].precision.integer = 3 params['inv_table_t'].precision.signed = False - + params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 params['inv_inp_t'].precision.signed = True - if params['implementation'] == 'stable': self.template += softmax_config_table_template_stable else: @@ -288,7 +285,7 @@ def format(self, node): params['exp_table_name'] = node.name + '_exp_table' params['inv_table_name'] = node.name + '_inv_table' - + return self.template.format(**params) From 7e2798a996af35614e261ff19839bef5c8515bc4 Mon Sep 17 00:00:00 2001 From: laurilaatu Date: Wed, 10 Jun 2026 20:37:45 +0100 Subject: [PATCH 20/40] pre-commit all --- .../firmware/nnet_utils/nnet_activation.h | 16 ++++---- .../nnet_utils/nnet_activation_stream.h | 27 +++++-------- hls4ml/writer/oneapi_writer.py | 39 ++++++++----------- 3 files changed, 34 insertions(+), 48 deletions(-) diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index 385457204d..c87d8a2c20 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -99,21 +99,19 @@ template void sigmoid(const data_ enum class softmax_implementation { latency = 0, legacy = 1, stable = 2, argmax = 3 }; - template inline unsigned softmax_stable_idx_from_real_val(const data_T x) { // Number of address bits for table static constexpr int N = ceillog2::val; // Slice the top N bits of the input [[intel::fpga_register]] ac_int y = x.template slc(x.width - N - 1); - + // If x is the most negative value, the slice will be 0, so we need to set the 0-th bit to ensure correctness if (x != 0 && y == 0) y[0] = 1; return y.to_uint(); } - template inline unsigned softmax_latency_idx_from_real_val(const data_T x) { // Number of address bits for table static constexpr int N = ceillog2::val; @@ -123,7 +121,6 @@ template inline unsigned softmax_latency_idx_f return y.to_uint(); } - template void softmax_stable(const data_T &data, res_T &res) { // Find maximum @@ -132,8 +129,7 @@ template void softmax_stable(cons reduce>(data.data(), op_max); // For the diffs, use the same type as the input but force rounding and saturation - [[intel::fpga_register]] - typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; + [[intel::fpga_register]] typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { d_xi_xmax[i] = data[i] - x_max; @@ -143,7 +139,9 @@ template void softmax_stable(cons [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - exp_res[i] = CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[i])]; //input_t, CONFIG_T + exp_res[i] = + CONFIG_T::exp_table[softmax_stable_idx_from_real_val( + d_xi_xmax[i])]; // input_t, CONFIG_T } // Explicitly sum previously calculated exponentials with an adder tree @@ -153,7 +151,8 @@ template void softmax_stable(cons // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_stable_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_stable_idx_from_real_val( + exp_sum)]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { @@ -161,7 +160,6 @@ template void softmax_stable(cons } } - // TODO - Improve accuracy template void softmax_latency(const data_T &data, res_T &res) { #include "activation_tables/exp_table_latency.tb" diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index d640f89f7e..c0400fb306 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -276,17 +276,13 @@ template void softmax_stabl using input_t = typename ExtractPipeType::value_type::value_type; constexpr unsigned input_arr_size = std::tuple_size{}; - - constexpr unsigned multiplier_limit = - DIV_ROUNDUP(input_arr_size, CONFIG_T::reuse_factor); + constexpr unsigned multiplier_limit = DIV_ROUNDUP(input_arr_size, CONFIG_T::reuse_factor); constexpr unsigned pipeline = input_arr_size / multiplier_limit; - [[intel::fpga_register]] input_t data_array[input_arr_size]; SoftmaxArrayLoop: - [[intel::initiation_interval(pipeline)]] - for (unsigned i = 0; i < CONFIG_T::n_in / input_arr_size; i++) { + [[intel::initiation_interval(pipeline)]] for (unsigned i = 0; i < CONFIG_T::n_in / input_arr_size; i++) { auto in_pack = data_pipe::read(); SoftmaxArrayPackLoop: @@ -297,11 +293,9 @@ template void softmax_stabl // Find the max and compute all delta(x_i, x_max) Op_max op_max; - [[intel::fpga_register]] - input_t x_max = reduce>(data_array, op_max); + [[intel::fpga_register]] input_t x_max = reduce>(data_array, op_max); - [[intel::fpga_register]] - typename CONFIG_T::inp_norm_t d_xi_xmax[input_arr_size]; + [[intel::fpga_register]] typename CONFIG_T::inp_norm_t d_xi_xmax[input_arr_size]; #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { @@ -309,24 +303,23 @@ template void softmax_stabl } // Calculate all the e^x's - [[intel::fpga_register]] - typename CONFIG_T::exp_table_t exp_res[input_arr_size]; + [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[input_arr_size]; #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { - exp_res[j] = - CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[j])]; + exp_res[j] = CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[j])]; } // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing Op_add op_add; [[intel::fpga_register]] typename CONFIG_T::inv_inp_t exp_sum = - reduce>(exp_res, op_add); + reduce>(exp_res, op_add); [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_stable_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_stable_idx_from_real_val( + exp_sum)]; typename ExtractPipeType::value_type out_pack; diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 320afa74db..56ee44d5f8 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -703,28 +703,27 @@ def __get_table_precision(self, model, activation, table_name='table_precision') return None # fp_bits, fp_integer, fp_signed - def __write_exp_table(self, model, path): for layer in model.get_layers(): - if 'softmax' in layer.name: - table_name = layer.name + '_exp_table' - table_size = int(layer.get_attr('exp_table_size'))//2 if ( - layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' - ) and layer.get_attr('exp_table_size') is not None else 1024 + table_size = ( + int(layer.get_attr('exp_table_size')) // 2 + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') + and layer.get_attr('exp_table_size') is not None + else 1024 + ) with open(f'{path}/{table_name}.h', 'w') as h_file: - header_name = table_name h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') - + ac_type = layer.get_attr('inp_norm_t') - + if ac_type is not None: try: fp_bits = ac_type.precision.integer + ac_type.precision.fractional @@ -738,7 +737,7 @@ def __write_exp_table(self, model, path): if fp_signed is False: raise Exception('Softmax types need to be signed') - + else: fp_bits = 16 fp_integer = 6 @@ -757,22 +756,22 @@ def __write_exp_table(self, model, path): real_val = f.exp_float() h_file.write(sep + str(real_val)) sep = ', ' - + h_file.write('};\n\n') h_file.write('#endif') - def __write_invert_table(self, model, path): for layer in model.get_layers(): if 'softmax' in layer.name: - table_name = layer.name + '_inv_table' - table_size = int(layer.get_attr('inv_table_size')) //2 if ( - layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' - ) and layer.get_attr('inv_table_size') is not None else 1024 - - with open(f'{path}/{table_name}.h', 'w') as h_file: + table_size = ( + int(layer.get_attr('inv_table_size')) // 2 + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') + and layer.get_attr('inv_table_size') is not None + else 1024 + ) + with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') @@ -814,7 +813,6 @@ def __write_invert_table(self, model, path): h_file.write('};\n\n') h_file.write('#endif') - def __write_exp_table_latency(self, model, path): table_name = 'exp_table_latency' table_size = self.__get_table_size(model, 'softmax') @@ -1033,6 +1031,3 @@ def write_hls(self, model): self.write_generated_code(model) self.write_yml(model) self.write_tar(model) - - - From bd4778e2a8c602c84a4d7e4e7651fa5d2d9b35b8 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 25 Jun 2026 19:15:49 +0100 Subject: [PATCH 21/40] softmax update --- .../backends/oneapi/passes/core_templates.py | 44 ++++++++++------- hls4ml/writer/oneapi_writer.py | 49 +++++++++---------- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index c6050dfb57..a0d78295c4 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -1,10 +1,8 @@ from hls4ml.backends.backend import get_backend from hls4ml.backends.oneapi.oneapi_template import StreamFunctionCallTemplate, TaskSequenceTemplate from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate -from hls4ml.model.types import FixedPrecisionType, RoundingMode, SaturationMode from hls4ml.model.layers import Activation, BatchNormalization, Dense, HardActivation, ParametrizedActivation, PReLU, Softmax -from hls4ml.utils.fixed_point_utils import FixedPointEmulator, ceil_log2, uint_to_binary -import numpy as np +from hls4ml.utils.fixed_point_utils import ceil_log2 # Dense templates @@ -197,6 +195,7 @@ def format(self, node): softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ static constexpr unsigned n_in = {n_in}; + static constexpr unsigned table_size = {table_size}; static constexpr unsigned exp_table_size = {exp_table_size}; static constexpr unsigned inv_table_size = {inv_table_size}; static constexpr unsigned io_type = nnet::{iotype}; @@ -207,16 +206,20 @@ def format(self, node): softmax_config_table_template = """ - static constexpr const exp_table_t *exp_table = &{exp_table_name}[0]; - static constexpr const inv_table_t *invert_table = &{inv_table_name}[0]; + using {exp_table_name}_arr_t = nnet::array; + using {inv_table_name}_arr_t = nnet::array; + static constexpr const {exp_table_name}_arr_t exp_table = {exp_table_name}; + static constexpr const {inv_table_name}_arr_t invert_table = {inv_table_name}; }};\n""" -softmax_config_table_template_stable = """ +softmax_config_table_template_stable = """ typedef {inv_inp_t.name} inv_inp_t; typedef {inp_norm_t.name} inp_norm_t; - static constexpr const exp_table_t *exp_table = &{exp_table_name}[0]; - static constexpr const inv_table_t *invert_table = &{inv_table_name}[0]; + using {exp_table_name}_arr_t = nnet::array; + using {inv_table_name}_arr_t = nnet::array; + static constexpr const {exp_table_name}_arr_t exp_table = {exp_table_name}; + static constexpr const {inv_table_name}_arr_t invert_table = {inv_table_name}; }};\n""" activ_function_template = 'nnet::{activation}<{input_t}, {output_t}, {config}>({input}, {output});' @@ -237,9 +240,8 @@ def __init__(self): def format(self, node): params = self._default_config_params(node) params['type'] = node.get_attr('activation') - - if params['type'] == 'softmax': + if params['type'] == 'softmax': if 'exp_table_size' in params: params['exp_table_size'] //= 2 else: @@ -248,23 +250,28 @@ def format(self, node): params['exp_table_t'].precision.width = ceil_log2(params['exp_table_size']) params['exp_table_t'].precision.integer = 3 params['exp_table_t'].precision.signed = False - + + params.setdefault('table_size', params['exp_table_size']) + if 'inp_norm_t' not in params: input_t = node.get_input_variable().type.precision width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 width, iwidth = width - signed, iwidth - signed import copy - params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) #assign type,later override - #this checks if table sizes will be default, if it is just use the table size to derive precision - if 'inv_table_size' not in params: + params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override + + # this checks if table sizes will be default, if it is just use the table size to derive precision + if 'inv_table_size' not in params: params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 params['inp_norm_t'].precision.signed = True params['inp_norm_t'].name = f'{node.name}_inp_norm_t' else: - params['inp_norm_t'].name = f'ac_fixed<{width},{iwidth},{'true' if signed else 'false'},AC_RND,AC_SAT_SYM>' - + params[ + 'inp_norm_t' + ].name = f'ac_fixed<{width},{iwidth},{"true" if signed else "false"},AC_RND,AC_SAT_SYM>' + node.set_attr('inp_norm_t', params['inp_norm_t']) if 'inv_table_size' in params: @@ -275,12 +282,11 @@ def format(self, node): params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) params['inv_table_t'].precision.integer = 3 params['inv_table_t'].precision.signed = False - + params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 params['inv_inp_t'].precision.signed = True - if params['implementation'] == 'stable': self.template += softmax_config_table_template_stable else: @@ -288,7 +294,7 @@ def format(self, node): params['exp_table_name'] = node.name + '_exp_table' params['inv_table_name'] = node.name + '_inv_table' - + return self.template.format(**params) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 320afa74db..3b866f3e83 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -306,7 +306,7 @@ def write_parameters(self, model): elif '// hls-fpga-machine-learning insert softmax tables' in line: newline = line for layer in model.get_layers(): - if 'softmax' in layer.name: + if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': newline += f'#include "nnet_utils/activation_tables/{layer.name}_exp_table.h"\n' newline += f'#include "nnet_utils/activation_tables/{layer.name}_inv_table.h"\n' @@ -703,28 +703,27 @@ def __get_table_precision(self, model, activation, table_name='table_precision') return None # fp_bits, fp_integer, fp_signed - def __write_exp_table(self, model, path): for layer in model.get_layers(): - - if 'softmax' in layer.name: - + if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': table_name = layer.name + '_exp_table' - table_size = int(layer.get_attr('exp_table_size'))//2 if ( - layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' - ) and layer.get_attr('exp_table_size') is not None else 1024 + table_size = ( + int(layer.get_attr('exp_table_size')) // 2 + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') + and layer.get_attr('exp_table_size') is not None + else 1024 + ) with open(f'{path}/{table_name}.h', 'w') as h_file: - header_name = table_name h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') - h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') - + h_file.write(f'static constexpr {layer.get_attr("exp_table_t").name} {table_name}[{table_size}] = {{') + ac_type = layer.get_attr('inp_norm_t') - + if ac_type is not None: try: fp_bits = ac_type.precision.integer + ac_type.precision.fractional @@ -738,7 +737,7 @@ def __write_exp_table(self, model, path): if fp_signed is False: raise Exception('Softmax types need to be signed') - + else: fp_bits = 16 fp_integer = 6 @@ -757,27 +756,27 @@ def __write_exp_table(self, model, path): real_val = f.exp_float() h_file.write(sep + str(real_val)) sep = ', ' - + h_file.write('};\n\n') h_file.write('#endif') - def __write_invert_table(self, model, path): for layer in model.get_layers(): - if 'softmax' in layer.name: - + if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': table_name = layer.name + '_inv_table' - table_size = int(layer.get_attr('inv_table_size')) //2 if ( - layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax' - ) and layer.get_attr('inv_table_size') is not None else 1024 - - with open(f'{path}/{table_name}.h', 'w') as h_file: + table_size = ( + int(layer.get_attr('inv_table_size')) // 2 + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') + and layer.get_attr('inv_table_size') is not None + else 1024 + ) + with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') - h_file.write(f'static constexpr {table_name}_t {table_name}[{table_size}] = {{') + h_file.write(f'static constexpr {layer.get_attr("exp_table_t").name} {table_name}[{table_size}] = {{') ac_type = layer.get_attr('inv_inp_t') @@ -814,7 +813,6 @@ def __write_invert_table(self, model, path): h_file.write('};\n\n') h_file.write('#endif') - def __write_exp_table_latency(self, model, path): table_name = 'exp_table_latency' table_size = self.__get_table_size(model, 'softmax') @@ -1033,6 +1031,3 @@ def write_hls(self, model): self.write_generated_code(model) self.write_yml(model) self.write_tar(model) - - - From 3946858bee70f7e4f19ed95da14912986c435101 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 25 Jun 2026 19:36:42 +0100 Subject: [PATCH 22/40] minor syntax fix --- hls4ml/writer/oneapi_writer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 3b866f3e83..29804ad173 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -720,7 +720,9 @@ def __write_exp_table(self, model, path): h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') - h_file.write(f'static constexpr {layer.get_attr("exp_table_t").name} {table_name}[{table_size}] = {{') + h_file.write( + f'static constexpr nnet::array<{layer.get_attr("exp_table_t").name},{table_size}> {table_name} = {{' + ) ac_type = layer.get_attr('inp_norm_t') @@ -776,7 +778,9 @@ def __write_invert_table(self, model, path): h_file.write(f'#ifndef {header_name.upper()}_H_\n') h_file.write(f'#define {header_name.upper()}_H_\n\n') - h_file.write(f'static constexpr {layer.get_attr("exp_table_t").name} {table_name}[{table_size}] = {{') + h_file.write( + f'static constexpr nnet::array<{layer.get_attr("inv_table_t").name},{table_size}> {table_name} = {{' + ) ac_type = layer.get_attr('inv_inp_t') From e0aba712d2f7f2c3a2b6fdac8e90aa664c8e8f71 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Sun, 28 Jun 2026 14:01:54 +0100 Subject: [PATCH 23/40] default case handling improvement --- .../backends/oneapi/passes/core_templates.py | 28 +++++++++++++------ .../nnet_utils/nnet_activation_stream.h | 6 ++-- hls4ml/writer/oneapi_writer.py | 19 ++++++++----- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index a0d78295c4..b178cf38cc 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -200,7 +200,9 @@ def format(self, node): static constexpr unsigned inv_table_size = {inv_table_size}; static constexpr unsigned io_type = nnet::{iotype}; static constexpr unsigned reuse_factor = {reuse}; + static constexpr nnet::softmax_implementation implementation = nnet::softmax_implementation::{implementation}; + typedef {smax_accum_t} accum_t; typedef {exp_table_t.name} exp_table_t; typedef {inv_table_t.name} inv_table_t;""" @@ -242,17 +244,27 @@ def format(self, node): params['type'] = node.get_attr('activation') if params['type'] == 'softmax': - if 'exp_table_size' in params: + # By dividing with 2 we only retain positive vals since e^x > 0, at the same precision, best we can do is + if 'exp_table_size' in params and params['exp_table_size'] is not None: params['exp_table_size'] //= 2 else: - params['exp_table_size'] = 1024 - + # Use the default precision + params['exp_table_size'] = 2 ** (params['table_t'].precision.width - 1) params['exp_table_t'].precision.width = ceil_log2(params['exp_table_size']) - params['exp_table_t'].precision.integer = 3 + params['exp_table_t'].precision.integer = params['table_t'].precision.integer - 1 params['exp_table_t'].precision.signed = False params.setdefault('table_size', params['exp_table_size']) + if params['accum_t'].name == 'model_default_t': + extra_bits_req = ceil_log2(params['n_in']) + s = 'true' if params['exp_table_t'].precision.signed else 'false' + w = params['exp_table_t'].precision.width + extra_bits_req + i = params['exp_table_t'].precision.integer + extra_bits_req + params['smax_accum_t'] = f'ac_fixed<{str(w)},{str(i)},{s}>' + else: + params['smax_accum_t'] = params['accum_t'].name + if 'inp_norm_t' not in params: input_t = node.get_input_variable().type.precision width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 @@ -277,10 +289,10 @@ def format(self, node): if 'inv_table_size' in params: params['inv_table_size'] //= 2 else: - params['inv_table_size'] = 1024 + params['inv_table_size'] = 2 ** (params['table_t'].precision.width - 1) params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) - params['inv_table_t'].precision.integer = 3 + params['inv_table_t'].precision.integer = params['table_t'].precision.integer - 1 params['inv_table_t'].precision.signed = False params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 @@ -288,9 +300,9 @@ def format(self, node): params['inv_inp_t'].precision.signed = True if params['implementation'] == 'stable': - self.template += softmax_config_table_template_stable + self.template = softmax_config_template + softmax_config_table_template_stable else: - self.template += softmax_config_table_template + self.template = softmax_config_template + softmax_config_table_template params['exp_table_name'] = node.name + '_exp_table' params['inv_table_name'] = node.name + '_inv_table' diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index c0400fb306..b263de4236 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -303,7 +303,7 @@ template void softmax_stabl } // Calculate all the e^x's - [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[input_arr_size]; + [[intel::fpga_register]] typename CONFIG_T::accum_t exp_res[input_arr_size]; #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { @@ -313,9 +313,9 @@ template void softmax_stabl // Explicitly sum the results with an adder tree. // Rounding & Saturation mode, which improve accuracy, prevent Vivado from expression balancing - Op_add op_add; + Op_add op_add; [[intel::fpga_register]] typename CONFIG_T::inv_inp_t exp_sum = - reduce>(exp_res, op_add); + reduce>(exp_res, op_add); [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = CONFIG_T::invert_table[softmax_stable_idx_from_real_val( diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 29804ad173..daf225cdf6 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -708,11 +708,11 @@ def __write_exp_table(self, model, path): for layer in model.get_layers(): if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': table_name = layer.name + '_exp_table' + table_size = ( int(layer.get_attr('exp_table_size')) // 2 - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') - and layer.get_attr('exp_table_size') is not None - else 1024 + if layer.get_attr('exp_table_size') is not None + else (2 ** layer.attributes['table_t'].precision.width) // 2 ) with open(f'{path}/{table_name}.h', 'w') as h_file: @@ -745,6 +745,12 @@ def __write_exp_table(self, model, path): fp_integer = 6 fp_signed = True + scale = ( + layer.attributes['exp_scale'] + if (('exp_scale' in layer.attributes) and (layer.attributes['exp_scale'] is not None)) + else 1.0 + ) + sep = '' N = ceil_log2(table_size) for i in range(table_size): @@ -755,7 +761,7 @@ def __write_exp_table(self, model, path): else: b.insert(0, 1) f.set_msb_bits(b) - real_val = f.exp_float() + real_val = f.exp_float() * scale h_file.write(sep + str(real_val)) sep = ', ' @@ -768,9 +774,8 @@ def __write_invert_table(self, model, path): table_name = layer.name + '_inv_table' table_size = ( int(layer.get_attr('inv_table_size')) // 2 - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') - and layer.get_attr('inv_table_size') is not None - else 1024 + if layer.get_attr('inv_table_size') is not None + else (2 ** layer.attributes['table_t'].precision.width) // 2 ) with open(f'{path}/{table_name}.h', 'w') as h_file: From 584c4f73b4fe92bcbb8bd646988c6a29674122bf Mon Sep 17 00:00:00 2001 From: bugracyln Date: Sun, 28 Jun 2026 16:51:06 +0100 Subject: [PATCH 24/40] minor improvements to default rollback and added comments --- .../backends/oneapi/passes/core_templates.py | 9 +++--- .../firmware/nnet_utils/nnet_activation.h | 6 ++-- .../nnet_utils/nnet_activation_stream.h | 4 --- hls4ml/writer/oneapi_writer.py | 31 ++++++++----------- 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index b178cf38cc..0712131750 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -244,7 +244,7 @@ def format(self, node): params['type'] = node.get_attr('activation') if params['type'] == 'softmax': - # By dividing with 2 we only retain positive vals since e^x > 0, at the same precision, best we can do is + # The lookup input (x - x_max) is always <= 0, so only the negative half if 'exp_table_size' in params and params['exp_table_size'] is not None: params['exp_table_size'] //= 2 else: @@ -254,8 +254,9 @@ def format(self, node): params['exp_table_t'].precision.integer = params['table_t'].precision.integer - 1 params['exp_table_t'].precision.signed = False - params.setdefault('table_size', params['exp_table_size']) + params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary + # Determine accumulator type if present, else derive it yourself based on the input size. if params['accum_t'].name == 'model_default_t': extra_bits_req = ceil_log2(params['n_in']) s = 'true' if params['exp_table_t'].precision.signed else 'false' @@ -273,7 +274,7 @@ def format(self, node): params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override - # this checks if table sizes will be default, if it is just use the table size to derive precision + # This checks if table sizes will be default, if it is just use the table size to derive precision if 'inv_table_size' not in params: params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 @@ -286,11 +287,11 @@ def format(self, node): node.set_attr('inp_norm_t', params['inp_norm_t']) + # Again we only look up 1/sum(e^x) which is >=0 so no need the entie address space if 'inv_table_size' in params: params['inv_table_size'] //= 2 else: params['inv_table_size'] = 2 ** (params['table_t'].precision.width - 1) - params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) params['inv_table_t'].precision.integer = params['table_t'].precision.integer - 1 params['inv_table_t'].precision.signed = False diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index c87d8a2c20..5d7f7fb071 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -136,7 +136,7 @@ template void softmax_stable(cons } // Calculate all the e^x's - [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; + [[intel::fpga_register]] typename CONFIG_T::accum_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { exp_res[i] = @@ -145,9 +145,9 @@ template void softmax_stable(cons } // Explicitly sum previously calculated exponentials with an adder tree - Op_add op_add; + Op_add op_add; [[intel::fpga_register]] typename CONFIG_T::inv_inp_t exp_sum = - reduce>(exp_res, op_add); + reduce>(exp_res, op_add); // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index b263de4236..a5777b8f7a 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -326,10 +326,6 @@ template void softmax_stabl SoftmaxInvPackLoop: #pragma unroll for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { - - // TODO - Find Quartus-equivalent pragma - // #pragma HLS ALLOCATION instances=mul limit=multiplier_limit operation - out_pack[j] = exp_res[j] * inv_exp_sum; } diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index daf225cdf6..a97ce2270e 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -709,6 +709,9 @@ def __write_exp_table(self, model, path): if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': table_name = layer.name + '_exp_table' + # The lookup input (x - x_max) is always <= 0, so only the negative half of + # the signed fixed-point input range is ever addressed. + # Therefore only half of the full address space is required. table_size = ( int(layer.get_attr('exp_table_size')) // 2 if layer.get_attr('exp_table_size') is not None @@ -733,24 +736,18 @@ def __write_exp_table(self, model, path): fp_signed = ac_type.precision.signed except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - fp_bits = 16 - fp_integer = 6 - fp_signed = True - + pass if fp_signed is False: raise Exception('Softmax types need to be signed') - else: - fp_bits = 16 - fp_integer = 6 - fp_signed = True - + # Copy scaling from attributes scale = ( layer.attributes['exp_scale'] if (('exp_scale' in layer.attributes) and (layer.attributes['exp_scale'] is not None)) else 1.0 ) + # Use the top bits if table_size < 2**bit_width sep = '' N = ceil_log2(table_size) for i in range(table_size): @@ -772,6 +769,11 @@ def __write_invert_table(self, model, path): for layer in model.get_layers(): if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': table_name = layer.name + '_inv_table' + + # Table size is halved since e^x >= 0 and we only lookup 1/sum(e^x) >= 0 + # since all values in the table are cast to inv_table_t, the precision + # is limited by that type (inv_inp_t is signed). Keeping the table size + # large has no advantage both in terms of space and in terms of precision. table_size = ( int(layer.get_attr('inv_table_size')) // 2 if layer.get_attr('inv_table_size') is not None @@ -796,18 +798,11 @@ def __write_invert_table(self, model, path): fp_signed = ac_type.precision.signed except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - fp_bits = 18 - fp_integer = 8 - fp_signed = True - + pass if fp_signed is False: raise Exception('Softmax types need to be signed') - else: - fp_bits = 18 - fp_integer = 8 - fp_signed = True - + # Use the top bits if table_size < 2**bit_width sep = '' N = ceil_log2(table_size) for i in range(table_size): From 6309fdbfb4d08d231889f50a9e53c884888e4c6b Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 2 Jul 2026 14:44:24 +0100 Subject: [PATCH 25/40] saving before merge --- hls4ml/backends/fpga/fpga_backend.py | 21 +- hls4ml/backends/oneapi/oneapi_backend.py | 6 +- .../backends/oneapi/passes/core_templates.py | 123 +++---- hls4ml/converters/keras_v3/hgq2/softmax.py | 21 +- .../firmware/nnet_utils/nnet_activation.h | 24 +- .../nnet_utils/nnet_activation_stream.h | 24 +- hls4ml/writer/oneapi_writer.py | 326 ++++++++++-------- 7 files changed, 306 insertions(+), 239 deletions(-) diff --git a/hls4ml/backends/fpga/fpga_backend.py b/hls4ml/backends/fpga/fpga_backend.py index db0f256172..1518f1a04b 100644 --- a/hls4ml/backends/fpga/fpga_backend.py +++ b/hls4ml/backends/fpga/fpga_backend.py @@ -129,22 +129,33 @@ def __init__(self, name): ConfigurableAttribute('skip', value_type=bool, default=False, description=descriptions.softmax_skip), TypeAttribute( 'exp_table', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), description=descriptions.table_type, ), TypeAttribute( 'inv_table', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), description=descriptions.table_type, ), TypeAttribute( 'inv_inp', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), + description='What the accumulated value is cast to before accessing the inversion table (only in stable)', ), TypeAttribute( - 'accum', - default=FixedPrecisionType(18, 8, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT), + 'inp_norm', + default=FixedPrecisionType( + 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT + ), + description='The internal width used for the exp table lookup (only in stable)', ), + TypeAttribute('accum', description=descriptions.accum_type), ] self.attribute_map[Softmax] = softmax_attrs diff --git a/hls4ml/backends/oneapi/oneapi_backend.py b/hls4ml/backends/oneapi/oneapi_backend.py index 94f26c9f1c..0da9f73f67 100644 --- a/hls4ml/backends/oneapi/oneapi_backend.py +++ b/hls4ml/backends/oneapi/oneapi_backend.py @@ -209,7 +209,11 @@ def build(self, model, build_type='fpga_emu', run=False): try: subprocess.run('which icpx', shell=True, cwd=builddir, check=True) except subprocess.CalledProcessError: - raise RuntimeError('Could not find icpx. Please configure oneAPI appropriately') + print("icpx not found. Trying ahls") + try: + subprocess.run('which ahls', shell=True, cwd=builddir, check=True) + except subprocess.CalledProcessError: + raise RuntimeError('Could not find icpx or ahls. Please configure oneAPI appropriately') subprocess.run('cmake ..', shell=True, cwd=builddir, check=True) subprocess.run(f'make {build_type}', shell=True, cwd=builddir, check=True) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 0712131750..f02e5a206e 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -244,69 +244,76 @@ def format(self, node): params['type'] = node.get_attr('activation') if params['type'] == 'softmax': - # The lookup input (x - x_max) is always <= 0, so only the negative half - if 'exp_table_size' in params and params['exp_table_size'] is not None: - params['exp_table_size'] //= 2 - else: - # Use the default precision - params['exp_table_size'] = 2 ** (params['table_t'].precision.width - 1) - params['exp_table_t'].precision.width = ceil_log2(params['exp_table_size']) - params['exp_table_t'].precision.integer = params['table_t'].precision.integer - 1 - params['exp_table_t'].precision.signed = False - - params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary - - # Determine accumulator type if present, else derive it yourself based on the input size. - if params['accum_t'].name == 'model_default_t': - extra_bits_req = ceil_log2(params['n_in']) - s = 'true' if params['exp_table_t'].precision.signed else 'false' - w = params['exp_table_t'].precision.width + extra_bits_req - i = params['exp_table_t'].precision.integer + extra_bits_req - params['smax_accum_t'] = f'ac_fixed<{str(w)},{str(i)},{s}>' - else: - params['smax_accum_t'] = params['accum_t'].name - if 'inp_norm_t' not in params: - input_t = node.get_input_variable().type.precision - width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 - width, iwidth = width - signed, iwidth - signed - import copy - - params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override + params['exp_table_name'] = node.name + '_exp_table' + params['inv_table_name'] = node.name + '_inv_table' + + if params['implementation'] == 'stable': + self.template = softmax_config_template + softmax_config_table_template_stable - # This checks if table sizes will be default, if it is just use the table size to derive precision + if 'exp_table_size' not in params: + params['exp_table_size'] = 2 ** params['inp_norm_t'].precision.width + node.set_attr('exp_table_size', params['exp_table_size']) + if 'inv_table_size' not in params: - params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 - params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 - params['inp_norm_t'].precision.signed = True - params['inp_norm_t'].name = f'{node.name}_inp_norm_t' - else: - params[ - 'inp_norm_t' - ].name = f'ac_fixed<{width},{iwidth},{"true" if signed else "false"},AC_RND,AC_SAT_SYM>' + params['inv_table_size'] = 2 ** params['inv_inp_t'].precision.width + node.set_attr('inv_table_size', params['inv_table_size']) - node.set_attr('inp_norm_t', params['inp_norm_t']) - - # Again we only look up 1/sum(e^x) which is >=0 so no need the entie address space - if 'inv_table_size' in params: - params['inv_table_size'] //= 2 - else: - params['inv_table_size'] = 2 ** (params['table_t'].precision.width - 1) - params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) - params['inv_table_t'].precision.integer = params['table_t'].precision.integer - 1 - params['inv_table_t'].precision.signed = False - - params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 - params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 - params['inv_inp_t'].precision.signed = True - - if params['implementation'] == 'stable': - self.template = softmax_config_template + softmax_config_table_template_stable + params['smax_accum_t'] = params['accum_t'].name + #params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary + else: - self.template = softmax_config_template + softmax_config_table_template + if 'exp_table_size' not in params: + params['exp_table_size'] = params['table_size'] + if 'inv_table_size' not in params: + params['inv_table_size'] = params['table_size'] + + ''' + # Determine accumulator type if present, else derive it yourself based on the input size. + if params['accum_t'].name == 'model_default_t': + extra_bits_req = ceil_log2(params['n_in']) + s = 'false'#'true' if params['exp_table_t'].precision.signed else 'false' + w = params['exp_table_t'].precision.width + extra_bits_req + i = params['exp_table_t'].precision.integer + extra_bits_req + params['smax_accum_t'] = f'ac_fixed<{str(w)},{str(i)},{s}>' + else: + params['smax_accum_t'] = params['accum_t'].name + + + if 'inp_norm_t' not in params: + input_t = node.get_input_variable().type.precision + width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 + width, iwidth = width - signed, iwidth - signed + import copy + + params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override + + # This checks if table sizes will be default, if it is just use the table size to derive precision + if 'inv_table_size' not in params: + params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 + params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 + params['inp_norm_t'].precision.signed = True + params['inp_norm_t'].name = f'{node.name}_inp_norm_t' + else: + params[ + 'inp_norm_t' + ].name = f'ac_fixed<{width},{iwidth},{"true" if signed else "false"},AC_RND,AC_SAT_SYM>' + + node.set_attr('inp_norm_t', params['inp_norm_t']) + + # Again we only look up 1/sum(e^x) which is >=0 so no need the entie address space + if 'inv_table_size' in params: + pass #params['inv_table_size'] //= 2 + else: + params['inv_table_size'] = 2 ** (params['table_t'].precision.width - 1) + params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) + params['inv_table_t'].precision.integer = params['table_t'].precision.integer - 1 + params['inv_table_t'].precision.signed = False - params['exp_table_name'] = node.name + '_exp_table' - params['inv_table_name'] = node.name + '_inv_table' + params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 + params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 + params['inv_inp_t'].precision.signed = True + ''' return self.template.format(**params) @@ -338,7 +345,7 @@ def format(self, node): class SoftmaxConfigTemplate(ActivationConfigTemplate): def __init__(self): super(ActivationConfigTemplate, self).__init__(Softmax) # Skip ActivationConfigTemplate's __init__ - self.template = softmax_config_template + self.template = softmax_config_template + softmax_config_table_template class ActivationFunctionTemplate(FunctionCallTemplate): diff --git a/hls4ml/converters/keras_v3/hgq2/softmax.py b/hls4ml/converters/keras_v3/hgq2/softmax.py index 136662beba..e461ad5521 100644 --- a/hls4ml/converters/keras_v3/hgq2/softmax.py +++ b/hls4ml/converters/keras_v3/hgq2/softmax.py @@ -12,10 +12,16 @@ from keras import KerasTensor -def fixed_quantizer_to_hls4ml_t(q: 'FixedPointQuantizerBase', take_max=False): +def fixed_quantizer_to_hls4ml_t(q: 'FixedPointQuantizerBase', take_max=False, force_unsigned=False): from keras import ops k, i, f = q.kif + + if force_unsigned: + k = 0. + i += 1. + f += 1. + k = ops.convert_to_numpy(k) i = ops.convert_to_numpy(i) f = ops.convert_to_numpy(f) @@ -74,14 +80,16 @@ def handle( exp_table_size = 2 ** int(ops.convert_to_numpy(ops.max(layer.exp_table.iq.quantizer.bits))) # type: ignore else: exp_table_size = None # Placeholder, will be overridden in bit-exact pass - + exp_oq = layer.exp_table.oq.quantizer inv_oq = layer.inv_table.oq.quantizer inv_iq = layer.inv_table.iq.quantizer + assert isinstance(exp_oq, FixedPointQuantizerBase), 'Only fixed-point quantizer is supported for exp_table' - exp_table_t = fixed_quantizer_to_hls4ml_t(exp_oq) - inv_table_t = fixed_quantizer_to_hls4ml_t(inv_oq) - inv_inp_t = fixed_quantizer_to_hls4ml_t(inv_iq) + # Enforce unsigned quantisers on all three types + exp_table_t = fixed_quantizer_to_hls4ml_t(exp_oq, force_unsigned=True) + inv_table_t = fixed_quantizer_to_hls4ml_t(inv_oq, force_unsigned=True) + inv_inp_t = fixed_quantizer_to_hls4ml_t(inv_iq, force_unsigned=True) exp_scale = layer.input_scaler inv_table_size = 2**inv_inp_t.width @@ -122,7 +130,8 @@ def handle( ) if layer.stable: - inp_norm_t = fixed_quantizer_to_hls4ml_t(layer.exp_table.iq.quantizer) + # Force unsigned since norm >= 0 + inp_norm_t = fixed_quantizer_to_hls4ml_t(layer.exp_table.iq.quantizer, force_unsigned=True) config['inp_norm_t'] = inp_norm_t return (config,) diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index 5d7f7fb071..c2e71028d3 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -99,7 +99,7 @@ template void sigmoid(const data_ enum class softmax_implementation { latency = 0, legacy = 1, stable = 2, argmax = 3 }; -template inline unsigned softmax_stable_idx_from_real_val(const data_T x) { +template inline unsigned softmax_idx_from_real_val_negative(const data_T x) { // Number of address bits for table static constexpr int N = ceillog2::val; @@ -112,9 +112,9 @@ template inline unsigned softmax_stable_idx_ return y.to_uint(); } -template inline unsigned softmax_latency_idx_from_real_val(const data_T x) { +template inline unsigned softmax_idx_from_real_val(const data_T x) { // Number of address bits for table - static constexpr int N = ceillog2::val; + static constexpr int N = ceillog2::val; // Slice the top N bits of the input [[intel::fpga_register]] ac_int y = x.template slc(x.width - N); @@ -132,7 +132,7 @@ template void softmax_stable(cons [[intel::fpga_register]] typename CONFIG_T::inp_norm_t d_xi_xmax[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - d_xi_xmax[i] = data[i] - x_max; + d_xi_xmax[i] = x_max - data[i]; } // Calculate all the e^x's @@ -140,7 +140,7 @@ template void softmax_stable(cons #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { exp_res[i] = - CONFIG_T::exp_table[softmax_stable_idx_from_real_val( + CONFIG_T::exp_table[softmax_idx_from_real_val( d_xi_xmax[i])]; // input_t, CONFIG_T } @@ -151,7 +151,7 @@ template void softmax_stable(cons // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_stable_idx_from_real_val( + CONFIG_T::invert_table[softmax_idx_from_real_val( exp_sum)]; #pragma unroll @@ -169,7 +169,7 @@ template void softmax_latency(con [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - exp_res[i] = exp_table_latency[softmax_latency_idx_from_real_val(data[i])]; + exp_res[i] = CONFIG_T::exp_table[softmax_idx_from_real_val(data[i])]; } // Explicitly sum the results with an adder tree. @@ -179,7 +179,7 @@ template void softmax_latency(con // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table_latency[softmax_latency_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_idx_from_real_val(exp_sum)]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { res[i] = exp_res[i] * inv_exp_sum; @@ -187,8 +187,8 @@ template void softmax_latency(con } template void softmax_legacy(const data_T &data, res_T &res) { -#include "activation_tables/exp_table_legacy.tb" -#include "activation_tables/invert_table_legacy.tb" +//#include "activation_tables/exp_table_legacy.tb" +//#include "activation_tables/invert_table_legacy.tb" [[intel::fpga_register]] int data_round[CONFIG_T::n_in]; New_loop: @@ -214,7 +214,7 @@ template void softmax_legacy(cons if (index > CONFIG_T::table_size - 1) index = CONFIG_T::table_size - 1; - typename CONFIG_T::exp_table_t temp_exp = exp_table_legacy[index]; + typename CONFIG_T::exp_table_t temp_exp = CONFIG_T::exp_table[index]; exp_res_temp += temp_exp; } } @@ -223,7 +223,7 @@ template void softmax_legacy(cons exp_res_index = 0; if (exp_res_index > CONFIG_T::table_size - 1) exp_res_index = CONFIG_T::table_size - 1; - res[ii] = invert_table_legacy[exp_res_index]; + res[ii] = CONFIG_T::invert_table[exp_res_index]; } } diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index a5777b8f7a..3b1f26b76b 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -299,7 +299,7 @@ template void softmax_stabl #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { - d_xi_xmax[j] = data_array[j] - x_max; + d_xi_xmax[j] = x_max - data_array[j]; } // Calculate all the e^x's @@ -307,7 +307,7 @@ template void softmax_stabl #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { - exp_res[j] = CONFIG_T::exp_table[softmax_stable_idx_from_real_val(d_xi_xmax[j])]; } @@ -318,7 +318,7 @@ template void softmax_stabl reduce>(exp_res, op_add); [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_stable_idx_from_real_val( + CONFIG_T::invert_table[softmax_idx_from_real_val( exp_sum)]; typename ExtractPipeType::value_type out_pack; @@ -334,8 +334,8 @@ template void softmax_stabl } template void softmax_latency_stream() { -#include "activation_tables/exp_table_latency.tb" -#include "activation_tables/invert_table_latency.tb" +//#include "activation_tables/exp_table_latency.tb" +//#include "activation_tables/invert_table_latency.tb" constexpr unsigned multiplier_limit = DIV_ROUNDUP(std::tuple_size::value_type>{}, CONFIG_T::reuse_factor); @@ -355,8 +355,8 @@ template void softmax_laten SoftmaxExpPackLoop: #pragma unroll for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { - exp_res[j] = exp_table_latency[softmax_latency_idx_from_real_val< - typename ExtractPipeType::value_type::value_type, CONFIG_T>(in_pack[j])]; + exp_res[j] = CONFIG_T::exp_table[softmax_idx_from_real_val< + typename ExtractPipeType::value_type::value_type, CONFIG_T::exp_table_size>(in_pack[j])]; } // Explicitly sum the results with an adder tree. @@ -367,7 +367,7 @@ template void softmax_laten // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - invert_table_latency[softmax_latency_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_idx_from_real_val(exp_sum)]; typename ExtractPipeType::value_type out_pack; SoftmaxInvPackLoop: @@ -382,8 +382,8 @@ template void softmax_laten } template void softmax_legacy_stream() { -#include "activation_tables/exp_table_legacy.tb" -#include "activation_tables/invert_table_legacy.tb" +//#include "activation_tables/exp_table_legacy.tb" +//#include "activation_tables/invert_table_legacy.tb" // Index into the lookup table based on data for exponentials [[intel::fpga_register]] @@ -421,7 +421,7 @@ template void softmax_legac index = 0; if (index > CONFIG_T::table_size - 1) index = CONFIG_T::table_size - 1; - exp_diff_res = exp_table_legacy[index]; + exp_diff_res = CONFIG_T::exp_table[index]; } exp_res[i] += exp_diff_res; } @@ -437,7 +437,7 @@ template void softmax_legac if (exp_res_index > CONFIG_T::table_size - 1) exp_res_index = CONFIG_T::table_size - 1; out_pack[j] = - static_cast::value_type::value_type>(invert_table_legacy[exp_res_index]); + static_cast::value_type::value_type>(CONFIG_T::invert_table[exp_res_index]); } res_pipe::write(out_pack); diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index a97ce2270e..220a8a7ba3 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -557,12 +557,12 @@ def write_nnet_utils(self, model): dstpath = f'{model.config.get_output_dir()}/src/firmware/{dst}' copyfile(srcpath, dstpath) - def __get_table_size(self, model, activation, table_name='table_size'): + def __get_table_size(self, model, activation, size_attr_name='table_size'): for layer in model.get_layers(): if ( layer.get_attr('activation') == activation or layer.get_attr('recurrent_activation') == activation - ) and layer.get_attr(table_name) is not None: - return int(layer.get_attr(table_name)) + ) and layer.get_attr(size_attr_name) is not None: + return int(layer.get_attr(size_attr_name)) return 1024 def __get_table_header(self, table_name, table_size, table_type='table_t'): @@ -706,17 +706,14 @@ def __get_table_precision(self, model, activation, table_name='table_precision') def __write_exp_table(self, model, path): for layer in model.get_layers(): - if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': - table_name = layer.name + '_exp_table' + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'stable': - # The lookup input (x - x_max) is always <= 0, so only the negative half of - # the signed fixed-point input range is ever addressed. - # Therefore only half of the full address space is required. - table_size = ( - int(layer.get_attr('exp_table_size')) // 2 - if layer.get_attr('exp_table_size') is not None - else (2 ** layer.attributes['table_t'].precision.width) // 2 - ) + + #import pdb; pdb.set_trace() + table_name = layer.name + '_exp_table' + table_size = int(layer.get_attr('exp_table_size')) with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -728,17 +725,12 @@ def __write_exp_table(self, model, path): ) ac_type = layer.get_attr('inp_norm_t') + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') + if fp_signed is True: + raise Exception('Softmax types need to be unsigned') # Copy scaling from attributes scale = ( @@ -746,19 +738,21 @@ def __write_exp_table(self, model, path): if (('exp_scale' in layer.attributes) and (layer.attributes['exp_scale'] is not None)) else 1.0 ) + + N = ceil_log2(table_size) + if N > 2**fp_bits: + raise Exception('Table size is bigger than what precision allows') + maxval = 2**fp_integer - 1 - # Use the top bits if table_size < 2**bit_width sep = '' - N = ceil_log2(table_size) + # Use the top bits if table_size < 2**bit_width for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + f = FixedPointEmulator(fp_bits, fp_integer, signed=False) b = uint_to_binary(i, N) - if i == 0: - b.insert(0, 0) - else: - b.insert(0, 1) f.set_msb_bits(b) - real_val = f.exp_float() * scale + real_val = (1.0 / f.exp_float()) * scale + if real_val > maxval: + real_val = maxval h_file.write(sep + str(real_val)) sep = ', ' @@ -767,18 +761,12 @@ def __write_exp_table(self, model, path): def __write_invert_table(self, model, path): for layer in model.get_layers(): - if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': - table_name = layer.name + '_inv_table' + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'stable': - # Table size is halved since e^x >= 0 and we only lookup 1/sum(e^x) >= 0 - # since all values in the table are cast to inv_table_t, the precision - # is limited by that type (inv_inp_t is signed). Keeping the table size - # large has no advantage both in terms of space and in terms of precision. - table_size = ( - int(layer.get_attr('inv_table_size')) // 2 - if layer.get_attr('inv_table_size') is not None - else (2 ** layer.attributes['table_t'].precision.width) // 2 - ) + table_name = layer.name + '_inv_table' + table_size = int(layer.get_attr('inv_table_size')) with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -790,27 +778,28 @@ def __write_invert_table(self, model, path): ) ac_type = layer.get_attr('inv_inp_t') + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') + if fp_signed is True: + raise Exception('Softmax types need to be unsigned') + + N = ceil_log2(table_size) + if N > 2**fp_bits: + raise Exception('Table size is bigger than what precision allows') + maxval = 2**fp_integer - 1 # Use the top bits if table_size < 2**bit_width sep = '' - N = ceil_log2(table_size) for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + # ALWAYS ASSUME ALL TYPES HERE ARE UNSIGNED + f = FixedPointEmulator(fp_bits, fp_integer, signed=False) b = uint_to_binary(i, N) - b.insert(0, 0) f.set_msb_bits(b) real_val = f.inv_float() + if real_val > maxval: + real_val = maxval h_file.write(sep + str(real_val)) sep = ', ' @@ -818,118 +807,165 @@ def __write_invert_table(self, model, path): h_file.write('#endif') def __write_exp_table_latency(self, model, path): - table_name = 'exp_table_latency' - table_size = self.__get_table_size(model, 'softmax') + for layer in model.get_layers(): + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'latency': - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) + table_name = layer.name + '_exp_table' + table_size = int(layer.get_attr('exp_table_size')) - # Default fixed point precision - # 6 bits for integer part, 10 bits for decimal - total, 16 - fp_bits = 16 - fp_integer = 6 - fp_signed = True + with open(f'{path}/{table_name}.h', 'w') as h_file: + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') - # Exp table should use the same precision as exp_table, as seen in Vivado code - # init_exp_table(exp_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_input_variable().type - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + h_file.write( + f'static constexpr nnet::array<{layer.get_attr("exp_table_t").name},{table_size}> {table_name} = {{' + ) - sep = '' - N = ceil_log2(table_size) - for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) - f.set_msb_bits(uint_to_binary(i, N)) - real_val = f.exp_float() - h_file.write(sep + str(real_val)) - sep = ', ' + # Default fixed point precision + # 6 bits for integer part, 10 bits for decimal - total, 16 + fp_bits = 16 + fp_integer = 6 + fp_signed = True - h_file.write('};\n') - h_file.close() + # Exp table should use the same precision as exp_table, as seen in Vivado code + # init_exp_table(exp_table); + for layer in model.get_layers(): + if layer.name == 'softmax': + ac_type = layer.get_input_variable().type + if ac_type is not None: + try: + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed + except Exception: + # FixedPrecisionType wasn't correctly stored in layer attributes, use default values + pass + + sep = '' + N = ceil_log2(table_size) + for i in range(table_size): + f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + f.set_msb_bits(uint_to_binary(i, N)) + real_val = f.exp_float() + h_file.write(sep + str(real_val)) + sep = ', ' + + h_file.write('};\n') + h_file.close() def __write_invert_table_latency(self, model, path): - table_name = 'invert_table_latency' - table_size = self.__get_table_size(model, 'softmax') + for layer in model.get_layers(): + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'latency': - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) + table_name = layer.name + '_inv_table' + table_size = int(layer.get_attr('inv_table_size')) - # Default fixed point precision, in case values from layer attributes cannot be extracted - # 8 bits for integer part, 10 bits for decimal - total, 18 - fp_bits = 18 - fp_integer = 8 - fp_signed = True + with open(f'{path}/{table_name}.h', 'w') as h_file: + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') - # Invert table should use the same precision as exp_table, as seen in Vivado code - # init_invert_table(invert_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_attr('exp_table_t') - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + h_file.write( + f'static constexpr nnet::array<{layer.get_attr("inv_table_t").name},{table_size}> {table_name} = {{' + ) - sep = '' - N = ceil_log2(table_size) - for i in range(table_size): - f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) - f.set_msb_bits(uint_to_binary(i, N)) - real_val = f.inv_float() - h_file.write(sep + str(real_val)) - sep = ', ' + # Default fixed point precision, in case values from layer attributes cannot be extracted + # 8 bits for integer part, 10 bits for decimal - total, 18 + fp_bits = 18 + fp_integer = 8 + fp_signed = True - h_file.write('};\n') - h_file.close() + # Invert table should use the same precision as exp_table, as seen in Vivado code + # init_invert_table(invert_table); + for layer in model.get_layers(): + if layer.name == 'softmax': + ac_type = layer.get_attr('exp_table_t') + if ac_type is not None: + try: + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed + except Exception: + # FixedPrecisionType wasn't correctly stored in layer attributes, use default values + pass + + sep = '' + N = ceil_log2(table_size) + for i in range(table_size): + f = FixedPointEmulator(fp_bits, fp_integer, signed=fp_signed) + f.set_msb_bits(uint_to_binary(i, N)) + real_val = f.inv_float() + h_file.write(sep + str(real_val)) + sep = ', ' + + h_file.write('};\n') + h_file.close() def __write_exp_table_legacy(self, model, path): - table_name = 'exp_table_legacy' - table_size = self.__get_table_size(model, 'softmax') - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) + for layer in model.get_layers(): + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'legacy': - sep = '' - for i in range(table_size): - in_val = 2 * 8.0 * (i - float(table_size) / 2.0) / float(table_size) - real_val = np.exp(in_val) - h_file.write(sep + str(real_val)) - sep = ', ' + import pdb; pdb.set_trace() + table_name = layer.name + '_exp_table' + table_size = int(layer.get_attr('exp_table_size')) # not sure if it works, have to test first - h_file.write('};\n') - h_file.close() + with open(f'{path}/{table_name}.h', 'w') as h_file: + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') + + h_file.write( + f'static constexpr nnet::array<{layer.get_attr('exp_table_t').name},{table_size}> {table_name} = {{' + ) + + sep = '' + for i in range(table_size): + in_val = 2 * 8.0 * (i - float(table_size) / 2.0) / float(table_size) + real_val = np.exp(in_val) + h_file.write(sep + str(real_val)) + sep = ', ' + + h_file.write('};\n') + h_file.close() def __write_invert_table_legacy(self, model, path): - table_name = 'invert_table_legacy' - table_size = self.__get_table_size(model, 'softmax') - h_file = open(f'{path}/{table_name}.tb', 'w') - h_file.write(self.__get_table_header(table_name, table_size)) + for layer in model.get_layers(): + # Last property is essential since it seperates layer with activation property from actual activation layers + if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ + and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'legacy': - sep = '' - for i in range(table_size): - real_val = 0 - in_val = 64.0 * i / float(table_size) - if in_val > 0.0: - real_val = 1.0 / in_val - h_file.write(sep + str(real_val)) - sep = ', ' + table_name = layer.name + '_inv_table' + table_size = int(layer.get_attr('inv_table_size')) - h_file.write('};\n') - h_file.close() + with open(f'{path}/{table_name}.h', 'w') as h_file: + header_name = table_name + h_file.write(f'#ifndef {header_name.upper()}_H_\n') + h_file.write(f'#define {header_name.upper()}_H_\n\n') + + h_file.write( + f'static constexpr nnet::array<{layer.get_attr('inv_table_t').name},{table_size}> {table_name} = {{' + ) + + sep = '' + for i in range(table_size): + real_val = 0 + in_val = 64.0 * i / float(table_size) + if in_val > 0.0: + real_val = 1.0 / in_val + h_file.write(sep + str(real_val)) + sep = ', ' + + h_file.write('};\n') + h_file.close() def write_activation_tables(self, model): """Write the lookup tables for activation functions From f990dbd2e63142b497d88d033c9463228787160d Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 2 Jul 2026 19:27:58 +0100 Subject: [PATCH 26/40] multidim softmax and other fixes --- .../backends/oneapi/passes/core_templates.py | 79 +++++---------- hls4ml/converters/keras_v3/hgq2/softmax.py | 20 ++-- .../firmware/nnet_utils/nnet_activation.h | 19 ++-- .../nnet_utils/nnet_activation_stream.h | 72 +++++++++++--- hls4ml/writer/oneapi_writer.py | 96 ++++++++++++++----- 5 files changed, 177 insertions(+), 109 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index f02e5a206e..8b6f5a0da8 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -2,7 +2,6 @@ from hls4ml.backends.oneapi.oneapi_template import StreamFunctionCallTemplate, TaskSequenceTemplate from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate from hls4ml.model.layers import Activation, BatchNormalization, Dense, HardActivation, ParametrizedActivation, PReLU, Softmax -from hls4ml.utils.fixed_point_utils import ceil_log2 # Dense templates @@ -195,7 +194,16 @@ def format(self, node): softmax_config_template = """struct {type}_config{index} : nnet::activ_config {{ static constexpr unsigned n_in = {n_in}; + + // For multi-dim softmax + static const unsigned n_slice = {n_slice}; + static const unsigned n_outer = {n_outer}; + static const unsigned n_inner = {n_inner}; + + // For legacy softmax + typedef {table_t.name} table_t; static constexpr unsigned table_size = {table_size}; + static constexpr unsigned exp_table_size = {exp_table_size}; static constexpr unsigned inv_table_size = {inv_table_size}; static constexpr unsigned io_type = nnet::{iotype}; @@ -243,77 +251,35 @@ def format(self, node): params = self._default_config_params(node) params['type'] = node.get_attr('activation') - if params['type'] == 'softmax': + if (params['type'] == 'softmax') or (params['type'] == 'softmax_multidim'): + params.setdefault('n_inner', 1) + params.setdefault('n_outer', 1) + n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] + params['n_slice'] = n_slice params['exp_table_name'] = node.name + '_exp_table' - params['inv_table_name'] = node.name + '_inv_table' - + params['inv_table_name'] = node.name + '_inv_table' + if params['implementation'] == 'stable': self.template = softmax_config_template + softmax_config_table_template_stable if 'exp_table_size' not in params: params['exp_table_size'] = 2 ** params['inp_norm_t'].precision.width node.set_attr('exp_table_size', params['exp_table_size']) - + if 'inv_table_size' not in params: params['inv_table_size'] = 2 ** params['inv_inp_t'].precision.width node.set_attr('inv_table_size', params['inv_table_size']) params['smax_accum_t'] = params['accum_t'].name - #params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary - + # params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary + else: + # TODO: For latency check the table sizes correctly, match them if 'exp_table_size' not in params: - params['exp_table_size'] = params['table_size'] + params['exp_table_size'] = params['table_size'] if 'inv_table_size' not in params: - params['inv_table_size'] = params['table_size'] - - ''' - # Determine accumulator type if present, else derive it yourself based on the input size. - if params['accum_t'].name == 'model_default_t': - extra_bits_req = ceil_log2(params['n_in']) - s = 'false'#'true' if params['exp_table_t'].precision.signed else 'false' - w = params['exp_table_t'].precision.width + extra_bits_req - i = params['exp_table_t'].precision.integer + extra_bits_req - params['smax_accum_t'] = f'ac_fixed<{str(w)},{str(i)},{s}>' - else: - params['smax_accum_t'] = params['accum_t'].name - - - if 'inp_norm_t' not in params: - input_t = node.get_input_variable().type.precision - width, iwidth, signed = input_t.width, input_t.integer, input_t.signed # noqa: F841 - width, iwidth = width - signed, iwidth - signed - import copy - - params['inp_norm_t'] = copy.deepcopy(params['exp_table_t']) # assign type,later override - - # This checks if table sizes will be default, if it is just use the table size to derive precision - if 'inv_table_size' not in params: - params['inp_norm_t'].precision.width = params['exp_table_t'].precision.width + 1 - params['inp_norm_t'].precision.integer = params['exp_table_t'].precision.integer + 1 - params['inp_norm_t'].precision.signed = True - params['inp_norm_t'].name = f'{node.name}_inp_norm_t' - else: - params[ - 'inp_norm_t' - ].name = f'ac_fixed<{width},{iwidth},{"true" if signed else "false"},AC_RND,AC_SAT_SYM>' - - node.set_attr('inp_norm_t', params['inp_norm_t']) - - # Again we only look up 1/sum(e^x) which is >=0 so no need the entie address space - if 'inv_table_size' in params: - pass #params['inv_table_size'] //= 2 - else: - params['inv_table_size'] = 2 ** (params['table_t'].precision.width - 1) - params['inv_table_t'].precision.width = ceil_log2(params['inv_table_size']) - params['inv_table_t'].precision.integer = params['table_t'].precision.integer - 1 - params['inv_table_t'].precision.signed = False - - params['inv_inp_t'].precision.width = params['inv_table_t'].precision.width + 1 - params['inv_inp_t'].precision.integer = params['inv_table_t'].precision.integer + 1 - params['inv_inp_t'].precision.signed = True - ''' + params['inv_table_size'] = params['table_size'] return self.template.format(**params) @@ -398,6 +364,7 @@ def format(self, node): params = self._default_function_params(node) params['activation'] = node.get_attr('activation').lower() params['config'] = f'{node.get_attr("activation")}_config{node.index}' + return self.template.format(**params) diff --git a/hls4ml/converters/keras_v3/hgq2/softmax.py b/hls4ml/converters/keras_v3/hgq2/softmax.py index e461ad5521..1c97555ebc 100644 --- a/hls4ml/converters/keras_v3/hgq2/softmax.py +++ b/hls4ml/converters/keras_v3/hgq2/softmax.py @@ -18,9 +18,9 @@ def fixed_quantizer_to_hls4ml_t(q: 'FixedPointQuantizerBase', take_max=False, fo k, i, f = q.kif if force_unsigned: - k = 0. - i += 1. - f += 1. + k = 0.0 + i += 1.0 + f += 1.0 k = ops.convert_to_numpy(k) i = ops.convert_to_numpy(i) @@ -80,16 +80,16 @@ def handle( exp_table_size = 2 ** int(ops.convert_to_numpy(ops.max(layer.exp_table.iq.quantizer.bits))) # type: ignore else: exp_table_size = None # Placeholder, will be overridden in bit-exact pass - + exp_oq = layer.exp_table.oq.quantizer inv_oq = layer.inv_table.oq.quantizer inv_iq = layer.inv_table.iq.quantizer - + assert isinstance(exp_oq, FixedPointQuantizerBase), 'Only fixed-point quantizer is supported for exp_table' # Enforce unsigned quantisers on all three types exp_table_t = fixed_quantizer_to_hls4ml_t(exp_oq, force_unsigned=True) inv_table_t = fixed_quantizer_to_hls4ml_t(inv_oq, force_unsigned=True) - inv_inp_t = fixed_quantizer_to_hls4ml_t(inv_iq, force_unsigned=True) + inv_inp_t = fixed_quantizer_to_hls4ml_t(inv_iq, force_unsigned=True) exp_scale = layer.input_scaler inv_table_size = 2**inv_inp_t.width @@ -107,13 +107,19 @@ def handle( else: raise ValueError(f'Too many inputs for softmax layer {layer.name}: expected 1 or 2, got {len(in_tensors)}') + # For masked implementation assume first input is the tensor we are operating on + activation = 'softmax' + if len(in_tensors[0].shape[1:]) > 1: + if (1 not in in_tensors[0].shape[1:]) or (len(in_tensors[0].shape[1:]) > 2): + activation = 'softmax_multidim' + config = {} config.update(self.default_config) config.update( { 'axis': ax, 'n_in': n_in, - 'activation': 'softmax', + 'activation': activation, 'n_outer': n_outer, 'n_inner': n_inner, 'implementation': impl, diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index c2e71028d3..c4d895102b 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -139,9 +139,8 @@ template void softmax_stable(cons [[intel::fpga_register]] typename CONFIG_T::accum_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - exp_res[i] = - CONFIG_T::exp_table[softmax_idx_from_real_val( - d_xi_xmax[i])]; // input_t, CONFIG_T + exp_res[i] = CONFIG_T::exp_table[softmax_idx_from_real_val( + d_xi_xmax[i])]; // input_t, CONFIG_T } // Explicitly sum previously calculated exponentials with an adder tree @@ -151,8 +150,7 @@ template void softmax_stable(cons // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_idx_from_real_val( - exp_sum)]; + CONFIG_T::invert_table[softmax_idx_from_real_val(exp_sum)]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { @@ -162,14 +160,15 @@ template void softmax_stable(cons // TODO - Improve accuracy template void softmax_latency(const data_T &data, res_T &res) { -#include "activation_tables/exp_table_latency.tb" -#include "activation_tables/invert_table_latency.tb" + //#include "activation_tables/exp_table_latency.tb" + //#include "activation_tables/invert_table_latency.tb" // Calculate all the e^x's [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; #pragma unroll for (unsigned i = 0; i < CONFIG_T::n_in; i++) { - exp_res[i] = CONFIG_T::exp_table[softmax_idx_from_real_val(data[i])]; + exp_res[i] = + CONFIG_T::exp_table[softmax_idx_from_real_val(data[i])]; } // Explicitly sum the results with an adder tree. @@ -187,8 +186,8 @@ template void softmax_latency(con } template void softmax_legacy(const data_T &data, res_T &res) { -//#include "activation_tables/exp_table_legacy.tb" -//#include "activation_tables/invert_table_legacy.tb" + //#include "activation_tables/exp_table_legacy.tb" + //#include "activation_tables/invert_table_legacy.tb" [[intel::fpga_register]] int data_round[CONFIG_T::n_in]; New_loop: diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index 3b1f26b76b..865536d273 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -307,8 +307,9 @@ template void softmax_stabl #pragma unroll for (unsigned j = 0; j < input_arr_size; j++) { - exp_res[j] = CONFIG_T::exp_table[softmax_idx_from_real_val(d_xi_xmax[j])]; + exp_res[j] = + CONFIG_T::exp_table[softmax_idx_from_real_val( + d_xi_xmax[j])]; } // Explicitly sum the results with an adder tree. @@ -334,8 +335,8 @@ template void softmax_stabl } template void softmax_latency_stream() { -//#include "activation_tables/exp_table_latency.tb" -//#include "activation_tables/invert_table_latency.tb" + //#include "activation_tables/exp_table_latency.tb" + //#include "activation_tables/invert_table_latency.tb" constexpr unsigned multiplier_limit = DIV_ROUNDUP(std::tuple_size::value_type>{}, CONFIG_T::reuse_factor); @@ -355,8 +356,9 @@ template void softmax_laten SoftmaxExpPackLoop: #pragma unroll for (unsigned j = 0; j < std::tuple_size::value_type>{}; j++) { - exp_res[j] = CONFIG_T::exp_table[softmax_idx_from_real_val< - typename ExtractPipeType::value_type::value_type, CONFIG_T::exp_table_size>(in_pack[j])]; + exp_res[j] = + CONFIG_T::exp_table[softmax_idx_from_real_val::value_type::value_type, + CONFIG_T::exp_table_size>(in_pack[j])]; } // Explicitly sum the results with an adder tree. @@ -367,7 +369,8 @@ template void softmax_laten // Multiply previously calculated exponetials with the reciprocal of the sum [[intel::fpga_register]] typename CONFIG_T::inv_table_t inv_exp_sum = - CONFIG_T::invert_table[softmax_idx_from_real_val(exp_sum)]; + CONFIG_T::invert_table[softmax_idx_from_real_val( + exp_sum)]; typename ExtractPipeType::value_type out_pack; SoftmaxInvPackLoop: @@ -382,8 +385,8 @@ template void softmax_laten } template void softmax_legacy_stream() { -//#include "activation_tables/exp_table_legacy.tb" -//#include "activation_tables/invert_table_legacy.tb" + //#include "activation_tables/exp_table_legacy.tb" + //#include "activation_tables/invert_table_legacy.tb" // Index into the lookup table based on data for exponentials [[intel::fpga_register]] @@ -436,8 +439,8 @@ template void softmax_legac exp_res_index = 0; if (exp_res_index > CONFIG_T::table_size - 1) exp_res_index = CONFIG_T::table_size - 1; - out_pack[j] = - static_cast::value_type::value_type>(CONFIG_T::invert_table[exp_res_index]); + out_pack[j] = static_cast::value_type::value_type>( + CONFIG_T::invert_table[exp_res_index]); } res_pipe::write(out_pack); @@ -491,6 +494,53 @@ template void softmax_strea } } +// ************************************************* +// Multidimensional Softmax +// ************************************************* +template inline void softmax_multidim_stream() { + + using data_pipe_T = typename ExtractPipeType::value_type; + using data_T = typename data_pipe_T::value_type; + using res_pipe_T = typename ExtractPipeType::value_type; + using res_T = typename res_pipe_T::value_type; + using in_slice_arr_T = nnet::array; + using out_slice_arr_T = nnet::array; + + using slice_config = softmax_multidim_slice_config; + + [[intel::fpga_register]] data_pipe_T buffer_in; + [[intel::fpga_register]] res_pipe_T buffer_out; + [[intel::fpga_register]] in_slice_arr_T smax_slice_in; + [[intel::fpga_register]] out_slice_arr_T smax_slice_out; + + buffer_in = data_pipe::read(); + + #pragma unroll + for (unsigned i = 0; i < CONFIG_T::n_outer; i++) { + unsigned outer_offset = i * CONFIG_T::n_slice * CONFIG_T::n_inner; + #pragma unroll + for (unsigned k = 0; k < CONFIG_T::n_inner; k++) { + + // TODO: Access might be inefficient consider data rearrangement + #pragma unroll + for (unsigned j = 0; j < CONFIG_T::n_slice; j++) { + unsigned idx = outer_offset + j * CONFIG_T::n_inner + k; + smax_slice_in[j] = buffer_in[idx]; + } + + nnet::softmax(smax_slice_in, smax_slice_out); + + #pragma unroll + for (unsigned j = 0; j < CONFIG_T::n_slice; j++) { + unsigned idx = outer_offset + j * CONFIG_T::n_inner + k; + buffer_out[idx] = smax_slice_out[j]; + } + } + } + + res_pipe::write(buffer_out); +} + // ************************************************* // TanH Activation // ************************************************* diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 220a8a7ba3..65c50aaf9e 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -306,7 +306,12 @@ def write_parameters(self, model): elif '// hls-fpga-machine-learning insert softmax tables' in line: newline = line for layer in model.get_layers(): - if layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax': + if ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ): newline += f'#include "nnet_utils/activation_tables/{layer.name}_exp_table.h"\n' newline += f'#include "nnet_utils/activation_tables/{layer.name}_inv_table.h"\n' @@ -707,13 +712,19 @@ def __write_exp_table(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'stable': - - #import pdb; pdb.set_trace() + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'stable' + ): table_name = layer.name + '_exp_table' - table_size = int(layer.get_attr('exp_table_size')) + table_size = int(layer.get_attr('exp_table_size')) with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -738,7 +749,7 @@ def __write_exp_table(self, model, path): if (('exp_scale' in layer.attributes) and (layer.attributes['exp_scale'] is not None)) else 1.0 ) - + N = ceil_log2(table_size) if N > 2**fp_bits: raise Exception('Table size is bigger than what precision allows') @@ -762,9 +773,17 @@ def __write_exp_table(self, model, path): def __write_invert_table(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'stable': + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'stable' + ): table_name = layer.name + '_inv_table' table_size = int(layer.get_attr('inv_table_size')) @@ -809,9 +828,16 @@ def __write_invert_table(self, model, path): def __write_exp_table_latency(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'latency': - + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'latency' + ): table_name = layer.name + '_exp_table' table_size = int(layer.get_attr('exp_table_size')) @@ -859,9 +885,16 @@ def __write_exp_table_latency(self, model, path): def __write_invert_table_latency(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'latency': - + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'latency' + ): table_name = layer.name + '_inv_table' table_size = int(layer.get_attr('inv_table_size')) @@ -910,12 +943,18 @@ def __write_exp_table_legacy(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'legacy': - - import pdb; pdb.set_trace() + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'legacy' + ): table_name = layer.name + '_exp_table' - table_size = int(layer.get_attr('exp_table_size')) # not sure if it works, have to test first + table_size = int(layer.get_attr('exp_table_size')) # not sure if it works, have to test first with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -923,7 +962,7 @@ def __write_exp_table_legacy(self, model, path): h_file.write(f'#define {header_name.upper()}_H_\n\n') h_file.write( - f'static constexpr nnet::array<{layer.get_attr('exp_table_t').name},{table_size}> {table_name} = {{' + f'static constexpr nnet::array<{layer.get_attr("exp_table_t").name},{table_size}> {table_name} = {{' ) sep = '' @@ -940,9 +979,16 @@ def __write_invert_table_legacy(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers - if (layer.get_attr('activation') == 'softmax' or layer.get_attr('recurrent_activation') == 'softmax') \ - and 'implementation' in layer.attributes and layer.get_attr('implementation') == 'legacy': - + if ( + ( + layer.get_attr('activation') == 'softmax' + or layer.get_attr('activation') == 'softmax_multidim' + or layer.get_attr('recurrent_activation') == 'softmax' + or layer.get_attr('recurrent_activation') == 'softmax_multidim' + ) + and 'implementation' in layer.attributes + and layer.get_attr('implementation') == 'legacy' + ): table_name = layer.name + '_inv_table' table_size = int(layer.get_attr('inv_table_size')) @@ -952,7 +998,7 @@ def __write_invert_table_legacy(self, model, path): h_file.write(f'#define {header_name.upper()}_H_\n\n') h_file.write( - f'static constexpr nnet::array<{layer.get_attr('inv_table_t').name},{table_size}> {table_name} = {{' + f'static constexpr nnet::array<{layer.get_attr("inv_table_t").name},{table_size}> {table_name} = {{' ) sep = '' From 5bee49a0c1da1a11cdb3af902c3bbf68357a4930 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 2 Jul 2026 20:00:28 +0100 Subject: [PATCH 27/40] quick fix to table include logic --- hls4ml/writer/oneapi_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 65c50aaf9e..0990a65400 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -311,7 +311,7 @@ def write_parameters(self, model): or layer.get_attr('recurrent_activation') == 'softmax' or layer.get_attr('activation') == 'softmax_multidim' or layer.get_attr('recurrent_activation') == 'softmax_multidim' - ): + ) and 'implementation' in layer.attributes: newline += f'#include "nnet_utils/activation_tables/{layer.name}_exp_table.h"\n' newline += f'#include "nnet_utils/activation_tables/{layer.name}_inv_table.h"\n' From 134787b37643cafd6f9a334851a76b6ac3643c85 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 2 Jul 2026 20:41:54 +0100 Subject: [PATCH 28/40] formatting change --- hls4ml/backends/oneapi/oneapi_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/backends/oneapi/oneapi_backend.py b/hls4ml/backends/oneapi/oneapi_backend.py index 0da9f73f67..d448cd41cc 100644 --- a/hls4ml/backends/oneapi/oneapi_backend.py +++ b/hls4ml/backends/oneapi/oneapi_backend.py @@ -209,7 +209,7 @@ def build(self, model, build_type='fpga_emu', run=False): try: subprocess.run('which icpx', shell=True, cwd=builddir, check=True) except subprocess.CalledProcessError: - print("icpx not found. Trying ahls") + print('icpx not found. Trying ahls') try: subprocess.run('which ahls', shell=True, cwd=builddir, check=True) except subprocess.CalledProcessError: From a90b79acdf9d04c3df3329138dd095d906bcda34 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Thu, 2 Jul 2026 22:36:30 +0100 Subject: [PATCH 29/40] suggested fixes and commented sections removed --- .../backends/oneapi/passes/core_templates.py | 35 ++++++++----------- .../firmware/nnet_utils/nnet_activation.h | 4 --- .../nnet_utils/nnet_activation_stream.h | 4 --- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 8b6f5a0da8..2918218a95 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -252,35 +252,30 @@ def format(self, node): params['type'] = node.get_attr('activation') if (params['type'] == 'softmax') or (params['type'] == 'softmax_multidim'): - params.setdefault('n_inner', 1) - params.setdefault('n_outer', 1) + # If no table size is specified, assume default size of 1024 + params.setdefault('exp_table_size', params['table_size']) + params.setdefault('inv_table_size', params['table_size']) + + # This is for non-quantised layers where table size is not a layer attribute + if node.get_attr('exp_table_size', -1) == -1: + node.set_attr('exp_table_size', params['exp_table_size']) + + if node.get_attr('inv_table_size', -1) == -1: + node.set_attr('inv_table_size', params['inv_table_size']) + + params.setdefault('exp_scale', 1.0) + params.setdefault('parallelization_factor', -1) + n_slice = params['n_in'] // params['n_inner'] // params['n_outer'] params['n_slice'] = n_slice params['exp_table_name'] = node.name + '_exp_table' params['inv_table_name'] = node.name + '_inv_table' + params['smax_accum_t'] = params['accum_t'].name if params['implementation'] == 'stable': self.template = softmax_config_template + softmax_config_table_template_stable - if 'exp_table_size' not in params: - params['exp_table_size'] = 2 ** params['inp_norm_t'].precision.width - node.set_attr('exp_table_size', params['exp_table_size']) - - if 'inv_table_size' not in params: - params['inv_table_size'] = 2 ** params['inv_inp_t'].precision.width - node.set_attr('inv_table_size', params['inv_table_size']) - - params['smax_accum_t'] = params['accum_t'].name - # params.setdefault('table_size', params['exp_table_size']) # Not sure if necessary - - else: - # TODO: For latency check the table sizes correctly, match them - if 'exp_table_size' not in params: - params['exp_table_size'] = params['table_size'] - if 'inv_table_size' not in params: - params['inv_table_size'] = params['table_size'] - return self.template.format(**params) diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index c4d895102b..618e35c082 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -160,8 +160,6 @@ template void softmax_stable(cons // TODO - Improve accuracy template void softmax_latency(const data_T &data, res_T &res) { - //#include "activation_tables/exp_table_latency.tb" - //#include "activation_tables/invert_table_latency.tb" // Calculate all the e^x's [[intel::fpga_register]] typename CONFIG_T::exp_table_t exp_res[CONFIG_T::n_in]; @@ -186,8 +184,6 @@ template void softmax_latency(con } template void softmax_legacy(const data_T &data, res_T &res) { - //#include "activation_tables/exp_table_legacy.tb" - //#include "activation_tables/invert_table_legacy.tb" [[intel::fpga_register]] int data_round[CONFIG_T::n_in]; New_loop: diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index 865536d273..8925cf6209 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -335,8 +335,6 @@ template void softmax_stabl } template void softmax_latency_stream() { - //#include "activation_tables/exp_table_latency.tb" - //#include "activation_tables/invert_table_latency.tb" constexpr unsigned multiplier_limit = DIV_ROUNDUP(std::tuple_size::value_type>{}, CONFIG_T::reuse_factor); @@ -385,8 +383,6 @@ template void softmax_laten } template void softmax_legacy_stream() { - //#include "activation_tables/exp_table_legacy.tb" - //#include "activation_tables/invert_table_legacy.tb" // Index into the lookup table based on data for exponentials [[intel::fpga_register]] From 07ad4a4bc9516f9c3301ecf157458b8d15bc3afe Mon Sep 17 00:00:00 2001 From: bugracyln Date: Fri, 3 Jul 2026 22:04:00 +0100 Subject: [PATCH 30/40] formatting quickfix --- hls4ml/backends/oneapi/passes/core_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 2918218a95..1d9660ae1a 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -249,7 +249,7 @@ def __init__(self): def format(self, node): params = self._default_config_params(node) - params['type'] = node.get_attr('activation') + params['type'] = node.get_attr('activation').lower() if (params['type'] == 'softmax') or (params['type'] == 'softmax_multidim'): # If no table size is specified, assume default size of 1024 From aa5af6bf99b804e72f019a8c3e3d633d0c8bf435 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Sat, 4 Jul 2026 17:32:23 +0100 Subject: [PATCH 31/40] minor fix to writer --- hls4ml/converters/keras_v3/hgq2/softmax.py | 1 - hls4ml/writer/oneapi_writer.py | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/hls4ml/converters/keras_v3/hgq2/softmax.py b/hls4ml/converters/keras_v3/hgq2/softmax.py index 1c97555ebc..324ded98be 100644 --- a/hls4ml/converters/keras_v3/hgq2/softmax.py +++ b/hls4ml/converters/keras_v3/hgq2/softmax.py @@ -20,7 +20,6 @@ def fixed_quantizer_to_hls4ml_t(q: 'FixedPointQuantizerBase', take_max=False, fo if force_unsigned: k = 0.0 i += 1.0 - f += 1.0 k = ops.convert_to_numpy(k) i = ops.convert_to_numpy(i) diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index 0990a65400..e1e5f509ae 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -738,10 +738,6 @@ def __write_exp_table(self, model, path): ac_type = layer.get_attr('inp_norm_t') fp_bits = ac_type.precision.integer + ac_type.precision.fractional fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - - if fp_signed is True: - raise Exception('Softmax types need to be unsigned') # Copy scaling from attributes scale = ( @@ -758,6 +754,8 @@ def __write_exp_table(self, model, path): sep = '' # Use the top bits if table_size < 2**bit_width for i in range(table_size): + # Norm type is always > 1 so if input quantiser is set to be signed for any reason, + # force unsigned but keep the width f = FixedPointEmulator(fp_bits, fp_integer, signed=False) b = uint_to_binary(i, N) f.set_msb_bits(b) @@ -799,10 +797,6 @@ def __write_invert_table(self, model, path): ac_type = layer.get_attr('inv_inp_t') fp_bits = ac_type.precision.integer + ac_type.precision.fractional fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - - if fp_signed is True: - raise Exception('Softmax types need to be unsigned') N = ceil_log2(table_size) if N > 2**fp_bits: @@ -812,7 +806,8 @@ def __write_invert_table(self, model, path): # Use the top bits if table_size < 2**bit_width sep = '' for i in range(table_size): - # ALWAYS ASSUME ALL TYPES HERE ARE UNSIGNED + # Norm type is always > 1 so if input quantiser is set to be signed for any reason, + # force unsigned but keep the width f = FixedPointEmulator(fp_bits, fp_integer, signed=False) b = uint_to_binary(i, N) f.set_msb_bits(b) From 3d7979aa3256f0c25cf275d4b582419c0aafe22f Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Tue, 7 Jul 2026 16:38:45 -0500 Subject: [PATCH 32/40] remove quartus reqirement for signed softmax --- hls4ml/writer/quartus_writer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hls4ml/writer/quartus_writer.py b/hls4ml/writer/quartus_writer.py index e0d6338ac3..f5b56d9f7c 100644 --- a/hls4ml/writer/quartus_writer.py +++ b/hls4ml/writer/quartus_writer.py @@ -1097,8 +1097,6 @@ def __write_exp_table(self, model, path): except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') sep = '' N = ceil_log2(table_size) @@ -1143,8 +1141,6 @@ def __write_invert_table(self, model, path): except Exception: # FixedPrecisionType wasn't correctly stored in layer attributes, use default values pass - if fp_signed is False: - raise Exception('Softmax types need to be signed') sep = '' N = ceil_log2(table_size) From aa678e07d26ee5a2a16198a18672a2649830fbf7 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Tue, 7 Jul 2026 18:39:31 -0500 Subject: [PATCH 33/40] add inp_norm_t inferrence --- hls4ml/backends/fpga/fpga_backend.py | 4 +--- hls4ml/model/optimizer/passes/infer_precision.py | 7 +++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/hls4ml/backends/fpga/fpga_backend.py b/hls4ml/backends/fpga/fpga_backend.py index 1518f1a04b..0c37085765 100644 --- a/hls4ml/backends/fpga/fpga_backend.py +++ b/hls4ml/backends/fpga/fpga_backend.py @@ -150,9 +150,7 @@ def __init__(self, name): ), TypeAttribute( 'inp_norm', - default=FixedPrecisionType( - 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT - ), + default=UnspecifiedPrecisionType(), description='The internal width used for the exp table lookup (only in stable)', ), TypeAttribute('accum', description=descriptions.accum_type), diff --git a/hls4ml/model/optimizer/passes/infer_precision.py b/hls4ml/model/optimizer/passes/infer_precision.py index 189ffb7dda..6151712ea8 100644 --- a/hls4ml/model/optimizer/passes/infer_precision.py +++ b/hls4ml/model/optimizer/passes/infer_precision.py @@ -626,6 +626,13 @@ def _infer_softmax_precision(self, node, types_to_infer): node.types['accum_t'].precision = FixedPrecisionType(exp_w + ceillog, exp_i + ceillog, signed=exp_s) inferred_types.append('accum_t') + if 'inp_norm_t' in types_to_infer: + in_type = node.get_input_variable().type.precision + inp_norm_width = in_type.width - in_type.signed + inp_norm_int = in_type.integer - in_type.signed + node.types['inp_norm_t'].precision = FixedPrecisionType(inp_norm_width, inp_norm_int, signed=False) + inferred_types.append('inp_norm_t') + return inferred_types From 5dadf1a4da01083a664615e86006adfa8105d5b8 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 8 Jul 2026 14:01:50 -0500 Subject: [PATCH 34/40] fix test_softmax --- test/pytest/test_softmax.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/pytest/test_softmax.py b/test/pytest/test_softmax.py index 418f64b558..e71b7de778 100644 --- a/test/pytest/test_softmax.py +++ b/test/pytest/test_softmax.py @@ -20,7 +20,7 @@ def generate_data(input_shape): @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult']) -@pytest.mark.parametrize('strategy', ['stable', 'latency', 'argmax']) +@pytest.mark.parametrize('implementation', ['stable', 'latency', 'argmax']) @pytest.mark.parametrize( 'input_bits,input_shape,table_bits,io_type,custom_accum', [ @@ -35,26 +35,24 @@ def generate_data(input_shape): ('16,6', (8, 8, 3), '18,8', 'io_stream', False), ], ) -def test_softmax(test_case_id, backend, strategy, generate_data, input_bits, input_shape, table_bits, io_type, custom_accum): +def test_softmax(test_case_id, backend, implementation, generate_data, input_bits, input_shape, table_bits, io_type, custom_accum): X = generate_data model = tf.keras.models.Sequential() model.add(tf.keras.layers.Activation(input_shape=input_shape, activation='softmax', name='softmax')) model.compile() - table_type = f'fixed<{table_bits}, RND, SAT>' + table_type = f'ufixed<{table_bits}, RND, SAT>' cfg = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) - cfg['LayerName']['softmax']['Strategy'] = strategy - cfg['LayerName']['softmax']['inv_table_t'] = table_type - cfg['LayerName']['softmax']['exp_table_t'] = table_type - cfg['LayerName']['softmax']['accum_t'] = table_type - cfg['LayerName']['softmax']['inv_inp_t'] = table_type + cfg['LayerName']['softmax']['Implementation'] = implementation + cfg['LayerName']['softmax']['Precision']['inv_table'] = table_type + cfg['LayerName']['softmax']['Precision']['exp_table'] = table_type + cfg['LayerName']['softmax']['Precision']['inv_inp'] = table_type if custom_accum: if backend not in ['Vivado', 'Vitis']: pytest.skip('Custom accumulators are only supported for Vivado and Vitis backends') W, I = map(int, input_bits.split(',')) # noqa: E741 - cfg['LayerName']['softmax']['accum_t'] = f'fixed<{W + 3},{I + 3}>' - cfg['LayerName']['softmax']['inv_inp_t'] = f'fixed<{W + 2},{I + 2}>' + cfg['LayerName']['softmax']['Precision']['inv_inp'] = f'ufixed<{W + 2},{I + 2}>' inp_layer_name = next(iter(cfg['LayerName'].keys())) cfg['LayerName'][inp_layer_name]['Precision']['result'] = f'fixed<{input_bits}>' From d8abde408e135be9c15818dc181237b269c2e625 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 8 Jul 2026 15:10:48 -0500 Subject: [PATCH 35/40] pre-commit fix --- test/pytest/test_softmax.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/pytest/test_softmax.py b/test/pytest/test_softmax.py index e71b7de778..8175785e10 100644 --- a/test/pytest/test_softmax.py +++ b/test/pytest/test_softmax.py @@ -35,7 +35,9 @@ def generate_data(input_shape): ('16,6', (8, 8, 3), '18,8', 'io_stream', False), ], ) -def test_softmax(test_case_id, backend, implementation, generate_data, input_bits, input_shape, table_bits, io_type, custom_accum): +def test_softmax( + test_case_id, backend, implementation, generate_data, input_bits, input_shape, table_bits, io_type, custom_accum +): X = generate_data model = tf.keras.models.Sequential() model.add(tf.keras.layers.Activation(input_shape=input_shape, activation='softmax', name='softmax')) From 55e7454dd441ff48cf611ec2bb161f5f983dc66e Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Wed, 8 Jul 2026 15:28:05 -0500 Subject: [PATCH 36/40] change randint to rand --- test/pytest/test_softmax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/test_softmax.py b/test/pytest/test_softmax.py index 8175785e10..6c666e7020 100644 --- a/test/pytest/test_softmax.py +++ b/test/pytest/test_softmax.py @@ -14,7 +14,7 @@ def generate_data(input_shape): shape = (5000, *input_shape) d = np.random.normal(0, 2, shape) - modify_entries = np.random.randint(0, 1, shape) < 0.05 + modify_entries = np.random.rand(*shape) < 0.05 d[modify_entries] = d[modify_entries] * 5 + 10 return np.clip(d, -32, 31) From 947f195c6ba67def843a7d95551d4fca6d4ebd77 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Thu, 9 Jul 2026 11:38:43 -0500 Subject: [PATCH 37/40] make sure softmax widths are transfered properly --- hls4ml/backends/fpga/fpga_backend.py | 4 +--- hls4ml/model/layers.py | 9 +++++++++ hls4ml/model/optimizer/passes/infer_precision.py | 5 +++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/hls4ml/backends/fpga/fpga_backend.py b/hls4ml/backends/fpga/fpga_backend.py index 0c37085765..eebb8d939f 100644 --- a/hls4ml/backends/fpga/fpga_backend.py +++ b/hls4ml/backends/fpga/fpga_backend.py @@ -143,9 +143,7 @@ def __init__(self, name): ), TypeAttribute( 'inv_inp', - default=FixedPrecisionType( - 18, 8, signed=False, rounding_mode=RoundingMode.RND, saturation_mode=SaturationMode.SAT - ), + default=UnspecifiedPrecisionType(), description='What the accumulated value is cast to before accessing the inversion table (only in stable)', ), TypeAttribute( diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index 42afbabd3c..625f62366d 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -965,6 +965,9 @@ def initialize(self): if 'n_in' not in self.attributes: self.set_attr('n_in', self.get_input_variable().size()) + # set the needed types if needed + self._set_type_t('table') + class ParametrizedActivation(Activation): _expected_attributes = [ @@ -1031,6 +1034,12 @@ class Softmax(Activation): def initialize(self): super().initialize() + # set the needed types if needed + self._set_type_t('exp_table') + self._set_type_t('inv_table') + self._set_type_t('inv_inp') + self._set_type_t('inv_norm') + class TernaryTanh(Activation): def initialize(self): diff --git a/hls4ml/model/optimizer/passes/infer_precision.py b/hls4ml/model/optimizer/passes/infer_precision.py index 6151712ea8..7d6c6f306e 100644 --- a/hls4ml/model/optimizer/passes/infer_precision.py +++ b/hls4ml/model/optimizer/passes/infer_precision.py @@ -626,6 +626,11 @@ def _infer_softmax_precision(self, node, types_to_infer): node.types['accum_t'].precision = FixedPrecisionType(exp_w + ceillog, exp_i + ceillog, signed=exp_s) inferred_types.append('accum_t') + if 'inv_inp_t' in types_to_infer: + # if not set, just choose the accumulator type + node.types['inv_inp_t'].precision = node.types['accum_t'].precision + inferred_types.append('inv_inp_t') + if 'inp_norm_t' in types_to_infer: in_type = node.get_input_variable().type.precision inp_norm_width = in_type.width - in_type.signed From 10532dadaa66df950fdfb82289d8343f86b4f409 Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Thu, 9 Jul 2026 12:55:46 -0500 Subject: [PATCH 38/40] Fix issues with skip, auto --- hls4ml/model/types.py | 3 +++ test/pytest/test_softmax.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hls4ml/model/types.py b/hls4ml/model/types.py index aa69130d9b..85ed1c8241 100644 --- a/hls4ml/model/types.py +++ b/hls4ml/model/types.py @@ -434,6 +434,9 @@ class UnspecifiedPrecisionType(PrecisionType): def __init__(self): super().__init__(width=0, signed=False) + def __str__(self): + return 'auto' + def find_minimum_width(data, signed=True): """ diff --git a/test/pytest/test_softmax.py b/test/pytest/test_softmax.py index 6c666e7020..45e4ad5e25 100644 --- a/test/pytest/test_softmax.py +++ b/test/pytest/test_softmax.py @@ -20,7 +20,7 @@ def generate_data(input_shape): @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult']) -@pytest.mark.parametrize('implementation', ['stable', 'latency', 'argmax']) +@pytest.mark.parametrize('implementation', ['stable', 'latency', 'argmax', 'stable']) @pytest.mark.parametrize( 'input_bits,input_shape,table_bits,io_type,custom_accum', [ @@ -38,6 +38,10 @@ def generate_data(input_shape): def test_softmax( test_case_id, backend, implementation, generate_data, input_bits, input_shape, table_bits, io_type, custom_accum ): + + if backend == 'Catapult' and implementation == 'argmax': + pytest.skip('Argmax is not supported in cataplut') + X = generate_data model = tf.keras.models.Sequential() model.add(tf.keras.layers.Activation(input_shape=input_shape, activation='softmax', name='softmax')) From 01d70a4d8ec475f50b78eb61a943e99686663fe3 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Sat, 11 Jul 2026 21:19:50 +0100 Subject: [PATCH 39/40] saving before testing --- test/pytest/test_softmax.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/pytest/test_softmax.py b/test/pytest/test_softmax.py index 45e4ad5e25..84925c445c 100644 --- a/test/pytest/test_softmax.py +++ b/test/pytest/test_softmax.py @@ -19,8 +19,8 @@ def generate_data(input_shape): return np.clip(d, -32, 31) -@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult']) -@pytest.mark.parametrize('implementation', ['stable', 'latency', 'argmax', 'stable']) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult', 'oneAPI']) +@pytest.mark.parametrize('implementation', ['stable', 'latency', 'argmax']) @pytest.mark.parametrize( 'input_bits,input_shape,table_bits,io_type,custom_accum', [ @@ -47,7 +47,7 @@ def test_softmax( model.add(tf.keras.layers.Activation(input_shape=input_shape, activation='softmax', name='softmax')) model.compile() - table_type = f'ufixed<{table_bits}, RND, SAT>' + table_type = f'ufixed<{table_bits}, RND, SAT>' if backend != 'oneAPI' else f'ac_fixed<{table_bits}, AC_RND, AC_SAT>' cfg = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) cfg['LayerName']['softmax']['Implementation'] = implementation @@ -59,6 +59,7 @@ def test_softmax( pytest.skip('Custom accumulators are only supported for Vivado and Vitis backends') W, I = map(int, input_bits.split(',')) # noqa: E741 cfg['LayerName']['softmax']['Precision']['inv_inp'] = f'ufixed<{W + 2},{I + 2}>' + inp_layer_name = next(iter(cfg['LayerName'].keys())) cfg['LayerName'][inp_layer_name]['Precision']['result'] = f'fixed<{input_bits}>' @@ -77,7 +78,7 @@ def test_softmax( assert acc_hls4ml >= 0.98 -@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult']) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'Catapult', 'oneAPI']) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) def test_softmax_skipped(test_case_id, backend, io_type): X = np.random.rand(100, 10) From 8d8bc8a2e54096d7f7a55d6cec6b5dd9929e2803 Mon Sep 17 00:00:00 2001 From: bugracyln Date: Mon, 13 Jul 2026 17:32:17 +0100 Subject: [PATCH 40/40] table accuracy improvement and bugfixes --- .../backends/oneapi/passes/core_templates.py | 2 + .../model/optimizer/passes/infer_precision.py | 12 ++- .../firmware/nnet_utils/nnet_activation.h | 18 ++-- .../nnet_utils/nnet_activation_stream.h | 18 ++-- hls4ml/writer/oneapi_writer.py | 82 ++++++------------- 5 files changed, 48 insertions(+), 84 deletions(-) diff --git a/hls4ml/backends/oneapi/passes/core_templates.py b/hls4ml/backends/oneapi/passes/core_templates.py index 1d9660ae1a..bab202a1f5 100644 --- a/hls4ml/backends/oneapi/passes/core_templates.py +++ b/hls4ml/backends/oneapi/passes/core_templates.py @@ -255,6 +255,8 @@ def format(self, node): # If no table size is specified, assume default size of 1024 params.setdefault('exp_table_size', params['table_size']) params.setdefault('inv_table_size', params['table_size']) + params['exp_table_size'] = min(params['exp_table_size'], params['table_size']) + params['inv_table_size'] = min(params['inv_table_size'], params['table_size']) # This is for non-quantised layers where table size is not a layer attribute if node.get_attr('exp_table_size', -1) == -1: diff --git a/hls4ml/model/optimizer/passes/infer_precision.py b/hls4ml/model/optimizer/passes/infer_precision.py index 7d6c6f306e..208224944d 100644 --- a/hls4ml/model/optimizer/passes/infer_precision.py +++ b/hls4ml/model/optimizer/passes/infer_precision.py @@ -629,12 +629,20 @@ def _infer_softmax_precision(self, node, types_to_infer): if 'inv_inp_t' in types_to_infer: # if not set, just choose the accumulator type node.types['inv_inp_t'].precision = node.types['accum_t'].precision + if node.model.config.backend.name == 'oneAPI': + # TODO - Can introduce this on onther backends since it benefits from only using required number of bits + # for the integer part. + 1 is for guarding + node.types['inv_inp_t'].precision.integer = 1 + math.ceil(math.log2(max(node.get_input_variable().shape))) inferred_types.append('inv_inp_t') if 'inp_norm_t' in types_to_infer: in_type = node.get_input_variable().type.precision - inp_norm_width = in_type.width - in_type.signed - inp_norm_int = in_type.integer - in_type.signed + if node.model.config.backend.name == 'oneAPI': + inp_norm_width = in_type.width + inp_norm_int = in_type.integer + else: + inp_norm_width = in_type.width - in_type.signed + inp_norm_int = in_type.integer - in_type.signed node.types['inp_norm_t'].precision = FixedPrecisionType(inp_norm_width, inp_norm_int, signed=False) inferred_types.append('inp_norm_t') diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h index 618e35c082..476e49c6b0 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation.h @@ -242,22 +242,14 @@ template void softmax_argmax(cons } template inline void softmax(const data_T &data, res_T &res) { - switch (CONFIG_T::implementation) { - case softmax_implementation::stable: - softmax_stable(data, res); - break; - case softmax_implementation::latency: + if constexpr (CONFIG_T::implementation == softmax_implementation::latency) { softmax_latency(data, res); - break; - case softmax_implementation::legacy: + } else if constexpr (CONFIG_T::implementation == softmax_implementation::argmax) { + softmax_argmax(data, res); + } else if constexpr (CONFIG_T::implementation == softmax_implementation::legacy) { softmax_legacy(data, res); - break; - default: + } else { softmax_stable(data, res); - break; - case softmax_implementation::argmax: - softmax_argmax(data, res); - break; } } diff --git a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h index 8925cf6209..cb65137927 100644 --- a/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h +++ b/hls4ml/templates/oneapi/firmware/nnet_utils/nnet_activation_stream.h @@ -471,22 +471,14 @@ template void softmax_argma } template void softmax_stream() { - switch (CONFIG_T::implementation) { - case softmax_implementation::latency: + if constexpr (CONFIG_T::implementation == softmax_implementation::latency) { softmax_latency_stream(); - break; - case softmax_implementation::stable: - softmax_stable_stream(); - break; - case softmax_implementation::legacy: - softmax_legacy_stream(); - break; - case softmax_implementation::argmax: + } else if constexpr (CONFIG_T::implementation == softmax_implementation::argmax) { + softmax_argmax_stream(); + } else if constexpr (CONFIG_T::implementation == softmax_implementation::legacy) { softmax_argmax_stream(); - break; - default: + } else { // Default to stable softmax_stable_stream(); - break; } } diff --git a/hls4ml/writer/oneapi_writer.py b/hls4ml/writer/oneapi_writer.py index e1e5f509ae..d5d769e22e 100644 --- a/hls4ml/writer/oneapi_writer.py +++ b/hls4ml/writer/oneapi_writer.py @@ -708,7 +708,7 @@ def __get_table_precision(self, model, activation, table_name='table_precision') return None # fp_bits, fp_integer, fp_signed - def __write_exp_table(self, model, path): + def __write_exp_tables_stable(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers @@ -724,7 +724,7 @@ def __write_exp_table(self, model, path): and layer.get_attr('implementation') == 'stable' ): table_name = layer.name + '_exp_table' - table_size = int(layer.get_attr('exp_table_size')) + table_size = min(int(layer.get_attr('table_size')), int(layer.get_attr('exp_table_size'))) with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -768,7 +768,7 @@ def __write_exp_table(self, model, path): h_file.write('};\n\n') h_file.write('#endif') - def __write_invert_table(self, model, path): + def __write_invert_tables_stable(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers @@ -783,7 +783,7 @@ def __write_invert_table(self, model, path): and layer.get_attr('implementation') == 'stable' ): table_name = layer.name + '_inv_table' - table_size = int(layer.get_attr('inv_table_size')) + table_size = min(int(layer.get_attr('table_size')), int(layer.get_attr('inv_table_size'))) with open(f'{path}/{table_name}.h', 'w') as h_file: header_name = table_name @@ -820,7 +820,7 @@ def __write_invert_table(self, model, path): h_file.write('};\n\n') h_file.write('#endif') - def __write_exp_table_latency(self, model, path): + def __write_exp_tables_latency(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers if ( @@ -845,25 +845,10 @@ def __write_exp_table_latency(self, model, path): f'static constexpr nnet::array<{layer.get_attr("exp_table_t").name},{table_size}> {table_name} = {{' ) - # Default fixed point precision - # 6 bits for integer part, 10 bits for decimal - total, 16 - fp_bits = 16 - fp_integer = 6 - fp_signed = True - - # Exp table should use the same precision as exp_table, as seen in Vivado code - # init_exp_table(exp_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_input_variable().type - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + ac_type = layer.get_input_variable().type + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed sep = '' N = ceil_log2(table_size) @@ -874,10 +859,10 @@ def __write_exp_table_latency(self, model, path): h_file.write(sep + str(real_val)) sep = ', ' - h_file.write('};\n') - h_file.close() + h_file.write('};\n\n') + h_file.write('#endif') - def __write_invert_table_latency(self, model, path): + def __write_invert_tables_latency(self, model, path): for layer in model.get_layers(): # Last property is essential since it seperates layer with activation property from actual activation layers if ( @@ -902,25 +887,10 @@ def __write_invert_table_latency(self, model, path): f'static constexpr nnet::array<{layer.get_attr("inv_table_t").name},{table_size}> {table_name} = {{' ) - # Default fixed point precision, in case values from layer attributes cannot be extracted - # 8 bits for integer part, 10 bits for decimal - total, 18 - fp_bits = 18 - fp_integer = 8 - fp_signed = True - - # Invert table should use the same precision as exp_table, as seen in Vivado code - # init_invert_table(invert_table); - for layer in model.get_layers(): - if layer.name == 'softmax': - ac_type = layer.get_attr('exp_table_t') - if ac_type is not None: - try: - fp_bits = ac_type.precision.integer + ac_type.precision.fractional - fp_integer = ac_type.precision.integer - fp_signed = ac_type.precision.signed - except Exception: - # FixedPrecisionType wasn't correctly stored in layer attributes, use default values - pass + ac_type = layer.get_attr('exp_table_t') + fp_bits = ac_type.precision.integer + ac_type.precision.fractional + fp_integer = ac_type.precision.integer + fp_signed = ac_type.precision.signed sep = '' N = ceil_log2(table_size) @@ -931,8 +901,8 @@ def __write_invert_table_latency(self, model, path): h_file.write(sep + str(real_val)) sep = ', ' - h_file.write('};\n') - h_file.close() + h_file.write('};\n\n') + h_file.write('#endif') def __write_exp_table_legacy(self, model, path): @@ -967,8 +937,8 @@ def __write_exp_table_legacy(self, model, path): h_file.write(sep + str(real_val)) sep = ', ' - h_file.write('};\n') - h_file.close() + h_file.write('};\n\n') + h_file.write('#endif') def __write_invert_table_legacy(self, model, path): @@ -1005,8 +975,8 @@ def __write_invert_table_legacy(self, model, path): h_file.write(sep + str(real_val)) sep = ', ' - h_file.write('};\n') - h_file.close() + h_file.write('};\n\n') + h_file.write('#endif') def write_activation_tables(self, model): """Write the lookup tables for activation functions @@ -1027,10 +997,10 @@ def write_activation_tables(self, model): self.__write_softplus_table(model, dstpath) self.__write_softsign_table(model, dstpath) self.__write_selu_table(model, dstpath) - self.__write_exp_table(model, dstpath) - self.__write_invert_table(model, dstpath) - self.__write_exp_table_latency(model, dstpath) - self.__write_invert_table_latency(model, dstpath) + self.__write_exp_tables_stable(model, dstpath) + self.__write_invert_tables_stable(model, dstpath) + self.__write_exp_tables_latency(model, dstpath) + self.__write_invert_tables_latency(model, dstpath) self.__write_exp_table_legacy(model, dstpath) self.__write_invert_table_legacy(model, dstpath)