diff --git a/Qwen-Qwen3-1.7B/QNN/README.md b/Qwen-Qwen3-1.7B/QNN/README.md new file mode 100644 index 000000000..752e58106 --- /dev/null +++ b/Qwen-Qwen3-1.7B/QNN/README.md @@ -0,0 +1,65 @@ +# Qwen3-1.7B Model Optimization + +This repository demonstrates the optimization of the [Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) model using **post-training quantization (PTQ)** techniques. + +### Quantization Python Environment Setup +Quantization is resource-intensive and requires GPU acceleration. In an x64 Python environment, install the required packages: + +```bash +pip install -r requirements.txt + +# Disable CUDA extension build (not required) +# Linux +export BUILD_CUDA_EXT=0 +# Windows +# set BUILD_CUDA_EXT=0 + +# Install GptqModel from source +pip install --no-build-isolation git+https://github.com/CodeLinaro/GPTQModel.git@rel_4.2.5 +``` + +### AOT Compilation Python Environment Setup +Model compilation using QNN Execution Provider requires a Python environment with onnxruntime-qnn installed. In a separate Python environment, install the required packages: + +```bash +# Install Olive +pip install olive-ai==0.13.0 + +# Install ONNX Runtime QNN +pip install onnxruntime==1.26.0 +pip install onnxruntime-qnn==2.4.0 +``` + +Replace `/path/to/qnn/env/bin` in the config file with the path to the directory containing your QNN environment's Python executable. This path can be found by running the following command in the environment: + +```bash +# Linux +command -v python +# Windows +# where python +``` + +This command will return the path to the Python executable. Set the parent directory of the executable as the `/path/to/qnn/env/bin` in the config file. + +### Run the Quantization + Compilation Config +Activate the **Quantization Python Environment** and run the workflow. + +For Snapdragon X Elite: + +```bash +olive run --config x_elite_config.json +``` + +For Snapdragon X2 Elite: + +```bash +olive run --config x2_elite_config.json +``` + +Olive will run the AOT compilation step in the **AOT Compilation Python Environment** specified in the config file using a subprocess. All other steps will run in the **Quantization Python Environment** natively. + +Optimized model saved in: `models/qwen3_1.7B/` + +> If optimization fails during context binary generation, rerun the command. The process will resume from the last completed step. + +> If the Static Quantization (SQ) pass fails with `Failed to allocate memory buffer of size...`, rerun the command without clearing the cache. Olive will resume from the last completed step and the pass will succeed. diff --git a/Qwen-Qwen3-1.7B/QNN/requirements.txt b/Qwen-Qwen3-1.7B/QNN/requirements.txt new file mode 100644 index 000000000..a920d997d --- /dev/null +++ b/Qwen-Qwen3-1.7B/QNN/requirements.txt @@ -0,0 +1,7 @@ +datasets +olive-ai==0.13.0 +onnxruntime-genai-cuda==0.14.1 +onnxruntime-gpu==1.26.0 +optimum +# newer transformers might have incompatibility with gptq passes +transformers==4.57.3 diff --git a/Qwen-Qwen3-1.7B/QNN/x2_elite_config.json b/Qwen-Qwen3-1.7B/QNN/x2_elite_config.json new file mode 100644 index 000000000..f6ee15bf2 --- /dev/null +++ b/Qwen-Qwen3-1.7B/QNN/x2_elite_config.json @@ -0,0 +1,100 @@ +{ + "input_model": { "type": "HfModel", "model_path": "Qwen/Qwen3-1.7B" }, + "systems": { + "qnn_system": { + "type": "PythonEnvironment", + "python_environment_path": "/path/to/qnn/env/bin", + "accelerators": [ { "execution_providers": [ "QNNExecutionProvider" ] } ] + } + }, + "data_configs": [ + { + "name": "wikitext2_train_joined", + "type": "HuggingfaceContainer", + "load_dataset_config": { "data_name": "wikitext", "subset": "wikitext-2-raw-v1", "split": "train" }, + "pre_process_data_config": { + "strategy": "join", + "add_special_tokens": false, + "max_seq_len": 4096, + "max_samples": 128 + } + }, + { + "name": "wikitext2_train_act", + "type": "HuggingfaceContainer", + "load_dataset_config": { "data_name": "wikitext", "subset": "wikitext-2-raw-v1", "split": "train" }, + "pre_process_data_config": { + "strategy": "line-by-line", + "add_special_tokens": true, + "max_samples": 256, + "max_seq_len": 4096 + } + } + ], + "passes": { + "q": { "type": "QuaRot" }, + "cs": { "type" : "CaptureSplitInfo", "num_splits": 1, "unique_embeds_lm_head_splits": true }, + "g": { + "type": "GptqModel", + "bits": 8, + "sym": true, + "group_size": -1, + "lm_head": true, + "device": "cuda", + "data_config": "wikitext2_train_joined", + "dynamic": { + "+:.*lm_head*": {"bits": 8, "sym": true, "group_size": 32, "desc_act": false } + } + }, + "mb": { + "type": "ModelBuilder", + "precision": "int4", + "int4_block_size": 32, + "int4_accuracy_level": 4, + "int4_op_types_to_quantize": [ "Gather" ] + }, + "mq": { + "type": "MatMulNBitsToQDQ", + "use_int4": true, + "add_zero_point": true, + "nodes_to_exclude": [ "/lm_head/MatMulNBits" ], + "save_as_external_data": true + }, + "gs": { + "type": "GraphSurgeries", + "surgeries": [ + { "surgeon": "AttentionMaskToSequenceLengths" }, + { "surgeon": "RemoveGidxFromMatMulNBits" }, + { "surgeon": "SimplifiedLayerNormToL2Norm" } + ], + "save_as_external_data": true + }, + "sq": { + "type": "OnnxStaticQuantization", + "data_config": "wikitext2_train_act", + "activation_type": "uint16", + "precision": "uint8", + "calibration_providers": [ "CUDAExecutionProvider" ], + "quant_preprocess": true, + "op_types_to_exclude": [ "GatherBlockQuantized", "GroupQueryAttention", "MatMulNBits" ], + "save_as_external_data": true + }, + "sp": { "type": "SplitModel" }, + "st": { "type": "StaticLLM", "batch_size": 1, "context_length": 64 }, + "cb": { + "type": "EPContextBinaryGenerator", + "provider_options": { + "htp_performance_mode": "burst", + "htp_graph_finalization_optimization_mode": "3", + "soc_model": "87" + }, + "weight_sharing": true + }, + "cp": { "type": "ComposeOnnxModels" } + }, + "target": "qnn_system", + "log_severity_level": 1, + "output_dir": "models/qwen3_1.7B", + "cache_dir": "cache", + "no_artifacts": true +} diff --git a/Qwen-Qwen3-1.7B/QNN/x_elite_config.json b/Qwen-Qwen3-1.7B/QNN/x_elite_config.json new file mode 100644 index 000000000..429f3d856 --- /dev/null +++ b/Qwen-Qwen3-1.7B/QNN/x_elite_config.json @@ -0,0 +1,100 @@ +{ + "input_model": { "type": "HfModel", "model_path": "Qwen/Qwen3-1.7B" }, + "systems": { + "qnn_system": { + "type": "PythonEnvironment", + "python_environment_path": "/path/to/qnn/env/bin", + "accelerators": [ { "execution_providers": [ "QNNExecutionProvider" ] } ] + } + }, + "data_configs": [ + { + "name": "wikitext2_train_joined", + "type": "HuggingfaceContainer", + "load_dataset_config": { "data_name": "wikitext", "subset": "wikitext-2-raw-v1", "split": "train" }, + "pre_process_data_config": { + "strategy": "join", + "add_special_tokens": false, + "max_seq_len": 4096, + "max_samples": 128 + } + }, + { + "name": "wikitext2_train_act", + "type": "HuggingfaceContainer", + "load_dataset_config": { "data_name": "wikitext", "subset": "wikitext-2-raw-v1", "split": "train" }, + "pre_process_data_config": { + "strategy": "line-by-line", + "add_special_tokens": true, + "max_samples": 256, + "max_seq_len": 4096 + } + } + ], + "passes": { + "q": { "type": "QuaRot" }, + "cs": { "type" : "CaptureSplitInfo", "num_splits": 1, "unique_embeds_lm_head_splits": true }, + "g": { + "type": "GptqModel", + "bits": 8, + "sym": true, + "group_size": -1, + "lm_head": true, + "device": "cuda", + "data_config": "wikitext2_train_joined", + "dynamic": { + "+:.*lm_head*": { "bits": 8, "sym": true, "group_size": 32, "desc_act": false } + } + }, + "mb": { + "type": "ModelBuilder", + "precision": "int4", + "int4_block_size": 32, + "int4_accuracy_level": 4, + "int4_op_types_to_quantize": [ "Gather" ] + }, + "mq": { + "type": "MatMulNBitsToQDQ", + "use_int4": true, + "add_zero_point": true, + "nodes_to_exclude": [ "/lm_head/MatMulNBits" ], + "save_as_external_data": true + }, + "gs": { + "type": "GraphSurgeries", + "surgeries": [ + { "surgeon": "AttentionMaskToSequenceLengths" }, + { "surgeon": "RemoveGidxFromMatMulNBits" }, + { "surgeon": "SimplifiedLayerNormToL2Norm" } + ], + "save_as_external_data": true + }, + "sq": { + "type": "OnnxStaticQuantization", + "data_config": "wikitext2_train_act", + "activation_type": "uint16", + "precision": "uint8", + "calibration_providers": [ "CUDAExecutionProvider" ], + "quant_preprocess": true, + "op_types_to_exclude": [ "GatherBlockQuantized", "GroupQueryAttention", "MatMulNBits" ], + "save_as_external_data": true + }, + "sp": { "type": "SplitModel" }, + "st": { "type": "StaticLLM", "batch_size": 1, "context_length": 64 }, + "cb": { + "type": "EPContextBinaryGenerator", + "provider_options": { + "htp_performance_mode": "burst", + "htp_graph_finalization_optimization_mode": "3", + "soc_model": "60" + }, + "weight_sharing": true + }, + "cp": { "type": "ComposeOnnxModels" } + }, + "target": "qnn_system", + "log_severity_level": 1, + "output_dir": "models/qwen3_1.7B", + "cache_dir": "cache", + "no_artifacts": true +}