Skip to content

fix: Qualcomm QNN AOT Pass #579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chenghuaWang merged 14 commits into from
Jan 5, 2026
Merged

fix: Qualcomm QNN AOT Pass #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
421265d
feat(qualcomm, qwen3 ptq): PTQ py script for quantize qwen3 for qualc…
chenghuaWang Dec 31, 2025
a18fe97
fix: add som comments and fix causal mask decoding bug.
chenghuaWang Dec 31, 2025
eeb5bac
Merge branch 'UbiquitousLearning:main' into wch-main
chenghuaWang Dec 31, 2025
ca55892
feat: add kvcache in attention
chenghuaWang Dec 31, 2025
dd58482
feat: Qualcomm Calibrate Things
chenghuaWang Jan 2, 2026
9ea8f9d
feat: update AOT Qualcomm Qwen3
chenghuaWang Jan 3, 2026
a99e5c6
fix: qnn aot, qwen3 silu int16.
chenghuaWang Jan 3, 2026
8774623
fix: Save Qnn Qwen3 AOT Model.
chenghuaWang Jan 3, 2026
00dd1b6
fix: PTQ pass in qualcomm AOT workflow.
chenghuaWang Jan 4, 2026
6d7b5b9
fix: tensor attaching view selective hashing
chenghuaWang Jan 5, 2026
ecbef68
fix: AOT Pipeline pass
chenghuaWang Jan 5, 2026
82900b1
fix: mismatched outputs and inputs
chenghuaWang Jan 5, 2026
54d9927
fix: typos
chenghuaWang Jan 5, 2026
fb40759
fix: typos
chenghuaWang Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 10 additions & 17 deletions examples/qwen3_qnn_aot/compile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,26 +37,13 @@ MLLM_MAIN({
auto model_cfg = mllm::models::qwen3::Qwen3Config(model_cfg_path.get());
auto model = mllm::models::qwen3::Qwen3ForCausalLM(model_cfg);
auto params = mllm::load(model_path.get(), mllm::ModelFileVersion::kV2);

// Gen sin and cos
{
auto inv = mllm::models::qwen3::makeRoPEInvFreq(model_cfg.head_dim, model_cfg.rope_theta);
auto position_ids = mllm::Tensor::empty({1, CL}, mllm::kInt64, mllm::kCPU).alloc();
auto position_ids_ptr = position_ids.ptr<int64_t>();
for (int b = 0; b < 1; ++b) {
for (int s = 0; s < CL; ++s) { position_ids_ptr[b * CL + s] = s; }
}
auto [rope_sin, rope_cos] = mllm::models::qwen3::makeRotaryPosEmbedding(position_ids, inv, 1.f);
params->push("rope_sin", rope_sin.to(mllm::kInt16PerTensorSym));
params->push("rope_cos", rope_cos.to(mllm::kInt16PerTensorSym));
}
model.load(params);

// Sequence: [B, N]
// past_key_i: [B, H, D, CL-N] for each layer i
// past_value_i: [B, H, CL-N, D] for each layer i
// causal_mask: [B, 1, N, CL]
auto sequence = mllm::Tensor::zeros({1, N}, mllm::kInt64);
auto sequence = mllm::Tensor::zeros({1, N}, mllm::kInt32);
auto causal_mask = mllm::Tensor::zeros({1, 1, N, CL}, mllm::kUInt16);

// Create KV cache inputs for all layers
Expand All @@ -74,8 +61,14 @@ MLLM_MAIN({
model_cfg.num_key_value_heads,
model_cfg.head_dim,
CL - N,
}, mllm::kInt8PerTensorSym);
trace_inputs[past_value_name] = mllm::Tensor::empty({1, model_cfg.num_key_value_heads, CL - N, model_cfg.head_dim}, mllm::kInt8PerTensorSym);
}, mllm::kUInt8PerTensorSym);
trace_inputs[past_value_name] = mllm::Tensor::empty({1, model_cfg.num_key_value_heads, CL - N, model_cfg.head_dim}, mllm::kUInt8PerTensorSym);

trace_inputs[past_key_name].attach("scale", params->pull("model.layers." + std::to_string(i) + ".self_attn.k_cast_to_int8_qdq.fake_quant.scale").impl(), true);
trace_inputs[past_key_name].attach("zero_point", params->pull("model.layers." + std::to_string(i) + ".self_attn.k_cast_to_int8_qdq.fake_quant.zero_point").impl(), true);

trace_inputs[past_value_name].attach("scale", params->pull("model.layers." + std::to_string(i) + ".self_attn.v_cast_to_int8_qdq.fake_quant.scale").impl(), true);
trace_inputs[past_value_name].attach("zero_point", params->pull("model.layers." + std::to_string(i) + ".self_attn.v_cast_to_int8_qdq.fake_quant.zero_point").impl(), true);
// clang-format on
}

Expand All @@ -86,7 +79,7 @@ MLLM_MAIN({
mllm::qnn::aot::parseQcomTargetMachineFromJSONFile(qnn_aot_cfg_files.get()));

mllm::ir::PassManager pm(ir["model"]);
pm.reg(mllm::qnn::aot::createQnnAOTLoweringPipeline(&qnn_aot_env, qnn_aot_cfg_files.get()));
pm.reg(mllm::qnn::aot::createQnnAOTLoweringPipeline(&qnn_aot_env, qnn_aot_cfg_files.get(), params));
pm.run();

mllm::redirect("qwen3_qnn_aot.mir", [&]() { mllm::print(ir["model"]); });
Expand Down
Loading