diff --git "a/build/webgpu/test.json" "b/build/webgpu/test.json" --- "a/build/webgpu/test.json" +++ "b/build/webgpu/test.json" @@ -18,7 +18,12 @@ "ort_sharedkv_rotary_multibatch_b2q1p8_h2kv1d16_input_queryT": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.1], "ort_quant_int8_pertensor_b1q4_h2kv1d8_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ort_quant_int4_pertensor_b1q4_h2kv1d8_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - "ort_quant_int8_multibatch_b2q4_h4kv2d16_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + "ort_quant_int8_multibatch_b2q4_h4kv2d16_input_pastKeyT": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_keyT": [90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 90, -90, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100, 100, -100], + "ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_valueT": [0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.25, 0.25, -0.25, -0.25, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75, 0.75, 0.75, -0.75, -0.75], + "ort_gqa_causal_default_b1q2_h1kv1d8_input_queryT": [1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], + "ort_gqa_causal_default_b1q2_h1kv1d8_input_keyT": [0, 0, 0, 0, 0, 0, 0, 0, 3.1073447968483734, 0, 0, 0, 0, 0, 0, 0], + "ort_gqa_causal_default_b1q2_h1kv1d8_input_valueT": [1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3] }, "cases": [ { @@ -52,7 +57,7 @@ { "name": "direct_f32_with_f16_present_cache", "provenance": { - "notes": "Connects the independent T and T_CACHE schema variables by narrowing a direct float32 K/V projection into float16 present-cache storage." + "notes": "Query/key/value are float32 while the present-key and present-value cache outputs are float16, narrowing on write. 2 query heads over 1 key/value head, head width 3." }, "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.5, "causal": 0 }, "inputs": { @@ -83,7 +88,7 @@ { "name": "direct_f16_with_f32_present_cache", "provenance": { - "notes": "Exercises the opposite independent T/T_CACHE conversion by widening a direct float16 K/V projection into float32 present-cache storage." + "notes": "Query/key/value are float16 while the present-key and present-value cache outputs are float32, widening on write. 2 query heads over 1 key/value head, head width 3." }, "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.5, "causal": 0 }, "inputs": { @@ -2230,7 +2235,7 @@ { "name": "qnorm_rotary_scalar_h8kv2_d256_q1", "provenance": { - "source": "scope coverage", + "source": "synthetic", "test": "headDim=256 scalar Q-norm and rotary specialization", "notes": "headDim=256 requires 34,048 bytes for the cooperative path, exceeding a 32 KiB workgroup-storage tier and exercising the thread-per-query Q-norm route." }, @@ -2285,7 +2290,7 @@ { "name": "f16_qnorm_rotary_scalar_f16wts_h8kv2_d256_q1", "provenance": { - "source": "scope coverage", + "source": "synthetic", "test": "headDim=256 float16-weight scalar Q-norm and rotary specialization", "notes": "Float16 counterpart to the thread-per-query Q-norm case. It verifies explicit f32 conversion of q_norm_weight, cos_cache, and sin_cache before scalar arithmetic." }, @@ -2950,7 +2955,7 @@ { "name": "q31_kv511_h8_kv2_d64_double_threshold_compact", "provenance": { - "notes": "Compact GQA lock immediately below both flash admission boundaries: qSeq*heads=31*8=248 (<256) and kvSeq=511 (<512). Preserves an 8:2 grouped-query layout and present-K/V outputs." + "notes": "A 31-token query against a 511-token key/value context (510 valid tokens via seqlensKT), with 8 query heads grouped 4:1 over 2 key/value heads and head width 64. Present-key and present-value outputs are requested." }, "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, "inputs": { @@ -2980,7 +2985,7 @@ }, { "name": "flash_cluster_q31_h8_kv8_d32_compact", - "provenance": { "notes": "Correctness lock for the vectorized headDim=32 GQA prefill cluster path." }, + "provenance": { "notes": "GQA prefill with head dimension 32 checks output across grouped query heads." }, "attrs": { "num_heads": 8, "kv_num_heads": 8, "scale": 0.17677669529663687, "causal": 0 }, "inputs": { "queryT": { @@ -3010,7 +3015,7 @@ { "name": "flash_splitk_decode_h8_kv8_d32_kv512_compact", "provenance": { - "notes": "Correctness lock for headDim=32 GQA decode split-K on subgroup and subgroup-free tiers." + "notes": "GQA decode with head dimension 32 and long context checks output on subgroup and portable tiers." }, "attrs": { "num_heads": 8, "kv_num_heads": 8, "scale": 0.17677669529663687, "causal": 0 }, "inputs": { @@ -3041,7 +3046,7 @@ { "name": "quant_int8_splitk_decode_h2_kv1_d64_kv1024", "provenance": { - "notes": "Correctness lock for buffer-sharing long-context INT8 decode split-K on subgroup and subgroup-free tiers, including per-channel cache scales, a partially filled physical cache, and an appended K/V token." + "notes": "Long-context int8 decode with a shared buffer, per-channel cache scales, a partially filled cache and an appended K/V token checks output across device tiers." }, "attrs": { "num_heads": 2, @@ -3220,7 +3225,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", - "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "test": "get_total_sequence_length / per-batch seqlens_k", "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." } }, @@ -3258,7 +3263,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", - "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "test": "get_total_sequence_length / per-batch seqlens_k", "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." } }, @@ -3312,7 +3317,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention.wgsl.template", - "test": "get_total_sequence_length / per-batch seqlens_k (PRs #29247, #29002)", + "test": "get_total_sequence_length / per-batch seqlens_k", "notes": "past_present_share_buffer semantics: the BNSH cache dim2 is capacity, seqlens_k[b]+1 is the active end per batch. Rows past the active end are storage and must not be attended; rotary positions derive from the active end, not the capacity." } }, @@ -3348,14 +3353,14 @@ "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", "test": "seqlens_k-driven total_sequence_length in the decode split path", - "notes": "Cache capacity 1024 with active end 512 selects the float decode split-K variant; the split ranges must cover only the active prefix." + "notes": "A cache capacity of 1,024 with an active end at 256; attention must cover only the active prefix." } }, { "name": "sharedkv_decode_tiny_headdim8_kv512", "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", - "test": "head_size=8 split-reduce OOB race class (PR #29593)", + "test": "head_size=8 split-reduce OOB race class", "notes": "Diverges from the upstream test's inputs (inputs.pastValueT fillFloat32 -> fillFloat32); the expected output is recomputed by the CPU reference for the new inputs. A head size below the flash minimum exercises the cooperative fallback and its tile-width-guarded reduction." }, "attrs": { "num_heads": 4, "kv_num_heads": 2 }, @@ -3390,7 +3395,7 @@ "name": "sharedkv_decode_tiny_headdim16_kv512", "provenance": { "source": "onnxruntime/contrib_ops/webgpu/bert/flash_attention_decode_qkv.wgsl.template", - "test": "head_size=16 split-reduce OOB race class (PR #29593)", + "test": "head_size=16 split-reduce OOB race class", "notes": "Diverges from the upstream test's inputs (inputs.pastValueT fillFloat32 -> fillFloat32); the expected output is recomputed by the CPU reference for the new inputs. Head size 16 exercises the cooperative fallback at the next four-wide tiny-head boundary." }, "attrs": { "num_heads": 4, "kv_num_heads": 2 }, @@ -3977,7 +3982,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "A capacity-8 windowed cache reaches `T = 6` without eviction, so its origin remains zero and unused rows must be cleared." } }, @@ -4012,7 +4017,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "At `T = 12` with capacity 8, appending one token evicts one row, writes at row 7, and advances the absolute cache origin to 4." } }, @@ -4047,7 +4052,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "Appending four tokens to a capacity-8 cache at `T = 10` evicts two rows, begins the append at row 4, and advances the absolute cache origin to 2." } }, @@ -4082,7 +4087,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "Two batches require independent cache origins: batch 0 reaches `T = 4` without eviction, while batch 1 reaches `T = 12`, evicts one row, and uses origin 4." } }, @@ -4117,7 +4122,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "A local window of four within a capacity-8 cache combines one-row eviction with a nonzero attention floor derived from the absolute cache origin." } }, @@ -4152,7 +4157,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "Cache capacity and local-window size are both four at `T = 10`, exercising a fully occupied sliding-window allocation with a nonzero origin." } }, @@ -4187,7 +4192,7 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", "notes": "At `T = 64` with capacity 16, the cache origin is 48. The four-to-one grouped-query layout and head size 16 exercise absolute-position translation far beyond the physical buffer." } }, @@ -4222,8 +4227,8 @@ }, "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", - "test": "GQA sliding_window_cache (ORT #29904), CPU/CUDA reference semantics", - "notes": "Diverges from the upstream test's inputs (inputs.pastValueT fillFloat32 -> fillFloat32); the expected output is recomputed by the CPU reference for the new inputs. Windowed cache at a capacity that clears CACHED_DECODE_MIN_KV_TOKENS, so the split-K decode variant is selected rather than the scalar fallback. T=4096 > C=1024, so the step genuinely evicts and the split-K kernel reads a shifted cache." + "test": "GQA sliding_window_cache, CPU/CUDA reference semantics", + "notes": "A sliding-window key/value cache capped at 1024 tokens decodes token 4096, past the window capacity, so the appended token evicts the oldest cached entry and the cache is read from a shifted offset. 2 query heads over 1 key/value head, head width 64." } }, { @@ -4406,7 +4411,7 @@ "provenance": { "source": "onnxruntime/contrib_ops/cpu/bert/group_query_attention.cc", "test": "GQA sliding_window_cache chunk prefill on the materialized subgroup-matrix route", - "notes": "T=384 > C=256 evicts 128 rows; w=256=C keeps the floor inactive, so this pins the shift + right-aligned causal path of the materialized window route at d=128 (two apply column tiles)." + "notes": "A 384-token sequence exceeds the 256-token cache and evicts 128 rows; a full-width window checks right-aligned causal attention after the shift." } }, { @@ -4634,7 +4639,7 @@ "presentValueT": { "dtype": "float32", "shape": [1, 1, 64, 256], "tolerance": 0.0001 } }, "provenance": { - "notes": "headDim-256 counterpart to the share-append capacity-slack case. It is above the f32 shared-memory cluster's register-geometry boundary and verifies consistent route admission for cached prefill while preserving the active-length versus capacity contract." + "notes": "Cached float32 prefill appends 32 tokens after 16 cached ones in a 64-slot key/value buffer (2 query heads over 1 key/value head, head width 256); the 16 unused slots must not affect the output." } }, { @@ -4872,9 +4877,9 @@ } }, { - "name": "splitk_decode_h8kv2_d64_kv2048_ramp_value_scale_lock", + "name": "splitk_decode_h8kv2_d64_kv2048_ramp_value_scale", "provenance": { - "notes": "Scale lock for present-KV split-K decode. A monotone V ramp keeps each output O(1) and dependent on the weighted key position, exposing errors in the softmax denominator, final divide, or cross-partition running-max rescale." + "notes": "Decoding a single query against a 2048-token key/value context (8 query heads grouped 4:1 over 2 key/value heads, head width 64). V ramps monotonically from 0.5 to 2.0 so the output stays dependent on key position, exposing normalization errors." }, "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.125, "causal": 0 }, "inputs": { @@ -4903,7 +4908,7 @@ } }, { - "name": "splitk_decode_h8kv2_d64_kv2048_f16_ramp_value_scale_lock", + "name": "splitk_decode_h8kv2_d64_kv2048_f16_ramp_value_scale", "provenance": { "notes": "A monotone value ramp keeps float16 decode outputs at order-one magnitude, making split-K accumulation, cross-partition rescaling, and the final normalization observable." }, @@ -4934,9 +4939,9 @@ } }, { - "name": "flash_prefill_h2kv1_d64_s128_f16_ramp_value_scale_lock", + "name": "flash_prefill_h2kv1_d64_s128_f16_ramp_value_scale", "provenance": { - "notes": "A monotone value ramp gives each of 128 causal float16 prefill rows a distinct order-one output, exposing the register-blocked q32 epilogue division and row selection." + "notes": "A monotone value ramp gives each of 128 noncausal float16 prefill rows an order-one output, with two query heads sharing one key/value head, exposing normalization errors." }, "attrs": { "num_heads": 2, "kv_num_heads": 1, "scale": 0.125, "causal": 0 }, "inputs": { @@ -4965,7 +4970,7 @@ } }, { - "name": "newkv_past_splitk_h8kv2_d64_q1p1024_ramp_value_scale_lock", + "name": "newkv_past_splitk_h8kv2_d64_q1p1024_ramp_value_scale", "provenance": { "notes": "Monotone cached and new value ramps make decode output an order-one function of the final 256 key positions. This exposes the local-window floor, split-K combine, softmax denominator, and cached/new seam." }, @@ -5002,7 +5007,7 @@ } }, { - "name": "newkv_past_f16_flash_h8kv2_d64_q32p32_ramp_value_scale_lock", + "name": "newkv_past_f16_flash_h8kv2_d64_q32p32_ramp_value_scale", "provenance": { "notes": "Monotone cached and new value ramps make each of 32 float16 prefill rows an order-one function of its 16-key causal window. Early rows cross the cached/new seam, exposing the window floor and flash normalization." }, @@ -5043,7 +5048,7 @@ } }, { - "name": "window_cache_decode_splitk_cap1024_b1q1_h2kv1d64_ramp_value_scale_lock", + "name": "window_cache_decode_splitk_cap1024_b1q1_h2kv1d64_ramp_value_scale", "provenance": { "notes": "A monotone past-cache ramp makes the decode output depend at order-one scale on the weighted slot within the shifted window. With `T = 4096` and capacity 1024, both the eviction offset and split-K combine affect the result." }, @@ -5076,7 +5081,7 @@ } }, { - "name": "quant_int8_scalar_prompt_ramp_value_scale_lock", + "name": "quant_int8_scalar_prompt_ramp_value_scale", "provenance": { "notes": "An INT8 value ramp from 0.5 to 2.0 gives each causal row a distinct order-one output. Tight expected values exercise value-scale dequantization, the softmax denominator, and the scalar apply division." }, @@ -5113,7 +5118,7 @@ } }, { - "name": "quant_int4_scalar_prompt_ramp_value_scale_lock", + "name": "quant_int4_scalar_prompt_ramp_value_scale", "provenance": { "notes": "An INT4 value ramp with scale 0.3 spans the signed nibble range and gives order-one outputs. Tight expected values exercise +8-biased nibble unpacking, dequantization, and normalization." }, @@ -5735,6 +5740,2430 @@ "provenance": { "notes": "Cached decode at head size 16 exercises the split-K route exactly at its minimum head-size boundary." } + }, + { + "name": "ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40", + "provenance": { + "notes": "ORT WebGPU_FP16_HighMagnitude_FlashPrefill. q = 100*sign[d], k = (90|100)*sign[d] by row parity, so every logit is ~1.0e5 -- far past the f16 max of 65504. Pins that scores accumulate in f32." + }, + "attrs": { "num_heads": 2, "kv_num_heads": 1 }, + "requires": { "features": ["shader-f16"] }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [1, 40, 256], "data": { "kind": "cycle", "values": [100.0, -100.0] } }, + "keyT": { + "dtype": "float16", + "shape": [1, 40, 128], + "data": { + "kind": "cycle", + "values": { "$ref": "#/fixtureArrays/ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_keyT" } + } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 40, 128], + "data": { + "kind": "cycle", + "values": { "$ref": "#/fixtureArrays/ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_valueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [39] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [40] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 40, 256], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 40, 128], "tolerance": 0.005 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 40, 128], "tolerance": 0.005 } + } + }, + { + "name": "ort_f16_high_magnitude_bidirectional_h2kv1_d128_s40", + "attrs": { "num_heads": 2, "kv_num_heads": 1, "causal": 0 }, + "requires": { "features": ["shader-f16"] }, + "inputs": { + "queryT": { "dtype": "float16", "shape": [1, 40, 256], "data": { "kind": "cycle", "values": [100.0, -100.0] } }, + "keyT": { + "dtype": "float16", + "shape": [1, 40, 128], + "data": { + "kind": "cycle", + "values": { "$ref": "#/fixtureArrays/ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_keyT" } + } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 40, 128], + "data": { + "kind": "cycle", + "values": { "$ref": "#/fixtureArrays/ort_f16_high_magnitude_causal_prefill_h2kv1_d128_s40_input_valueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [39] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [40] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 40, 256], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 1, 40, 128], "tolerance": 0.005 }, + "presentValueT": { "dtype": "float16", "shape": [1, 1, 40, 128], "tolerance": 0.005 } + } + }, + { + "name": "ort_quant_int8_decode_ragged_batch_b2q1cap1024_h2kv1d64", + "provenance": { + "notes": "Adapted from onnxruntime's WebGPU_TurboQuant_Decode_MultiBatch_UsesPerBatchSeqlensK test. Batch 0 and batch 1 use different past lengths (700 and 900 tokens) within a shared 901-token total and distinct int8 cache scales (0.05, 0.055), so per-batch seqlens indexing is verified rather than assumed identical." + }, + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "scale": 0.125, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.12 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 1, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.12 } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 1, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.019, "cosStep": 0.037, "scale": 0.12 } + }, + "pastKeyT": { + "dtype": "int8", + "shape": [2, 1, 1024, 64], + "data": { "kind": "cycle", "values": [-8, -3, 0, 2, 7, 4, -6] } + }, + "pastValueT": { + "dtype": "int8", + "shape": [2, 1, 1024, 64], + "data": { "kind": "cycle", "values": [5, -7, 1, 0, 6, -2, 3] } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [700, 900] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [901] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.05] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.055] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 1, 128], "tolerance": 0.0022, "relTolerance": 0.005 }, + "presentKeyT": { "dtype": "int8", "shape": [2, 1, 1024, 64], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [2, 1, 1024, 64], "tolerance": 0 } + } + }, + { + "name": "ort_quant_int8_prompt_right_padded_b2q4_h2kv1d64", + "attrs": { + "num_heads": 2, + "kv_num_heads": 1, + "kv_cache_bit_width": 8, + "k_quant_type": "PER_TENSOR", + "v_quant_type": "PER_TENSOR" + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 4, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.12 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 4, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.027, "scale": 0.12 } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 4, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.033, "scale": 0.12 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [1, 3] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "kScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.004] } }, + "vScaleT": { "dtype": "float32", "shape": [1], "data": { "kind": "values", "values": [0.004] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 4, 128], "tolerance": 0.01 }, + "presentKeyT": { "dtype": "int8", "shape": [2, 1, 4, 64], "tolerance": 0 }, + "presentValueT": { "dtype": "int8", "shape": [2, 1, 4, 64], "tolerance": 0 } + } + }, + { + "name": "ort_gqa_causal_default_b1q2_h1kv1d8", + "provenance": { + "notes": "ORT CausalMaskDefaultsToEnabled_CPU. key[hidden_size] = sqrt(head_size) * ln(3) makes the single live logit exactly ln 3, so the causal answer (1.0, 2.5) and the bidirectional answer (2.5, 2.5) are distinguishable by hand." + }, + "attrs": { "num_heads": 1, "kv_num_heads": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_queryT" } + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_keyT" } + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_valueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 2, 8], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 2, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 2, 8], "tolerance": 0.000001 } + } + }, + { + "name": "ort_gqa_bidirectional_b1q2_h1kv1d8", + "attrs": { "num_heads": 1, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_queryT" } + } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_keyT" } + } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 2, 8], + "data": { + "kind": "values", + "values": { "$ref": "#/fixtureArrays/ort_gqa_causal_default_b1q2_h1kv1d8_input_valueT" } + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [1] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [2] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 2, 8], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 2, 8], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 2, 8], "tolerance": 0.000001 } + } + }, + { + "name": "ort_gqa_bidirectional_share_append_b2q2cap8_h4kv2d64", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 2, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 2, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 2, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.5 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 2, 8, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.25 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 2, 8, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.25 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [5, 5] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [6] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 2, 256], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 2, 8, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 2, 8, 64], "tolerance": 0.0001 } + } + }, + { + "name": "bidir-shared-cache-flash-prefill-f32-tail-q33", + "provenance": { + "source": "Bidirectional shared-capacity cache flash-prefill boundary", + "notes": "Thirty-three queries cross a 32-query tile; every query must see all 64 live keys after the append." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 1024], "tolerance": 0.0005 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 64, 128], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 64, 128], "tolerance": 0.0001 } + } + }, + { + "name": "bidir-shared-cache-flash-prefill-f16-ragged-q32", + "provenance": { + "source": "Bidirectional shared-capacity cache flash-prefill boundary", + "notes": "Two batches have different live key ranges while appending a full 32-query tile." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2, "causal": 0 }, + "requires": { "features": ["shader-f16"] }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [2, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float16", + "shape": [2, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float16", + "shape": [2, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float16", + "shape": [2, 2, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [2, 2, 64, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [63, 47] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [2, 32, 256], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float16", "shape": [2, 2, 64, 64], "tolerance": 0.005 }, + "presentValueT": { "dtype": "float16", "shape": [2, 2, 64, 64], "tolerance": 0.005 } + } + }, + { + "name": "ort_window_cache_bias_absolute_column_b1q1cap2_h1kv1d16", + "provenance": { + "notes": "ORT WindowedCacheAttentionBiasWithPositionIds_CPU. attention_bias last dim is total_sequence_length (5), NOT the cache capacity (2): resident cache row 0 is absolute column 3. Expected output is 17.5 in all 16 channels; present_value is [10 x16, 20 x16]." + }, + "attrs": { "num_heads": 1, "kv_num_heads": 1, "do_rotary": 1, "local_window_size": 2, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "constant", "value": 0.0 } }, + "keyT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "constant", "value": 0.0 } }, + "valueT": { "dtype": "float32", "shape": [1, 1, 16], "data": { "kind": "constant", "value": 20.0 } }, + "pastKeyT": { "dtype": "float32", "shape": [1, 1, 2, 16], "data": { "kind": "constant", "value": 0.0 } }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 1, 2, 16], + "data": { + "kind": "values", + "values": [5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0] + } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [4] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [5] } }, + "cosCacheT": { "dtype": "float32", "shape": [5, 8], "data": { "kind": "constant", "value": 1.0 } }, + "sinCacheT": { "dtype": "float32", "shape": [5, 8], "data": { "kind": "constant", "value": 0.0 } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 1, 1, 5], + "data": { "kind": "values", "values": [-100.0, -100.0, -100.0, 0.0, 1.0986122886681098] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 16], "tolerance": 0.001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 2, 16], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 2, 16], "tolerance": 0.001 } + } + }, + { + "name": "ort_window_cache_rotary_decode_cap384_b1q1_h8kv2d64", + "attrs": { + "num_heads": 8, + "kv_num_heads": 2, + "do_rotary": 1, + "local_window_size": 128, + "sliding_window_cache": 1 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotaryCos", "thetaStart": 0.0, "thetaStep": 0.017 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotarySin", "thetaStart": 0.0, "thetaStep": 0.017 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + } + }, + { + "name": "ort_window_cache_headsink_gptoss_decode_cap384_b1q1_h8kv2d64", + "attrs": { + "num_heads": 8, + "kv_num_heads": 2, + "do_rotary": 1, + "local_window_size": 128, + "sliding_window_cache": 1 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotaryCos", "thetaStart": 0.0, "thetaStep": 0.017 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotarySin", "thetaStart": 0.0, "thetaStep": 0.017 } + }, + "headSinkT": { + "dtype": "float32", + "shape": [8], + "data": { "kind": "values", "values": [0.25, -0.5, 1.0, 0.125, -0.75, 0.5, 0.0, 0.375] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + } + }, + { + "name": "window_cache_headsink_decode_cap384_b1q1_h8kv2d64", + "provenance": { + "source": "synthetic", + "test": "windowed cache with a head sink and no rotary", + "notes": "The rotary-free twin of ort_window_cache_headsink_gptoss_decode_cap384_b1q1_h8kv2d64: head_sink does not require RoPE, so the two windowed head-sink routes (window_shift_headsink_append here, window_shift_rotary_headsink_append there) each need a fixture. Same capacity 384 / window 128 / total 600 eviction geometry." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "local_window_size": 128, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "headSinkT": { + "dtype": "float32", + "shape": [8], + "data": { "kind": "values", "values": [0.25, -0.5, 1.0, 0.125, -0.75, 0.5, 0.0, 0.375] } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + } + }, + { + "name": "window_cache_bias_absolute_column_decode_cap384_b1q1_h8kv2d64", + "provenance": { + "source": "synthetic", + "test": "windowed cache with an absolute-column attention bias and no rotary", + "notes": "The rotary-free twin of ort_window_cache_bias_absolute_column_b1q1cap2_h1kv1d16, at a shape whose output varies per channel so the bias route has a non-degenerate proof. attention_bias last dim is total_sequence_length (600), not the cache capacity (384): with 216 evicted positions, resident cache row j is absolute column 216 + j, and the live window covers columns 472..599." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "local_window_size": 128, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 1, 1, 600], + "data": { "kind": "fillFloat32", "sinStep": 0.019, "cosStep": 0.043, "scale": 1.5 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + } + }, + { + "name": "causal_prefill_no_past_flash_b1q64_h4kv2d64", + "provenance": { + "source": "synthetic", + "test": "causal prefill without a past cache at a flash-eligible shape", + "notes": "causal=1 with local_window_size=-1 and no past cache is the first-prompt spelling onnxruntime's GroupQueryAttention tests use; 64 queries x 4 heads clears the flash route's 248-row floor so qkv_present_flash_causal (subgroup tiers) is exercised as well as the scalar twin." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 64, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 64, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 64, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q31_h8kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 31, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 31, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 31, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [30] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 31, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 31, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 31, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q32_h8kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 32, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 32, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q33_h8kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 33, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 33, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 33, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 33, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q63_h4kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 63, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 63, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 63, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [62] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 63, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 63, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 63, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q64_h4kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 64, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 64, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [63] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 64, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 64, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q65_h4kv2d64", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 65, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 65, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 65, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 65, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 65, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 65, 64], "tolerance": 0.000001 } + } + }, + { + "name": "causal_prefill_no_past_f16_b1q65_h4kv2d128", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 4, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 65, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 65, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 65, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [64] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 65, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 65, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 65, 128], "tolerance": 0.000001 } + } + }, + { + "name": "causal_contract_f16-h8kv2-d64-q512", + "provenance": { + "source": "synthetic", + "test": "Causal half-precision prompt prefill", + "notes": "Each query attends only itself and earlier key positions. Verify the complete attention output and exact retention of both key/value inputs in the present caches." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 512, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.0625 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 512, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.0625 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 512, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.0625 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [511] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [512] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 512, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 512, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 512, 64], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q31_h8kv1d32", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 31, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 31, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [30] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 31, 32], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 31, 32], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q32_h8kv1d32", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 32], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 32], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q33_h8kv1d32", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 32], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 32], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 32], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q31_h8kv1d128", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 31, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 31, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [30] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 31, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 31, 128], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q32_h8kv1d128", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 128], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q33_h8kv1d128", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q31_h8kv1d256", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 31, 2048], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 31, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 31, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [30] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 31, 2048], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 31, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 31, 256], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q32_h8kv1d256", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 32, 2048], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 32, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 32, 2048], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 32, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 32, 256], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_query_tile_boundary_f32_b1q33_h8kv1d256", + "provenance": { + "source": "synthetic", + "test": "bidirectional attention around a query-tile boundary", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 2048], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 2048], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 256], "tolerance": 0.000001 } + } + }, + { + "name": "prefill_f32_b1q33_h8kv1d64_prefill_lanes_per_query_2", + "provenance": { + "source": "synthetic", + "test": "portable attention with alternate reduction geometry", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 64], "tolerance": 0.000001 } + }, + "tunables": { "PREFILL_LANES_PER_QUERY": 2 } + }, + { + "name": "prefill_f32_b1q33_h8kv1d64_prefill_lanes_per_query_4", + "provenance": { + "source": "synthetic", + "test": "portable attention with alternate reduction geometry", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 64], "tolerance": 0.000001 } + }, + "tunables": { "PREFILL_LANES_PER_QUERY": 4 } + }, + { + "name": "prefill_f32_b1q33_h8kv1d128_no_sg_tile_k_max_1", + "provenance": { + "source": "synthetic", + "test": "portable attention with alternate reduction geometry", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 } + }, + "tunables": { "NO_SG_TILE_K_MAX": 1 } + }, + { + "name": "prefill_f32_b1q33_h8kv1d128_no_sg_tile_k_max_3", + "provenance": { + "source": "synthetic", + "test": "portable attention with alternate reduction geometry", + "notes": "Complete attention output and present-cache parity across narrow and wide heads." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 1, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 33, 1024], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.023, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 33, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.031, "scale": 0.5 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [33] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 33, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 1, 33, 128], "tolerance": 0.000001 } + }, + "tunables": { "NO_SG_TILE_K_MAX": 3 } + }, + { + "name": "storage_fit_window_decode_32h8kv_d128_cap256_w256", + "attrs": { "num_heads": 32, "kv_num_heads": 8, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 4096], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 8, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 8, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 4096], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 8, 256, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 8, 256, 128], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d122_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 488], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 244], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 244], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 122], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 122], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 488], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 122], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 122], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d124_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 496], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 248], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 248], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 124], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 124], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 496], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 124], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 124], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d128_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 128], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 128], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d246_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 984], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 492], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 492], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 246], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 246], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 984], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 246], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 246], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d248_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 992], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 496], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 496], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 248], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 248], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 992], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 248], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 248], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d256_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 1024], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 256], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 256], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d512_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 512], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 2048], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 512], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 512], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d1024_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 4096], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 1024], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 4096], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 1024], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 1024], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d1364_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 5456], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 2728], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 2728], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 1364], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 1364], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 5456], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 1364], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 1364], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d1366_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 5464], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 2732], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 2732], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 1366], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 1366], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 5464], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 1366], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 1366], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d2046_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 8184], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 4092], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 4092], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 2046], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 2046], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 8184], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 2046], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 2046], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d2048_cap256", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 8192], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 4096], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 4096], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 8192], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 2048], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 2048], "tolerance": 0.000001 } + } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg0", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg1", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 1 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg3", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 3 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg16", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 16 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg33", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 33 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg64", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 256], "tolerance": 0.0001 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 64 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg3_f16", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 256], "tolerance": 0.00001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 3 } + }, + { + "name": "storage_fit_cooperative_window_h4kv2_d64_cap256_wg64_f16", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "sliding_window_cache": 1, "local_window_size": 256 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { + "dtype": "float16", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.031, "cosStep": 0.007 } + }, + "valueT": { + "dtype": "float16", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.037, "cosStep": 0.005 } + }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 256, 64], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65535] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [65536] } } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 256], "tolerance": 0.00001 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 256, 64], "tolerance": 0.000001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 256, 64], "tolerance": 0.000001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 64 } + }, + { + "name": "storage_fit_forced_qnorm_rotary_scalar_h8kv2_d256_q1", + "provenance": { + "source": "synthetic", + "test": "headDim=256 scalar Q-norm and rotary specialization", + "notes": "headDim=256 requires 34,048 bytes for the cooperative path, exceeding a 32 KiB workgroup-storage tier and exercising the thread-per-query Q-norm route." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.0625, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float32", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float32", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float32", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float32", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 2048], "tolerance": 0.0003 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 32, 256], "tolerance": 0.0001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } + }, + { + "name": "storage_fit_forced_f16_qnorm_rotary_scalar_f16wts_h8kv2_d256_q1", + "provenance": { + "source": "synthetic", + "test": "headDim=256 float16-weight scalar Q-norm and rotary specialization", + "notes": "Float16 counterpart to the thread-per-query Q-norm case. It verifies explicit f32 conversion of q_norm_weight, cos_cache, and sin_cache before scalar arithmetic." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "scale": 0.0625, "do_rotary": 1, "qk_norm_epsilon": 0.00001 }, + "inputs": { + "queryT": { + "dtype": "float16", + "shape": [1, 1, 2048], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.011, "cosStep": 0.029 } + }, + "keyT": { "dtype": "float16", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "valueT": { "dtype": "float16", "shape": [1, 0, 512], "data": { "kind": "values", "values": [] } }, + "pastKeyT": { + "dtype": "float16", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.017, "cosStep": 0.019 } + }, + "pastValueT": { + "dtype": "float16", + "shape": [1, 2, 32, 256], + "data": { "kind": "fillFloat32", "scale": 0.2, "sinStep": 0.023, "cosStep": 0.013 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [31] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [32] } }, + "cosCacheT": { + "dtype": "float16", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.031, "cosStep": 0.007 } + }, + "sinCacheT": { + "dtype": "float16", + "shape": [32, 128], + "data": { "kind": "fillFloat32", "scale": 0.5, "sinStep": 0.013, "cosStep": 0.037 } + }, + "qNormWeightT": { + "dtype": "float16", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + }, + "kNormWeightT": { + "dtype": "float16", + "shape": [256], + "data": { "kind": "fillFloat32", "scale": 0.3, "sinStep": 0.05, "cosStep": 0.09 } + } + }, + "outputs": { + "outputT": { "dtype": "float16", "shape": [1, 1, 2048], "tolerance": 0.0003 }, + "presentKeyT": { "dtype": "float16", "shape": [1, 2, 32, 256], "tolerance": 0.001 }, + "presentValueT": { "dtype": "float16", "shape": [1, 2, 32, 256], "tolerance": 0.001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } + }, + { + "name": "ort_gqa_bidirectional_share_append_b2q2cap8_h4kv2d64_scalar", + "attrs": { "num_heads": 4, "kv_num_heads": 2, "causal": 0 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [2, 2, 256], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.5 } + }, + "keyT": { + "dtype": "float32", + "shape": [2, 2, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.5 } + }, + "valueT": { + "dtype": "float32", + "shape": [2, 2, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.5 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [2, 2, 8, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.25 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [2, 2, 8, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.25 } + }, + "seqlensKT": { "dtype": "int32", "shape": [2], "data": { "kind": "values", "values": [5, 5] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [6] } } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [2, 2, 256], "tolerance": 0.005 }, + "presentKeyT": { "dtype": "float32", "shape": [2, 2, 8, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [2, 2, 8, 64], "tolerance": 0.0001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } + }, + { + "name": "ort_window_cache_rotary_decode_cap384_b1q1_h8kv2d64_scalar", + "attrs": { + "num_heads": 8, + "kv_num_heads": 2, + "do_rotary": 1, + "local_window_size": 128, + "sliding_window_cache": 1 + }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "cosCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotaryCos", "thetaStart": 0.0, "thetaStep": 0.017 } + }, + "sinCacheT": { + "dtype": "float32", + "shape": [1024, 32], + "data": { "kind": "rotarySin", "thetaStart": 0.0, "thetaStep": 0.017 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } + }, + { + "name": "window_cache_bias_absolute_column_decode_cap384_b1q1_h8kv2d64_scalar", + "provenance": { + "source": "synthetic", + "test": "windowed cache with an absolute-column attention bias and no rotary", + "notes": "The rotary-free twin of ort_window_cache_bias_absolute_column_b1q1cap2_h1kv1d16, at a shape whose output varies per channel so the bias route has a non-degenerate proof. attention_bias last dim is total_sequence_length (600), not the cache capacity (384): with 216 evicted positions, resident cache row j is absolute column 216 + j, and the live window covers columns 472..599." + }, + "attrs": { "num_heads": 8, "kv_num_heads": 2, "local_window_size": 128, "sliding_window_cache": 1 }, + "inputs": { + "queryT": { + "dtype": "float32", + "shape": [1, 1, 512], + "data": { "kind": "fillFloat32", "sinStep": 0.011, "cosStep": 0.023, "scale": 0.2 } + }, + "keyT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.013, "cosStep": 0.029, "scale": 0.2 } + }, + "valueT": { + "dtype": "float32", + "shape": [1, 1, 128], + "data": { "kind": "fillFloat32", "sinStep": 0.017, "cosStep": 0.031, "scale": 0.2 } + }, + "pastKeyT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.007, "cosStep": 0.037, "scale": 0.2 } + }, + "pastValueT": { + "dtype": "float32", + "shape": [1, 2, 384, 64], + "data": { "kind": "fillFloat32", "sinStep": 0.005, "cosStep": 0.041, "scale": 0.2 } + }, + "seqlensKT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [599] } }, + "totalSequenceLengthT": { "dtype": "int32", "shape": [1], "data": { "kind": "values", "values": [600] } }, + "attentionBiasT": { + "dtype": "float32", + "shape": [1, 1, 1, 600], + "data": { "kind": "fillFloat32", "sinStep": 0.019, "cosStep": 0.043, "scale": 1.5 } + } + }, + "outputs": { + "outputT": { "dtype": "float32", "shape": [1, 1, 512], "tolerance": 0.00047 }, + "presentKeyT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 }, + "presentValueT": { "dtype": "float32", "shape": [1, 2, 384, 64], "tolerance": 0.0001 } + }, + "tunables": { "COOPERATIVE_WORKGROUP_SIZE": 0 } } ] }