void0x14 commited on
Commit
c2e902a
·
unverified ·
1 Parent(s): 33bb7d3

test: define qwen35 pruning contract

Browse files
MVP/tests/fixtures/qwen35_metadata.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hidden_size": 1024,
3
+ "intermediate_size": 3584,
4
+ "vocab_size": 248320,
5
+ "tie_word_embeddings": true,
6
+ "layer_types": [
7
+ "linear_attention", "linear_attention", "linear_attention", "full_attention",
8
+ "linear_attention", "linear_attention", "linear_attention", "full_attention",
9
+ "linear_attention", "linear_attention", "linear_attention", "full_attention",
10
+ "linear_attention", "linear_attention", "linear_attention", "full_attention",
11
+ "linear_attention", "linear_attention", "linear_attention", "full_attention",
12
+ "linear_attention", "linear_attention", "linear_attention", "full_attention"
13
+ ],
14
+ "embedding_params": 254279680,
15
+ "linear_attention_params": 21555360,
16
+ "full_attention_params": 18352640,
17
+ "final_norm_params": 1024,
18
+ "all_named_params": 873438784,
19
+ "text_backbone_params": 752393024
20
+ }
MVP/tests/test_pruning_contract.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import pytest
5
+
6
+ from MVP.qwen35_prune import (
7
+ ParameterReport,
8
+ build_text_config,
9
+ choose_prefix,
10
+ translate_text_key,
11
+ )
12
+ from MVP.validate_checkpoint import validate_state_dict_keys
13
+
14
+
15
+ FIXTURE = Path(__file__).parent / "fixtures" / "qwen35_metadata.json"
16
+
17
+
18
+ def load_fixture():
19
+ return json.loads(FIXTURE.read_text())
20
+
21
+
22
+ def test_measured_n4_prefix_is_inside_required_interval():
23
+ data = load_fixture()
24
+ report = ParameterReport(
25
+ embedding_params=data["embedding_params"],
26
+ layer_params=tuple(
27
+ [data["linear_attention_params"]] * 3
28
+ + [data["full_attention_params"]]
29
+ + [data["linear_attention_params"]] * 3
30
+ + [data["full_attention_params"]] * 5
31
+ ),
32
+ layer_types=tuple(data["layer_types"]),
33
+ final_norm_params=data["final_norm_params"],
34
+ all_named_params=data["all_named_params"],
35
+ )
36
+
37
+ choice = choose_prefix(report, 330_000_000, 350_000_000)
38
+
39
+ assert choice.layer_count == 4
40
+ assert choice.parameter_count == 337_299_424
41
+ assert 330_000_000 <= choice.parameter_count <= 350_000_000
42
+
43
+
44
+ def test_prefix_selection_rejects_incomplete_hybrid_block():
45
+ data = load_fixture()
46
+ report = ParameterReport(
47
+ embedding_params=data["embedding_params"],
48
+ layer_params=(data["linear_attention_params"],) * 24,
49
+ layer_types=tuple(data["layer_types"]),
50
+ final_norm_params=data["final_norm_params"],
51
+ all_named_params=data["all_named_params"],
52
+ )
53
+
54
+ with pytest.raises(ValueError, match="complete hybrid"):
55
+ choose_prefix(report, 330_000_000, 350_000_000, requested_layers=3)
56
+
57
+
58
+ def test_text_prefix_translation_drops_vision_and_mtp():
59
+ assert (
60
+ translate_text_key("model.language_model.layers.3.linear_attn.A_log")
61
+ == "model.layers.3.linear_attn.A_log"
62
+ )
63
+ assert translate_text_key("model.visual.patch_embed.proj.weight") is None
64
+ assert translate_text_key("mtp.layers.0.mlp.down_proj.weight") is None
65
+
66
+
67
+ def test_text_config_is_standalone_and_keeps_live_attention_fields():
68
+ full = {
69
+ "model_type": "qwen3_5",
70
+ "tie_word_embeddings": True,
71
+ "vision_config": {"hidden_size": 512},
72
+ "text_config": {
73
+ "model_type": "qwen3_5_text",
74
+ "hidden_size": 1024,
75
+ "intermediate_size": 3584,
76
+ "num_hidden_layers": 24,
77
+ "layer_types": list(load_fixture()["layer_types"]),
78
+ "linear_num_key_heads": 16,
79
+ "linear_num_value_heads": 16,
80
+ "linear_key_head_dim": 128,
81
+ "linear_value_head_dim": 128,
82
+ "linear_conv_kernel_dim": 4,
83
+ "vocab_size": 248320,
84
+ },
85
+ }
86
+
87
+ text = build_text_config(full, 4)
88
+
89
+ assert text["model_type"] == "qwen3_5_text"
90
+ assert text["num_hidden_layers"] == 4
91
+ assert text["layer_types"] == load_fixture()["layer_types"][:4]
92
+ assert text["linear_num_value_heads"] == 16
93
+ assert text["tie_word_embeddings"] is True
94
+ assert "vision_config" not in text
95
+
96
+
97
+ def test_validator_rejects_vision_mtp_and_duplicate_tied_head():
98
+ config = {
99
+ "model_type": "qwen3_5_text",
100
+ "tie_word_embeddings": True,
101
+ "num_hidden_layers": 4,
102
+ "layer_types": load_fixture()["layer_types"][:4],
103
+ }
104
+ keys = {
105
+ "model.embed_tokens.weight": (248320, 1024),
106
+ "model.layers.0.input_layernorm.weight": (1024,),
107
+ "model.layers.1.input_layernorm.weight": (1024,),
108
+ "model.layers.2.input_layernorm.weight": (1024,),
109
+ "model.layers.3.input_layernorm.weight": (1024,),
110
+ "model.norm.weight": (1024,),
111
+ "lm_head.weight": (248320, 1024),
112
+ "model.visual.patch_embed.proj.weight": (1, 1),
113
+ "mtp.fc.weight": (1, 1),
114
+ }
115
+
116
+ with pytest.raises(ValueError, match="vision|MTP|tied"):
117
+ validate_state_dict_keys(keys, config, 330_000_000, 350_000_000)
118
+