aceOfDiamonds commited on
Commit
cb73e1a
·
1 Parent(s): 899a3bd

Add Vulkan fp16 variants (image and text)

Browse files

Both encoders lower to Vulkan with eager attention, which keeps the graph in
one delegate blob for vision and three for text instead of the 25 and 26 that
sdpa produces. Attention runs rank-3, folding the singleton batch into the
head dim, which is bit-identical and drops a third of the materialised view
dispatches.

The text encoder needs its token embedding kept off texture storage for fp16
to be correct (pytorch/executorch#22333); the vision tower has no token
embedding and is unaffected.

Requires the ExecuTorch 1.4.1 Vulkan runtime from the v0.10.0-libs-1.4.1
artifacts.

.gitattributes CHANGED
@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  *.pte filter=lfs diff=lfs merge=lfs -text
 
 
 
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  *.pte filter=lfs diff=lfs merge=lfs -text
37
+ vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
38
+ vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c683ed2d1b14fa197d22f33f29cee685f64b72048f7c0f900fab74cfb9544cfe
3
+ size 175823106
vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f077eb561dc8ea7c72c3f4023190fec2521e132c86b918e7e5e1dd40f4c72f1
3
+ size 126991872
vulkan/config.json ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "clip_vit_base_patch32",
4
+ "family": "clip",
5
+ "capabilities": [
6
+ "text-embedding",
7
+ "image-embedding"
8
+ ],
9
+ "backend": "vulkan",
10
+ "license": "mit",
11
+ "tokenizer": "../tokenizer.json",
12
+ "variants": [
13
+ {
14
+ "file": null,
15
+ "components": {
16
+ "image": "clip_vit_base_patch32_image_vulkan_fp16.pte",
17
+ "text": "clip_vit_base_patch32_text_vulkan_fp16.pte"
18
+ },
19
+ "precision": "fp16",
20
+ "quantized": false,
21
+ "default": true,
22
+ "methods": {
23
+ "image": {
24
+ "inputs": [
25
+ {
26
+ "shape": [
27
+ 1,
28
+ 3,
29
+ 224,
30
+ 224
31
+ ],
32
+ "dtype": "float32"
33
+ }
34
+ ],
35
+ "outputs": [
36
+ {
37
+ "shape": [
38
+ 1,
39
+ 512
40
+ ],
41
+ "dtype": "float32"
42
+ }
43
+ ]
44
+ },
45
+ "text": {
46
+ "inputs": [
47
+ {
48
+ "shape": [
49
+ 1,
50
+ 74
51
+ ],
52
+ "dtype": "int64"
53
+ },
54
+ {
55
+ "shape": [
56
+ 1,
57
+ 74
58
+ ],
59
+ "dtype": "int64"
60
+ }
61
+ ],
62
+ "outputs": [
63
+ {
64
+ "shape": [
65
+ 1,
66
+ 512
67
+ ],
68
+ "dtype": "float32"
69
+ }
70
+ ]
71
+ }
72
+ }
73
+ }
74
+ ]
75
+ }