Commit ·
cb73e1a
1
Parent(s): 899a3bd
Add Vulkan fp16 variants (image and text)
Browse filesBoth encoders lower to Vulkan with eager attention, which keeps the graph in
one delegate blob for vision and three for text instead of the 25 and 26 that
sdpa produces. Attention runs rank-3, folding the singleton batch into the
head dim, which is bit-identical and drops a third of the materialised view
dispatches.
The text encoder needs its token embedding kept off texture storage for fp16
to be correct (pytorch/executorch#22333); the vision tower has no token
embedding and is unaffected.
Requires the ExecuTorch 1.4.1 Vulkan runtime from the v0.10.0-libs-1.4.1
artifacts.
.gitattributes
CHANGED
|
@@ -34,3 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
*.pte filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
*.pte filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
|
vulkan/clip_vit_base_patch32_image_vulkan_fp16.pte
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c683ed2d1b14fa197d22f33f29cee685f64b72048f7c0f900fab74cfb9544cfe
|
| 3 |
+
size 175823106
|
vulkan/clip_vit_base_patch32_text_vulkan_fp16.pte
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4f077eb561dc8ea7c72c3f4023190fec2521e132c86b918e7e5e1dd40f4c72f1
|
| 3 |
+
size 126991872
|
vulkan/config.json
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
|
| 3 |
+
"model": "clip_vit_base_patch32",
|
| 4 |
+
"family": "clip",
|
| 5 |
+
"capabilities": [
|
| 6 |
+
"text-embedding",
|
| 7 |
+
"image-embedding"
|
| 8 |
+
],
|
| 9 |
+
"backend": "vulkan",
|
| 10 |
+
"license": "mit",
|
| 11 |
+
"tokenizer": "../tokenizer.json",
|
| 12 |
+
"variants": [
|
| 13 |
+
{
|
| 14 |
+
"file": null,
|
| 15 |
+
"components": {
|
| 16 |
+
"image": "clip_vit_base_patch32_image_vulkan_fp16.pte",
|
| 17 |
+
"text": "clip_vit_base_patch32_text_vulkan_fp16.pte"
|
| 18 |
+
},
|
| 19 |
+
"precision": "fp16",
|
| 20 |
+
"quantized": false,
|
| 21 |
+
"default": true,
|
| 22 |
+
"methods": {
|
| 23 |
+
"image": {
|
| 24 |
+
"inputs": [
|
| 25 |
+
{
|
| 26 |
+
"shape": [
|
| 27 |
+
1,
|
| 28 |
+
3,
|
| 29 |
+
224,
|
| 30 |
+
224
|
| 31 |
+
],
|
| 32 |
+
"dtype": "float32"
|
| 33 |
+
}
|
| 34 |
+
],
|
| 35 |
+
"outputs": [
|
| 36 |
+
{
|
| 37 |
+
"shape": [
|
| 38 |
+
1,
|
| 39 |
+
512
|
| 40 |
+
],
|
| 41 |
+
"dtype": "float32"
|
| 42 |
+
}
|
| 43 |
+
]
|
| 44 |
+
},
|
| 45 |
+
"text": {
|
| 46 |
+
"inputs": [
|
| 47 |
+
{
|
| 48 |
+
"shape": [
|
| 49 |
+
1,
|
| 50 |
+
74
|
| 51 |
+
],
|
| 52 |
+
"dtype": "int64"
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"shape": [
|
| 56 |
+
1,
|
| 57 |
+
74
|
| 58 |
+
],
|
| 59 |
+
"dtype": "int64"
|
| 60 |
+
}
|
| 61 |
+
],
|
| 62 |
+
"outputs": [
|
| 63 |
+
{
|
| 64 |
+
"shape": [
|
| 65 |
+
1,
|
| 66 |
+
512
|
| 67 |
+
],
|
| 68 |
+
"dtype": "float32"
|
| 69 |
+
}
|
| 70 |
+
]
|
| 71 |
+
}
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
]
|
| 75 |
+
}
|