Update tiny models for VibeVoiceForConditionalGeneration

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,51 @@
+{%- set system_prompt = system_prompt | default(" Transform the text provided by various speakers into speech output, utilizing the distinct voice of each respective speaker.
+") -%}
+{{ system_prompt -}}
+{%- set audio_bos_token = audio_bos_token | default("<|vision_start|>") %}
+{%- set audio_eos_token = audio_eos_token | default("<|vision_end|>") %}
+{%- set audio_token = audio_token | default("<|vision_pad|>") %}
+{%- set eos_token = eos_token | default("<|endoftext|>") %}
+{#- Training regime (`add_generation_prompt=False`): the last audio of the conversation is the target speech and is
+    placed after "Speech output:", followed by the audio EOS and text EOS tokens; any preceding audio is used as a
+    voice prompt. Generation regime (`add_generation_prompt=True`): all audios are voice prompts. #}
+{%- set ns = namespace(num_audio=0, num_seen=0, speakers_with_audio="") %}
+{%- for message in messages %}
+    {%- set ns.num_audio = ns.num_audio + (message['content'] | selectattr('type', 'equalto', 'audio') | list | length) %}
+{%- endfor %}
+{%- set has_target_audio = not add_generation_prompt and ns.num_audio > 0 %}
+{%- set num_voice_prompts = ns.num_audio - 1 if has_target_audio else ns.num_audio %}
+{%- for message in messages %}
+    {%- set role = message['role'] %}
+    {%- set audio_count = message['content'] | selectattr('type', 'equalto', 'audio') | list | length %}
+    {%- if audio_count > 0 and ns.num_seen < num_voice_prompts and role not in ns.speakers_with_audio %}
+        {%- set ns.speakers_with_audio = ns.speakers_with_audio + role + "," %}
+    {%- endif %}
+    {%- set ns.num_seen = ns.num_seen + audio_count %}
+{%- endfor %}
+
+{%- if ns.speakers_with_audio %}
+{{ " Voice input:
+" }}
+{%- for speaker in ns.speakers_with_audio.rstrip(',').split(',') %}
+{%- if speaker %}
+ Speaker {{ speaker }}:{{ audio_bos_token }}{{ audio_token }}{{ audio_eos_token }}{{ "
+" }}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+ Text input:{{ "
+" }}
+
+{%- for message in messages %}
+    {%- set role = message['role'] %}
+    {%- set text_items = message['content'] | selectattr('type', 'equalto', 'text') | list %}
+    {%- for item in text_items %}
+ Speaker {{ role }}: {{ item['text'] }}{{ "
+" }}
+    {%- endfor %}
+{%- endfor %}
+ Speech output:{{ "
+" }}{{ audio_bos_token }}
+{%- if not add_generation_prompt %}
+{%- if has_target_audio %}{{ audio_token }}{{ audio_eos_token }}{% endif %}{{ eos_token }}
+{%- endif %}

config.json

@@ -0,0 +1,100 @@
+{
+  "architectures": [
+    "VibeVoiceForConditionalGeneration"
+  ],
+  "audio_bos_token_id": 3,
+  "audio_config": {
+    "channels": 1,
+    "depths": [
+      1,
+      1
+    ],
+    "downsampling_ratios": [
+      2
+    ],
+    "ffn_expansion": 4,
+    "hidden_act": "gelu",
+    "hidden_size": 16,
+    "initializer_range": 0.01,
+    "kernel_size": 3,
+    "layer_scale_init_value": 1e-06,
+    "model_type": "vibevoice_acoustic_tokenizer",
+    "num_filters": 4,
+    "rms_norm_eps": 1e-05,
+    "vae_std": 0.625
+  },
+  "audio_eos_token_id": 4,
+  "audio_token_id": 5,
+  "diffusion_head_config": {
+    "diffusion_max_period": 10000,
+    "frequency_embedding_size": 8,
+    "hidden_act": "silu",
+    "hidden_size": 32,
+    "intermediate_size": 16,
+    "latent_size": 16,
+    "mlp_bias": false,
+    "model_type": "",
+    "num_hidden_layers": 2,
+    "rms_norm_eps": 1e-05
+  },
+  "diffusion_loss_weight": 0.5,
+  "dtype": "float32",
+  "eos_token_id": 151643,
+  "model_type": "vibevoice",
+  "pad_token_id": 151643,
+  "semantic_model_config": {
+    "channels": 1,
+    "depths": [
+      1,
+      1
+    ],
+    "downsampling_ratios": [
+      2
+    ],
+    "ffn_expansion": 4,
+    "hidden_act": "gelu",
+    "hidden_size": 32,
+    "initializer_range": 0.01,
+    "kernel_size": 3,
+    "layer_scale_init_value": 1e-06,
+    "model_type": "vibevoice_acoustic_tokenizer_encoder",
+    "num_filters": 4,
+    "rms_norm_eps": 1e-05,
+    "vae_std": 0.625
+  },
+  "text_config": {
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "eos_token_id": 151643,
+    "hidden_act": "silu",
+    "hidden_size": 32,
+    "initializer_range": 0.02,
+    "intermediate_size": 36,
+    "layer_types": [
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 200,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 4,
+    "num_hidden_layers": 2,
+    "num_key_value_heads": 4,
+    "pad_token_id": 151643,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    },
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "use_labels": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 151665
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.19.0.dev0",
+  "vocab_size": 151665
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 151643,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 151643,
+  "transformers_version": "5.19.0.dev0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3828af79c0196e3704d22240cc0ca9de7126426f12ee07bcfaab6d1b2d865247
+size 38995604

preprocessor_config.json

@@ -0,0 +1,12 @@
+{
+  "eps": 1e-06,
+  "feature_extractor_type": "VibeVoiceAcousticTokenizerFeatureExtractor",
+  "feature_size": 1,
+  "normalize_audio": true,
+  "pad_to_multiple_of": 3200,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 24000,
+  "target_dB_FS": -25
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding": true,
+  "padding_side": "left",
+  "processor_class": "VibeVoiceProcessor",
+  "return_tensors": "pt",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}