| |
|
|
| from pathlib import Path |
|
|
| import alpaca_lora_4bit.autograd_4bit as autograd_4bit |
| from alpaca_lora_4bit.amp_wrapper import AMPWrapper |
| from alpaca_lora_4bit.autograd_4bit import ( |
| Autograd4bitQuantLinear, |
| load_llama_model_4bit_low_ram |
| ) |
| from alpaca_lora_4bit.models import Linear4bitLt |
| from alpaca_lora_4bit.monkeypatch.peft_tuners_lora_monkey_patch import ( |
| replace_peft_model_with_int4_lora_model |
| ) |
|
|
| from modules import shared |
| from modules.GPTQ_loader import find_quantized_model_file |
|
|
| replace_peft_model_with_int4_lora_model() |
|
|
|
|
| def load_model_llama(model_name): |
| config_path = str(Path(f'{shared.args.model_dir}/{model_name}')) |
| model_path = str(find_quantized_model_file(model_name)) |
| model, tokenizer = load_llama_model_4bit_low_ram(config_path, model_path, groupsize=shared.args.groupsize, is_v1_model=False) |
| for _, m in model.named_modules(): |
| if isinstance(m, Autograd4bitQuantLinear) or isinstance(m, Linear4bitLt): |
| if m.is_v1_model: |
| m.zeros = m.zeros.half() |
| m.scales = m.scales.half() |
| m.bias = m.bias.half() |
|
|
| autograd_4bit.auto_switch = True |
|
|
| model.half() |
| wrapper = AMPWrapper(model) |
| wrapper.apply_generate() |
|
|
| return model, tokenizer |
|
|
