simcop2387/AutoGPTQ
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity

Update example script to include desc_act

Browse source 
It will help with people unwittingly making incompatible models.
This commit is contained in:
Forkoz 2023-05-15 11:26:22 +00:00 • committed by GitHub
parent 570867c109
commit eaac7a7b76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
examples/quantization/quant_with_alpaca.py
View file

@ -77,6 +77,7 @@ def main():
parser.add_argument("--quantized_model_dir", type=str, default=None)
parser.add_argument("--bits", type=int, default=4, choices=[2, 3, 4, 8])
parser.add_argument("--group_size", type=int, default=128)
parser.add_argument("--desc_act", action="store_true", help="whether to quantize with desc_act")
parser.add_argument("--num_samples", type=int, default=128, help="how many samples will be used to quantize model")
parser.add_argument("--save_and_reload", action="store_true", help="whether save quantized model to disk and reload back")
parser.add_argument("--fast_tokenizer", action="store_true", help="whether use fast tokenizer")
@ -104,7 +105,7 @@ def main():
)
model = AutoGPTQForCausalLM.from_pretrained(
args.pretrained_model_dir,
quantize_config=BaseQuantizeConfig(bits=args.bits, group_size=args.group_size),
quantize_config=BaseQuantizeConfig(bits=args.bits, group_size=args.group_size, desc_act=args.desc_act),
max_memory=max_memory
)