update another example
This commit is contained in:
parent
922ec02998
commit
2d0aaa423f
1 changed files with 1 additions and 0 deletions
|
@ -137,6 +137,7 @@ def main():
|
|||
quantize_config = BaseQuantizeConfig(
|
||||
bits=4, # quantize model to 4-bit
|
||||
group_size=128, # it is recommended to set the value to 128
|
||||
desc_act=False, # desc_act and group size only works on triton
|
||||
)
|
||||
|
||||
# load un-quantized model, the model will always be force loaded into cpu
|
||||
|
|
Loading…
Add table
Reference in a new issue