apps/FastDeploy
1
0
Fork 0
mirror of https://github.com/PaddlePaddle/FastDeploy.git synced 2025-10-06 00:57:33 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix machete pre quant (#4295)

Browse Source
This commit is contained in:
Sunny-bot1
2025-09-28 16:11:09 +08:00
committed by GitHub
parent 7b6cb72ab2
commit aa1cc09c5b
2 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fastdeploy/model_executor/layers/linear.py
View File

@@ -129,6 +129,7 @@ class LinearBase(nn.Layer):
self.with_bias = with_bias self.with_bias = with_bias
self.add_bias = add_bias self.add_bias = add_bias
self.prefix = prefix self.prefix = prefix
self.is_quantized = fd_config.model_config.is_quantized
# key # key
if weight_key: if weight_key:
self.weight_key = f"{prefix}.{weight_key}" self.weight_key = f"{prefix}.{weight_key}"

5
fastdeploy/model_executor/layers/quantization/weight_only.py
View File

@@ -20,6 +20,7 @@ from typing import Optional
import paddle import paddle
from paddle.nn.quant import weight_quantize from paddle.nn.quant import weight_quantize
from paddleformers.utils.log import logger
from fastdeploy import envs from fastdeploy import envs
from fastdeploy.model_executor.layers.linear import ( from fastdeploy.model_executor.layers.linear import (
@@ -159,9 +160,11 @@ class WeightOnlyConfig(QuantConfigBase):
if ( if (
_ENABLE_MACHETE _ENABLE_MACHETE
and envs.FD_USE_MACHETE == "1" and envs.FD_USE_MACHETE == "1"
and not layer.is_quantized
and layer.weight_shape[1] and layer.weight_shape[1]
and layer.weight_shape[1] % 128 == 0 and layer.weight_shape[1] % 128 == 0
): ):
logger.info("Using Machete kernel for WeightOnlyLinearMethod")
return MacheteWeightOnlyLinearMethod(self) return MacheteWeightOnlyLinearMethod(self)
return GPUWeightOnlyLinearMethod(self) return GPUWeightOnlyLinearMethod(self)
@@ -399,7 +402,7 @@ class MacheteWeightOnlyLinearMethod(WeightOnlyLinearMethod):
super().__init__(quant_config) super().__init__(quant_config)
def process_prequanted_weights(self, layer, state_dict) -> None: def process_prequanted_weights(self, layer, state_dict) -> None:
pass raise NotImplementedError("Machete kernel doesn't support prequant. Please set FD_USE_MACHETE to 0.")
def process_loaded_weights(self, layer, weight) -> None: def process_loaded_weights(self, layer, weight) -> None:
from fastdeploy.model_executor.layers.quantization.ops import ( from fastdeploy.model_executor.layers.quantization.ops import (