[Optimize] Support WINT8 and group scale for Machete (#3905)

fastdeploy/model_executor/layers/quantization/ops/machete_mm.py

@@ -85,7 +85,7 @@ def quantize_weights(
         w_s: Scales (None if `group_size` is None).
     """
     assert paddle.is_floating_point(w), "w must be float type"
-    assert quant_type in ["uint4", "uint4b8"], "only support quant_type = uint4, uint4b8"
+    assert quant_type in ["uint4b8", "uint8b128"], "only support quant_type = uint4b8, uint8b128"
 
     orig_device = w.place
     size_k, size_n = w.shape
@@ -103,8 +103,12 @@ def quantize_weights(
     max_val = paddle.max(w, axis=0, keepdim=True)
     min_val = paddle.min(w, axis=0, keepdim=True)
 
-    max_q_val = float(7.0)
-    min_q_val = float(-8.0)
+    if quant_type == "uint4b8":
+        max_q_val = float(7.0)
+        min_q_val = float(-8.0)
+    else:
+        max_q_val = float(127.0)
+        min_q_val = float(-128.0)
 
     w_s = paddle.ones([1], dtype=paddle.float32)  # unscaled case
 
@@ -124,6 +128,8 @@ def quantize_weights(
     # w_q += quant_type.bias
     if quant_type == "uint4b8":
         w_q += 8
+    else:
+        w_q += 128
 
     # Restore original shapes
     if group_size is not None and group_size < size_k:
@@ -131,11 +137,11 @@ def quantize_weights(
         def reshape_w(w_tensor):
             w_tensor = w_tensor.reshape([group_size, -1, size_n])
             w_tensor = w_tensor.transpose([1, 0, 2])
-            w_tensor = w_tensor.reshape([size_k, size_n])
+            w_tensor = w_tensor.reshape([size_k, size_n]).contiguous()
             return w_tensor
 
         w_q = reshape_w(w_q)
-        w_s = w_s.reshape([-1, size_n])
+        w_s = w_s.reshape([-1, size_n]).contiguous()
 
     # Move tensors back to original device
     w_q = w_q.to(orig_device)
@@ -153,7 +159,8 @@ def machete_quantize_and_pack(
     group_size: int = -1,
 ):
     w_q, w_s = quantize_weights(w, group_size, quant_type=quant_type)
-    w_q = pack_rows(w_q, 4, *w_q.shape)
+    num_bits = 4 if quant_type == "uint4b8" else 8
+    w_q = pack_rows(w_q, num_bits, *w_q.shape)
     w_q_col = w_q.transpose([1, 0]).contiguous()  # convert to col major
     w_q_prepack = machete_prepack_B(
         w_q_col,

fastdeploy/model_executor/layers/quantization/weight_only.py

@@ -142,11 +142,11 @@ class WeightOnlyConfig(QuantConfigBase):
                 )
 
                 if (
-                    self.name() == "wint4"
-                    and _ENABLE_MACHETE
+                    _ENABLE_MACHETE
                     and envs.FD_USE_MACHETE == "1"
                     and layer.weight_shape[1]
                     and layer.weight_shape[1] % 128 == 0
+                    and not layer.add_bias
                 ):
                     return MacheteWeightOnlyLinearMethod(self)
                 return GPUWeightOnlyLinearMethod(self)
@@ -230,6 +230,8 @@ class WeightOnlyLinearMethod(QuantMethodBase):
                 weight_scale_shape = [1, layer.weight_shape[1]]
                 if self.quant_config.name() == "wint4":
                     layer.weight_shape[0] //= 8
+                else:
+                    layer.weight_shape[0] //= 4
                 layer.weight_dtype = "int32"
             else:
                 # The scale shape should be equal to the output dim of weight using Per-Channel Quantization.
@@ -282,7 +284,7 @@ class WeightOnlyLinearMethod(QuantMethodBase):
             quanted_weight_tensor, weight_scale_tensor = machete_quantize_and_pack(
                 w=layer.weight,
                 atype=layer._dtype,
-                quant_type="uint4b8",
+                quant_type="uint4b8" if self.quant_config.name() == "wint4" else "uint8b128",
             )
         else:
             quanted_weight_tensor, weight_scale_tensor = weight_quantize(
@@ -387,7 +389,7 @@ class MacheteWeightOnlyLinearMethod(WeightOnlyLinearMethod):
         quanted_weight_tensor, weight_scale_tensor = machete_quantize_and_pack(
             w=weight,
             atype=layer._dtype,
-            quant_type="uint4b8",
+            quant_type="uint4b8" if self.quant_config.name() == "wint4" else "uint8b128",
         )
         layer.weight.set_value(quanted_weight_tensor)
         layer.weight_scale.set_value(weight_scale_tensor.astype(paddle.get_default_dtype()))
@@ -400,7 +402,7 @@ class MacheteWeightOnlyLinearMethod(WeightOnlyLinearMethod):
             x,
             w_prepack=layer.weight,
             w_g_s=layer.weight_scale,
-            weight_dtype="uint4b8",
+            weight_dtype="uint4b8" if self.quant_config.name() == "wint4" else "uint8b128",
         )
 
         return linear_out