mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-05 08:37:06 +08:00
polish code with new pre-commit rule (#2923)
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import list
|
||||
@@ -25,6 +26,7 @@ class KVCacheSpec:
|
||||
"""
|
||||
A base class for specifying the KV cache format of one layer.
|
||||
"""
|
||||
|
||||
# number of tokens in a block
|
||||
block_size: int
|
||||
# the memory size used by each block in bytes.
|
||||
@@ -37,10 +39,9 @@ class KVCacheSpec:
|
||||
"""
|
||||
# check list
|
||||
assert all(
|
||||
(spec.block_size == specs[0].block_size
|
||||
and spec.block_memory_used == specs[0].block_memory_used)
|
||||
for spec in specs[1:]), (
|
||||
"All layers in the model must share the same block_size.")
|
||||
(spec.block_size == specs[0].block_size and spec.block_memory_used == specs[0].block_memory_used)
|
||||
for spec in specs[1:]
|
||||
), "All layers in the model must share the same block_size."
|
||||
|
||||
return copy.deepcopy(specs[0])
|
||||
|
||||
@@ -48,6 +49,7 @@ class KVCacheSpec:
|
||||
@dataclass
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
""" """
|
||||
|
||||
num_kv_heads: int
|
||||
head_size: int
|
||||
dtype: str
|
||||
|
Reference in New Issue
Block a user