From 07a82afcae626cf345e3879ba55f0ef2ff33c9cd Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 11 Nov 2025 10:46:35 +0800 Subject: [PATCH] add tie_word_embeddings for lmhead (#4916) --- fastdeploy/model_executor/models/qwen2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fastdeploy/model_executor/models/qwen2.py b/fastdeploy/model_executor/models/qwen2.py index 501d30e38..7dd6dd040 100644 --- a/fastdeploy/model_executor/models/qwen2.py +++ b/fastdeploy/model_executor/models/qwen2.py @@ -311,7 +311,7 @@ class Qwen2ForCausalLM(ModelForCasualLM): self.qwen2 = Qwen2Model(fd_config=fd_config) self.ori_vocab_size = fd_config.model_config.ori_vocab_size - + self.tie_word_embeddings = fd_config.model_config.tie_word_embeddings self.lm_head = ParallelLMHead( fd_config=fd_config, embedding_dim=fd_config.model_config.hidden_size, @@ -376,6 +376,8 @@ class Qwen2ForCausalLM(ModelForCasualLM): weight_loader(param, loaded_weight) model_sublayer_name = re.sub(r"\.(weight)$", "", model_param_name) process_weights_after_loading_fn(model_sublayer_name, param) + if self.tie_word_embeddings: + self.lm_head.load_state_dict({self.lm_head.weight_key: self.qwen2.embed_tokens.embeddings.weight}) @classmethod def name(self):