mirror of
https://github.com/PaddlePaddle/FastDeploy.git
synced 2025-10-05 08:37:06 +08:00
polish code with new pre-commit rule (#2923)
This commit is contained in:
@@ -14,55 +14,50 @@
|
||||
# limitations under the License.
|
||||
"""
|
||||
|
||||
import time
|
||||
import os
|
||||
import multiprocessing
|
||||
import os
|
||||
import time
|
||||
|
||||
from fastdeploy.entrypoints.llm import LLM
|
||||
from fastdeploy.engine.sampling_params import SamplingParams
|
||||
|
||||
|
||||
|
||||
model_name_or_path = "baidu/ERNIE-4.5-21B-A3B-Paddle"
|
||||
|
||||
|
||||
|
||||
def start_decode(model_name_or_path):
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
|
||||
os.environ["FD_LOG_DIR"] = "log_decode"
|
||||
llm_decode = LLM(
|
||||
model=model_name_or_path,
|
||||
tensor_parallel_size=1,
|
||||
model=model_name_or_path,
|
||||
tensor_parallel_size=1,
|
||||
splitwise_role="decode",
|
||||
engine_worker_queue_port=6678,
|
||||
innode_prefill_ports=[6676],
|
||||
cache_queue_port=55668
|
||||
)
|
||||
cache_queue_port=55668,
|
||||
)
|
||||
return llm_decode
|
||||
|
||||
|
||||
def start_prefill(model_name_or_path):
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
|
||||
os.environ["FD_LOG_DIR"] = "log_prefill"
|
||||
llm_prefill = LLM(
|
||||
model=model_name_or_path,
|
||||
tensor_parallel_size=1,
|
||||
LLM(
|
||||
model=model_name_or_path,
|
||||
tensor_parallel_size=1,
|
||||
splitwise_role="prefill",
|
||||
engine_worker_queue_port=6677,
|
||||
engine_worker_queue_port=6677,
|
||||
cache_queue_port=55667,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
prefill = multiprocessing.Process(
|
||||
target=start_prefill,
|
||||
args=(model_name_or_path,)).start()
|
||||
prefill = multiprocessing.Process(target=start_prefill, args=(model_name_or_path,)).start()
|
||||
time.sleep(10)
|
||||
llm_decode = start_decode(model_name_or_path)
|
||||
|
||||
output = llm_decode.generate(prompts=["who are you?", "what can you do?"], use_tqdm=True)
|
||||
print(output)
|
||||
|
||||
decode.join()
|
||||
prefill.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
Reference in New Issue
Block a user