bash
docker run -d --gpus device=3 \
    -v /data/xiedong/LLaMA-Factory/output/checkpoint-20600:/data/xiedong/LLaMA-Factory/output/checkpoint-20600 \
    -p 8005:8000 \
    --ipc=host \
    dockerpull.org/vllm/vllm-openai:latest \
    --model /data/xiedong/LLaMA-Factory/output/checkpoint-20600 --gpu_memory_utilization=0.4 

离线推理：

python
from transformers import AutoTokenizer
from vllm import LLM, SamplingParams

# 定义模型的路径
model_path = r"/data/xiedong/LLaMA-Factory/output/checkpoint-20600"

# 初始化分词器
tokenizer = AutoTokenizer.from_pretrained(model_path)

# 设置模型的默认解码超参数
sampling_params = SamplingParams(
    temperature=0.3,
    top_p=0.3,
    repetition_penalty=1.05,
    max_tokens=512
)

# 初始化LLM，传入模型路径
llm = LLM(model=model_path,
          enforce_eager=True,
          )

# 准备聊天消息
messages = [
    {"role": "system", "content": "你是Prompt提示词生成器，擅长将用户提供的提示词优化为更合适的表达。你只能输出英文。"},
    {"role": "user", "content": "现在请你开始优化提示词，用户输入为：一个女人，动漫风格"}
]

# 对聊天消息进行分词
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)

# 使用LLM生成输出
outputs = llm.generate([text], sampling_params)

# 打印生成的输出
for output in outputs:
    prompt = output.prompt
    generated_text = output.outputs[0].text
    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

api访问：

python
from openai import OpenAI

client = OpenAI(
    api_key="xxx",
    base_url="http://10.136.8.66:8005/v1/"
)

chat_completion = client.chat.completions.create(
    messages=[
        {"role": "system",
         "content": "你是Prompt提示词生成器，擅长将用户提供的提示词优化为更合适的表达。你只能输出英文。"},
        {"role": "user", "content": "现在请你开始优化提示词，用户输入为：一个女人，动漫风格"}
    ],
    model="/data/xiedong/LLaMA-Factory/output/checkpoint-20600",
)
print(chat_completion)

generated_text = chat_completion.choices[0].message.content
print(generated_text)