온라인 서빙

온라인 서빙 (Online Serving)

examples/basic/online_serving 디렉토리는 vllm serve로 띄운 vLLM API 서버를 OpenAI 클라이언트로 호출하는 예제들을 담고 있습니다. 채팅 완성(chat completion)·완성(completion) 클라이언트와 워터마크 감지(watermark detection) 서버 예제가 포함되어 있습니다.

출처: 문서

본문

소스: https://github.com/vllm-project/vllm/tree/main/examples/basic/online_serving

OpenAI 채팅 완성 클라이언트 (OpenAI Chat Completion Client)

vllm serve로 채팅 완성 모델 서버를 띄운 뒤 OpenAI 클라이언트로 /v1/chat/completions를 호출하는 최소 예제입니다. 스트리밍 여부도 --stream으로 조절할 수 있습니다.

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Example Python client for OpenAI Chat Completion using vLLM API server
NOTE: start a supported chat completion model server with `vllm serve`, e.g.
    vllm serve meta-llama/Llama-2-7b-chat-hf
"""

import argparse

from openai import OpenAI

# Modify OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Who won the world series in 2020?"},
    {
        "role": "assistant",
        "content": "The Los Angeles Dodgers won the World Series in 2020.",
    },
    {"role": "user", "content": "Where was it played?"},
]

def parse_args():
    parser = argparse.ArgumentParser(description="Client for vLLM API server")
    parser.add_argument(
        "--stream", action="store_true", help="Enable streaming response"
    )
    return parser.parse_args()

def main(args):
    client = OpenAI(
        # defaults to os.environ.get("OPENAI_API_KEY")
        api_key=openai_api_key,
        base_url=openai_api_base,
    )

    models = client.models.list()
    model = models.data[0].id

    # Chat Completion API
    chat_completion = client.chat.completions.create(
        messages=messages,
        model=model,
        stream=args.stream,
    )

    print("-" * 50)
    print("Chat completion results:")
    if args.stream:
        for c in chat_completion:
            print(c)
    else:
        print(chat_completion)
    print("-" * 50)

if __name__ == "__main__":
    args = parse_args()
    main(args)

OpenAI 완성 클라이언트 (OpenAI Completion Client)

/v1/completions를 호출하는 최소 예제입니다. 프롬프트·n(생성 수)·logprobs·스트리밍 등을 지정해 결과를 받습니다.

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import argparse

from openai import OpenAI

# Modify OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://localhost:8000/v1"

def parse_args():
    parser = argparse.ArgumentParser(description="Client for vLLM API server")
    parser.add_argument(
        "--stream", action="store_true", help="Enable streaming response"
    )
    return parser.parse_args()

def main(args):
    client = OpenAI(
        # defaults to os.environ.get("OPENAI_API_KEY")
        api_key=openai_api_key,
        base_url=openai_api_base,
    )

    models = client.models.list()
    model = models.data[0].id

    # Completion API
    completion = client.completions.create(
        model=model,
        prompt="A robot may not injure a human being",
        echo=False,
        n=2,
        stream=args.stream,
        logprobs=3,
    )

    print("-" * 50)
    print("Completion results:")
    if args.stream:
        for c in completion:
            print(c)
    else:
        print(completion)
    print("-" * 50)

if __name__ == "__main__":
    args = parse_args()
    main(args)

워터마크 감지 서버 (Watermark Detection Server)

Gumbel 워터마크 감지기를 위한 최소 참조 서버입니다. 텍스트를 받아 워터마크 점수(score)·p-value·스코어링된 토큰 수·워터마크 여부를 반환합니다.

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""Minimal reference server for watermark detection."""

import argparse

import uvicorn
from fastapi import FastAPI
from pydantic import BaseModel

from vllm.tokenizers import TokenizerLike, cached_get_tokenizer
from vllm.v1.watermarking import GumbelWatermarkDetector

app = FastAPI()
tokenizer: TokenizerLike | None = None
detector: GumbelWatermarkDetector | None = None

class DetectionRequest(BaseModel):
    text: str

class DetectionResponse(BaseModel):
    score: float
    p_value: float
    num_scored_tokens: int
    is_watermarked: bool

@app.post("/detect")
def detect(request: DetectionRequest) -> DetectionResponse:
    assert tokenizer is not None
    assert detector is not None
    token_ids = tokenizer.encode(request.text, add_special_tokens=False)
    result = detector.detect(token_ids)
    return DetectionResponse(
        score=result.score,
        p_value=result.p_value,
        num_scored_tokens=result.num_scored_tokens,
        is_watermarked=result.is_watermarked,
    )

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--tokenizer", required=True)
    parser.add_argument("--key", required=True, type=int)
    parser.add_argument("--prf", choices=("philox",), default="philox")
    parser.add_argument("--context-width", type=int, default=4)
    parser.add_argument("--p-value-threshold", type=float, default=0.01)
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--port", type=int, default=8000)
    return parser.parse_args()

def main(args: argparse.Namespace) -> None:
    global tokenizer, detector
    tokenizer = cached_get_tokenizer(args.tokenizer)
    detector = GumbelWatermarkDetector(
        key=args.key,
        context_width=args.context_width,
        p_value_threshold=args.p_value_threshold,
        prf=args.prf,
    )
    uvicorn.run(app, host=args.host, port=args.port)

if __name__ == "__main__":
    main(parse_args())

더 알아보기 (Learn more)