Hanzo vLLM

vllm/
  Cargo.toml              # candle-vllm v0.2.1, feature flags
  build.rs
  LICENSE                 # MIT
  README.md
  README-CN.md
  src/
    main.rs               # CLI entry, Axum server setup
    lib.rs                 # Core library exports
    openai/                # OpenAI-compatible API layer
      openai_server.rs     # Axum routes (/v1/chat/completions, /v1/models)
      communicator.rs      # Request/response coordination
      distributed.rs       # Multi-GPU/multi-node communication
      streaming.rs         # SSE streaming responses
      requests.rs          # Request types
      responses.rs         # Response types
      sampling_params.rs   # Temperature, top-k, top-p, penalties
      logits_processor.rs  # Token sampling logic
      utils.rs
      conversation/        # Chat template handling
      models/              # Model implementations (17 files)
        llama.rs, mistral.rs, phi2.rs, phi3.rs
        qwen.rs, yi.rs, stable_lm.rs, gemma.rs, gemma3.rs
        glm4.rs, deepseek.rs
        quantized_llama.rs, quantized_phi3.rs
        quantized_qwen.rs, quantized_glm4.rs
        linear.rs          # Shared linear layer (Marlin/GPTQ)
        mod.rs             # Model registry and loading
      pipelines/           # Inference pipeline
        pipeline.rs        # Model loading, weight management
        llm_engine.rs      # Continuous batching engine
        mod.rs
    backend/               # Low-level inference backend
      paged_attention.rs   # PagedAttention implementation
      cache.rs             # KV cache management
      gguf.rs              # GGUF format loader
      gptq.rs              # GPTQ format loader
      heartbeat.rs         # Health monitoring
      progress.rs          # Loading progress bars
      custom_ops/          # Custom operations
      mod.rs
    paged_attention/       # PagedAttention core algorithms
    scheduler/             # Request scheduling (continuous batching)
  kernels/                 # CUDA kernels
    Cargo.toml
    build.rs               # CUDA kernel compilation
    src/
  metal-kernels/           # Metal kernels (Apple Silicon)
    Cargo.toml
    src/
  examples/
    chat.py                # Python chat client
    benchmark.py           # Batch throughput benchmark
    convert_marlin.py      # GPTQ to Marlin conversion
    convert_awq_marlin.py  # AWQ to Marlin conversion
    llama.py               # Simple LLAMA example
  tests/

# Clone
git clone git@github.com:hanzoai/vllm.git
cd vllm

# Apple Silicon (Metal)
cargo build --release --features metal

# CUDA single-node (single or multi-GPU)
export PATH=$PATH:/usr/local/cuda/bin/
cargo build --release --features cuda,nccl

# CUDA with flash attention (faster for long context)
cargo build --release --features cuda,nccl,flash-attn

# CUDA multi-node (MPI)
sudo apt install libopenmpi-dev openmpi-bin clang libclang-dev -y
cargo build --release --features cuda,nccl,mpi

# From local path
target/release/candle-vllm --port 2000 \
  --weight-path /home/DeepSeek-R1-Distill-Llama-8B/ llama3 \
  --temperature 0. --penalty 1.0

# From HuggingFace
target/release/candle-vllm \
  --model-id deepseek-ai/DeepSeek-R1-0528-Qwen3-8B qwen3

# Apple Silicon
cargo run --release --features metal -- --port 2000 --dtype bf16 \
  --weight-file /home/qwq-32b-q4_k_m.gguf qwen2 \
  --quant gguf --temperature 0. --penalty 1.0

# From HuggingFace
target/release/candle-vllm \
  --model-id unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF \
  --weight-file DeepSeek-R1-0528-Qwen3-8B-Q2_K.gguf qwen3 --quant gguf

# Load unquantized model as quantized
target/release/candle-vllm --port 2000 \
  --weight-path /home/Meta-Llama-3.1-8B-Instruct/ llama3 --quant q4k

target/release/candle-vllm --dtype bf16 --port 2000 \
  --weight-path /home/model-GPTQ-4bit qwen2 \
  --quant gptq --temperature 0. --penalty 1.0

# Multi-process mode (recommended)
cargo run --release --features cuda,nccl -- \
  --multi-process --dtype bf16 --port 2000 \
  --device-ids "0,1" --weight-path /home/QwQ-32B/ qwen2 \
  --penalty 1.0 --temperature 0.

# GPU count must be power of 2 (2, 4, 8)

# Convert AWQ to Marlin format
python3 examples/convert_awq_marlin.py \
  --src /data/DeepSeek-R1-AWQ/ --dst /data/DeepSeek-R1-AWQ-Marlin/

# Run on 8x A100 with 120/256 experts offloaded to CPU
cargo run --release --features cuda,nccl -- \
  --log --multi-process --dtype bf16 --port 2000 \
  --device-ids "0,1,2,3,4,5,6,7" \
  --weight-path /data/DeepSeek-R1-AWQ-Marlin/ deep-seek \
  --quant awq --temperature 0. --penalty 1.0 \
  --num-experts-offload-per-rank 15

curl -X POST "http://127.0.0.1:2000/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -d '{
    "model": "llama",
    "messages": [{"role": "user", "content": "Hello"}],
    "temperature": 0.7,
    "max_tokens": 128
  }'

import openai
openai.api_key = "EMPTY"
openai.base_url = "http://localhost:2000/v1/"

completion = openai.chat.completions.create(
    model="llama",
    messages=[{"role": "user", "content": "Explain Rust."}],
    max_tokens=64,
)
print(completion.choices[0].message.content)

pip install openai rich click
python3 examples/chat.py                           # Plain text
python3 examples/chat.py --thinking True            # Reasoning models
python3 examples/chat.py --live                     # Markdown rendering

python3 examples/benchmark.py --batch 16 --max_tokens 1024