LiteLLM

An AI gateway that routes OpenAI-compatible requests to upstream model providers through one unified endpoint.

LLM InferenceAdvanced

Versionlatest

LicenseMIT License

Python3.11

EnvironmentPython (uv)

InterfaceAPI

GPUOptional

One-Click Install on Laboratory OS

GitHub

Securely deploy LiteLLM on Laboratory OS.

LiteLLM

Laboratory OS Installation Details

Environment

Working Directory/workspace/litellm

Python3.11

Env Manageruv

Environment Variables

APP_DIRECTORY

/workspace/litellm

OPENLAB_LITELLM_PORT

OPENLAB_LITELLM_CONFIG_PATH

${APP_DIRECTORY}/config.yaml

OPENLAB_LITELLM_MODEL_NAMEModel alias exposed by LiteLLM on /v1/models.

my-custom-model

OPENLAB_LITELLM_UPSTREAM_MODELProvider/model string for the OpenAI-compatible backend.

openai/nvidia/llama-3.2-nv-embedqa-1b-v2

OPENLAB_LITELLM_API_BASEBase URL for the upstream OpenAI-compatible API.

http://my-service.svc.cluster.local:8000/v1

OPENLAB_LITELLM_API_KEYAPI key used by LiteLLM when forwarding requests upstream.

sk-1234

Install & Run

install-and-run.sh

#!/bin/bash
set -e

export PATH="/opt/conda/bin:/root/.local/bin:$PATH"

# Create the directory if it doesn't exist
mkdir -p "/workspace/litellm"
  
# Change to the app directory
cd "/workspace/litellm"

# Install the requested Python version and create a virtual environment when needed
uv python install 3.11
if [ ! -d ".venv" ]; then
  uv venv --seed --python 3.11 .venv
else
  echo "Virtual environment already exists at /workspace/litellm/.venv"
fi

# Activate the virtual environment
source .venv/bin/activate


# Environment setup complete
echo "Environment setup complete!"

# Non-interactive install of LiteLLM proxy dependencies.
uv pip install --upgrade "litellm[proxy]"

mkdir -p "$APP_DIRECTORY" "$(dirname "$OPENLAB_LITELLM_CONFIG_PATH")"

if [[ "$OPENLAB_LITELLM_UPSTREAM_MODEL" == */* ]]; then
  OPENLAB_LITELLM_EFFECTIVE_UPSTREAM_MODEL="$OPENLAB_LITELLM_UPSTREAM_MODEL"
else
  OPENLAB_LITELLM_EFFECTIVE_UPSTREAM_MODEL="openai/$OPENLAB_LITELLM_UPSTREAM_MODEL"
fi

cat > "$OPENLAB_LITELLM_CONFIG_PATH" << MODELS_EOF
model_list:
  - model_name: $OPENLAB_LITELLM_MODEL_NAME
    litellm_params:
      model: $OPENLAB_LITELLM_EFFECTIVE_UPSTREAM_MODEL
      api_base: $OPENLAB_LITELLM_API_BASE
      api_key: "$OPENLAB_LITELLM_API_KEY"
MODELS_EOF

exec litellm --config "$OPENLAB_LITELLM_CONFIG_PATH" --host 0.0.0.0 --port "$OPENLAB_LITELLM_PORT" $OPENLAB_EXTRA_ARGS