deer-flow/docker/docker-compose-dev.yaml

# DeerFlow Development Environment
#
# Requires Docker Compose >= 2.24. The env_file entries below use the long-form
# path/required syntax so a missing ../.env or ../frontend/.env does not abort
# Compose on Windows ("file not found" / "Le fichier spécifique est
# introuvable"). Clients older than 2.24 do not understand that field and
# reject this file with:
#   validating docker-compose-dev.yaml: services.gateway.env_file.0 must be a string
# make docker-start checks the version and says so; direct `docker compose`
# callers get the message above instead.
#
# Supported entry: from the repository root, run `make docker-start`. That
# wrapper checks the Compose version, creates missing .env files, exports
# DEER_FLOW_ROOT, and invokes Compose from this directory with a relative
# filename.
#
# Running Compose yourself works too, but export DEER_FLOW_ROOT as the absolute
# checkout path first (see CONTRIBUTING.md for the Windows form). Unset, the
# DEER_FLOW_HOST_BASE_DIR and THREADS_HOST_PATH values below render as
# /backend/.deer-flow, so the provisioner bind-mounts the wrong host paths:
#   DEER_FLOW_ROOT="$PWD" docker compose -f docker/docker-compose-dev.yaml up --build
# The -f path resolves against your shell's directory, not against this file:
# from the repository root use docker/docker-compose-dev.yaml; from inside
# docker/ use the bare docker-compose-dev.yaml, as make docker-start does.
#
# Services:
#   - nginx: Reverse proxy (port 2026)
#   - frontend: Frontend Next.js dev server (port 3000)
#   - gateway: Backend Gateway API + agent runtime (port 8001)
#   - redis: Redis Streams backend for cross-worker SSE stream bridge
#   - provisioner (optional): Sandbox provisioner (creates Pods in host Kubernetes)
#
# Prerequisites:
#   - Kubernetes cluster + kubeconfig are only required when using provisioner mode.
#
# Access: http://localhost:2026

services:
  # ── Redis Stream Bridge ────────────────────────────────────────────────
  redis:
    image: redis:7-alpine
    container_name: deer-flow-redis
    command: ["redis-server", "--appendonly", "yes"]
    volumes:
      - redis-data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 10
    networks:
      - deer-flow-dev
    restart: unless-stopped

  # ── Sandbox Provisioner ────────────────────────────────────────────────
  # Manages per-sandbox Pod + Service lifecycle in the host Kubernetes
  # cluster via the K8s API.
  # Backend accesses sandboxes directly via host.docker.internal:{NodePort}.
  provisioner:
    build:
      context: ./provisioner
      dockerfile: Dockerfile
      args:
        APT_MIRROR: ${APT_MIRROR:-}
    container_name: deer-flow-provisioner
    volumes:
      - ~/.kube/config:/root/.kube/config:ro
    environment:
      - K8S_NAMESPACE=deer-flow
      - SANDBOX_IMAGE=enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest
      # Optional lark-cli init image (Pattern A). Empty ⇒ legacy runtime mount.
      # Set to a published tag (e.g. deer-flow/lark-cli-init:v1.0.65) to provision
      # the sandbox lark-cli runtime via an init container + emptyDir.
      - LARK_CLI_INIT_IMAGE=${LARK_CLI_INIT_IMAGE:-}
      # Optional lark-cli broker image (Pattern B, issue #4338). Empty ⇒ broker
      # off. Supersedes LARK_CLI_INIT_IMAGE when both are set.
      - LARK_CLI_BROKER_IMAGE=${LARK_CLI_BROKER_IMAGE:-}
      # Host paths for K8s HostPath volumes (must be absolute paths accessible by K8s node)
      # On Docker Desktop/OrbStack, use your actual host paths like /Users/username/...
      # Set these in your shell before running docker-compose:
      #   export DEER_FLOW_ROOT=/absolute/path/to/deer-flow
      # Deliberately left without a ${DEER_FLOW_ROOT:-...} fallback. The obvious
      # candidate, $PWD, is exported by POSIX shells but not by PowerShell or
      # cmd, so the default would resolve to an empty string on Windows — the
      # platform this variable exists to get right. A wrong-but-plausible host
      # path mounts an empty directory instead of failing, so we require the
      # caller to be explicit; make docker-start sets it for you.
      - THREADS_HOST_PATH=${DEER_FLOW_ROOT}/backend/.deer-flow/threads
      # Per-user data base directory for user-scoped skill mounts
      - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow
      # Production: use PVC instead of hostPath to avoid data loss on node failure.
      # When set, hostPath vars above are ignored for the corresponding volume.
      # USERDATA_PVC_NAME uses subPath (deer-flow/users/{user_id}/threads/{thread_id}/user-data) automatically.
      # - SKILLS_PVC_NAME=deer-flow-skills-pvc
      # - USERDATA_PVC_NAME=deer-flow-userdata-pvc
      - KUBECONFIG_PATH=/root/.kube/config
      - NODE_HOST=host.docker.internal
      # Override K8S API server URL since kubeconfig uses 127.0.0.1
      # which is unreachable from inside the container
      - K8S_API_SERVER=https://host.docker.internal:26443
      # Optional: set PROVISIONER_API_KEY in .env to enable provisioner auth.
      # The same value must be set on the gateway side via config.yaml sandbox.provisioner_api_key.
      - PROVISIONER_API_KEY=${PROVISIONER_API_KEY:-}
    env_file:
      - path: ../.env
        required: false
    extra_hosts:
      - "host.docker.internal:host-gateway"
    networks:
      - deer-flow-dev
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
      interval: 10s
      timeout: 5s
      retries: 6
      start_period: 15s

  # ── Reverse Proxy ──────────────────────────────────────────────────────
  # Routes API traffic to gateway and (optionally) provisioner.
  nginx:
    image: nginx:alpine
    container_name: deer-flow-nginx
    # Loopback-only by default; see the note in docker-compose.yaml. Override
    # with BIND_HOST when you deliberately need the dev stack reachable from
    # another machine.
    ports:
      - "${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026"
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf.template:ro
    command:
      - sh
      - -c
      - |
        set -e
        cp /etc/nginx/nginx.conf.template /etc/nginx/nginx.conf
        test -e /proc/net/if_inet6 || sed -i '/^[[:space:]]*listen[[:space:]]\+\[::\]:2026[[:space:]]/d' /etc/nginx/nginx.conf
        exec nginx -g 'daemon off;'
    depends_on:
      - frontend
      - gateway
    networks:
      - deer-flow-dev
    restart: unless-stopped

  # Frontend - Next.js Development Server
  frontend:
    build:
      context: ../
      dockerfile: frontend/Dockerfile
      target: dev
      args:
        PNPM_STORE_PATH: ${PNPM_STORE_PATH:-/root/.local/share/pnpm/store}
        NPM_REGISTRY: ${NPM_REGISTRY:-}
    container_name: deer-flow-frontend
    command: sh -c "cd frontend && pnpm run dev > /app/logs/frontend.log 2>&1"
    volumes:
      - ../frontend/src:/app/frontend/src
      - ../frontend/public:/app/frontend/public
      - ../frontend/next.config.js:/app/frontend/next.config.js:ro
      - ../logs:/app/logs
      # Mount pnpm store for caching
      - ${PNPM_STORE_PATH:-~/.local/share/pnpm/store}:/root/.local/share/pnpm/store
    working_dir: /app
    environment:
      - NODE_ENV=development
      - WATCHPACK_POLLING=true
      - CI=true
      - DEER_FLOW_INTERNAL_GATEWAY_BASE_URL=http://gateway:8001
      - DEER_FLOW_DEV_ALLOWED_ORIGINS=${DEER_FLOW_DEV_ALLOWED_ORIGINS:-127.0.0.1,::1}
    env_file:
      - path: ../frontend/.env
        required: false
    networks:
      - deer-flow-dev
    restart: unless-stopped

  # Backend - Gateway API
  gateway:
    build:
      context: ../
      dockerfile: backend/Dockerfile
      target: dev
      # cache_from disabled - requires manual setup: mkdir -p /tmp/docker-cache-gateway
      args:
        APT_MIRROR: ${APT_MIRROR:-}
        UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:0.11.1}
        UV_INDEX_URL: ${UV_INDEX_URL:-https://pypi.org/simple}
        NPM_REGISTRY: ${NPM_REGISTRY:-}
        LARK_CLI_NPM_VERSION: ${LARK_CLI_NPM_VERSION:-1.0.65}
    container_name: deer-flow-gateway
    # Startup logic lives in docker/dev-entrypoint.sh — UV_EXTRAS validation,
    # `uv sync --all-packages`, .venv self-heal, and uvicorn handoff. Keeps
    # this file readable and lets the script be linted (shellcheck-clean).
    # See PR #2767 / Issue #2754.
    command: ["sh", "/usr/local/bin/dev-entrypoint.sh"]
    # Docker's 10s default SIGKILLed the 30s memory queue flush
    # (memory.shutdown_flush_timeout_seconds) on every `make docker-restart`;
    # dev-entrypoint.sh bounds uvicorn's graceful shutdown to 10s inside this.
    # Must cover that bound plus the lifespan's worst case (~61s, see the
    # gateway comment in docker-compose.yaml) with slack.
    stop_grace_period: 90s
    volumes:
      # Mount the dev entrypoint as a read-only file so edits to the script
      # take effect on `make docker-restart` without requiring an image rebuild.
      - ./dev-entrypoint.sh:/usr/local/bin/dev-entrypoint.sh:ro
      - ../backend/:/app/backend/
      # Preserve the .venv built during Docker image build — mounting the full backend/
      # directory above would otherwise shadow it with the (empty) host directory.
      - gateway-venv:/app/backend/.venv
      # Mount the project directory instead of its mutable config files individually.
      # Host editors commonly replace files on save; a directory bind keeps those
      # replacements visible inside Docker Desktop/WSL containers.
      - ../:/app/project
      - ../skills:/app/skills
      - ../logs:/app/logs
      # Use a Docker-managed uv cache volume instead of a host bind mount.
      # On macOS/Docker Desktop, uv may fail to create symlinks inside shared
      # host directories, which causes startup-time `uv sync` to crash.
      - gateway-uv-cache:/root/.cache/uv
      # DooD: the host Docker socket is NOT mounted by default. It is added only
      # for aio (pure-DooD) sandbox mode via the opt-in docker-compose.dood.yaml
      # overlay (appended by scripts/docker.sh). See backend/docs/CONFIGURATION.md 
      
      # CLI auth dirs (Claude Code / Codex) are NOT mounted by default: they
      # expose the entire ~/.claude and ~/.codex (history, projects, global
      # config, credentials) into the container. Mount them only when you use
      # the Claude/Codex CLI login as a model provider or ACP agent, via the
      # opt-in docker-compose.cli-auth.yaml overlay. Prefer an env token
      # (CLAUDE_CODE_OAUTH_TOKEN, see .env.example / backend/docs/CONFIGURATION.md).
    working_dir: /app
    environment:
      - CI=true
      - DEER_FLOW_PROJECT_ROOT=/app
      - DEER_FLOW_HOME=/app/backend/.deer-flow
      - DEER_FLOW_CONFIG_PATH=/app/project/config.yaml
      - DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/project/extensions_config.json
      - DEER_FLOW_STREAM_BRIDGE_REDIS_URL=${DEER_FLOW_STREAM_BRIDGE_REDIS_URL:-redis://redis:6379/0}
      - DEER_FLOW_CHANNELS_LANGGRAPH_URL=${DEER_FLOW_CHANNELS_LANGGRAPH_URL:-http://gateway:8001/api}
      - DEER_FLOW_CHANNELS_GATEWAY_URL=${DEER_FLOW_CHANNELS_GATEWAY_URL:-http://gateway:8001}
      - DEER_FLOW_INTERNAL_AUTH_TOKEN=${DEER_FLOW_INTERNAL_AUTH_TOKEN:-}
      - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow
      - DEER_FLOW_SANDBOX_HOST=host.docker.internal
      # The login lockout counts failures per client IP, but every browser
      # request reaches the Gateway from nginx. Trust nginx's X-Real-IP (it
      # overwrites the header on every Gateway route) so one client's wrong
      # passwords cannot lock everyone out. Hostname, re-resolved every 10s.
      # `:-` on purpose: an empty value also means nginx, because every browser
      # request arrives through it, so direct mode would only bring back the
      # shared lockout. Set another list in .env to change whom to trust.
      - AUTH_TRUSTED_PROXIES=${AUTH_TRUSTED_PROXIES:-nginx}
      # Pass PROVISIONER_API_KEY into the gateway container so config.yaml can reference it
      # as sandbox.provisioner_api_key: $PROVISIONER_API_KEY
      - PROVISIONER_API_KEY=${PROVISIONER_API_KEY:-}
      # Proxy values (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) are inherited from ../.env via env_file.
      # Only NO_PROXY is declared here so internal service hostnames are always exempt from the proxy.
      - NO_PROXY=${NO_PROXY:-}${NO_PROXY:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal
      - no_proxy=${no_proxy:-}${no_proxy:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal
    env_file:
      - path: ../.env
        required: false
    extra_hosts:
      # For Linux: map host.docker.internal to host gateway
      - "host.docker.internal:host-gateway"
    depends_on:
      redis:
        condition: service_healthy
    networks:
      - deer-flow-dev
    restart: unless-stopped

volumes:
  # Persist .venv across container restarts so dependencies installed during
  # image build are not shadowed by the host backend/ directory mount.
  gateway-venv:
  gateway-uv-cache:
  redis-data:

networks:
  deer-flow-dev:
    driver: bridge
    ipam:
      config:
        - subnet: 192.168.200.0/24