mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
35f6a98cd9
* feat(dev): add a two-Gateway multi-instance harness script There was no way to run the deployment.multi_instance topology locally: scripts/serve.sh pins one Gateway to port 8001, the local nginx config has a single upstream, and getting two processes past the multi-instance startup gate needs Postgres, Redis, a shared DEER_FLOW_HOME, shared auth secrets and a per-process DeerMem retrieval index by hand. scripts/dev_multi_instance.sh (make dev-multi / dev-multi-check / dev-multi-down) starts throwaway Postgres and Redis containers on loopback, renders a config from the developer's config.yaml (or config.example.yaml), and runs Gateway A on 8001 and Gateway B on 8011 with one shared home, one copy of extensions_config.json and generated AUTH_JWT_SECRET, DEER_FLOW_INTERNAL_AUTH_TOKEN and DEER_FLOW_CREDENTIALS_KEY values, plus an optional round-robin nginx on 2027. B starts after A answers /health: Alembic migrations are advisory-locked, but the LangGraph checkpointer/store setup() that follows is not. scripts/dev_multi_instance.py holds the testable parts: the config overlay (neutralizing and reporting settings the gate refuses), the secrets file, the nginx config, and `check`, which automates the scriptable cross-instance acceptance checks: readiness with stream_bridge ok, A-issued sessions and the internal token accepted by B, thread/upload/artifact visibility, a skill toggle seen by B, nginx balancing, and SSE join on B with Last-Event-ID resume on A. Tests pin that the rendered config passes the real _enforce_postgres_for_multi_worker gate. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(changelog): link the two-Gateway harness entries to #6613 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(dev): keep the multi-instance harness inside its own state Address review findings on the two-Gateway harness (#6613): - Containers carry a deerflow.harness.state-dir label; up/down remove only containers of their own state dir and refuse those of another checkout or DEERFLOW_MI_STATE_DIR instead of force-removing them. - Containers are removed with `docker rm -f -v`, so the anonymous volumes the postgres and redis images declare no longer leak on every up/down. - `up` installs an EXIT trap once it starts containers, so any later failure (including B's port being taken after preflight) tears down A and both containers; start_gateway returns failures instead of exiting the script, and the Gateway is disowned so stopping it prints no job notice. - The overlay removes any checkpointer section (a postgres one would write checkpoints and the Store into the developer's database), points an explicit sandbox.ownership and a Redis checkpoint_cache at the harness Redis (keeping ownership timing), drops DeerMem storage_path (both spellings) and the blob_storage root so data lands in the harness home, gives a local AIO sandbox a state-dir-specific container_prefix, and reports a non-DeerMem memory backend it keeps sharing. - nginx paths are rendered as quoted string literals ('$' and control characters are refused), so a state dir with spaces works. - `check` skips the run-based SSE check when model discovery did not succeed, unless --with-llm, and enforces the SSE time budget per raw line so heartbeat-only traffic cannot hold it open. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(scripts): keep the harness guidance under the AGENTS.md soft limit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
243 lines
11 KiB
Makefile
243 lines
11 KiB
Makefile
# DeerFlow - Unified Development Environment
|
|
|
|
.PHONY: help config config-upgrade check check-agent-guidance install extension-install extension-upgrade extension-list extension-enable extension-disable extension-remove setup doctor support-bundle detect-thread-boundaries detect-blocking-io dev dev-daemon dev-multi dev-multi-check dev-multi-down start start-daemon nginx stop up down prod-logs clean docker-init docker-start docker-stop docker-logs docker-logs-frontend docker-logs-gateway docker-logs-redis setup-sandbox
|
|
|
|
BASH ?= bash
|
|
BACKEND_UV_RUN = cd backend && uv run
|
|
|
|
# Detect OS for Windows compatibility
|
|
ifeq ($(OS),Windows_NT)
|
|
SHELL := cmd.exe
|
|
PYTHON ?= python
|
|
# Run repo shell scripts through Git Bash when Make is launched from cmd.exe / PowerShell.
|
|
RUN_SHELL_SCRIPT = call scripts\run-with-git-bash.cmd
|
|
else
|
|
PYTHON ?= python3
|
|
# Invoke repo shell scripts through an explicit interpreter, so recipes keep
|
|
# working in checkouts that lost the executable bit (zip download,
|
|
# core.fileMode=false, non-POSIX filesystem).
|
|
RUN_SHELL_SCRIPT = $(BASH)
|
|
endif
|
|
|
|
FRONTEND_PNPM = $(PYTHON) ../scripts/pnpm.py
|
|
# What `make clean` deletes; shared by its help line and deletion notice.
|
|
RUNTIME_DATA_CONTENTS = database, users, threads, uploads, memory, secrets
|
|
|
|
help:
|
|
@echo "DeerFlow Development Commands:"
|
|
@echo " make setup - Interactive setup wizard (recommended for new users)"
|
|
@echo " make doctor - Check configuration and system requirements"
|
|
@echo " make support-bundle - Create a redacted issue summary, AI draft, and evidence bundle"
|
|
@echo " make config - Generate local config files (aborts if config already exists)"
|
|
@echo " make config-upgrade - Merge new fields from config.example.yaml into config.yaml"
|
|
@echo " make check - Check if all required tools are installed"
|
|
@echo " make check-agent-guidance - Validate scoped AGENTS.md file and chain budgets"
|
|
@echo " make detect-thread-boundaries - Inventory backend executor/thread/event-loop boundaries"
|
|
@echo " make detect-blocking-io - Inventory blocking IO that may block the backend event loop"
|
|
@echo " make install - Install all dependencies (frontend + backend + pre-commit hooks)"
|
|
@echo " make extension-install SOURCE=... - Install and enable a trusted Python extension"
|
|
@echo " make extension-upgrade SOURCE=... - Replace an installed extension and keep its config"
|
|
@echo " make extension-list - List configured Python extensions"
|
|
@echo " make extension-enable NAME=... - Enable an installed extension"
|
|
@echo " make extension-disable NAME=... - Disable an extension without uninstalling it"
|
|
@echo " make extension-remove NAME=... - Uninstall a managed extension"
|
|
@echo " make setup-sandbox - Pre-pull sandbox container image (recommended)"
|
|
@echo " make dev - Start all services in development mode (with hot-reloading)"
|
|
@echo " make dev-daemon - Start dev services in background (daemon mode)"
|
|
@echo " make start - Start all services in production mode (optimized, no hot-reloading)"
|
|
@echo " make start-daemon - Start prod services in background (daemon mode)"
|
|
@echo " make nginx - Start nginx alone in the foreground (local dev config)"
|
|
@echo " make stop - Stop all running services"
|
|
@echo " make dev-multi - Start two Gateways as a local multi-instance cluster (Postgres + Redis in Docker)"
|
|
@echo " make dev-multi-check - Run the cross-instance checks against that pair"
|
|
@echo " make dev-multi-down - Stop the pair and delete its containers and data"
|
|
@echo " make clean - Stop local services and DELETE local runtime data (backend/.deer-flow: $(RUNTIME_DATA_CONTENTS)) and logs"
|
|
@echo ""
|
|
@echo "Docker Production Commands:"
|
|
@echo " make up - Build and start production Docker services (localhost:2026)"
|
|
@echo " make prod-logs - Follow production Docker logs (stack started by 'make up')"
|
|
@echo " make down - Stop and remove production Docker containers"
|
|
@echo ""
|
|
@echo "Docker Development Commands:"
|
|
@echo " make docker-init - Pull the sandbox image"
|
|
@echo " make docker-start - Start Docker services (mode-aware from config.yaml, localhost:2026)"
|
|
@echo " make docker-stop - Stop Docker development services"
|
|
@echo " make docker-logs - View Docker development logs"
|
|
@echo " make docker-logs-frontend - View Docker frontend logs"
|
|
@echo " make docker-logs-gateway - View Docker gateway logs"
|
|
@echo " make docker-logs-redis - View Docker Redis logs"
|
|
|
|
## Setup & Diagnosis
|
|
setup:
|
|
@$(BACKEND_UV_RUN) python ../scripts/setup_wizard.py
|
|
|
|
doctor:
|
|
@$(BACKEND_UV_RUN) python ../scripts/doctor.py
|
|
|
|
support-bundle:
|
|
@$(BACKEND_UV_RUN) python ../scripts/support_bundle.py --include-doctor
|
|
|
|
detect-thread-boundaries:
|
|
@$(BACKEND_UV_RUN) python ../scripts/detect_thread_boundaries.py --json-output ../.deer-flow/thread-boundary-inventory.json
|
|
|
|
detect-blocking-io:
|
|
@$(MAKE) -C backend detect-blocking-io
|
|
|
|
config:
|
|
@$(PYTHON) ./scripts/configure.py
|
|
|
|
config-upgrade:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/config-upgrade.sh
|
|
|
|
# Check required tools
|
|
check:
|
|
@$(PYTHON) ./scripts/check.py
|
|
|
|
check-agent-guidance:
|
|
@$(PYTHON) ./scripts/check_agent_guidance.py
|
|
|
|
# Install all dependencies
|
|
install:
|
|
@echo "Installing backend dependencies..."
|
|
@cd backend && uv sync --locked
|
|
@echo "Installing frontend dependencies..."
|
|
@cd frontend && $(FRONTEND_PNPM) install
|
|
@echo "Installing pre-commit hooks..."
|
|
@uv tool install pre-commit
|
|
@uv tool run pre-commit install --overwrite
|
|
@echo "✓ All dependencies installed"
|
|
@echo ""
|
|
@echo "=========================================="
|
|
@echo " Optional: Pre-pull Sandbox Image"
|
|
@echo "=========================================="
|
|
@echo ""
|
|
@echo "If you plan to use Docker/Container-based sandbox, you can pre-pull the image:"
|
|
@echo " make setup-sandbox"
|
|
@echo ""
|
|
|
|
extension-install: export DEER_FLOW_EXTENSION_SOURCE := $(value SOURCE)
|
|
extension-install:
|
|
$(if $(and $(filter command line,$(origin SOURCE)),$(strip $(value SOURCE))),,$(error usage: make extension-install SOURCE=<package|git-url|dir>))
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions install --source-env __deerflow_extension_source__
|
|
|
|
extension-upgrade: export DEER_FLOW_EXTENSION_SOURCE := $(value SOURCE)
|
|
extension-upgrade:
|
|
$(if $(and $(filter command line,$(origin SOURCE)),$(strip $(value SOURCE))),,$(error usage: make extension-upgrade SOURCE=<package|git-url|dir>))
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions upgrade --source-env __deerflow_extension_source__
|
|
|
|
extension-list:
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions list
|
|
|
|
extension-enable: export DEER_FLOW_EXTENSION_NAME := $(value NAME)
|
|
extension-enable:
|
|
$(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-enable NAME=<extension>))
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions enable --name-env __deerflow_extension_name__
|
|
|
|
extension-disable: export DEER_FLOW_EXTENSION_NAME := $(value NAME)
|
|
extension-disable:
|
|
$(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-disable NAME=<extension>))
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions disable --name-env __deerflow_extension_name__
|
|
|
|
extension-remove: export DEER_FLOW_EXTENSION_NAME := $(value NAME)
|
|
extension-remove:
|
|
$(if $(and $(filter command line,$(origin NAME)),$(strip $(value NAME))),,$(error usage: make extension-remove NAME=<extension>))
|
|
@cd backend && uv run --frozen --no-group extensions deerflow extensions remove --name-env __deerflow_extension_name__
|
|
|
|
# Pre-pull sandbox Docker image (optional but recommended)
|
|
setup-sandbox:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/setup-sandbox.sh
|
|
|
|
# Start all services in development mode (with hot-reloading)
|
|
dev:
|
|
@$(PYTHON) ./scripts/check.py
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev
|
|
|
|
# Start all services in production mode (with optimizations).
|
|
# SKIP_FRONTEND_BUILD=1 reuses the existing frontend build instead of running
|
|
# `next build`; see scripts/serve.sh --skip-frontend-build.
|
|
start:
|
|
@$(PYTHON) ./scripts/check.py
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
|
|
|
# Start all services in daemon mode (background)
|
|
dev-daemon:
|
|
@$(PYTHON) ./scripts/check.py
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev --daemon
|
|
|
|
# Start prod services in daemon mode (background)
|
|
start-daemon:
|
|
@$(PYTHON) ./scripts/check.py
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod --daemon $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
|
|
|
# Start nginx alone in the foreground with the local dev config
|
|
nginx:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/nginx.sh
|
|
|
|
# Stop all services
|
|
stop:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --stop
|
|
|
|
# Two Gateways on shared Postgres, Redis and DEER_FLOW_HOME (scripts/dev_multi_instance.sh --help)
|
|
dev-multi:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/dev_multi_instance.sh up
|
|
|
|
dev-multi-check:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/dev_multi_instance.sh check
|
|
|
|
dev-multi-down:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/dev_multi_instance.sh down
|
|
|
|
# Clean up: deletes local runtime data, not just temporary files. The guard
|
|
# runs before stop, which would otherwise stop a running stack's sandboxes.
|
|
clean:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/check-data-not-in-use.sh
|
|
@$(MAKE) --no-print-directory stop
|
|
@echo "Deleting local runtime data in backend/.deer-flow ($(RUNTIME_DATA_CONTENTS)) and logs/*.log..."
|
|
@-rm -rf backend/.deer-flow 2>/dev/null || true
|
|
@-rm -rf logs/*.log 2>/dev/null || true
|
|
@echo "✓ Cleanup complete"
|
|
|
|
# ==========================================
|
|
# Docker Development Commands
|
|
# ==========================================
|
|
|
|
# Initialize Docker containers and install dependencies
|
|
docker-init:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh init
|
|
|
|
# Start Docker development environment
|
|
docker-start:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh start
|
|
|
|
# Stop Docker development environment
|
|
docker-stop:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh stop
|
|
|
|
# View Docker development logs
|
|
docker-logs:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs
|
|
|
|
# View Docker development logs
|
|
docker-logs-frontend:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --frontend
|
|
docker-logs-gateway:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --gateway
|
|
docker-logs-redis:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --redis
|
|
|
|
# ==========================================
|
|
# Production Docker Commands
|
|
# ==========================================
|
|
|
|
# Build and start production services
|
|
up:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh
|
|
|
|
# Stop and remove production containers
|
|
down:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh down
|
|
|
|
# Follow production container logs (stack started by `make up`)
|
|
prod-logs:
|
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --prod
|