mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 13:39:26 +00:00
fix(scripts): invoke repo shell scripts through an explicit interpreter (#5031)
Recipes and scripts ran sibling shell scripts bare (./scripts/x.sh), so
any checkout that lost the executable bit -- zip/tarball download,
core.fileMode=false, non-POSIX filesystem -- failed with:
make: ./scripts/docker.sh: Permission denied
make: *** [Makefile:181: docker-start] Error 127
The tracked modes are already 100755, so chmod cannot fix it. Name the
interpreter instead: the POSIX branch of RUN_SHELL_SCRIPT (renamed from
RUN_WITH_GIT_BASH) now expands to $(BASH) rather than nothing, and the
five script-to-script call sites are prefixed with bash.
Fixes #2903
Co-authored-by: zaoshangduziteng <309590849+zaoshangduziteng@users.noreply.github.com>
This commit is contained in:
parent
2eba65449f
commit
b41354d75f
41
Makefile
41
Makefile
@ -10,10 +10,13 @@ ifeq ($(OS),Windows_NT)
|
|||||||
SHELL := cmd.exe
|
SHELL := cmd.exe
|
||||||
PYTHON ?= python
|
PYTHON ?= python
|
||||||
# Run repo shell scripts through Git Bash when Make is launched from cmd.exe / PowerShell.
|
# Run repo shell scripts through Git Bash when Make is launched from cmd.exe / PowerShell.
|
||||||
RUN_WITH_GIT_BASH = call scripts\run-with-git-bash.cmd
|
RUN_SHELL_SCRIPT = call scripts\run-with-git-bash.cmd
|
||||||
else
|
else
|
||||||
PYTHON ?= python3
|
PYTHON ?= python3
|
||||||
RUN_WITH_GIT_BASH =
|
# Invoke repo shell scripts through an explicit interpreter, so recipes keep
|
||||||
|
# working in checkouts that lost the executable bit (zip download,
|
||||||
|
# core.fileMode=false, non-POSIX filesystem).
|
||||||
|
RUN_SHELL_SCRIPT = $(BASH)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
FRONTEND_PNPM = $(PYTHON) ../scripts/pnpm.py
|
FRONTEND_PNPM = $(PYTHON) ../scripts/pnpm.py
|
||||||
@ -77,7 +80,7 @@ config:
|
|||||||
@$(PYTHON) ./scripts/configure.py
|
@$(PYTHON) ./scripts/configure.py
|
||||||
|
|
||||||
config-upgrade:
|
config-upgrade:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/config-upgrade.sh
|
@$(RUN_SHELL_SCRIPT) ./scripts/config-upgrade.sh
|
||||||
|
|
||||||
# Check required tools
|
# Check required tools
|
||||||
check:
|
check:
|
||||||
@ -130,37 +133,37 @@ extension-remove:
|
|||||||
|
|
||||||
# Pre-pull sandbox Docker image (optional but recommended)
|
# Pre-pull sandbox Docker image (optional but recommended)
|
||||||
setup-sandbox:
|
setup-sandbox:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/setup-sandbox.sh
|
@$(RUN_SHELL_SCRIPT) ./scripts/setup-sandbox.sh
|
||||||
|
|
||||||
# Start all services in development mode (with hot-reloading)
|
# Start all services in development mode (with hot-reloading)
|
||||||
dev:
|
dev:
|
||||||
@$(PYTHON) ./scripts/check.py
|
@$(PYTHON) ./scripts/check.py
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --dev
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev
|
||||||
|
|
||||||
# Start all services in production mode (with optimizations).
|
# Start all services in production mode (with optimizations).
|
||||||
# SKIP_FRONTEND_BUILD=1 reuses the existing frontend build instead of running
|
# SKIP_FRONTEND_BUILD=1 reuses the existing frontend build instead of running
|
||||||
# `next build`; see scripts/serve.sh --skip-frontend-build.
|
# `next build`; see scripts/serve.sh --skip-frontend-build.
|
||||||
start:
|
start:
|
||||||
@$(PYTHON) ./scripts/check.py
|
@$(PYTHON) ./scripts/check.py
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --prod $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||||
|
|
||||||
# Start all services in daemon mode (background)
|
# Start all services in daemon mode (background)
|
||||||
dev-daemon:
|
dev-daemon:
|
||||||
@$(PYTHON) ./scripts/check.py
|
@$(PYTHON) ./scripts/check.py
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --dev --daemon
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev --daemon
|
||||||
|
|
||||||
# Start prod services in daemon mode (background)
|
# Start prod services in daemon mode (background)
|
||||||
start-daemon:
|
start-daemon:
|
||||||
@$(PYTHON) ./scripts/check.py
|
@$(PYTHON) ./scripts/check.py
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --prod --daemon $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod --daemon $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||||
|
|
||||||
# Start nginx alone in the foreground with the local dev config
|
# Start nginx alone in the foreground with the local dev config
|
||||||
nginx:
|
nginx:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/nginx.sh
|
@$(RUN_SHELL_SCRIPT) ./scripts/nginx.sh
|
||||||
|
|
||||||
# Stop all services
|
# Stop all services
|
||||||
stop:
|
stop:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --stop
|
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --stop
|
||||||
|
|
||||||
# Clean up
|
# Clean up
|
||||||
clean: stop
|
clean: stop
|
||||||
@ -175,27 +178,27 @@ clean: stop
|
|||||||
|
|
||||||
# Initialize Docker containers and install dependencies
|
# Initialize Docker containers and install dependencies
|
||||||
docker-init:
|
docker-init:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh init
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh init
|
||||||
|
|
||||||
# Start Docker development environment
|
# Start Docker development environment
|
||||||
docker-start:
|
docker-start:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh start
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh start
|
||||||
|
|
||||||
# Stop Docker development environment
|
# Stop Docker development environment
|
||||||
docker-stop:
|
docker-stop:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh stop
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh stop
|
||||||
|
|
||||||
# View Docker development logs
|
# View Docker development logs
|
||||||
docker-logs:
|
docker-logs:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs
|
||||||
|
|
||||||
# View Docker development logs
|
# View Docker development logs
|
||||||
docker-logs-frontend:
|
docker-logs-frontend:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --frontend
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --frontend
|
||||||
docker-logs-gateway:
|
docker-logs-gateway:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --gateway
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --gateway
|
||||||
docker-logs-redis:
|
docker-logs-redis:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --redis
|
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --redis
|
||||||
|
|
||||||
# ==========================================
|
# ==========================================
|
||||||
# Production Docker Commands
|
# Production Docker Commands
|
||||||
@ -203,8 +206,8 @@ docker-logs-redis:
|
|||||||
|
|
||||||
# Build and start production services
|
# Build and start production services
|
||||||
up:
|
up:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/deploy.sh
|
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh
|
||||||
|
|
||||||
# Stop and remove production containers
|
# Stop and remove production containers
|
||||||
down:
|
down:
|
||||||
@$(RUN_WITH_GIT_BASH) ./scripts/deploy.sh down
|
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh down
|
||||||
|
|||||||
@ -344,6 +344,11 @@ If you prefer running services locally:
|
|||||||
Prerequisite: complete the "Configuration" steps above first (`make setup`). `make dev` requires a valid `config.yaml` in the project root. Set `DEER_FLOW_PROJECT_ROOT` to define that root explicitly, or `DEER_FLOW_CONFIG_PATH` to point at a specific config file. Runtime state defaults to `.deer-flow` under the project root and can be moved with `DEER_FLOW_HOME`; skills default to `skills/` under the project root and can be moved with `DEER_FLOW_SKILLS_PATH`. Run `make doctor` to verify your setup before starting.
|
Prerequisite: complete the "Configuration" steps above first (`make setup`). `make dev` requires a valid `config.yaml` in the project root. Set `DEER_FLOW_PROJECT_ROOT` to define that root explicitly, or `DEER_FLOW_CONFIG_PATH` to point at a specific config file. Runtime state defaults to `.deer-flow` under the project root and can be moved with `DEER_FLOW_HOME`; skills default to `skills/` under the project root and can be moved with `DEER_FLOW_SKILLS_PATH`. Run `make doctor` to verify your setup before starting.
|
||||||
On Windows, run the local development flow from Git Bash. Native `cmd.exe` and PowerShell shells are not supported for the bash-based service scripts, and WSL is not guaranteed because some scripts rely on Git for Windows utilities such as `cygpath`.
|
On Windows, run the local development flow from Git Bash. Native `cmd.exe` and PowerShell shells are not supported for the bash-based service scripts, and WSL is not guaranteed because some scripts rely on Git for Windows utilities such as `cygpath`.
|
||||||
|
|
||||||
|
The documented root `make` commands invoke repository `.sh` files through Bash
|
||||||
|
explicitly. They therefore continue to work from source archives or filesystems
|
||||||
|
that do not preserve POSIX executable bits. When calling a script directly from
|
||||||
|
such a checkout, use `bash ./scripts/<name>.sh ...`.
|
||||||
|
|
||||||
1. **Check prerequisites**:
|
1. **Check prerequisites**:
|
||||||
```bash
|
```bash
|
||||||
make check # Verifies Node.js 22+, pnpm, uv, nginx
|
make check # Verifies Node.js 22+, pnpm, uv, nginx
|
||||||
|
|||||||
110
backend/tests/test_makefile_shell_script_invocation.py
Normal file
110
backend/tests/test_makefile_shell_script_invocation.py
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
"""Regression test keeping repo shell scripts callable without the executable bit.
|
||||||
|
|
||||||
|
The root ``Makefile`` drives every shell script under ``scripts/`` through the
|
||||||
|
``RUN_SHELL_SCRIPT`` variable. On Windows that variable is the Git Bash shim; on
|
||||||
|
POSIX it used to expand to nothing, so recipes invoked the script directly::
|
||||||
|
|
||||||
|
make: ./scripts/docker.sh: Permission denied
|
||||||
|
make: *** [Makefile:181: docker-start] Error 127
|
||||||
|
|
||||||
|
Git tracks the executable bit, so a normal ``git clone`` is fine. It is lost in
|
||||||
|
the working tree whenever the checkout does not carry POSIX modes: source zip or
|
||||||
|
tarball downloads from the Releases/Code page, ``core.fileMode=false`` clones,
|
||||||
|
and non-POSIX filesystems (some Windows/WSL and network mounts). Nothing in the
|
||||||
|
repository can restore the bit for those users.
|
||||||
|
|
||||||
|
Naming the interpreter removes the dependency entirely -- ``bash ./x.sh`` ignores
|
||||||
|
the mode. Every script under ``scripts/`` uses ``#!/usr/bin/env bash``, so a
|
||||||
|
single ``$(BASH)`` default is correct for all of them, and the Windows branch is
|
||||||
|
untouched.
|
||||||
|
|
||||||
|
This test pins both halves of the invariant: no recipe may invoke a shell script
|
||||||
|
bare, and the POSIX definition of ``RUN_SHELL_SCRIPT`` must name an interpreter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
MAKEFILE = REPO_ROOT / "Makefile"
|
||||||
|
|
||||||
|
# A recipe line is a tab-indented command, optionally silenced with "@".
|
||||||
|
_RECIPE_SHELL_SCRIPT = re.compile(r"^\t@?(?P<command>.*\./scripts/[\w.-]+\.sh.*)$")
|
||||||
|
|
||||||
|
|
||||||
|
def _recipe_lines_invoking_shell_scripts() -> list[tuple[int, str]]:
|
||||||
|
lines = MAKEFILE.read_text(encoding="utf-8").splitlines()
|
||||||
|
found = []
|
||||||
|
for number, line in enumerate(lines, start=1):
|
||||||
|
match = _RECIPE_SHELL_SCRIPT.match(line)
|
||||||
|
if match:
|
||||||
|
found.append((number, match.group("command")))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_makefile_never_invokes_shell_scripts_bare() -> None:
|
||||||
|
"""Every ``scripts/*.sh`` recipe goes through an interpreter variable."""
|
||||||
|
invocations = _recipe_lines_invoking_shell_scripts()
|
||||||
|
assert invocations, "expected the Makefile to invoke scripts/*.sh somewhere"
|
||||||
|
|
||||||
|
bare = [f"Makefile:{number}: {command}" for number, command in invocations if not command.startswith("$(RUN_SHELL_SCRIPT)")]
|
||||||
|
assert not bare, "these recipes invoke a shell script without an interpreter, so they fail with 'Permission denied' in checkouts that lost the executable bit; prefix them with $(RUN_SHELL_SCRIPT):\n " + "\n ".join(bare)
|
||||||
|
|
||||||
|
|
||||||
|
def test_posix_run_shell_script_names_an_interpreter() -> None:
|
||||||
|
"""The non-Windows branch must expand to an interpreter, not to nothing."""
|
||||||
|
text = MAKEFILE.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert re.search(r"^BASH \?= bash$", text, re.MULTILINE), "expected 'BASH ?= bash' so operators can override the interpreter"
|
||||||
|
|
||||||
|
assignments = re.findall(r"^\s*RUN_SHELL_SCRIPT\s*=\s*(.*)$", text, re.MULTILINE)
|
||||||
|
assert len(assignments) == 2, f"expected one RUN_SHELL_SCRIPT definition per platform branch, got {assignments}"
|
||||||
|
windows, posix = assignments
|
||||||
|
assert "run-with-git-bash.cmd" in windows, windows
|
||||||
|
assert posix.strip() == "$(BASH)", f"the POSIX branch must name an interpreter; an empty value makes recipes depend on the executable bit, got {posix!r}"
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPTS_DIR = REPO_ROOT / "scripts"
|
||||||
|
|
||||||
|
# Any reference to a shell script, however the path is spelled: "./scripts/x.sh",
|
||||||
|
# "$REPO_ROOT/scripts/x.sh", "$SCRIPT_DIR/x.sh".
|
||||||
|
_SCRIPT_REFERENCE = re.compile(r"\S*[\w.-]+\.sh\b")
|
||||||
|
|
||||||
|
# A script name inside a usage string or hint is prose, not an invocation.
|
||||||
|
_MESSAGE_COMMAND = re.compile(r"\b(echo|printf|cat)\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _sibling_script_invocations() -> list[tuple[str, int, str]]:
|
||||||
|
"""Every line in scripts/*.sh that actually runs another shell script."""
|
||||||
|
found = []
|
||||||
|
for script in sorted(SCRIPTS_DIR.glob("*.sh")):
|
||||||
|
for number, raw in enumerate(script.read_text(encoding="utf-8").splitlines(), start=1):
|
||||||
|
line = raw.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
for match in _SCRIPT_REFERENCE.finditer(line):
|
||||||
|
reference = match.group(0)
|
||||||
|
before = line[: match.start()]
|
||||||
|
if "://" in reference: # a download URL, not a local script
|
||||||
|
continue
|
||||||
|
if reference.endswith(script.name): # usage string / log prefix
|
||||||
|
continue
|
||||||
|
if _MESSAGE_COMMAND.search(before): # a hint printed to the user
|
||||||
|
continue
|
||||||
|
if not re.search(r"\bbash\s+$", before):
|
||||||
|
found.append((script.name, number, line))
|
||||||
|
break
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_scripts_never_invoke_sibling_scripts_bare() -> None:
|
||||||
|
"""A script calling another script must name the interpreter too.
|
||||||
|
|
||||||
|
``make docker-stop`` reaches ``scripts/cleanup-containers.sh`` through
|
||||||
|
``scripts/docker.sh``, so fixing only the Makefile would leave the same
|
||||||
|
``Permission denied`` failure one level deeper.
|
||||||
|
"""
|
||||||
|
bare = [f"scripts/{name}:{number}: {line}" for name, number, line in _sibling_script_invocations()]
|
||||||
|
assert not bare, "these lines run another shell script without an interpreter, so they fail with 'Permission denied' in checkouts that lost the executable bit; prefix them with 'bash':\n " + "\n ".join(bare)
|
||||||
@ -6,6 +6,15 @@ synchronized environment with `uv run --no-sync`. Production Compose probes
|
|||||||
Gateway `/health`, and `deploy.sh` waits for all services before reporting
|
Gateway `/health`, and `deploy.sh` waits for all services before reporting
|
||||||
success; failures print Compose status and recent Gateway logs.
|
success; failures print Compose status and recent Gateway logs.
|
||||||
|
|
||||||
|
## Shell Script Invocation Contract
|
||||||
|
|
||||||
|
Root Makefile recipes must invoke repository `.sh` files through
|
||||||
|
`RUN_SHELL_SCRIPT`. On POSIX this expands to `$(BASH)`; on Windows it uses the
|
||||||
|
Git Bash wrapper. Shell scripts that invoke sibling repository scripts must
|
||||||
|
likewise prefix the target with `bash`. This keeps documented `make` commands
|
||||||
|
working when a source archive, `core.fileMode=false`, or a non-POSIX filesystem
|
||||||
|
does not preserve executable bits.
|
||||||
|
|
||||||
## Backend Static Analysis Commands
|
## Backend Static Analysis Commands
|
||||||
|
|
||||||
The root `detect-thread-boundaries` target statically inventories execution
|
The root `detect-thread-boundaries` target statically inventories execution
|
||||||
|
|||||||
@ -456,7 +456,7 @@ stop() {
|
|||||||
echo "Stopping Docker development services..."
|
echo "Stopping Docker development services..."
|
||||||
cd "$DOCKER_DIR" && $COMPOSE_CMD down
|
cd "$DOCKER_DIR" && $COMPOSE_CMD down
|
||||||
echo "Cleaning up sandbox containers..."
|
echo "Cleaning up sandbox containers..."
|
||||||
"$SCRIPT_DIR/cleanup-containers.sh" deer-flow-sandbox 2>/dev/null || true
|
bash "$SCRIPT_DIR/cleanup-containers.sh" deer-flow-sandbox 2>/dev/null || true
|
||||||
echo -e "${GREEN}✓ Docker services stopped${NC}"
|
echo -e "${GREEN}✓ Docker services stopped${NC}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -268,7 +268,7 @@ stop_all() {
|
|||||||
_kill_repo_port 8001
|
_kill_repo_port 8001
|
||||||
_kill_repo_port 3000
|
_kill_repo_port 3000
|
||||||
_kill_repo_port 2026
|
_kill_repo_port 2026
|
||||||
./scripts/cleanup-containers.sh deer-flow-sandbox 2>/dev/null || true
|
bash ./scripts/cleanup-containers.sh deer-flow-sandbox 2>/dev/null || true
|
||||||
echo "✓ All services stopped"
|
echo "✓ All services stopped"
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -377,7 +377,7 @@ if ! { \
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
"$REPO_ROOT/scripts/config-upgrade.sh"
|
bash "$REPO_ROOT/scripts/config-upgrade.sh"
|
||||||
|
|
||||||
# ── Install dependencies ────────────────────────────────────────────────────
|
# ── Install dependencies ────────────────────────────────────────────────────
|
||||||
|
|
||||||
@ -471,7 +471,7 @@ run_service() {
|
|||||||
sh -c "$cmd" &
|
sh -c "$cmd" &
|
||||||
fi
|
fi
|
||||||
|
|
||||||
./scripts/wait-for-port.sh "$port" "$timeout" "$name" || {
|
bash ./scripts/wait-for-port.sh "$port" "$timeout" "$name" || {
|
||||||
local logfile="logs/$(echo "$name" | tr '[:upper:]' '[:lower:]' | tr ' ' '-').log"
|
local logfile="logs/$(echo "$name" | tr '[:upper:]' '[:lower:]' | tr ' ' '-').log"
|
||||||
echo "✗ $name failed to start."
|
echo "✗ $name failed to start."
|
||||||
[ -f "$logfile" ] && tail -20 "$logfile"
|
[ -f "$logfile" ] && tail -20 "$logfile"
|
||||||
|
|||||||
@ -6,4 +6,4 @@
|
|||||||
# Kept for backward compatibility.
|
# Kept for backward compatibility.
|
||||||
|
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
exec "$REPO_ROOT/scripts/serve.sh" --dev --daemon "$@"
|
exec bash "$REPO_ROOT/scripts/serve.sh" --dev --daemon "$@"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user