mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 13:39:26 +00:00
fix(scripts): invoke repo shell scripts through an explicit interpreter (#5031)
Recipes and scripts ran sibling shell scripts bare (./scripts/x.sh), so
any checkout that lost the executable bit -- zip/tarball download,
core.fileMode=false, non-POSIX filesystem -- failed with:
make: ./scripts/docker.sh: Permission denied
make: *** [Makefile:181: docker-start] Error 127
The tracked modes are already 100755, so chmod cannot fix it. Name the
interpreter instead: the POSIX branch of RUN_SHELL_SCRIPT (renamed from
RUN_WITH_GIT_BASH) now expands to $(BASH) rather than nothing, and the
five script-to-script call sites are prefixed with bash.
Fixes #2903
Co-authored-by: zaoshangduziteng <309590849+zaoshangduziteng@users.noreply.github.com>
This commit is contained in:
早上肚子疼
GitHub
parent
2eba65449f
commit
b41354d75f
41
Makefile
41
Makefile
@ -10,10 +10,13 @@ ifeq ($(OS),Windows_NT)
|
||||
SHELL := cmd.exe
|
||||
PYTHON ?= python
|
||||
# Run repo shell scripts through Git Bash when Make is launched from cmd.exe / PowerShell.
|
||||
RUN_WITH_GIT_BASH = call scripts\run-with-git-bash.cmd
|
||||
RUN_SHELL_SCRIPT = call scripts\run-with-git-bash.cmd
|
||||
else
|
||||
PYTHON ?= python3
|
||||
RUN_WITH_GIT_BASH =
|
||||
# Invoke repo shell scripts through an explicit interpreter, so recipes keep
|
||||
# working in checkouts that lost the executable bit (zip download,
|
||||
# core.fileMode=false, non-POSIX filesystem).
|
||||
RUN_SHELL_SCRIPT = $(BASH)
|
||||
endif
|
||||
|
||||
FRONTEND_PNPM = $(PYTHON) ../scripts/pnpm.py
|
||||
@ -77,7 +80,7 @@ config:
|
||||
@$(PYTHON) ./scripts/configure.py
|
||||
|
||||
config-upgrade:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/config-upgrade.sh
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/config-upgrade.sh
|
||||
|
||||
# Check required tools
|
||||
check:
|
||||
@ -130,37 +133,37 @@ extension-remove:
|
||||
|
||||
# Pre-pull sandbox Docker image (optional but recommended)
|
||||
setup-sandbox:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/setup-sandbox.sh
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/setup-sandbox.sh
|
||||
|
||||
# Start all services in development mode (with hot-reloading)
|
||||
dev:
|
||||
@$(PYTHON) ./scripts/check.py
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --dev
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev
|
||||
|
||||
# Start all services in production mode (with optimizations).
|
||||
# SKIP_FRONTEND_BUILD=1 reuses the existing frontend build instead of running
|
||||
# `next build`; see scripts/serve.sh --skip-frontend-build.
|
||||
start:
|
||||
@$(PYTHON) ./scripts/check.py
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --prod $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||
|
||||
# Start all services in daemon mode (background)
|
||||
dev-daemon:
|
||||
@$(PYTHON) ./scripts/check.py
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --dev --daemon
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --dev --daemon
|
||||
|
||||
# Start prod services in daemon mode (background)
|
||||
start-daemon:
|
||||
@$(PYTHON) ./scripts/check.py
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --prod --daemon $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --prod --daemon $(if $(filter 1,$(SKIP_FRONTEND_BUILD)),--skip-frontend-build)
|
||||
|
||||
# Start nginx alone in the foreground with the local dev config
|
||||
nginx:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/nginx.sh
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/nginx.sh
|
||||
|
||||
# Stop all services
|
||||
stop:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/serve.sh --stop
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/serve.sh --stop
|
||||
|
||||
# Clean up
|
||||
clean: stop
|
||||
@ -175,27 +178,27 @@ clean: stop
|
||||
|
||||
# Initialize Docker containers and install dependencies
|
||||
docker-init:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh init
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh init
|
||||
|
||||
# Start Docker development environment
|
||||
docker-start:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh start
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh start
|
||||
|
||||
# Stop Docker development environment
|
||||
docker-stop:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh stop
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh stop
|
||||
|
||||
# View Docker development logs
|
||||
docker-logs:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs
|
||||
|
||||
# View Docker development logs
|
||||
docker-logs-frontend:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --frontend
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --frontend
|
||||
docker-logs-gateway:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --gateway
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --gateway
|
||||
docker-logs-redis:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/docker.sh logs --redis
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/docker.sh logs --redis
|
||||
|
||||
# ==========================================
|
||||
# Production Docker Commands
|
||||
@ -203,8 +206,8 @@ docker-logs-redis:
|
||||
|
||||
# Build and start production services
|
||||
up:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/deploy.sh
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh
|
||||
|
||||
# Stop and remove production containers
|
||||
down:
|
||||
@$(RUN_WITH_GIT_BASH) ./scripts/deploy.sh down
|
||||
@$(RUN_SHELL_SCRIPT) ./scripts/deploy.sh down
|
||||
|
||||
@ -344,6 +344,11 @@ If you prefer running services locally:
|
||||
Prerequisite: complete the "Configuration" steps above first (`make setup`). `make dev` requires a valid `config.yaml` in the project root. Set `DEER_FLOW_PROJECT_ROOT` to define that root explicitly, or `DEER_FLOW_CONFIG_PATH` to point at a specific config file. Runtime state defaults to `.deer-flow` under the project root and can be moved with `DEER_FLOW_HOME`; skills default to `skills/` under the project root and can be moved with `DEER_FLOW_SKILLS_PATH`. Run `make doctor` to verify your setup before starting.
|
||||
On Windows, run the local development flow from Git Bash. Native `cmd.exe` and PowerShell shells are not supported for the bash-based service scripts, and WSL is not guaranteed because some scripts rely on Git for Windows utilities such as `cygpath`.
|
||||
|
||||
The documented root `make` commands invoke repository `.sh` files through Bash
|
||||
explicitly. They therefore continue to work from source archives or filesystems
|
||||
that do not preserve POSIX executable bits. When calling a script directly from
|
||||
such a checkout, use `bash ./scripts/<name>.sh ...`.
|
||||
|
||||
1. **Check prerequisites**:
|
||||
```bash
|
||||
make check # Verifies Node.js 22+, pnpm, uv, nginx
|
||||
|
||||
110
backend/tests/test_makefile_shell_script_invocation.py
Normal file
110
backend/tests/test_makefile_shell_script_invocation.py
Normal file
@ -0,0 +1,110 @@
|
||||
"""Regression test keeping repo shell scripts callable without the executable bit.
|
||||
|
||||
The root ``Makefile`` drives every shell script under ``scripts/`` through the
|
||||
``RUN_SHELL_SCRIPT`` variable. On Windows that variable is the Git Bash shim; on
|
||||
POSIX it used to expand to nothing, so recipes invoked the script directly::
|
||||
|
||||
make: ./scripts/docker.sh: Permission denied
|
||||
make: *** [Makefile:181: docker-start] Error 127
|
||||
|
||||
Git tracks the executable bit, so a normal ``git clone`` is fine. It is lost in
|
||||
the working tree whenever the checkout does not carry POSIX modes: source zip or
|
||||
tarball downloads from the Releases/Code page, ``core.fileMode=false`` clones,
|
||||
and non-POSIX filesystems (some Windows/WSL and network mounts). Nothing in the
|
||||
repository can restore the bit for those users.
|
||||
|
||||
Naming the interpreter removes the dependency entirely -- ``bash ./x.sh`` ignores
|
||||
the mode. Every script under ``scripts/`` uses ``#!/usr/bin/env bash``, so a
|
||||
single ``$(BASH)`` default is correct for all of them, and the Windows branch is
|
||||
untouched.
|
||||
|
||||
This test pins both halves of the invariant: no recipe may invoke a shell script
|
||||
bare, and the POSIX definition of ``RUN_SHELL_SCRIPT`` must name an interpreter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
MAKEFILE = REPO_ROOT / "Makefile"
|
||||
|
||||
# A recipe line is a tab-indented command, optionally silenced with "@".
|
||||
_RECIPE_SHELL_SCRIPT = re.compile(r"^\t@?(?P<command>.*\./scripts/[\w.-]+\.sh.*)$")
|
||||
|
||||
|
||||
def _recipe_lines_invoking_shell_scripts() -> list[tuple[int, str]]:
|
||||
lines = MAKEFILE.read_text(encoding="utf-8").splitlines()
|
||||
found = []
|
||||
for number, line in enumerate(lines, start=1):
|
||||
match = _RECIPE_SHELL_SCRIPT.match(line)
|
||||
if match:
|
||||
found.append((number, match.group("command")))
|
||||
return found
|
||||
|
||||
|
||||
def test_makefile_never_invokes_shell_scripts_bare() -> None:
|
||||
"""Every ``scripts/*.sh`` recipe goes through an interpreter variable."""
|
||||
invocations = _recipe_lines_invoking_shell_scripts()
|
||||
assert invocations, "expected the Makefile to invoke scripts/*.sh somewhere"
|
||||
|
||||
bare = [f"Makefile:{number}: {command}" for number, command in invocations if not command.startswith("$(RUN_SHELL_SCRIPT)")]
|
||||
assert not bare, "these recipes invoke a shell script without an interpreter, so they fail with 'Permission denied' in checkouts that lost the executable bit; prefix them with $(RUN_SHELL_SCRIPT):\n " + "\n ".join(bare)
|
||||
|
||||
|
||||
def test_posix_run_shell_script_names_an_interpreter() -> None:
|
||||
"""The non-Windows branch must expand to an interpreter, not to nothing."""
|
||||
text = MAKEFILE.read_text(encoding="utf-8")
|
||||
|
||||
assert re.search(r"^BASH \?= bash$", text, re.MULTILINE), "expected 'BASH ?= bash' so operators can override the interpreter"
|
||||
|
||||
assignments = re.findall(r"^\s*RUN_SHELL_SCRIPT\s*=\s*(.*)$", text, re.MULTILINE)
|
||||
assert len(assignments) == 2, f"expected one RUN_SHELL_SCRIPT definition per platform branch, got {assignments}"
|
||||
windows, posix = assignments
|
||||
assert "run-with-git-bash.cmd" in windows, windows
|
||||
assert posix.strip() == "$(BASH)", f"the POSIX branch must name an interpreter; an empty value makes recipes depend on the executable bit, got {posix!r}"
|
||||
|
||||
|
||||
SCRIPTS_DIR = REPO_ROOT / "scripts"
|
||||
|
||||
# Any reference to a shell script, however the path is spelled: "./scripts/x.sh",
|
||||
# "$REPO_ROOT/scripts/x.sh", "$SCRIPT_DIR/x.sh".
|
||||
_SCRIPT_REFERENCE = re.compile(r"\S*[\w.-]+\.sh\b")
|
||||
|
||||
# A script name inside a usage string or hint is prose, not an invocation.
|
||||
_MESSAGE_COMMAND = re.compile(r"\b(echo|printf|cat)\b")
|
||||
|
||||
|
||||
def _sibling_script_invocations() -> list[tuple[str, int, str]]:
|
||||
"""Every line in scripts/*.sh that actually runs another shell script."""
|
||||
found = []
|
||||
for script in sorted(SCRIPTS_DIR.glob("*.sh")):
|
||||
for number, raw in enumerate(script.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
for match in _SCRIPT_REFERENCE.finditer(line):
|
||||
reference = match.group(0)
|
||||
before = line[: match.start()]
|
||||
if "://" in reference: # a download URL, not a local script
|
||||
continue
|
||||
if reference.endswith(script.name): # usage string / log prefix
|
||||
continue
|
||||
if _MESSAGE_COMMAND.search(before): # a hint printed to the user
|
||||
continue
|
||||
if not re.search(r"\bbash\s+$", before):
|
||||
found.append((script.name, number, line))
|
||||
break
|
||||
return found
|
||||
|
||||
|
||||
def test_scripts_never_invoke_sibling_scripts_bare() -> None:
|
||||
"""A script calling another script must name the interpreter too.
|
||||
|
||||
``make docker-stop`` reaches ``scripts/cleanup-containers.sh`` through
|
||||
``scripts/docker.sh``, so fixing only the Makefile would leave the same
|
||||
``Permission denied`` failure one level deeper.
|
||||
"""
|
||||
bare = [f"scripts/{name}:{number}: {line}" for name, number, line in _sibling_script_invocations()]
|
||||
assert not bare, "these lines run another shell script without an interpreter, so they fail with 'Permission denied' in checkouts that lost the executable bit; prefix them with 'bash':\n " + "\n ".join(bare)
|
||||
@ -6,6 +6,15 @@ synchronized environment with `uv run --no-sync`. Production Compose probes
|
||||
Gateway `/health`, and `deploy.sh` waits for all services before reporting
|
||||
success; failures print Compose status and recent Gateway logs.
|
||||
|
||||
## Shell Script Invocation Contract
|
||||
|
||||
Root Makefile recipes must invoke repository `.sh` files through
|
||||
`RUN_SHELL_SCRIPT`. On POSIX this expands to `$(BASH)`; on Windows it uses the
|
||||
Git Bash wrapper. Shell scripts that invoke sibling repository scripts must
|
||||
likewise prefix the target with `bash`. This keeps documented `make` commands
|
||||
working when a source archive, `core.fileMode=false`, or a non-POSIX filesystem
|
||||
does not preserve executable bits.
|
||||
|
||||
## Backend Static Analysis Commands
|
||||
|
||||
The root `detect-thread-boundaries` target statically inventories execution
|
||||
|
||||
@ -456,7 +456,7 @@ stop() {
|
||||
echo "Stopping Docker development services..."
|
||||
cd "$DOCKER_DIR" && $COMPOSE_CMD down
|
||||
echo "Cleaning up sandbox containers..."
|
||||
"$SCRIPT_DIR/cleanup-containers.sh" deer-flow-sandbox 2>/dev/null || true
|
||||
bash "$SCRIPT_DIR/cleanup-containers.sh" deer-flow-sandbox 2>/dev/null || true
|
||||
echo -e "${GREEN}✓ Docker services stopped${NC}"
|
||||
}
|
||||
|
||||
|
||||
@ -268,7 +268,7 @@ stop_all() {
|
||||
_kill_repo_port 8001
|
||||
_kill_repo_port 3000
|
||||
_kill_repo_port 2026
|
||||
./scripts/cleanup-containers.sh deer-flow-sandbox 2>/dev/null || true
|
||||
bash ./scripts/cleanup-containers.sh deer-flow-sandbox 2>/dev/null || true
|
||||
echo "✓ All services stopped"
|
||||
}
|
||||
|
||||
@ -377,7 +377,7 @@ if ! { \
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$REPO_ROOT/scripts/config-upgrade.sh"
|
||||
bash "$REPO_ROOT/scripts/config-upgrade.sh"
|
||||
|
||||
# ── Install dependencies ────────────────────────────────────────────────────
|
||||
|
||||
@ -471,7 +471,7 @@ run_service() {
|
||||
sh -c "$cmd" &
|
||||
fi
|
||||
|
||||
./scripts/wait-for-port.sh "$port" "$timeout" "$name" || {
|
||||
bash ./scripts/wait-for-port.sh "$port" "$timeout" "$name" || {
|
||||
local logfile="logs/$(echo "$name" | tr '[:upper:]' '[:lower:]' | tr ' ' '-').log"
|
||||
echo "✗ $name failed to start."
|
||||
[ -f "$logfile" ] && tail -20 "$logfile"
|
||||
|
||||
@ -6,4 +6,4 @@
|
||||
# Kept for backward compatibility.
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
exec "$REPO_ROOT/scripts/serve.sh" --dev --daemon "$@"
|
||||
exec bash "$REPO_ROOT/scripts/serve.sh" --dev --daemon "$@"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user