Merge pull request #188 from CagesThrottleUs/cages/parallelise-scripts-default-mode

feat(scripts): add opt-in parallel execution to convert & install
2026-04-25 11:18:05 +00:00 · 2026-03-15 15:06:01 -05:00 · 2026-03-15 15:06:01 -05:00 · 40963d5402
commit 40963d5402
parent 5c669c28e6 37c220bd7c
3 changed files with 173 additions and 29 deletions
--- a/README.md
+++ b/README.md
@ -507,11 +507,13 @@ The Agency works natively with Claude Code, and ships conversion + install scrip
 **Step 1 -- Generate integration files:**
 ```bash
 ./scripts/convert.sh
+# Faster (parallel, output order may vary): ./scripts/convert.sh --parallel
 ```

 **Step 2 -- Install (interactive, auto-detects your tools):**
 ```bash
 ./scripts/install.sh
+# Faster (parallel, output order may vary): ./scripts/install.sh --no-interactive --parallel
 ```

 The installer scans your system for installed tools, shows a checkbox UI, and lets you pick exactly what to install:
@ -551,6 +553,16 @@ The installer scans your system for installed tools, shows a checkbox UI, and le
 ./scripts/install.sh --no-interactive --tool all
 ```

+**Faster runs (parallel)** — On multi-core machines, use `--parallel` so each tool is processed in parallel. Output order across tools is non-deterministic. Works with both interactive and non-interactive install: e.g. `./scripts/install.sh --interactive --parallel` (pick tools, then install in parallel) or `./scripts/install.sh --no-interactive --parallel`. Job count defaults to `nproc` (Linux), `sysctl -n hw.ncpu` (macOS), or 4; override with `--jobs N`.
+
+```bash
+./scripts/convert.sh --parallel                    # convert all tools in parallel
+./scripts/convert.sh --parallel --jobs 8           # cap parallel jobs
+./scripts/install.sh --no-interactive --parallel   # install all detected tools in parallel
+./scripts/install.sh --interactive --parallel      # pick tools, then install in parallel
+./scripts/install.sh --no-interactive --parallel --jobs 4
+```
+
 ---

 ### Tool-Specific Instructions
@ -741,8 +753,9 @@ cd /your/project
 When you add new agents or edit existing ones, regenerate all integration files:

 ```bash
-./scripts/convert.sh        # regenerate all
-./scripts/convert.sh --tool cursor   # regenerate just one tool
+./scripts/convert.sh                    # regenerate all (serial)
+./scripts/convert.sh --parallel         # regenerate all in parallel (faster)
+./scripts/convert.sh --tool cursor      # regenerate just one tool
 ```

 ---
--- a/scripts/convert.sh
+++ b/scripts/convert.sh
@ -7,7 +7,7 @@
 # integration files after adding or modifying agents.
 #
 # Usage:
-#   ./scripts/convert.sh [--tool <name>] [--out <dir>] [--help]
+#   ./scripts/convert.sh [--tool <name>] [--out <dir>] [--parallel] [--jobs N] [--help]
 #
 # Tools:
 #   antigravity  — Antigravity skill files (~/.gemini/antigravity/skills/)
@ -22,6 +22,9 @@
 #
 # Output is written to integrations/<tool>/ relative to the repo root.
 # This script never touches user config dirs — see install.sh for that.
+#
+#   --parallel       When tool is 'all', run independent tools in parallel (output order may vary).
+#   --jobs N         Max parallel jobs when using --parallel (default: nproc or 4).

 set -euo pipefail

@ -37,6 +40,20 @@ warn()    { printf "${YELLOW}[!!]${RESET}  %s\n" "$*"; }
 error()   { printf "${RED}[ERR]${RESET} %s\n" "$*" >&2; }
 header()  { echo -e "\n${BOLD}$*${RESET}"; }

+# Progress bar: [=======>    ] 3/8 (tqdm-style)
+progress_bar() {
+  local current="$1" total="$2" width="${3:-20}" i filled empty
+  (( total > 0 )) || return
+  filled=$(( width * current / total ))
+  empty=$(( width - filled ))
+  printf "\r  ["
+  for (( i=0; i<filled; i++ )); do printf "="; done
+  if (( filled < width )); then printf ">"; (( empty-- )); fi
+  for (( i=0; i<empty; i++ )); do printf " "; done
+  printf "] %s/%s" "$current" "$total"
+  [[ -t 1 ]] || printf "\n"
+}
+
 # --- Paths ---
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
@ -50,10 +67,18 @@ AGENT_DIRS=(

 # --- Usage ---
 usage() {
-  sed -n '3,22p' "$0" | sed 's/^# \{0,1\}//'
+  sed -n '3,26p' "$0" | sed 's/^# \{0,1\}//'
  exit 0
 }

+# Default parallel job count (nproc on Linux; sysctl on macOS when nproc missing)
+parallel_jobs_default() {
+  local n
+  n=$(nproc 2>/dev/null) && [[ -n "$n" ]] && echo "$n" && return
+  n=$(sysctl -n hw.ncpu 2>/dev/null) && [[ -n "$n" ]] && echo "$n" && return
+  echo 4
+}
+
 # --- Frontmatter helpers ---

 # Extract a single field value from YAML frontmatter block.
@ -460,13 +485,18 @@ run_conversions() {

 main() {
  local tool="all"
+  local use_parallel=false
+  local parallel_jobs
+  parallel_jobs="$(parallel_jobs_default)"

  while [[ $# -gt 0 ]]; do
    case "$1" in
-      --tool) tool="${2:?'--tool requires a value'}"; shift 2 ;;
-      --out)  OUT_DIR="${2:?'--out requires a value'}"; shift 2 ;;
-      --help|-h) usage ;;
-      *) error "Unknown option: $1"; usage ;;
+      --tool)     tool="${2:?'--tool requires a value'}"; shift 2 ;;
+      --out)      OUT_DIR="${2:?'--out requires a value'}"; shift 2 ;;
+      --parallel) use_parallel=true; shift ;;
+      --jobs)     parallel_jobs="${2:?'--jobs requires a value'}"; shift 2 ;;
+      --help|-h)  usage ;;
+      *)          error "Unknown option: $1"; usage ;;
    esac
  done

@ -483,6 +513,9 @@ main() {
  echo "  Output: $OUT_DIR"
  echo "  Tool:   $tool"
  echo "  Date:   $TODAY"
+  if $use_parallel && [[ "$tool" == "all" ]]; then
+    info "Parallel mode: output buffered so each tool's output stays together."
+  fi

  local tools_to_run=()
  if [[ "$tool" == "all" ]]; then
@ -492,26 +525,60 @@ main() {
  fi

  local total=0
-  for t in "${tools_to_run[@]}"; do
-    header "Converting: $t"
-    local count
-    count="$(run_conversions "$t")"
-    total=$(( total + count ))

-    # Gemini CLI also needs the extension manifest
-    if [[ "$t" == "gemini-cli" ]]; then
-      mkdir -p "$OUT_DIR/gemini-cli"
-      cat > "$OUT_DIR/gemini-cli/gemini-extension.json" <<'HEREDOC'
+  local n_tools=${#tools_to_run[@]}
+
+  if $use_parallel && [[ "$tool" == "all" ]]; then
+    # Tools that write to separate dirs can run in parallel; buffer output so each tool's output stays together
+    local parallel_tools=(antigravity gemini-cli opencode cursor openclaw qwen)
+    local parallel_out_dir
+    parallel_out_dir="$(mktemp -d)"
+    info "Converting: ${#parallel_tools[@]}/${n_tools} tools in parallel (output buffered per tool)..."
+    export AGENCY_CONVERT_OUT_DIR="$parallel_out_dir"
+    export AGENCY_CONVERT_SCRIPT="$SCRIPT_DIR/convert.sh"
+    export AGENCY_CONVERT_OUT="$OUT_DIR"
+    printf '%s\n' "${parallel_tools[@]}" | xargs -P "$parallel_jobs" -I {} sh -c '"$AGENCY_CONVERT_SCRIPT" --tool "{}" --out "$AGENCY_CONVERT_OUT" > "$AGENCY_CONVERT_OUT_DIR/{}" 2>&1'
+    for t in "${parallel_tools[@]}"; do
+      [[ -f "$parallel_out_dir/$t" ]] && cat "$parallel_out_dir/$t"
+    done
+    rm -rf "$parallel_out_dir"
+    local idx=7
+    for t in aider windsurf; do
+      progress_bar "$idx" "$n_tools"
+      printf "\n"
+      header "Converting: $t ($idx/$n_tools)"
+      local count
+      count="$(run_conversions "$t")"
+      total=$(( total + count ))
+      info "Converted $count agents for $t"
+      (( idx++ )) || true
+    done
+  else
+    local i=0
+    for t in "${tools_to_run[@]}"; do
+      (( i++ )) || true
+      progress_bar "$i" "$n_tools"
+      printf "\n"
+      header "Converting: $t ($i/$n_tools)"
+      local count
+      count="$(run_conversions "$t")"
+      total=$(( total + count ))
+
+      # Gemini CLI also needs the extension manifest (written by this process when --tool gemini-cli)
+      if [[ "$t" == "gemini-cli" ]]; then
+        mkdir -p "$OUT_DIR/gemini-cli"
+        cat > "$OUT_DIR/gemini-cli/gemini-extension.json" <<'HEREDOC'
 {
  "name": "agency-agents",
  "version": "1.0.0"
 }
 HEREDOC
-      info "Wrote gemini-extension.json"
-    fi
+        info "Wrote gemini-extension.json"
+      fi

-    info "Converted $count agents for $t"
-  done
+      info "Converted $count agents for $t"
+    done
+  fi

  # Write single-file outputs after accumulation
  if [[ "$tool" == "all" || "$tool" == "aider" ]]; then
@ -526,7 +593,11 @@ HEREDOC
  fi

  echo ""
-  info "Done. Total conversions: $total"
+  if $use_parallel && [[ "$tool" == "all" ]]; then
+    info "Done. $n_tools tools (parallel; total conversions not aggregated)."
+  else
+    info "Done. Total conversions: $total"
+  fi
 }

 main "$@"
--- a/scripts/install.sh
+++ b/scripts/install.sh
@ -7,7 +7,7 @@
 # is missing or stale.
 #
 # Usage:
-#   ./scripts/install.sh [--tool <name>] [--interactive] [--no-interactive] [--help]
+#   ./scripts/install.sh [--tool <name>] [--interactive] [--no-interactive] [--parallel] [--jobs N] [--help]
 #
 # Tools:
 #   claude-code  -- Copy agents to ~/.claude/agents/
@ -26,6 +26,8 @@
 #   --tool <name>     Install only the specified tool
 #   --interactive     Show interactive selector (default when run in a terminal)
 #   --no-interactive  Skip interactive selector, install all detected tools
+#   --parallel        Run install for each selected tool in parallel (output order may vary)
+#   --jobs N          Max parallel jobs when using --parallel (default: nproc or 4)
 #   --help            Show this help
 #
 # Platform support:
@ -54,6 +56,20 @@ err()    { printf "${C_RED}[ERR]${C_RESET} %s\n" "$*" >&2; }
 header() { printf "\n${C_BOLD}%s${C_RESET}\n" "$*"; }
 dim()    { printf "${C_DIM}%s${C_RESET}\n" "$*"; }

+# Progress bar: [=======>    ] 3/8 (tqdm-style)
+progress_bar() {
+  local current="$1" total="$2" width="${3:-20}" i filled empty
+  (( total > 0 )) || return
+  filled=$(( width * current / total ))
+  empty=$(( width - filled ))
+  printf "\r  ["
+  for (( i=0; i<filled; i++ )); do printf "="; done
+  if (( filled < width )); then printf ">"; (( empty-- )); fi
+  for (( i=0; i<empty; i++ )); do printf " "; done
+  printf "] %s/%s" "$current" "$total"
+  [[ -t 1 ]] || printf "\n"
+}
+
 # ---------------------------------------------------------------------------
 # Box drawing -- pure ASCII, fixed 52-char wide
 #   box_top / box_mid / box_bot  -- structural lines
@ -91,10 +107,18 @@ ALL_TOOLS=(claude-code copilot antigravity gemini-cli opencode openclaw cursor a
 # Usage
 # ---------------------------------------------------------------------------
 usage() {
-  sed -n '3,28p' "$0" | sed 's/^# \{0,1\}//'
+  sed -n '3,32p' "$0" | sed 's/^# \{0,1\}//'
  exit 0
 }

+# Default parallel job count (nproc on Linux; sysctl on macOS when nproc missing)
+parallel_jobs_default() {
+  local n
+  n=$(nproc 2>/dev/null) && [[ -n "$n" ]] && echo "$n" && return
+  n=$(sysctl -n hw.ncpu 2>/dev/null) && [[ -n "$n" ]] && echo "$n" && return
+  echo 4
+}
+
 # ---------------------------------------------------------------------------
 # Preflight
 # ---------------------------------------------------------------------------
@ -465,12 +489,17 @@ install_tool() {
 main() {
  local tool="all"
  local interactive_mode="auto"
+  local use_parallel=false
+  local parallel_jobs
+  parallel_jobs="$(parallel_jobs_default)"

  while [[ $# -gt 0 ]]; do
    case "$1" in
      --tool)            tool="${2:?'--tool requires a value'}"; shift 2; interactive_mode="no" ;;
      --interactive)     interactive_mode="yes"; shift ;;
      --no-interactive)  interactive_mode="no"; shift ;;
+      --parallel)        use_parallel=true; shift ;;
+      --jobs)            parallel_jobs="${2:?'--jobs requires a value'}"; shift 2 ;;
      --help|-h)         usage ;;
      *)                 err "Unknown option: $1"; usage ;;
    esac
@ -527,17 +556,48 @@ main() {
    exit 0
  fi

+  # When parent runs install.sh --parallel, it spawns workers with AGENCY_INSTALL_WORKER=1
+  # so each worker only runs install_tool(s) and skips header/done box (avoids duplicate output).
+  if [[ -n "${AGENCY_INSTALL_WORKER:-}" ]]; then
+    local t
+    for t in "${SELECTED_TOOLS[@]}"; do
+      install_tool "$t"
+    done
+    return 0
+  fi
+
  printf "\n"
  header "The Agency -- Installing agents"
  printf "  Repo:       %s\n" "$REPO_ROOT"
+  local n_selected=${#SELECTED_TOOLS[@]}
  printf "  Installing: %s\n" "${SELECTED_TOOLS[*]}"
+  if $use_parallel; then
+    ok "Installing $n_selected tools in parallel (output buffered per tool)."
+  fi
  printf "\n"

-  local installed=0 t
-  for t in "${SELECTED_TOOLS[@]}"; do
-    install_tool "$t"
-    (( installed++ )) || true
-  done
+  local installed=0 t i=0
+  if $use_parallel; then
+    local install_out_dir
+    install_out_dir="$(mktemp -d)"
+    export AGENCY_INSTALL_OUT_DIR="$install_out_dir"
+    export AGENCY_INSTALL_SCRIPT="$SCRIPT_DIR/install.sh"
+    printf '%s\n' "${SELECTED_TOOLS[@]}" | xargs -P "$parallel_jobs" -I {} sh -c 'AGENCY_INSTALL_WORKER=1 "$AGENCY_INSTALL_SCRIPT" --tool "{}" --no-interactive > "$AGENCY_INSTALL_OUT_DIR/{}" 2>&1'
+    for t in "${SELECTED_TOOLS[@]}"; do
+      [[ -f "$install_out_dir/$t" ]] && cat "$install_out_dir/$t"
+    done
+    rm -rf "$install_out_dir"
+    installed=$n_selected
+  else
+    for t in "${SELECTED_TOOLS[@]}"; do
+      (( i++ )) || true
+      progress_bar "$i" "$n_selected"
+      printf "\n"
+      printf "  ${C_DIM}[%s/%s]${C_RESET} %s\n" "$i" "$n_selected" "$t"
+      install_tool "$t"
+      (( installed++ )) || true
+    done
+  fi

  # Done box
  local msg="  Done!  Installed $installed tool(s)."