Merge pull request #519 from LaansDole/main

Feat/Dev tools enhancement

.env

@@ -1,4 +0,0 @@
-BASE_URL=
-API_KEY=
-SERPER_DEV_API_KEY=
-JINA_API_KEY=

.env.example

@@ -0,0 +1,24 @@
+# ============================================================================
+# LLM Provider Configuration
+# ============================================================================
+# BASE_URL and API_KEY are the standard configurations for model call authentication.
+# These variables support OpenAI, Gemini, LM Studio, Ollama, and other providers.
+
+BASE_URL=https://api.openai.com/v1
+API_KEY=sk-your-openai-api-key-here
+
+# Example BASE_URL values:
+# - OpenAI:    https://api.openai.com/v1
+# - Gemini:    https://generativelanguage.googleapis.com/v1beta/openai/
+# - LM Studio: http://localhost:1234/v1
+# - Ollama:    http://localhost:11434/v1
+
+# ============================================================================
+# Optional: Web Search and Reading Tools
+# ============================================================================
+
+# SERPER_DEV_API_KEY=your-serper-api-key-here
+# Get from: https://serper.dev
+
+# JINA_API_KEY=your-jina-api-key-here
+# Get from: https://jina.ai

.github/workflows/validate-yamls.yml

@@ -0,0 +1,65 @@
+name: Validate YAML Workflows
+
+on:
+  pull_request:
+    paths:
+      - 'yaml_instance/**/*.yaml'
+      - '.github/workflows/**/*.yml'
+      - 'tools/validate_all_yamls.py'
+      - 'check/**/*.py'
+  
+  push:
+    branches:
+      - main
+    paths:
+      - 'yaml_instance/**/*.yaml'
+      - '.github/workflows/**/*.yml'
+      - 'tools/validate_all_yamls.py'
+      - 'check/**/*.py'
+  
+  workflow_dispatch:
+
+jobs:
+  validate:
+    name: Validate YAML Configuration Files
+    runs-on: ubuntu-latest
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      
+      - name: Cache uv dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+      
+      - name: Install dependencies
+        run: uv sync
+      
+      - name: Run YAML validation
+        run: uv run python tools/validate_all_yamls.py
+      
+      - name: Report validation results
+        if: always()
+        run: |
+          if [ $? -eq 0 ]; then
+            echo "All YAML workflow files passed validation"
+          else
+            echo "YAML validation failed - check the logs above for details"
+            exit 1
+          fi

.gitignore

@@ -1,18 +1,29 @@
-*.pyc
-.DS_Store
-.idea
-.vscode
+# Python
 __pycache__/
-.env/
+*.pyc
+
+# Virtual environments
 .venv/
-env/
 venv/
-.idea
-.venv
-.uv-cache
-logs
-node_modules
-frontend/.vscode
-WareHouse/
+env/
+
+# uv
+.uv-cache/
+
+# IDEs
+.idea/
+.vscode/
+frontend/.vscode/
+
+# OS
+.DS_Store
+
+# Environment
+.env
+
+# Project Specific
+logs/
+node_modules/
 data/
-temp/
+temp/
+WareHouse/

Makefile

@@ -0,0 +1,42 @@
+# ==============================================================================
+# Development Commands
+# ==============================================================================
+
+.PHONY: dev
+dev: server client ## Run both backend and frontend development servers
+
+.PHONY: server
+server: ## Start the backend server in the background
+	@echo "Starting server in background..."
+	@uv run python server_main.py --port 6400 --reload &
+
+.PHONY: client
+client: ## Start the frontend development server
+	@cd frontend && VITE_API_BASE_URL=http://localhost:6400 npm run dev
+
+.PHONY: stop
+stop: ## Stop backend and frontend servers
+	@echo "Stopping backend server (port 6400)..."
+	@lsof -t -i:6400 | xargs kill -9 2>/dev/null || echo "Backend server not found on port 6400."
+	@echo "Stopping frontend server (port 5173)..."
+	@lsof -t -i:5173 | xargs kill -9 2>/dev/null || echo "Frontend server not found on port 5173."
+
+# ==============================================================================
+# Tools & Maintenance
+# ==============================================================================
+
+.PHONY: sync
+sync: ## Sync Vue graphs to the server database
+	@uv run python tools/sync_vuegraphs.py
+
+.PHONY: validate-yamls
+validate-yamls: ## Validate all YAML configuration files
+	@uv run python tools/validate_all_yamls.py
+
+# ==============================================================================
+# Help
+# ==============================================================================
+
+.PHONY: help
+help: ## Display this help message
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'

README.md

@@ -120,9 +120,30 @@ See our paper in [Multi-Agent Collaboration via Evolving Orchestration](https://
     cd frontend && npm install
     ```
 
+### 🔑 Configuration
+
+*   **Environment Variables**:
+    ```bash
+    cp .env.example .env
+    ```
+*   **Model Keys**: Set `API_KEY` and `BASE_URL` in `.env` for your LLM provider.
+*   **YAML placeholders**: Use `${VAR}`（e.g., `${API_KEY}`）in configuration files to reference these variables.
+
 ### ⚡️ Run the Application
 
-1.  **Start Backend** :
+
+#### Using Makefile (Recommended)
+
+**Start both Backend and Frontent**:
+```bash
+make dev
+```
+
+> Then access the Web Console at **[http://localhost:5173](http://localhost:5173)**.
+
+#### Manual Commands
+
+1.  **Start Backend**:
     ```bash
     # Run from the project root
     uv run python server_main.py --port 6400 --reload
@@ -143,12 +164,25 @@ See our paper in [Multi-Agent Collaboration via Evolving Orchestration](https://
     > * **Backend**: start with `--port 6401`
     > * **Frontend**: set `VITE_API_BASE_URL=http://localhost:6401`
 
+#### Utility Commands
 
-### 🔑 Configuration
+*   **Help command**:
+    ```bash
+    make help
+    ```
+
+*   **Sync YAML workflows to frontend**:
+    ```bash
+    make sync
+    ```
+    Uploads all workflow files from `yaml_instance/` to the database.
+
+*   **Validate all YAML workflows**:
+    ```bash
+    make validate-yamls
+    ```
+    Checks all YAML files for syntax and schema errors.
 
-*   **Environment Variables**: Create a `.env` file in the project root.
-*   **Model Keys**: Set `API_KEY` and `BASE_URL` in `.env` for your LLM provider.
-*   **YAML placeholders**: Use `${VAR}`（e.g., `${API_KEY}`）in configuration files to reference these variables.
 
 ---
 

check/check.py

@@ -118,4 +118,4 @@ def check_config(yaml_content: Any) -> str:
     except Exception as e:
         return str(e)
 
-    return ""
+    return ""

tools/sync_vuegraphs.py

@@ -0,0 +1,65 @@
+"""
+Synchronize YAML Configurations to VueGraph Database
+
+This tool uploads local YAML workflow configurations from the yaml_instance/
+directory to the VueGraph database via the API endpoint. This is essential for
+making workflow configurations available to the frontend visualization system.
+
+Purpose:
+- Ensures the database reflects the latest YAML configurations
+- Required after modifying workflow YAML files to see changes in the UI
+- Useful for development and deployment workflows
+
+Usage:
+    python tools/sync_vuegraphs.py
+    # or via Makefile:
+    make sync
+"""
+
+import os
+import glob
+import requests
+import yaml
+from pathlib import Path
+
+# Configuration
+API_URL = "http://localhost:6400/api/vuegraphs/upload/content"
+YAML_DIR = "yaml_instance"
+
+
+def sync_yaml_to_vuegraphs():
+    """Reads all YAML files and uploads them to the VueGraph database."""
+    print(f"Syncing YAML files from {YAML_DIR} to {API_URL}...")
+
+    yaml_files = glob.glob(os.path.join(YAML_DIR, "*.yaml"))
+
+    for file_path in yaml_files:
+        try:
+            filename = Path(file_path).stem  # simulation_hospital_lmstudio
+
+            with open(file_path, "r") as f:
+                content = f.read()
+
+            # Basic validation to ensure it's a valid YAML
+            try:
+                yaml.safe_load(content)
+            except yaml.YAMLError as e:
+                print(f"Skipping {filename}: Invalid YAML - {e}")
+                continue
+
+            # Upload to VueGraph API
+            payload = {"filename": filename, "content": content}
+
+            response = requests.post(API_URL, json=payload)
+
+            if response.status_code == 200:
+                print(f"Synced: {filename}")
+            else:
+                print(f"Failed: {filename} - {response.status_code} {response.text}")
+
+        except Exception as e:
+            print(f"Error processing {file_path}: {e}")
+
+
+if __name__ == "__main__":
+    sync_yaml_to_vuegraphs()

tools/validate_all_yamls.py

@@ -0,0 +1,104 @@
+"""
+Validate All YAML Workflow Configurations
+
+This tool performs strict validation on all YAML workflow configuration files
+in the yaml_instance/ directory. It ensures configuration integrity and prevents
+runtime errors by catching issues early in the development process.
+
+Purpose:
+- Validates YAML syntax and schema compliance for all workflow configurations
+- Prevents invalid configurations from causing runtime failures
+- Essential for CI/CD pipelines to ensure code quality
+- Provides detailed error reporting for debugging
+
+Usage:
+    python tools/validate_all_yamls.py
+    # or via Makefile:
+    make validate-yamls
+"""
+
+import sys
+import subprocess
+from pathlib import Path
+
+
+def validate_all():
+    base_dir = Path("yaml_instance")
+    if not base_dir.exists():
+        print(f"Directory {base_dir} not found.")
+        sys.exit(1)
+
+    # Recursive search for all .yaml files
+    files = sorted(list(base_dir.rglob("*.yaml")))
+
+    if not files:
+        print("No YAML files found.")
+        return
+
+    print(
+        f"Found {len(files)} YAML files. Running FULL validation via check.check...\n"
+    )
+
+    passed = 0
+    failed = 0
+    failed_files = []
+
+    for yaml_file in files:
+        # Use relative path for cleaner output
+        try:
+            rel_path = yaml_file.relative_to(Path.cwd())
+        except ValueError:
+            rel_path = yaml_file
+
+        # NOW we run check.check, which we just patched to have a main()
+        # This performs the stricter load_config() validation
+        cmd = [sys.executable, "-m", "check.check", "--path", str(yaml_file)]
+
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            if result.returncode == 0:
+                print(f"{rel_path}")
+                passed += 1
+            else:
+                print(f"{rel_path}")
+                # Indent error output
+                if result.stdout:
+                    print("  stdout:", result.stdout.strip().replace("\n", "\n  "))
+                # Validation errors usually print to stdout/stderr depending on impl
+                # Our new main prints to stdout for success/failure message
+                failed += 1
+                failed_files.append(str(rel_path))
+        except Exception as e:
+            print(f"{rel_path} (Execution Failed)")
+            print(f"  Error: {e}")
+            failed += 1
+            failed_files.append(str(rel_path))
+
+    print("\n" + "=" * 40)
+    print(f"YAML Validation Summary")
+    print("=" * 40)
+    print(f"Total Files: {len(files)}")
+    print(f"Passed:      {passed}")
+    print(f"Failed:      {failed}")
+
+    if failed > 0:
+        print("\nFailed Files:")
+        for f in failed_files:
+            print(f"- {f}")
+
+    # Overall validation status
+    print("\n" + "=" * 40)
+    print("Overall Validation Status")
+    print("=" * 40)
+
+    if failed > 0:
+        print("YAML validation: FAILED")
+        sys.exit(1)
+    else:
+        print("All validations passed successfully.")
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    validate_all()