Add staged MCP release imports

dev/test_athena_operator.py

@@ -29,7 +29,8 @@ class OperatorTests(unittest.TestCase):
         root = Path(self.temporary.name)
         self.repo, self.stack = root / "repository", root / "stack"
         self.models, self.state = root / "models", root / "state"
-        for path in (self.repo, self.stack, self.models, self.state):
+        self.staging = root / "staging"
+        for path in (self.repo, self.stack, self.models, self.state, self.staging):
             path.mkdir()
         (self.repo / ".git").mkdir()
         (self.repo / "docs").mkdir()
@@ -40,6 +41,7 @@ class OperatorTests(unittest.TestCase):
         self.module.STACK = self.stack.resolve()
         self.module.MODELS = self.models.resolve()
         self.module.STATE = self.state.resolve()
+        self.module.STAGING_ROOTS = (self.staging.resolve(),)
         self.module.audit = lambda *args, **kwargs: None
         self.module.run = lambda argv, **kwargs: {"argv": argv, "exit_code": 0, "output": "ok"}
 
@@ -124,6 +126,40 @@ class OperatorTests(unittest.TestCase):
         self.assertEqual(payload["paths"], ["docs/test.md"])
         self.assertIn("ONE MCP RELEASE", preview)
 
+    def test_mcp_release_imports_reviewed_staging_file_by_sha(self):
+        source = self.staging / "server.py"
+        source.write_text("print('reviewed')\n", encoding="utf-8")
+        digest = self.module.sha(source.read_bytes())
+        payload, preview = self.module.normalise_operation("mcp_release", {
+            "imports": [{"source": str(source), "path": "server.py", "expected_source_sha256": digest}],
+            "services": ["mcp-example"], "message": "Import reviewed MCP source",
+            "checks": ["operator-tests"], "create_recovery": False, "hermes_sync": True,
+        })
+        self.assertEqual(payload["files"][0]["content"], "print('reviewed')\n")
+        self.assertTrue(payload["hermes_sync"])
+        self.assertIn(f"sha256={digest}", preview)
+
+    def test_staged_import_rejects_wrong_sha_and_unapproved_path(self):
+        source = self.staging / "server.py"
+        source.write_text("safe\n", encoding="utf-8")
+        with self.assertRaises(RuntimeError):
+            self.module.staged_file({"source": str(source), "path": "server.py", "expected_source_sha256": "0" * 64})
+        outside = Path(self.temporary.name) / "outside.py"
+        outside.write_text("unsafe\n", encoding="utf-8")
+        with self.assertRaises(PermissionError):
+            self.module.staged_file({"source": str(outside), "path": "server.py", "expected_source_sha256": self.module.sha(outside.read_bytes())})
+
+    def test_search_source_has_python_fallback_without_rg(self):
+        (self.repo / "docs" / "needle.md").write_text("alpha\nneedle here\nomega\n", encoding="utf-8")
+        original = self.module.shutil.which
+        self.module.shutil.which = lambda name: None
+        try:
+            result = self.module.search_source({"query": "needle"})
+        finally:
+            self.module.shutil.which = original
+        self.assertEqual(result["engine"], "python")
+        self.assertIn("docs/needle.md:2:needle here", result["matches"])
+
     def test_structured_power_operations_do_not_exist(self):
         self.assertNotIn("shell", self.module.ALLOWED_OPERATIONS)
         for operation in ("shutdown", "reboot", "ssh", "network", "command"):

docs/CURRENT_REFERENCE.md

@@ -358,11 +358,14 @@ sind serverseitig blockiert.
 Ein separater allgemeiner Shell-MCP wird nicht benötigt; die breite Fähigkeit
 ist portabel im Athena Operator auf VPN-Port 8202 enthalten.
 
-Seit Operator 2.2 werden kleine Änderungen als SHA-geschützte Unified Diffs
+Seit Operator 2.3 werden kleine Änderungen als SHA-geschützte Unified Diffs
 über `patch_update` übertragen. `mcp_release` fasst den üblichen vollständigen
 MCP-Ablauf in einem bestätigten Auftrag zusammen: Patch, Tests, benannter
-Deploy, OpenWebUI-Sync, selektiver Git-Publish und Recovery. Damit muss das
-Modell keine kompletten Compose- oder Installationsdateien rekonstruieren.
+Deploy, OpenWebUI-/Hermes-Sync, selektiver Git-Publish und Recovery. Geprüfte
+Staging-Dateien werden per Pfad und SHA importiert. Damit muss das Modell weder
+lange MCP-Quellen noch komplette Compose- oder Installationsdateien
+rekonstruieren. Die Quellensuche besitzt einen Python-Fallback, falls `rg` im
+Executor-Image fehlt.
 
 ## Bekannte Probleme des alten Hosts
 

docs/PLATFORM_CONTEXT_MCP.md

@@ -98,6 +98,11 @@ Der verbindliche Ablauf für dauerhafte Änderungen lautet:
    vorbereiten und nach separater Benutzerfreigabe ausführen. Es bündelt
    Prüfungen, benannten Compose-Deploy, OpenWebUI-Sync, selektiven Git-Publish
    und Recovery.
+   Bereits geprüfte lange Quelldateien werden mit `imports` plus exakter
+   SHA-256-Prüfsumme aus einem freigegebenen Staging-Verzeichnis übernommen;
+   sie werden nicht als Chattext oder Full-File-Payload nachgebaut. Änderungen
+   an `platform/hermes/config.yaml` verwenden `hermes_sync: true`. Für rein
+   interne MCPs wird WireGuard nicht geändert.
 3. Einzeloperationen `run_checks`, `compose_deploy`, `git_publish` und
    `recovery` nur für Diagnose oder bewusst partielle Wartung verwenden.
    Fremde Dirty-Worktree-Dateien bleiben unberührt.

docs/PLATFORM_OVERVIEW.md

@@ -171,5 +171,12 @@ Dateiersatz. Ein normaler MCP-Release erfolgt über `mcp_release`, das den
 versionierten Gesamtweg von Patch und Tests bis Deploy, Client-Sync, selektivem
 Git-Publish und Recovery kapselt.
 
+Seit Operator 2.3 übernimmt `mcp_release.imports` bereits geprüfte UTF-8-Dateien
+aus freigegebenen Staging-Verzeichnissen anhand ihrer SHA-256-Prüfsumme. Das
+Modell muss lange vorbereitete MCP-Quellen weder erneut lesen noch im Chat
+rekonstruieren. `hermes_sync: true` verteilt eine geänderte verwaltete
+Hermes-Konfiguration ohne Containerneustart. Ein interner MCP benötigt keinen
+neuen VPN-Port: Hermes und OpenWebUI erreichen ihn per Docker-DNS im Toolnetz.
+
 Secrets unter `/etc/mike-ai` werden ausschließlich verschlüsselt gesichert und
 gehören nie in Git, ein Wissensdokument oder einen Modellkontext.

docs/QWEN_OPERATOR_CONTEXT.md

@@ -107,8 +107,12 @@ Arbeitsweg vorgesehen: `patch_update` ändert kleine Stellen als SHA-geschützte
 Unified Diff im kanonischen Working Tree und in der ausgerollten Kopie;
 `file_update` ist neuen oder vollständig ersetzten Dateien vorbehalten. Für
 einen vollständigen MCP-Lifecycle bündelt `mcp_release` Patch, Tests, benannten
-Deploy, OpenWebUI-Sync, selektiven Git-Publish und Recovery in einem bestätigten
-Ablauf. Vollständige Compose-Dateien oder Base64-Kopien sind dafür unnötig.
+Deploy, OpenWebUI-/Hermes-Sync, selektiven Git-Publish und Recovery in einem
+bestätigten Ablauf. Bereits geprüfte Staging-Dateien müssen über `imports` mit
+exakter SHA-256-Prüfsumme übernommen werden; ihr Inhalt wird nicht erneut
+erzeugt. Vollständige Compose-Dateien oder Base64-Kopien sind unnötig. Interne
+MCPs verwenden Docker-DNS und benötigen ohne ausdrücklichen Auftrag weder einen
+VPN-Port noch eine WireGuard-Änderung.
 `run_checks` prüft, `compose_deploy`
 rollt nur benannte Dienste aus, `git_publish` veröffentlicht nur ausdrücklich
 ausgewählte Pfade und `recovery` erneuert den Recovery-Koffer. Lege niemals

platform/hermes/skills/athena-operator/SKILL.md

@@ -4,7 +4,7 @@ description: Operate and extend the Athena AI platform safely.
 license: MIT
 metadata:
   hermes:
-    version: 0.1.0
+    version: 0.2.0
     author: Michael Roll, Hermes Agent
     platforms: [linux, macos, windows]
     tags: [athena, operations, docker, mcp, models, recovery]
@@ -86,6 +86,34 @@ repository and use live measurements only as evidence of current state.
    recovery bundle, then check maintenance status.
    Completion: source commit, deployed state, docs, and recovery agree.
 
+## Compact MCP Release Recipe
+
+Use this route for a new self-written MCP. Do not rediscover the platform file
+by file.
+
+1. Inspect Athena once and check the service catalogue so an existing backend
+   is reused rather than duplicated.
+2. Treat an already reviewed artifact in an approved staging directory as an
+   input. Calculate its SHA once and pass it to `mcp_release.imports`; never
+   reproduce a long staged source file in chat or a `file_update` payload.
+3. Patch only the actual integration sources: `platform/mcp/compose.yaml`, the
+   managed Hermes config in `platform/hermes/config.yaml`, OpenWebUI's
+   versioned connector sync/seed, the env example, tests and relevant docs.
+4. Set `hermes_sync: true` when the managed Hermes MCP list changes and
+   `openwebui_sync: true` when OpenWebUI's connector list changes. Neither sync
+   restarts Hermes, Router, Qwen, WireGuard, or the complete stack.
+5. Do not add a VPN port or edit WireGuard for an ordinary in-stack MCP. Hermes
+   and OpenWebUI use Docker DNS on the private tool network. Add external VPN
+   publication only when the user explicitly asks for access by outside MCP
+   clients.
+6. One `mcp_release` should import/patch, test, deploy only the named MCP,
+   synchronize clients, publish selected paths and create recovery. Then verify
+   handshake plus one bounded non-writing function.
+
+A tool-call budget that ends "at a checkpoint" means: report a compact status,
+then continue the same approved task with a fresh budget. It does not mean
+abandon the requested implementation after reconnaissance.
+
 ## Persistent Versus Temporary Work
 
 - "Use" a missing helper for one task: place it in a task-specific temporary

platform/mcp/athena_operator_mcp.py

@@ -92,8 +92,11 @@ TOOLS = [
             "request or copy an SSH key. "
             "patch_update payload: {files:[{path,patch,expected_sha256?}]} using standard unified diffs; "
             "file_update payload: {files:[{path,content,expected_sha256?}]}; "
-            "mcp_release payload: {files:[patch entries],services,checks?,message,paths?,build?,"
-            "openwebui_sync?,create_recovery?,recovery_label?}. It applies drift-protected patches, runs checks, "
+            "mcp_release payload: {files?:[patch entries],imports?:[{source,path,expected_source_sha256,"
+            "expected_target_sha256?}],services,checks?,message,paths?,build?,openwebui_sync?,hermes_sync?,"
+            "create_recovery?,recovery_label?}. imports copies already reviewed UTF-8 staging files by exact SHA "
+            "from an approved staging root, so never reproduce a long staged source file in a payload. It applies "
+            "drift-protected patches/imports, runs checks, "
             "deploys only named MCP services, syncs OpenWebUI, publishes only selected paths and creates recovery. "
             "Use mcp_release instead of manually chaining all those operations. run_checks: "
             "{checks:[operator-tests,openwebui-filter-tests,platform-verify,compose-main,compose-mcp]}; "

platform/operator/athena_operatord.py

@@ -28,7 +28,7 @@ from pathlib import Path
 from typing import Any
 
 
-VERSION = "2.2.0"
+VERSION = "2.3.0"
 STACK = Path(os.environ.get("ATHENA_OPERATOR_STACK", "/opt/mike-ai/stack")).resolve()
 REPOSITORY = Path(os.environ.get("ATHENA_OPERATOR_REPOSITORY", "/data/mike-ai-operator/repository")).resolve()
 STATE = Path(os.environ.get("ATHENA_OPERATOR_STATE", "/data/mike-ai-operator/state")).resolve()
@@ -39,6 +39,14 @@ MAX_FILE_BYTES = 1_000_000
 MAX_FILES = 24
 MAX_OUTPUT = 30_000
 LOCK = threading.RLock()
+STAGING_ROOTS = tuple(
+    Path(value).resolve()
+    for value in os.environ.get(
+        "ATHENA_OPERATOR_STAGING_ROOTS",
+        "/data/mike-ai-operator/staging:/data/hermes/deemix-mcp-build",
+    ).split(":")
+    if value
+)
 
 SAFE_PATH = re.compile(r"^[A-Za-z0-9_.+/-]{1,240}$")
 SAFE_NAME = re.compile(r"^[A-Za-z0-9_.-]{1,100}$")
@@ -296,8 +304,59 @@ def search_source(arguments: dict[str, Any]) -> dict[str, Any]:
     query = str(arguments.get("query", ""))
     if not query or len(query) > 200 or any(x in query for x in ("\x00", "\n", "\r")):
         raise ValueError("invalid query")
+    if shutil.which("rg"):
         result = run(["rg", "-n", "--hidden", "--glob", "!.git/**", "--", query, "."], cwd=REPOSITORY, timeout=20)
-    return {"query": query, "matches": result["output"], "exit_code": result["exit_code"]}
+        return {"query": query, "matches": result["output"], "exit_code": result["exit_code"], "engine": "rg"}
+    try:
+        pattern = re.compile(query)
+    except re.error as exc:
+        raise ValueError(f"invalid search expression: {exc}") from exc
+    matches: list[str] = []
+    for path in sorted(REPOSITORY.rglob("*")):
+        if not path.is_file() or ".git" in path.parts or path.stat().st_size > MAX_FILE_BYTES:
+            continue
+        try:
+            lines = path.read_text(encoding="utf-8", errors="strict").splitlines()
+        except (UnicodeDecodeError, OSError):
+            continue
+        relative = path.relative_to(REPOSITORY)
+        for number, line in enumerate(lines, 1):
+            if pattern.search(line):
+                matches.append(f"{relative}:{number}:{line}")
+                if len(matches) >= 200:
+                    return {"query": query, "matches": "\n".join(matches), "exit_code": 0, "engine": "python", "truncated": True}
+    return {"query": query, "matches": "\n".join(matches), "exit_code": 0 if matches else 1, "engine": "python", "truncated": False}
+
+
+def staged_file(item: dict[str, Any]) -> dict[str, Any]:
+    source = Path(str(item.get("source", "")))
+    if not source.is_absolute() or not source.is_file():
+        raise FileNotFoundError("staged source file not found")
+    resolved = source.resolve()
+    if not any(resolved.is_relative_to(root) for root in STAGING_ROOTS):
+        raise PermissionError("staged source is outside approved staging roots")
+    raw = resolved.read_bytes()
+    if len(raw) > MAX_FILE_BYTES:
+        raise ValueError("staged source exceeds limit")
+    expected = str(item.get("expected_source_sha256", ""))
+    actual = sha(raw)
+    if not expected or expected != actual:
+        raise RuntimeError("staged source checksum is missing or does not match")
+    try:
+        content = raw.decode("utf-8", errors="strict")
+    except UnicodeDecodeError as exc:
+        raise ValueError("staged source must be UTF-8 text") from exc
+    relative = safe_relative(str(item.get("path", "")))
+    target = source_file(REPOSITORY, relative)
+    before = target.read_text(encoding="utf-8", errors="strict") if target.is_file() else ""
+    before_sha = sha(before.encode())
+    expected_target = str(item.get("expected_target_sha256", ""))
+    if expected_target and expected_target != before_sha:
+        raise RuntimeError(f"source drift for {relative}")
+    return {
+        "path": str(relative), "content": content, "before_sha256": before_sha,
+        "after_sha256": actual, "staged_source": str(resolved),
+    }
 
 
 def terminal(arguments: dict[str, Any]) -> dict[str, Any]:
@@ -362,7 +421,19 @@ def normalise_operation(operation: str, payload: dict[str, Any]) -> tuple[dict[s
         files, preview = normalise_patches(payload.get("files"))
         return {"files": files}, preview
     if operation == "mcp_release":
-        files, patch_preview = normalise_patches(payload.get("files"))
+        patch_items = payload.get("files") or []
+        import_items = payload.get("imports") or []
+        if not isinstance(patch_items, list) or not isinstance(import_items, list):
+            raise ValueError("files and imports must be lists")
+        if not patch_items and not import_items:
+            raise ValueError("mcp_release requires at least one patch or staged import")
+        files, patch_preview = normalise_patches(patch_items) if patch_items else ([], "")
+        imports = [staged_file(item) for item in import_items]
+        files.extend(imports)
+        import_preview = "\n".join(
+            f"IMPORT {item['staged_source']} -> {item['path']} sha256={item['after_sha256']}"
+            for item in imports
+        )
         checks = payload.get("checks") or ["operator-tests", "compose-mcp"]
         if not isinstance(checks, list) or not checks or any(name not in ALLOWED_CHECKS for name in checks):
             raise ValueError("unknown release check suite")
@@ -386,14 +457,17 @@ def normalise_operation(operation: str, payload: dict[str, Any]) -> tuple[dict[s
             "files": files, "checks": checks, "compose_file": compose_file,
             "services": [str(x) for x in services], "build": bool(payload.get("build", True)),
             "openwebui_sync": bool(payload.get("openwebui_sync", True)),
+            "hermes_sync": bool(payload.get("hermes_sync", False)),
             "message": message, "paths": selected,
             "create_recovery": bool(payload.get("create_recovery", True)), "recovery_label": label,
         }
         preview = (
             f"ONE MCP RELEASE\nServices: {', '.join(normal['services'])}\n"
             f"Checks: {', '.join(checks)}\nOpenWebUI sync: {normal['openwebui_sync']}\n"
+            f"Hermes sync: {normal['hermes_sync']}\n"
             f"Selective commit: {message}\nPaths: {', '.join(selected)}\n"
-            f"Recovery: {normal['create_recovery']} ({label})\n\n{patch_preview}"
+            f"Recovery: {normal['create_recovery']} ({label})\n\n"
+            + "\n".join(part for part in (import_preview, patch_preview) if part)
         )
         return normal, preview
     if operation == "run_checks":
@@ -579,6 +653,7 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d
             published = False
             deployed = False
             synced = False
+            hermes_synced = False
             try:
                 changed = apply_files(payload["files"], backup)
                 checks = []
@@ -595,10 +670,14 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d
                 if payload["openwebui_sync"]:
                     sync = run(["bash", str(STACK / "platform/openwebui/install-filters.sh")], cwd=STACK, timeout=1800, check=True)
                     synced = True
+                hermes_sync = None
+                if payload["hermes_sync"]:
+                    hermes_sync = run(["bash", str(STACK / "platform/hermes/install-hermes.sh")], cwd=STACK, timeout=1800, check=True)
+                    hermes_synced = True
                 publication = publish_paths(payload["message"], payload["paths"])
                 published = True
                 recovery = perform_recovery(payload["recovery_label"]) if payload["create_recovery"] else None
-                return {"changed": changed, "checks": checks, "deploy": deploy, "openwebui_sync": sync, "publication": publication, "recovery": recovery, "containers": run(["docker", "ps", "--format", "{{.Names}}\t{{.Status}}"])}
+                return {"changed": changed, "checks": checks, "deploy": deploy, "openwebui_sync": sync, "hermes_sync": hermes_sync, "publication": publication, "recovery": recovery, "containers": run(["docker", "ps", "--format", "{{.Names}}\t{{.Status}}"])}
             except Exception:
                 if not published:
                     restore_files(payload["files"], backup)
@@ -608,6 +687,8 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d
                         run(rollback_argv, cwd=STACK, timeout=3600)
                     if synced:
                         run(["bash", str(STACK / "platform/openwebui/install-filters.sh")], cwd=STACK, timeout=1800)
+                    if hermes_synced:
+                        run(["bash", str(STACK / "platform/hermes/install-hermes.sh")], cwd=STACK, timeout=1800)
                 raise
         return start_job(ticket, operation, release)
     if operation == "run_checks":