diff --git a/dev/test_athena_operator.py b/dev/test_athena_operator.py index 17702d3..342d986 100644 --- a/dev/test_athena_operator.py +++ b/dev/test_athena_operator.py @@ -29,7 +29,8 @@ class OperatorTests(unittest.TestCase): root = Path(self.temporary.name) self.repo, self.stack = root / "repository", root / "stack" self.models, self.state = root / "models", root / "state" - for path in (self.repo, self.stack, self.models, self.state): + self.staging = root / "staging" + for path in (self.repo, self.stack, self.models, self.state, self.staging): path.mkdir() (self.repo / ".git").mkdir() (self.repo / "docs").mkdir() @@ -40,6 +41,7 @@ class OperatorTests(unittest.TestCase): self.module.STACK = self.stack.resolve() self.module.MODELS = self.models.resolve() self.module.STATE = self.state.resolve() + self.module.STAGING_ROOTS = (self.staging.resolve(),) self.module.audit = lambda *args, **kwargs: None self.module.run = lambda argv, **kwargs: {"argv": argv, "exit_code": 0, "output": "ok"} @@ -124,6 +126,40 @@ class OperatorTests(unittest.TestCase): self.assertEqual(payload["paths"], ["docs/test.md"]) self.assertIn("ONE MCP RELEASE", preview) + def test_mcp_release_imports_reviewed_staging_file_by_sha(self): + source = self.staging / "server.py" + source.write_text("print('reviewed')\n", encoding="utf-8") + digest = self.module.sha(source.read_bytes()) + payload, preview = self.module.normalise_operation("mcp_release", { + "imports": [{"source": str(source), "path": "server.py", "expected_source_sha256": digest}], + "services": ["mcp-example"], "message": "Import reviewed MCP source", + "checks": ["operator-tests"], "create_recovery": False, "hermes_sync": True, + }) + self.assertEqual(payload["files"][0]["content"], "print('reviewed')\n") + self.assertTrue(payload["hermes_sync"]) + self.assertIn(f"sha256={digest}", preview) + + def test_staged_import_rejects_wrong_sha_and_unapproved_path(self): + source = self.staging / "server.py" + source.write_text("safe\n", encoding="utf-8") + with self.assertRaises(RuntimeError): + self.module.staged_file({"source": str(source), "path": "server.py", "expected_source_sha256": "0" * 64}) + outside = Path(self.temporary.name) / "outside.py" + outside.write_text("unsafe\n", encoding="utf-8") + with self.assertRaises(PermissionError): + self.module.staged_file({"source": str(outside), "path": "server.py", "expected_source_sha256": self.module.sha(outside.read_bytes())}) + + def test_search_source_has_python_fallback_without_rg(self): + (self.repo / "docs" / "needle.md").write_text("alpha\nneedle here\nomega\n", encoding="utf-8") + original = self.module.shutil.which + self.module.shutil.which = lambda name: None + try: + result = self.module.search_source({"query": "needle"}) + finally: + self.module.shutil.which = original + self.assertEqual(result["engine"], "python") + self.assertIn("docs/needle.md:2:needle here", result["matches"]) + def test_structured_power_operations_do_not_exist(self): self.assertNotIn("shell", self.module.ALLOWED_OPERATIONS) for operation in ("shutdown", "reboot", "ssh", "network", "command"): diff --git a/docs/CURRENT_REFERENCE.md b/docs/CURRENT_REFERENCE.md index d0272dd..3d0399f 100644 --- a/docs/CURRENT_REFERENCE.md +++ b/docs/CURRENT_REFERENCE.md @@ -358,11 +358,14 @@ sind serverseitig blockiert. Ein separater allgemeiner Shell-MCP wird nicht benötigt; die breite Fähigkeit ist portabel im Athena Operator auf VPN-Port 8202 enthalten. -Seit Operator 2.2 werden kleine Änderungen als SHA-geschützte Unified Diffs +Seit Operator 2.3 werden kleine Änderungen als SHA-geschützte Unified Diffs über `patch_update` übertragen. `mcp_release` fasst den üblichen vollständigen MCP-Ablauf in einem bestätigten Auftrag zusammen: Patch, Tests, benannter -Deploy, OpenWebUI-Sync, selektiver Git-Publish und Recovery. Damit muss das -Modell keine kompletten Compose- oder Installationsdateien rekonstruieren. +Deploy, OpenWebUI-/Hermes-Sync, selektiver Git-Publish und Recovery. Geprüfte +Staging-Dateien werden per Pfad und SHA importiert. Damit muss das Modell weder +lange MCP-Quellen noch komplette Compose- oder Installationsdateien +rekonstruieren. Die Quellensuche besitzt einen Python-Fallback, falls `rg` im +Executor-Image fehlt. ## Bekannte Probleme des alten Hosts diff --git a/docs/PLATFORM_CONTEXT_MCP.md b/docs/PLATFORM_CONTEXT_MCP.md index a065a41..4355a49 100644 --- a/docs/PLATFORM_CONTEXT_MCP.md +++ b/docs/PLATFORM_CONTEXT_MCP.md @@ -98,6 +98,11 @@ Der verbindliche Ablauf für dauerhafte Änderungen lautet: vorbereiten und nach separater Benutzerfreigabe ausführen. Es bündelt Prüfungen, benannten Compose-Deploy, OpenWebUI-Sync, selektiven Git-Publish und Recovery. + Bereits geprüfte lange Quelldateien werden mit `imports` plus exakter + SHA-256-Prüfsumme aus einem freigegebenen Staging-Verzeichnis übernommen; + sie werden nicht als Chattext oder Full-File-Payload nachgebaut. Änderungen + an `platform/hermes/config.yaml` verwenden `hermes_sync: true`. Für rein + interne MCPs wird WireGuard nicht geändert. 3. Einzeloperationen `run_checks`, `compose_deploy`, `git_publish` und `recovery` nur für Diagnose oder bewusst partielle Wartung verwenden. Fremde Dirty-Worktree-Dateien bleiben unberührt. diff --git a/docs/PLATFORM_OVERVIEW.md b/docs/PLATFORM_OVERVIEW.md index 472f84d..292eee0 100644 --- a/docs/PLATFORM_OVERVIEW.md +++ b/docs/PLATFORM_OVERVIEW.md @@ -171,5 +171,12 @@ Dateiersatz. Ein normaler MCP-Release erfolgt über `mcp_release`, das den versionierten Gesamtweg von Patch und Tests bis Deploy, Client-Sync, selektivem Git-Publish und Recovery kapselt. +Seit Operator 2.3 übernimmt `mcp_release.imports` bereits geprüfte UTF-8-Dateien +aus freigegebenen Staging-Verzeichnissen anhand ihrer SHA-256-Prüfsumme. Das +Modell muss lange vorbereitete MCP-Quellen weder erneut lesen noch im Chat +rekonstruieren. `hermes_sync: true` verteilt eine geänderte verwaltete +Hermes-Konfiguration ohne Containerneustart. Ein interner MCP benötigt keinen +neuen VPN-Port: Hermes und OpenWebUI erreichen ihn per Docker-DNS im Toolnetz. + Secrets unter `/etc/mike-ai` werden ausschließlich verschlüsselt gesichert und gehören nie in Git, ein Wissensdokument oder einen Modellkontext. diff --git a/docs/QWEN_OPERATOR_CONTEXT.md b/docs/QWEN_OPERATOR_CONTEXT.md index 1890dde..6cb7a78 100644 --- a/docs/QWEN_OPERATOR_CONTEXT.md +++ b/docs/QWEN_OPERATOR_CONTEXT.md @@ -107,8 +107,12 @@ Arbeitsweg vorgesehen: `patch_update` ändert kleine Stellen als SHA-geschützte Unified Diff im kanonischen Working Tree und in der ausgerollten Kopie; `file_update` ist neuen oder vollständig ersetzten Dateien vorbehalten. Für einen vollständigen MCP-Lifecycle bündelt `mcp_release` Patch, Tests, benannten -Deploy, OpenWebUI-Sync, selektiven Git-Publish und Recovery in einem bestätigten -Ablauf. Vollständige Compose-Dateien oder Base64-Kopien sind dafür unnötig. +Deploy, OpenWebUI-/Hermes-Sync, selektiven Git-Publish und Recovery in einem +bestätigten Ablauf. Bereits geprüfte Staging-Dateien müssen über `imports` mit +exakter SHA-256-Prüfsumme übernommen werden; ihr Inhalt wird nicht erneut +erzeugt. Vollständige Compose-Dateien oder Base64-Kopien sind unnötig. Interne +MCPs verwenden Docker-DNS und benötigen ohne ausdrücklichen Auftrag weder einen +VPN-Port noch eine WireGuard-Änderung. `run_checks` prüft, `compose_deploy` rollt nur benannte Dienste aus, `git_publish` veröffentlicht nur ausdrücklich ausgewählte Pfade und `recovery` erneuert den Recovery-Koffer. Lege niemals diff --git a/platform/hermes/skills/athena-operator/SKILL.md b/platform/hermes/skills/athena-operator/SKILL.md index 798d96d..2e8988b 100644 --- a/platform/hermes/skills/athena-operator/SKILL.md +++ b/platform/hermes/skills/athena-operator/SKILL.md @@ -4,7 +4,7 @@ description: Operate and extend the Athena AI platform safely. license: MIT metadata: hermes: - version: 0.1.0 + version: 0.2.0 author: Michael Roll, Hermes Agent platforms: [linux, macos, windows] tags: [athena, operations, docker, mcp, models, recovery] @@ -86,6 +86,34 @@ repository and use live measurements only as evidence of current state. recovery bundle, then check maintenance status. Completion: source commit, deployed state, docs, and recovery agree. +## Compact MCP Release Recipe + +Use this route for a new self-written MCP. Do not rediscover the platform file +by file. + +1. Inspect Athena once and check the service catalogue so an existing backend + is reused rather than duplicated. +2. Treat an already reviewed artifact in an approved staging directory as an + input. Calculate its SHA once and pass it to `mcp_release.imports`; never + reproduce a long staged source file in chat or a `file_update` payload. +3. Patch only the actual integration sources: `platform/mcp/compose.yaml`, the + managed Hermes config in `platform/hermes/config.yaml`, OpenWebUI's + versioned connector sync/seed, the env example, tests and relevant docs. +4. Set `hermes_sync: true` when the managed Hermes MCP list changes and + `openwebui_sync: true` when OpenWebUI's connector list changes. Neither sync + restarts Hermes, Router, Qwen, WireGuard, or the complete stack. +5. Do not add a VPN port or edit WireGuard for an ordinary in-stack MCP. Hermes + and OpenWebUI use Docker DNS on the private tool network. Add external VPN + publication only when the user explicitly asks for access by outside MCP + clients. +6. One `mcp_release` should import/patch, test, deploy only the named MCP, + synchronize clients, publish selected paths and create recovery. Then verify + handshake plus one bounded non-writing function. + +A tool-call budget that ends "at a checkpoint" means: report a compact status, +then continue the same approved task with a fresh budget. It does not mean +abandon the requested implementation after reconnaissance. + ## Persistent Versus Temporary Work - "Use" a missing helper for one task: place it in a task-specific temporary diff --git a/platform/mcp/athena_operator_mcp.py b/platform/mcp/athena_operator_mcp.py index b316583..1d6d52c 100755 --- a/platform/mcp/athena_operator_mcp.py +++ b/platform/mcp/athena_operator_mcp.py @@ -92,8 +92,11 @@ TOOLS = [ "request or copy an SSH key. " "patch_update payload: {files:[{path,patch,expected_sha256?}]} using standard unified diffs; " "file_update payload: {files:[{path,content,expected_sha256?}]}; " - "mcp_release payload: {files:[patch entries],services,checks?,message,paths?,build?," - "openwebui_sync?,create_recovery?,recovery_label?}. It applies drift-protected patches, runs checks, " + "mcp_release payload: {files?:[patch entries],imports?:[{source,path,expected_source_sha256," + "expected_target_sha256?}],services,checks?,message,paths?,build?,openwebui_sync?,hermes_sync?," + "create_recovery?,recovery_label?}. imports copies already reviewed UTF-8 staging files by exact SHA " + "from an approved staging root, so never reproduce a long staged source file in a payload. It applies " + "drift-protected patches/imports, runs checks, " "deploys only named MCP services, syncs OpenWebUI, publishes only selected paths and creates recovery. " "Use mcp_release instead of manually chaining all those operations. run_checks: " "{checks:[operator-tests,openwebui-filter-tests,platform-verify,compose-main,compose-mcp]}; " diff --git a/platform/operator/athena_operatord.py b/platform/operator/athena_operatord.py index 473a762..2148377 100755 --- a/platform/operator/athena_operatord.py +++ b/platform/operator/athena_operatord.py @@ -28,7 +28,7 @@ from pathlib import Path from typing import Any -VERSION = "2.2.0" +VERSION = "2.3.0" STACK = Path(os.environ.get("ATHENA_OPERATOR_STACK", "/opt/mike-ai/stack")).resolve() REPOSITORY = Path(os.environ.get("ATHENA_OPERATOR_REPOSITORY", "/data/mike-ai-operator/repository")).resolve() STATE = Path(os.environ.get("ATHENA_OPERATOR_STATE", "/data/mike-ai-operator/state")).resolve() @@ -39,6 +39,14 @@ MAX_FILE_BYTES = 1_000_000 MAX_FILES = 24 MAX_OUTPUT = 30_000 LOCK = threading.RLock() +STAGING_ROOTS = tuple( + Path(value).resolve() + for value in os.environ.get( + "ATHENA_OPERATOR_STAGING_ROOTS", + "/data/mike-ai-operator/staging:/data/hermes/deemix-mcp-build", + ).split(":") + if value +) SAFE_PATH = re.compile(r"^[A-Za-z0-9_.+/-]{1,240}$") SAFE_NAME = re.compile(r"^[A-Za-z0-9_.-]{1,100}$") @@ -296,8 +304,59 @@ def search_source(arguments: dict[str, Any]) -> dict[str, Any]: query = str(arguments.get("query", "")) if not query or len(query) > 200 or any(x in query for x in ("\x00", "\n", "\r")): raise ValueError("invalid query") - result = run(["rg", "-n", "--hidden", "--glob", "!.git/**", "--", query, "."], cwd=REPOSITORY, timeout=20) - return {"query": query, "matches": result["output"], "exit_code": result["exit_code"]} + if shutil.which("rg"): + result = run(["rg", "-n", "--hidden", "--glob", "!.git/**", "--", query, "."], cwd=REPOSITORY, timeout=20) + return {"query": query, "matches": result["output"], "exit_code": result["exit_code"], "engine": "rg"} + try: + pattern = re.compile(query) + except re.error as exc: + raise ValueError(f"invalid search expression: {exc}") from exc + matches: list[str] = [] + for path in sorted(REPOSITORY.rglob("*")): + if not path.is_file() or ".git" in path.parts or path.stat().st_size > MAX_FILE_BYTES: + continue + try: + lines = path.read_text(encoding="utf-8", errors="strict").splitlines() + except (UnicodeDecodeError, OSError): + continue + relative = path.relative_to(REPOSITORY) + for number, line in enumerate(lines, 1): + if pattern.search(line): + matches.append(f"{relative}:{number}:{line}") + if len(matches) >= 200: + return {"query": query, "matches": "\n".join(matches), "exit_code": 0, "engine": "python", "truncated": True} + return {"query": query, "matches": "\n".join(matches), "exit_code": 0 if matches else 1, "engine": "python", "truncated": False} + + +def staged_file(item: dict[str, Any]) -> dict[str, Any]: + source = Path(str(item.get("source", ""))) + if not source.is_absolute() or not source.is_file(): + raise FileNotFoundError("staged source file not found") + resolved = source.resolve() + if not any(resolved.is_relative_to(root) for root in STAGING_ROOTS): + raise PermissionError("staged source is outside approved staging roots") + raw = resolved.read_bytes() + if len(raw) > MAX_FILE_BYTES: + raise ValueError("staged source exceeds limit") + expected = str(item.get("expected_source_sha256", "")) + actual = sha(raw) + if not expected or expected != actual: + raise RuntimeError("staged source checksum is missing or does not match") + try: + content = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError("staged source must be UTF-8 text") from exc + relative = safe_relative(str(item.get("path", ""))) + target = source_file(REPOSITORY, relative) + before = target.read_text(encoding="utf-8", errors="strict") if target.is_file() else "" + before_sha = sha(before.encode()) + expected_target = str(item.get("expected_target_sha256", "")) + if expected_target and expected_target != before_sha: + raise RuntimeError(f"source drift for {relative}") + return { + "path": str(relative), "content": content, "before_sha256": before_sha, + "after_sha256": actual, "staged_source": str(resolved), + } def terminal(arguments: dict[str, Any]) -> dict[str, Any]: @@ -362,7 +421,19 @@ def normalise_operation(operation: str, payload: dict[str, Any]) -> tuple[dict[s files, preview = normalise_patches(payload.get("files")) return {"files": files}, preview if operation == "mcp_release": - files, patch_preview = normalise_patches(payload.get("files")) + patch_items = payload.get("files") or [] + import_items = payload.get("imports") or [] + if not isinstance(patch_items, list) or not isinstance(import_items, list): + raise ValueError("files and imports must be lists") + if not patch_items and not import_items: + raise ValueError("mcp_release requires at least one patch or staged import") + files, patch_preview = normalise_patches(patch_items) if patch_items else ([], "") + imports = [staged_file(item) for item in import_items] + files.extend(imports) + import_preview = "\n".join( + f"IMPORT {item['staged_source']} -> {item['path']} sha256={item['after_sha256']}" + for item in imports + ) checks = payload.get("checks") or ["operator-tests", "compose-mcp"] if not isinstance(checks, list) or not checks or any(name not in ALLOWED_CHECKS for name in checks): raise ValueError("unknown release check suite") @@ -386,14 +457,17 @@ def normalise_operation(operation: str, payload: dict[str, Any]) -> tuple[dict[s "files": files, "checks": checks, "compose_file": compose_file, "services": [str(x) for x in services], "build": bool(payload.get("build", True)), "openwebui_sync": bool(payload.get("openwebui_sync", True)), + "hermes_sync": bool(payload.get("hermes_sync", False)), "message": message, "paths": selected, "create_recovery": bool(payload.get("create_recovery", True)), "recovery_label": label, } preview = ( f"ONE MCP RELEASE\nServices: {', '.join(normal['services'])}\n" f"Checks: {', '.join(checks)}\nOpenWebUI sync: {normal['openwebui_sync']}\n" + f"Hermes sync: {normal['hermes_sync']}\n" f"Selective commit: {message}\nPaths: {', '.join(selected)}\n" - f"Recovery: {normal['create_recovery']} ({label})\n\n{patch_preview}" + f"Recovery: {normal['create_recovery']} ({label})\n\n" + + "\n".join(part for part in (import_preview, patch_preview) if part) ) return normal, preview if operation == "run_checks": @@ -579,6 +653,7 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d published = False deployed = False synced = False + hermes_synced = False try: changed = apply_files(payload["files"], backup) checks = [] @@ -595,10 +670,14 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d if payload["openwebui_sync"]: sync = run(["bash", str(STACK / "platform/openwebui/install-filters.sh")], cwd=STACK, timeout=1800, check=True) synced = True + hermes_sync = None + if payload["hermes_sync"]: + hermes_sync = run(["bash", str(STACK / "platform/hermes/install-hermes.sh")], cwd=STACK, timeout=1800, check=True) + hermes_synced = True publication = publish_paths(payload["message"], payload["paths"]) published = True recovery = perform_recovery(payload["recovery_label"]) if payload["create_recovery"] else None - return {"changed": changed, "checks": checks, "deploy": deploy, "openwebui_sync": sync, "publication": publication, "recovery": recovery, "containers": run(["docker", "ps", "--format", "{{.Names}}\t{{.Status}}"])} + return {"changed": changed, "checks": checks, "deploy": deploy, "openwebui_sync": sync, "hermes_sync": hermes_sync, "publication": publication, "recovery": recovery, "containers": run(["docker", "ps", "--format", "{{.Names}}\t{{.Status}}"])} except Exception: if not published: restore_files(payload["files"], backup) @@ -608,6 +687,8 @@ def execute_operation(ticket: str, operation: str, payload: dict[str, Any]) -> d run(rollback_argv, cwd=STACK, timeout=3600) if synced: run(["bash", str(STACK / "platform/openwebui/install-filters.sh")], cwd=STACK, timeout=1800) + if hermes_synced: + run(["bash", str(STACK / "platform/hermes/install-hermes.sh")], cwd=STACK, timeout=1800) raise return start_job(ticket, operation, release) if operation == "run_checks":