Hub-and-Spoke Umbau: Multi-Tenant Zentrale + Satellite-Agent
- Backend: Customer/Satellite Models, customer_id auf Server/Job/Audit - Satellite-API: heartbeat, poll (atomares Claiming), logs, result, scan-result, health-report - Auth via X-Api-Key (SHA-256 gehasht) - Job-Queue: pending/claimed/running/success/failed + Stale-Janitor - Batch-Trigger: ein Job pro Server, Satellite arbeitet sequenziell ab - Credentials bleiben lokal: nur symbolische credential_ref zentral - Neues Paket satellite/: Pull-Loop, WinRM/SSH/CAU/Scanner, PyInstaller-tauglich - Frontend: Kunden-Switcher, Satelliten-View, Polling statt WebSocket - Entfernt: WebSocket/Socket.io, Redis, zentrale Credentials, JobRunner - Docs: README/AGENTS/PROMPT auf neue Architektur aktualisiert
This commit is contained in:
B0rbor4d
@@ -25,6 +25,7 @@ class AuditService:
|
||||
result: str = "success",
|
||||
details: dict[str, Any] | None = None,
|
||||
ip_address: str | None = None,
|
||||
customer_id: int | None = None,
|
||||
) -> AuditLog:
|
||||
entry = AuditLog(
|
||||
username=username,
|
||||
@@ -33,6 +34,7 @@ class AuditService:
|
||||
result=result,
|
||||
details=json.dumps(details) if details else None,
|
||||
ip_address=ip_address,
|
||||
customer_id=customer_id,
|
||||
)
|
||||
self.db.add(entry)
|
||||
await self.db.flush()
|
||||
@@ -42,5 +44,6 @@ class AuditService:
|
||||
action=action,
|
||||
target=target,
|
||||
result=result,
|
||||
customer_id=customer_id,
|
||||
)
|
||||
return entry
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
"""CAU service: Cluster-Aware Updating via PowerShell remoting (WinRM)."""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from app.core.exceptions import CAUError
|
||||
from app.core.logging import get_logger
|
||||
from app.services.winrm import WinRMCredentials, WinRMService
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class CAUService:
|
||||
"""Orchestrates Invoke-CauRun against a Windows failover cluster."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cluster_name: str,
|
||||
access_node: str,
|
||||
port: int = 5985,
|
||||
credentials: WinRMCredentials | None = None,
|
||||
) -> None:
|
||||
self.cluster_name = cluster_name
|
||||
self.winrm = WinRMService(access_node, port=port, credentials=credentials)
|
||||
|
||||
async def get_cluster_nodes(self) -> list[dict[str, str]]:
|
||||
"""List cluster nodes and their state."""
|
||||
script = f"""
|
||||
Import-Module FailoverClusters
|
||||
Get-ClusterNode -Cluster {self.cluster_name} |
|
||||
Select-Object Name, State |
|
||||
ForEach-Object {{ "$($_.Name)|$($_.State)" }}
|
||||
"""
|
||||
output = await self.winrm.run_powershell(script)
|
||||
nodes = []
|
||||
for line in output.splitlines():
|
||||
if "|" in line:
|
||||
name, state = line.split("|", 1)
|
||||
nodes.append({"name": name.strip(), "state": state.strip()})
|
||||
return nodes
|
||||
|
||||
async def get_cau_status(self) -> str:
|
||||
"""Return the last CAU run summary."""
|
||||
script = f"""
|
||||
Import-Module ClusterAwareUpdating
|
||||
Get-CauRun -ClusterName {self.cluster_name} |
|
||||
Select-Object -First 1 |
|
||||
Format-List | Out-String
|
||||
"""
|
||||
return await self.winrm.run_powershell(script)
|
||||
|
||||
async def test_cluster(self) -> tuple[bool, str]:
|
||||
"""Verify the cluster is reachable and CAU module present."""
|
||||
try:
|
||||
nodes = await self.get_cluster_nodes()
|
||||
if not nodes:
|
||||
return False, "Keine Cluster-Knoten gefunden"
|
||||
return True, f"Cluster OK — {len(nodes)} Knoten"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return False, str(exc)
|
||||
|
||||
async def invoke_cau_run(self) -> AsyncIterator[str]:
|
||||
"""Start a CAU run and stream per-node progress lines.
|
||||
|
||||
Scaffold implementation: starts Invoke-CauRun and polls Get-CauReport.
|
||||
TODO: true live streaming of per-node phases via event log polling.
|
||||
"""
|
||||
script = f"""
|
||||
Import-Module ClusterAwareUpdating
|
||||
Invoke-CauRun -ClusterName {self.cluster_name} `
|
||||
-Force -Confirm:$false -WaitForCompletion |
|
||||
Out-String
|
||||
"""
|
||||
try:
|
||||
async for line in self.winrm.stream_powershell(script):
|
||||
yield line
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise CAUError(f"CAU-Lauf fehlgeschlagen: {exc}") from exc
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Stale job janitor: marks claimed/running jobs as failed when their
|
||||
satellite stops reporting (e.g. satellite offline, job crashed).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import async_session_factory
|
||||
from app.core.logging import get_logger
|
||||
from app.models.update_job import JobStatus, UpdateJob
|
||||
|
||||
settings = get_settings()
|
||||
logger = get_logger(__name__)
|
||||
|
||||
CHECK_INTERVAL = 60 # seconds
|
||||
|
||||
|
||||
async def run_janitor() -> None:
|
||||
"""Background task; runs until cancelled."""
|
||||
while True:
|
||||
try:
|
||||
await _sweep()
|
||||
except Exception as exc: # noqa: BLE001 - janitor must never die
|
||||
logger.error("janitor.error", error=str(exc))
|
||||
await asyncio.sleep(CHECK_INTERVAL)
|
||||
|
||||
|
||||
async def _sweep() -> None:
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=settings.job_stale_timeout)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(
|
||||
select(UpdateJob).where(
|
||||
UpdateJob.status.in_([JobStatus.CLAIMED, JobStatus.RUNNING]),
|
||||
UpdateJob.last_report_at < cutoff,
|
||||
)
|
||||
)
|
||||
stale = list(result.scalars().all())
|
||||
for job in stale:
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = "Satellite meldet sich nicht mehr (Timeout)"
|
||||
job.finished_at = datetime.now(UTC)
|
||||
logger.warning("janitor.job_stale", job_id=job.id, satellite_id=job.satellite_id)
|
||||
if stale:
|
||||
await db.commit()
|
||||
@@ -1,138 +0,0 @@
|
||||
"""Job runner: executes update jobs in background tasks, streams logs to WS + DB."""
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.core.database import async_session_factory
|
||||
from app.core.logging import get_logger
|
||||
from app.models.credential import Credential, CredentialType
|
||||
from app.models.server import Server, ServerType
|
||||
from app.models.update_job import JobStatus, UpdateJob, UpdateLog
|
||||
from app.services.cau import CAUService
|
||||
from app.services.ssh import SSHCredentials, SSHService
|
||||
from app.services.winrm import WinRMCredentials, WinRMService
|
||||
from app.websocket import handlers as ws
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class JobRunner:
|
||||
"""Manages asyncio tasks for running update jobs."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tasks: dict[int, asyncio.Task] = {} # type: ignore[type-arg]
|
||||
|
||||
async def start(self, job_id: int) -> None:
|
||||
task = asyncio.create_task(self._run(job_id), name=f"job-{job_id}")
|
||||
self._tasks[job_id] = task
|
||||
task.add_done_callback(lambda _t: self._tasks.pop(job_id, None))
|
||||
|
||||
async def cancel(self, job_id: int) -> bool:
|
||||
task = self._tasks.get(job_id)
|
||||
if not task:
|
||||
return False
|
||||
task.cancel()
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _run(self, job_id: int) -> None:
|
||||
async with async_session_factory() as db:
|
||||
job = await db.get(UpdateJob, job_id)
|
||||
if not job:
|
||||
logger.error("job.not_found", job_id=job_id)
|
||||
return
|
||||
server = await db.get(Server, job.server_id)
|
||||
if not server:
|
||||
await self._finish(db, job, JobStatus.FAILED, "Server nicht gefunden")
|
||||
return
|
||||
|
||||
job.status = JobStatus.RUNNING
|
||||
job.started_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
|
||||
await ws.emit_job_start(job.id, server.id, job.type.value)
|
||||
started = datetime.now(UTC)
|
||||
|
||||
try:
|
||||
await self._dispatch(db, job, server)
|
||||
await self._finish(db, job, JobStatus.SUCCESS, None, started)
|
||||
except asyncio.CancelledError:
|
||||
await self._finish(db, job, JobStatus.CANCELLED, "Vom Benutzer abgebrochen", started)
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("job.failed", job_id=job_id, error=str(exc))
|
||||
await self._finish(db, job, JobStatus.FAILED, str(exc), started)
|
||||
|
||||
async def _dispatch(self, db, job: UpdateJob, server: Server) -> None: # type: ignore[no-untyped-def]
|
||||
credential = await db.get(Credential, server.credential_id) if server.credential_id else None
|
||||
|
||||
if server.type == ServerType.LINUX:
|
||||
creds = self._ssh_creds(credential)
|
||||
service = SSHService(server.hostname, port=server.port, credentials=creds)
|
||||
async for line in service.stream_updates():
|
||||
await self._log(db, job, line)
|
||||
|
||||
elif server.type == ServerType.WINDOWS:
|
||||
creds = self._winrm_creds(credential)
|
||||
service = WinRMService(server.hostname, port=server.port, credentials=creds)
|
||||
async for line in service.install_updates():
|
||||
await self._log(db, job, line)
|
||||
|
||||
elif server.type == ServerType.CAU_CLUSTER:
|
||||
creds = self._winrm_creds(credential)
|
||||
service = CAUService(server.hostname, access_node=server.hostname, port=server.port, credentials=creds)
|
||||
async for line in service.invoke_cau_run():
|
||||
await self._log(db, job, line)
|
||||
|
||||
async def _log(self, db, job: UpdateJob, line: str, level: str = "info") -> None: # type: ignore[no-untyped-def]
|
||||
db.add(UpdateLog(job_id=job.id, level=level, line=line))
|
||||
await db.commit()
|
||||
await ws.emit_job_log(job.id, line, level)
|
||||
|
||||
async def _finish( # type: ignore[no-untyped-def]
|
||||
self, db, job: UpdateJob, status: JobStatus, error: str | None, started: datetime | None = None
|
||||
) -> None:
|
||||
job.status = status
|
||||
job.error = error
|
||||
job.finished_at = datetime.now(UTC)
|
||||
if status == JobStatus.SUCCESS:
|
||||
job.progress_percent = 100
|
||||
await db.commit()
|
||||
duration = (
|
||||
(job.finished_at - started).total_seconds() if started else None
|
||||
)
|
||||
await ws.emit_job_complete(job.id, status.value, duration)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _winrm_creds(credential: Credential | None) -> WinRMCredentials | None:
|
||||
if not credential or credential.type != CredentialType.WINRM_USERPASS:
|
||||
return None
|
||||
from app.core.security import decrypt
|
||||
|
||||
return WinRMCredentials(
|
||||
username=credential.username,
|
||||
password=decrypt(credential.password_encrypted or ""),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _ssh_creds(credential: Credential | None) -> SSHCredentials | None:
|
||||
if not credential:
|
||||
return None
|
||||
from app.core.security import decrypt
|
||||
|
||||
return SSHCredentials(
|
||||
username=credential.username,
|
||||
password=decrypt(credential.password_encrypted) if credential.password_encrypted else None,
|
||||
private_key=decrypt(credential.private_key_encrypted)
|
||||
if credential.private_key_encrypted
|
||||
else None,
|
||||
passphrase=decrypt(credential.key_passphrase_encrypted)
|
||||
if credential.key_passphrase_encrypted
|
||||
else None,
|
||||
)
|
||||
|
||||
|
||||
job_runner = JobRunner()
|
||||
@@ -1,118 +0,0 @@
|
||||
"""SSH service: connect to Linux hosts via asyncssh, run updates with sudo."""
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.exceptions import SSHError
|
||||
from app.core.logging import get_logger
|
||||
|
||||
settings = get_settings()
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SSHCredentials:
|
||||
username: str
|
||||
password: str | None = None
|
||||
private_key: str | None = None
|
||||
passphrase: str | None = None
|
||||
|
||||
|
||||
class SSHService:
|
||||
"""Async SSH operations via asyncssh."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hostname: str,
|
||||
port: int = 22,
|
||||
credentials: SSHCredentials | None = None,
|
||||
) -> None:
|
||||
self.hostname = hostname
|
||||
self.port = port
|
||||
self.credentials = credentials
|
||||
|
||||
def _connect_kwargs(self) -> dict:
|
||||
kwargs: dict = {
|
||||
"host": self.hostname,
|
||||
"port": self.port,
|
||||
"known_hosts": None, # scaffold: accept any host key
|
||||
"connect_timeout": settings.ssh_timeout,
|
||||
}
|
||||
if self.credentials:
|
||||
kwargs["username"] = self.credentials.username
|
||||
if self.credentials.password:
|
||||
kwargs["password"] = self.credentials.password
|
||||
if self.credentials.private_key:
|
||||
kwargs["client_keys"] = [
|
||||
__import__("asyncssh").import_private_key(
|
||||
self.credentials.private_key,
|
||||
passphrase=self.credentials.passphrase,
|
||||
)
|
||||
]
|
||||
return kwargs
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
import asyncssh
|
||||
|
||||
try:
|
||||
async with asyncssh.connect(**self._connect_kwargs()) as conn:
|
||||
result = await conn.run("hostname", check=True)
|
||||
return True, f"Verbunden mit {result.stdout.strip()}" # type: ignore[union-attr]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return False, str(exc)
|
||||
|
||||
async def run_command(self, command: str, sudo: bool = False) -> str:
|
||||
"""Run a command, optionally via sudo -S with the stored password."""
|
||||
import asyncssh
|
||||
|
||||
if sudo:
|
||||
password = (self.credentials.password if self.credentials else None) or ""
|
||||
command = f"echo '{password}' | sudo -S {command}"
|
||||
|
||||
async with asyncssh.connect(**self._connect_kwargs()) as conn:
|
||||
result = await conn.run(command, check=False)
|
||||
if result.returncode != 0:
|
||||
raise SSHError(
|
||||
f"Command failed (exit {result.returncode}): {str(result.stderr).strip()}"
|
||||
)
|
||||
return str(result.stdout)
|
||||
|
||||
async def detect_package_manager(self) -> str:
|
||||
"""Detect apt, dnf, or yum on the target."""
|
||||
import asyncssh
|
||||
|
||||
async with asyncssh.connect(**self._connect_kwargs()) as conn:
|
||||
for pm in ("apt-get", "dnf", "yum"):
|
||||
result = await conn.run(f"command -v {pm}", check=False)
|
||||
if result.returncode == 0:
|
||||
return pm
|
||||
raise SSHError("Kein unterstützter Paketmanager gefunden (apt/dnf/yum)")
|
||||
|
||||
async def stream_updates(self, reboot_if_required: bool = False) -> AsyncIterator[str]:
|
||||
"""Run the distro update command and yield output lines live."""
|
||||
import asyncssh
|
||||
|
||||
pm = await self.detect_package_manager()
|
||||
if pm == "apt-get":
|
||||
cmd = "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"
|
||||
else:
|
||||
cmd = f"{pm} update -y"
|
||||
|
||||
password = (self.credentials.password if self.credentials else None) or ""
|
||||
full_cmd = f"echo '{password}' | sudo -S sh -c '{cmd}'"
|
||||
|
||||
async with asyncssh.connect(**self._connect_kwargs()) as conn:
|
||||
async with conn.create_process(full_cmd) as process:
|
||||
async for line in process.stdout: # type: ignore[union-attr]
|
||||
yield str(line).rstrip()
|
||||
await process.wait()
|
||||
if process.returncode != 0:
|
||||
raise SSHError(f"Update fehlgeschlagen (exit {process.returncode})")
|
||||
|
||||
if reboot_if_required:
|
||||
async with asyncssh.connect(**self._connect_kwargs()) as conn:
|
||||
check = await conn.run("test -f /var/run/reboot-required", check=False)
|
||||
if check.returncode == 0:
|
||||
yield "REBOOT erforderlich — wird ausgeführt..."
|
||||
await conn.run(f"echo '{password}' | sudo -S reboot", check=False)
|
||||
@@ -1,122 +0,0 @@
|
||||
"""WinRM service: connect to Windows hosts, run PowerShell, stream output.
|
||||
|
||||
Uses python-winrm (pywinrm). All blocking calls run in a thread pool
|
||||
so the async event loop is never blocked.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.exceptions import WinRMError
|
||||
from app.core.logging import get_logger
|
||||
|
||||
settings = get_settings()
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WinRMCredentials:
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class WinRMService:
|
||||
"""Wraps pywinrm for async usage."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hostname: str,
|
||||
port: int = 5985,
|
||||
credentials: WinRMCredentials | None = None,
|
||||
transport: str | None = None,
|
||||
) -> None:
|
||||
self.hostname = hostname
|
||||
self.port = port
|
||||
self.credentials = credentials
|
||||
self.transport = transport or settings.winrm_transport
|
||||
|
||||
def _build_session(self): # type: ignore[no-untyped-def]
|
||||
import winrm # pywinrm
|
||||
|
||||
scheme = "https" if self.port == 5986 else "http"
|
||||
endpoint = f"{scheme}://{self.hostname}:{self.port}/wsman"
|
||||
kwargs: dict = {
|
||||
"transport": self.transport,
|
||||
"server_cert_validation": settings.winrm_cert_validation,
|
||||
"operation_timeout_sec": settings.winrm_operation_timeout,
|
||||
"read_timeout_sec": settings.winrm_read_timeout,
|
||||
}
|
||||
if self.credentials:
|
||||
kwargs["username"] = self.credentials.username
|
||||
kwargs["password"] = self.credentials.password
|
||||
return winrm.Session(endpoint, **kwargs)
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
"""Run a trivial command to verify connectivity."""
|
||||
|
||||
def _run() -> tuple[bool, str]:
|
||||
try:
|
||||
session = self._build_session()
|
||||
result = session.run_ps("$env:COMPUTERNAME")
|
||||
if result.status_code == 0:
|
||||
name = result.std_out.decode(errors="replace").strip()
|
||||
return True, f"Verbunden mit {name}"
|
||||
return False, result.std_err.decode(errors="replace").strip()
|
||||
except Exception as exc: # noqa: BLE001 - surface any transport error
|
||||
return False, str(exc)
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
async def run_powershell(self, script: str) -> str:
|
||||
"""Run a PowerShell script and return stdout. Raises WinRMError on failure."""
|
||||
|
||||
def _run() -> str:
|
||||
session = self._build_session()
|
||||
result = session.run_ps(script)
|
||||
if result.status_code != 0:
|
||||
err = result.std_err.decode(errors="replace").strip()
|
||||
raise WinRMError(f"PowerShell exit {result.status_code}: {err}")
|
||||
return result.std_out.decode(errors="replace")
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
async def stream_powershell(self, script: str) -> AsyncIterator[str]:
|
||||
"""Yield output lines as they arrive (scaffold: runs to completion, then yields).
|
||||
|
||||
TODO: switch to winrm Protocol with shell polling for true streaming.
|
||||
"""
|
||||
output = await self.run_powershell(script)
|
||||
for line in output.splitlines():
|
||||
yield line
|
||||
await asyncio.sleep(0) # keep the loop responsive
|
||||
|
||||
async def get_pending_updates(self) -> list[str]:
|
||||
"""Query Windows Update for pending updates (titles only)."""
|
||||
script = """
|
||||
$session = New-Object -ComObject Microsoft.Update.Session
|
||||
$searcher = $session.CreateUpdateSearcher()
|
||||
$result = $searcher.Search("IsInstalled=0")
|
||||
$result.Updates | ForEach-Object { $_.Title }
|
||||
"""
|
||||
output = await self.run_powershell(script)
|
||||
return [line.strip() for line in output.splitlines() if line.strip()]
|
||||
|
||||
async def install_updates(self) -> AsyncIterator[str]:
|
||||
"""Install all pending Windows Updates, yielding progress lines."""
|
||||
script = """
|
||||
$session = New-Object -ComObject Microsoft.Update.Session
|
||||
$searcher = $session.CreateUpdateSearcher()
|
||||
$result = $searcher.Search("IsInstalled=0")
|
||||
Write-Output "Gefundene Updates: $($result.Updates.Count)"
|
||||
$toInstall = New-Object -ComObject Microsoft.Update.UpdateColl
|
||||
$result.Updates | ForEach-Object { $toInstall.Add($_) | Out-Null }
|
||||
$installer = $session.CreateUpdateInstaller()
|
||||
$installer.Updates = $toInstall
|
||||
$installResult = $installer.Install()
|
||||
Write-Output "ResultCode: $($installResult.ResultCode)"
|
||||
Write-Output "RebootRequired: $($installResult.RebootRequired)"
|
||||
"""
|
||||
async for line in self.stream_powershell(script):
|
||||
yield line
|
||||
Reference in New Issue
Block a user