B0rbor4d
cc4c3fcecb
- Backend: Customer/Satellite Models, customer_id auf Server/Job/Audit - Satellite-API: heartbeat, poll (atomares Claiming), logs, result, scan-result, health-report - Auth via X-Api-Key (SHA-256 gehasht) - Job-Queue: pending/claimed/running/success/failed + Stale-Janitor - Batch-Trigger: ein Job pro Server, Satellite arbeitet sequenziell ab - Credentials bleiben lokal: nur symbolische credential_ref zentral - Neues Paket satellite/: Pull-Loop, WinRM/SSH/CAU/Scanner, PyInstaller-tauglich - Frontend: Kunden-Switcher, Satelliten-View, Polling statt WebSocket - Entfernt: WebSocket/Socket.io, Redis, zentrale Credentials, JobRunner - Docs: README/AGENTS/PROMPT auf neue Architektur aktualisiert
48 lines
1.5 KiB
Python
48 lines
1.5 KiB
Python
"""Stale job janitor: marks claimed/running jobs as failed when their
|
|
satellite stops reporting (e.g. satellite offline, job crashed).
|
|
"""
|
|
|
|
import asyncio
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from app.core.config import get_settings
|
|
from app.core.database import async_session_factory
|
|
from app.core.logging import get_logger
|
|
from app.models.update_job import JobStatus, UpdateJob
|
|
|
|
settings = get_settings()
|
|
logger = get_logger(__name__)
|
|
|
|
CHECK_INTERVAL = 60 # seconds
|
|
|
|
|
|
async def run_janitor() -> None:
|
|
"""Background task; runs until cancelled."""
|
|
while True:
|
|
try:
|
|
await _sweep()
|
|
except Exception as exc: # noqa: BLE001 - janitor must never die
|
|
logger.error("janitor.error", error=str(exc))
|
|
await asyncio.sleep(CHECK_INTERVAL)
|
|
|
|
|
|
async def _sweep() -> None:
|
|
cutoff = datetime.now(UTC) - timedelta(seconds=settings.job_stale_timeout)
|
|
async with async_session_factory() as db:
|
|
result = await db.execute(
|
|
select(UpdateJob).where(
|
|
UpdateJob.status.in_([JobStatus.CLAIMED, JobStatus.RUNNING]),
|
|
UpdateJob.last_report_at < cutoff,
|
|
)
|
|
)
|
|
stale = list(result.scalars().all())
|
|
for job in stale:
|
|
job.status = JobStatus.FAILED
|
|
job.error = "Satellite meldet sich nicht mehr (Timeout)"
|
|
job.finished_at = datetime.now(UTC)
|
|
logger.warning("janitor.job_stale", job_id=job.id, satellite_id=job.satellite_id)
|
|
if stale:
|
|
await db.commit()
|