"""Stale job janitor: marks claimed/running jobs as failed when their satellite stops reporting (e.g. satellite offline, job crashed). """ import asyncio from datetime import UTC, datetime, timedelta from sqlalchemy import select from app.core.config import get_settings from app.core.database import async_session_factory from app.core.logging import get_logger from app.models.update_job import JobStatus, UpdateJob settings = get_settings() logger = get_logger(__name__) CHECK_INTERVAL = 60 # seconds async def run_janitor() -> None: """Background task; runs until cancelled.""" while True: try: await _sweep() except Exception as exc: # noqa: BLE001 - janitor must never die logger.error("janitor.error", error=str(exc)) await asyncio.sleep(CHECK_INTERVAL) async def _sweep() -> None: cutoff = datetime.now(UTC) - timedelta(seconds=settings.job_stale_timeout) async with async_session_factory() as db: result = await db.execute( select(UpdateJob).where( UpdateJob.status.in_([JobStatus.CLAIMED, JobStatus.RUNNING]), UpdateJob.last_report_at < cutoff, ) ) stale = list(result.scalars().all()) for job in stale: job.status = JobStatus.FAILED job.error = "Satellite meldet sich nicht mehr (Timeout)" job.finished_at = datetime.now(UTC) logger.warning("janitor.job_stale", job_id=job.id, satellite_id=job.satellite_id) if stale: await db.commit()