auto-stock/scripts/health_watch.py

#!/usr/bin/env python3
"""/health 주기 감시 (NAS cron: 평일 장중 5분). 표준 라이브러리만 사용.

  */5 9-15 * * 1-5  python3 /path/scripts/health_watch.py   (KST 기준 cron)

판정: status != healthy / 스케줄러 alive=False·stale / db_worker alive=False / dlq.manual>0 /
      응답 불가(연결 실패·비JSON) 가 FAIL_THRESHOLD 회 연속이면 텔레그램 경보(상태 전이 시 1회, 복구 시 1회).
환경변수: HEALTH_URL(기본 http://127.0.0.1:8000/health), TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID,
          HEALTH_WATCH_STATE(기본 ~/.cache/auto-stock/health_watch.json), HEALTH_WATCH_FAILS(기본 2)
텔레그램 미설정이면 stderr 에 ERROR 를 남기고 exit 1 (cron 메일/로그로 식별).
종료코드: 0 정상, 1 이상(현재 회차 기준).
"""
import json
import os
import sys
import urllib.parse
import urllib.request


def problems_v1(health) -> list:
    """/health 응답 dict 에서 문제 목록을 만든다(순수 함수)."""
    if not isinstance(health, dict):
        return ["응답이 JSON 객체가 아님"]
    out = []
    if health.get("status") != "healthy":
        out.append(f"status={health.get('status')}")
    for name, v in (health.get("schedulers") or {}).items():
        if not v.get("alive"):
            out.append(f"scheduler {name} 사망")
        elif v.get("stale"):
            out.append(f"scheduler {name} stale({v.get('last_cycle_age')}s)")
    dbw = health.get("db_worker") or {}
    if "alive" in dbw and not dbw.get("alive"):
        out.append("db_worker 사망")
    if ((health.get("dlq") or {}).get("manual") or 0) > 0:
        out.append(f"DLQ manual={health['dlq']['manual']}")
    reasons = health.get("degraded_reasons")
    if out and isinstance(reasons, list) and reasons:
        out.append("degraded_reasons=" + ",".join(str(r) for r in reasons))
    return out


def next_state_v1(state: dict, problems: list, threshold: int):
    """(새 상태, 행동) — 행동: None | 'alert' | 'recover'."""
    fails = int(state.get("fails", 0))
    alerted = bool(state.get("alerted", False))
    if problems:
        fails += 1
        if fails >= threshold and not alerted:
            return {"fails": fails, "alerted": True}, "alert"
        return {"fails": fails, "alerted": alerted}, None
    if alerted:
        return {"fails": 0, "alerted": False}, "recover"
    return {"fails": 0, "alerted": False}, None


def _send_telegram(text: str) -> bool:
    token, chat = os.environ.get("TELEGRAM_BOT_TOKEN", ""), os.environ.get("TELEGRAM_CHAT_ID", "")
    if not token or not chat:
        print("ERROR: TELEGRAM_BOT_TOKEN/CHAT_ID 미설정 — 경보 전송 불가", file=sys.stderr)
        return False
    data = urllib.parse.urlencode({"chat_id": chat, "text": text}).encode()
    try:
        with urllib.request.urlopen(f"https://api.telegram.org/bot{token}/sendMessage", data=data, timeout=10) as r:
            return r.status == 200
    except Exception as e:  # 전송 실패는 경보 재시도를 위해 False 로 보고
        print(f"ERROR: 텔레그램 전송 실패: {type(e).__name__}", file=sys.stderr)
        return False


def _fetch(url: str):
    try:
        with urllib.request.urlopen(url, timeout=10) as r:
            return json.loads(r.read().decode())
    except Exception as e:
        return {"__error__": f"/health 조회 실패: {type(e).__name__}"}


def main() -> int:
    url = os.environ.get("HEALTH_URL", "http://127.0.0.1:8000/health")
    threshold = int(os.environ.get("HEALTH_WATCH_FAILS", "2"))
    path = os.environ.get("HEALTH_WATCH_STATE", os.path.expanduser("~/.cache/auto-stock/health_watch.json"))
    h = _fetch(url)
    probs = [h["__error__"]] if isinstance(h, dict) and "__error__" in h else problems_v1(h)
    try:
        with open(path) as f:
            state = json.load(f)
    except (OSError, ValueError):
        state = {}
    new, action = next_state_v1(state, probs, threshold)
    if action == "alert":
        if not _send_telegram("[auto-stock] /health 이상 연속 " + str(new["fails"]) + "회: " + "; ".join(probs)):
            new["alerted"] = False  # 전송 실패 → 다음 회차에 재시도
    elif action == "recover":
        _send_telegram("[auto-stock] /health 복구")
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w") as f:
        json.dump(new, f)
    for p in probs:
        print("PROBLEM:", p, file=sys.stderr)
    return 1 if probs else 0


if __name__ == "__main__":
    sys.exit(main())