#!/usr/bin/env python3 """/health 주기 감시 (NAS cron: 평일 장중 5분). 표준 라이브러리만 사용. */5 9-15 * * 1-5 python3 /path/scripts/health_watch.py (KST 기준 cron) 판정: status != healthy / 스케줄러 alive=False·stale / db_worker alive=False / dlq.manual>0 / 응답 불가(연결 실패·비JSON) 가 FAIL_THRESHOLD 회 연속이면 텔레그램 경보(상태 전이 시 1회, 복구 시 1회). 환경변수: HEALTH_URL(기본 http://127.0.0.1:8000/health), TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID, HEALTH_WATCH_STATE(기본 ~/.cache/auto-stock/health_watch.json), HEALTH_WATCH_FAILS(기본 2) 텔레그램 미설정이면 stderr 에 ERROR 를 남기고 exit 1 (cron 메일/로그로 식별). 종료코드: 0 정상, 1 이상(현재 회차 기준). """ import json import os import sys import urllib.parse import urllib.request def problems_v1(health) -> list: """/health 응답 dict 에서 문제 목록을 만든다(순수 함수).""" if not isinstance(health, dict): return ["응답이 JSON 객체가 아님"] out = [] if health.get("status") != "healthy": out.append(f"status={health.get('status')}") for name, v in (health.get("schedulers") or {}).items(): if not v.get("alive"): out.append(f"scheduler {name} 사망") elif v.get("stale"): out.append(f"scheduler {name} stale({v.get('last_cycle_age')}s)") dbw = health.get("db_worker") or {} if "alive" in dbw and not dbw.get("alive"): out.append("db_worker 사망") if ((health.get("dlq") or {}).get("manual") or 0) > 0: out.append(f"DLQ manual={health['dlq']['manual']}") reasons = health.get("degraded_reasons") if out and isinstance(reasons, list) and reasons: out.append("degraded_reasons=" + ",".join(str(r) for r in reasons)) return out def next_state_v1(state: dict, problems: list, threshold: int): """(새 상태, 행동) — 행동: None | 'alert' | 'recover'.""" fails = int(state.get("fails", 0)) alerted = bool(state.get("alerted", False)) if problems: fails += 1 if fails >= threshold and not alerted: return {"fails": fails, "alerted": True}, "alert" return {"fails": fails, "alerted": alerted}, None if alerted: return {"fails": 0, "alerted": False}, "recover" return {"fails": 0, "alerted": False}, None def _send_telegram(text: str) -> bool: token, chat = os.environ.get("TELEGRAM_BOT_TOKEN", ""), os.environ.get("TELEGRAM_CHAT_ID", "") if not token or not chat: print("ERROR: TELEGRAM_BOT_TOKEN/CHAT_ID 미설정 — 경보 전송 불가", file=sys.stderr) return False data = urllib.parse.urlencode({"chat_id": chat, "text": text}).encode() try: with urllib.request.urlopen(f"https://api.telegram.org/bot{token}/sendMessage", data=data, timeout=10) as r: return r.status == 200 except Exception as e: # 전송 실패는 경보 재시도를 위해 False 로 보고 print(f"ERROR: 텔레그램 전송 실패: {type(e).__name__}", file=sys.stderr) return False def _fetch(url: str): try: with urllib.request.urlopen(url, timeout=10) as r: return json.loads(r.read().decode()) except Exception as e: return {"__error__": f"/health 조회 실패: {type(e).__name__}"} def main() -> int: url = os.environ.get("HEALTH_URL", "http://127.0.0.1:8000/health") threshold = int(os.environ.get("HEALTH_WATCH_FAILS", "2")) path = os.environ.get("HEALTH_WATCH_STATE", os.path.expanduser("~/.cache/auto-stock/health_watch.json")) h = _fetch(url) probs = [h["__error__"]] if isinstance(h, dict) and "__error__" in h else problems_v1(h) try: with open(path) as f: state = json.load(f) except (OSError, ValueError): state = {} new, action = next_state_v1(state, probs, threshold) if action == "alert": if not _send_telegram("[auto-stock] /health 이상 연속 " + str(new["fails"]) + "회: " + "; ".join(probs)): new["alerted"] = False # 전송 실패 → 다음 회차에 재시도 elif action == "recover": _send_telegram("[auto-stock] /health 복구") os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as f: json.dump(new, f) for p in probs: print("PROBLEM:", p, file=sys.stderr) return 1 if probs else 0 if __name__ == "__main__": sys.exit(main())