"""하드닝 R1: 스케줄러 heartbeat/health/done_callback 회귀 테스트.""" import asyncio import types import pytest from app import main as m from app.core import heartbeat as hb @pytest.fixture(autouse=True) def _clean(monkeypatch): hb.reset() monkeypatch.setattr(m, "_scheduler_tasks", {}) monkeypatch.setattr(m, "_scheduler_restarts", {}) monkeypatch.setattr(m, "_shutting_down", False) monkeypatch.setattr(m, "_restart_handles", {}) monkeypatch.setattr(m, "_HB_SCHEDULER_RESTART_BACKOFF_SEC", (0.0,)) # 텔레그램 실호출 차단: 예외 발생형이 아닌 기록형 대체(알림 경로 자체가 검증 대상) calls = [] monkeypatch.setattr(m, "_notify_system_warning_bg", lambda c, msg: calls.append((c, msg))) yield calls hb.reset() def _app(): return types.SimpleNamespace(state=types.SimpleNamespace()) def test_stale_only_in_market_hours(): app = _app() class T: def done(self): return False app.state.swing_trade_task = T() m._scheduler_tasks["swing"] = "swing_trade_task" hb.beat("swing", "x") far = hb._beats["swing"]["at"] + m._HB_SCHEDULER_STALE_SECONDS + 5 assert m._scheduler_health_v2(app, now=far, market_open=True)["swing"]["stale"] is True assert m._scheduler_health_v2(app, now=far, market_open=False)["swing"]["stale"] is False def test_no_heartbeat_not_stale_but_dead_task_reported(): app = _app() class D: def done(self): return True app.state.day_trade_task = D() m._scheduler_tasks["daytrading"] = "day_trade_task" h = m._scheduler_health_v2(app, market_open=True)["daytrading"] assert h["alive"] is False and h["last_cycle_age"] is None @pytest.mark.asyncio async def test_crash_triggers_alert_and_restart(_clean): app = _app() runs = [] async def boom(): runs.append(1) if len(runs) == 1: raise RuntimeError("x") await asyncio.sleep(60) m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom) await asyncio.sleep(0.05) assert len(runs) == 2 assert any(c[0] == "SCHEDULER" for c in _clean) app.state.swing_trade_task.cancel() @pytest.mark.asyncio async def test_restart_limit(_clean, monkeypatch): app = _app() monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 2) runs = [] async def boom(): runs.append(1) raise RuntimeError("x") m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom) await asyncio.sleep(0.1) monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 3600.0) await asyncio.sleep(0.1) assert len(runs) == 3 # 최초 + 재시작 2회 assert any("한도" in c[1] for c in _clean) for h in m._restart_handles.values(): h.cancel() @pytest.mark.asyncio async def test_cancel_does_not_restart(_clean): app = _app() runs = [] async def ok(): runs.append(1) await asyncio.sleep(60) m._spawn_scheduler_v2(app, "swing_trade_task", "swing", ok) await asyncio.sleep(0.01) app.state.swing_trade_task.cancel() await asyncio.sleep(0.02) assert len(runs) == 1 and not _clean # ── 하드닝 R2 ────────────────────────────────────────────────────────── def test_reset_daily_loss_does_not_touch_mdd(monkeypatch): """MDD reset 계열은 예외 발생형 목 — 호출되면 즉시 실패. DT daily_pnl 만 0 이 된다.""" from app.services import mdd_manager as mm from app.services.daytrading.position_manager import day_trading_state def boom(*a, **k): raise AssertionError("MDD reset must not be called") for n in ("reset", "reset_v2"): monkeypatch.setattr(mm.mdd_manager, n, boom, raising=False) day_trading_state.daily_pnl = -12345.0 out = asyncio.run(m.reset_daily_loss(api_key="x")) assert day_trading_state.daily_pnl == 0.0 assert out["previous_daily_pnl"] == -12345.0 def test_failed_trade_retention_is_long(): assert m._FAILED_TRADE_RETENTION_DAYS >= 30 def test_dt_scheduler_beat_makes_stale_detectable(): from app.services.daytrading import scheduler as dts app = _app() class T: def done(self): return False app.state.day_trade_task = T() m._scheduler_tasks["daytrading"] = "day_trade_task" dts._beat_v2("loop") far = hb._beats["daytrading"]["at"] + m._HB_SCHEDULER_STALE_SECONDS + 5 assert m._scheduler_health_v2(app, now=far, market_open=True)["daytrading"]["stale"] is True @pytest.mark.asyncio async def test_dt_exit_consecutive_failures_alert(monkeypatch): from app.services.daytrading import scheduler as dts sent = [] monkeypatch.setattr(dts, "_notify_dt_warning_bg_v2", lambda msg: sent.append(msg)) dts._record_dt_exit_result_v2(True) for _ in range(dts.DT_EXIT_FAILURE_ALERT_THRESHOLD - 1): dts._record_dt_exit_result_v2(False, "timeout") assert sent == [] dts._record_dt_exit_result_v2(False, "timeout") assert len(sent) == 1 dts._record_dt_exit_result_v2(True) assert dts._dt_exit_health["consecutive_failures"] == 0 # ── 하드닝 라운드1: 백오프 / 한도 후 복구 / dead+age None / KIS 스트림 ────── @pytest.mark.asyncio async def test_restart_uses_backoff_delay(monkeypatch): app = _app() monkeypatch.setattr(m, "_HB_SCHEDULER_RESTART_BACKOFF_SEC", (0.2, 0.4)) runs = [] async def boom(): runs.append(1) raise RuntimeError("x") m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom) await asyncio.sleep(0.05) assert len(runs) == 1 # 지연 중 즉시 재호출 없음 await asyncio.sleep(0.25) assert len(runs) == 2 for h in m._restart_handles.values(): h.cancel() @pytest.mark.asyncio async def test_restart_resumes_after_limit_cooldown(monkeypatch): import time as _real_time app = _app() monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 1) monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 0.05) offset = {"v": 0.0} monkeypatch.setattr(m, "time", types.SimpleNamespace( monotonic=lambda: _real_time.monotonic() + offset["v"])) runs = [] async def boom(): runs.append(1) raise RuntimeError("x") # 한도(1회) 소진 상태에서 시작: 창이 지나기 전엔 재시도해도 재시작하지 않는다 m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom) await asyncio.sleep(0.03) assert len(runs) == 2 assert "swing" in m._restart_handles # 영구 포기가 아니라 재시도 예약됨 n_before = len(runs) offset["v"] = 4000.0 # 1시간 창 경과 가정 -> 다음 쿨다운 재시도가 통과 # 이미 예약된 핸들은 ~3600s 뒤라 직접 재예약: 핸들 취소 후 새 크래시 유발 대신 상태 검증 h = m._restart_handles["swing"] assert h.when() - asyncio.get_running_loop().time() > 100 # 창이 풀릴 때까지 대기(백오프 아님) h.cancel() m._restart_handles.clear() m._shutting_down = True assert len(runs) == n_before def test_dead_watched_scheduler_with_no_heartbeat_is_flagged_in_market(): app = _app() class D: def done(self): return True app.state.swing_trade_task = D() m._scheduler_tasks["swing"] = "swing_trade_task" h = m._scheduler_health_v2(app, market_open=True)["swing"] assert h["alive"] is False and h["last_cycle_age"] is None and h["stale"] is True assert m._scheduler_health_v2(app, market_open=False)["swing"]["stale"] is False def test_kis_stream_silent_and_probe(monkeypatch): monkeypatch.setattr(m, "_kis_stream_last_msg_mono", None) class T: def __init__(self, d): self._d = d def done(self): return self._d app = _app() app.state.kis_stream_task = None assert m._kis_stream_health_v2(app, market_open=True) is None app.state.kis_stream_task = T(False) monkeypatch.setattr(m, "_kis_stream_last_msg_mono", 100.0) limit = m._HB_KIS_STREAM_SILENT_SECONDS assert m._kis_stream_health_v2(app, now=100.0 + limit + 1, market_open=True)["silent"] is True assert m._kis_stream_health_v2(app, now=100.0 + limit + 1, market_open=False)["silent"] is False assert m._kis_stream_health_v2(app, now=101.0, market_open=True)["silent"] is False app.state.kis_stream_task = T(True) # 태스크 사망 + 장중 assert m._kis_stream_health_v2(app, now=101.0, market_open=True)["silent"] is True seen = [] class S: def _enqueue_realtime(self, t, c, p): seen.append((t, c)) s = S() assert m._install_kis_stream_probe_v2(s) is True assert m._install_kis_stream_probe_v2(s) is False # 멱등 monkeypatch.setattr(m, "_kis_stream_last_msg_mono", None) s._enqueue_realtime("tick", "005930", {}) assert seen == [("tick", "005930")] and m._kis_stream_last_msg_mono is not None @pytest.mark.asyncio async def test_kis_stream_done_callback_alerts(_clean): async def boom(): raise RuntimeError("ws") t = asyncio.create_task(boom()) t.add_done_callback(m._on_kis_stream_done_v2) await asyncio.sleep(0.02) assert any(c[0] == "KIS_STREAM" for c in _clean) @pytest.mark.asyncio async def test_limit_exhausted_scheduler_restarts_when_window_frees(monkeypatch): import time as _t app = _app() monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 1) monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 0.05) m._scheduler_restarts["swing"] = [_t.monotonic() - 3599.5] # 0.5초 뒤 창에서 빠짐 runs = [] async def work(): runs.append(1) if len(runs) == 1: raise RuntimeError("x") await asyncio.sleep(60) m._spawn_scheduler_v2(app, "swing_trade_task", "swing", work) await asyncio.sleep(0.1) assert len(runs) == 1 # 한도 소진 — 아직 대기 await asyncio.sleep(1.8) assert len(runs) == 2 # 창이 풀리자 자동 복구 app.state.swing_trade_task.cancel()