- 스윙 체결가 확인 전 executed_price NULL + fill_sweep, 진입가 resolve(ActiveTrade↔Portfolio), 0체결/부분체결 매수 장부 원복 - DT 고아 감지(근거 한정)·0체결 매도 복원·stale 점검 확대, 시간손절 -0.3% 원복 - 폴백 손절 ATR 캐시, 거래량부족-차단 신호 매수 차단, 현금 5% 유보 재적용, MDD peak/reset 수정 - KIS 서킷브레이커 청산 우회·경보, db_worker 재큐잉 순서, heartbeat·health-watch 사이드카 - 실시간 WS 재구독·41건 우선순위·다건 프레임 분할, 차트 샘플 제거·당일봉 합성, WS 서브프로토콜 인증 - 프론트: 미관리 그룹·확인 모달·차트 실시간 반영·예수금 표시 정정 - 전문가 에이전트 6종 추가(.claude/agents) - 유닛 스위트 2667 passed (소켓 차단 하), tsc OK Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y83Za98XsNw9zuA4FTUytN
288 lines
10 KiB
Python
288 lines
10 KiB
Python
"""하드닝 R1: 스케줄러 heartbeat/health/done_callback 회귀 테스트."""
|
|
import asyncio
|
|
import types
|
|
|
|
import pytest
|
|
|
|
from app import main as m
|
|
from app.core import heartbeat as hb
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean(monkeypatch):
|
|
hb.reset()
|
|
monkeypatch.setattr(m, "_scheduler_tasks", {})
|
|
monkeypatch.setattr(m, "_scheduler_restarts", {})
|
|
monkeypatch.setattr(m, "_shutting_down", False)
|
|
monkeypatch.setattr(m, "_restart_handles", {})
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_RESTART_BACKOFF_SEC", (0.0,))
|
|
# 텔레그램 실호출 차단: 예외 발생형이 아닌 기록형 대체(알림 경로 자체가 검증 대상)
|
|
calls = []
|
|
monkeypatch.setattr(m, "_notify_system_warning_bg", lambda c, msg: calls.append((c, msg)))
|
|
yield calls
|
|
hb.reset()
|
|
|
|
|
|
def _app():
|
|
return types.SimpleNamespace(state=types.SimpleNamespace())
|
|
|
|
|
|
def test_stale_only_in_market_hours():
|
|
app = _app()
|
|
|
|
class T:
|
|
def done(self):
|
|
return False
|
|
app.state.swing_trade_task = T()
|
|
m._scheduler_tasks["swing"] = "swing_trade_task"
|
|
hb.beat("swing", "x")
|
|
far = hb._beats["swing"]["at"] + m._HB_SCHEDULER_STALE_SECONDS + 5
|
|
assert m._scheduler_health_v2(app, now=far, market_open=True)["swing"]["stale"] is True
|
|
assert m._scheduler_health_v2(app, now=far, market_open=False)["swing"]["stale"] is False
|
|
|
|
|
|
def test_no_heartbeat_not_stale_but_dead_task_reported():
|
|
app = _app()
|
|
|
|
class D:
|
|
def done(self):
|
|
return True
|
|
app.state.day_trade_task = D()
|
|
m._scheduler_tasks["daytrading"] = "day_trade_task"
|
|
h = m._scheduler_health_v2(app, market_open=True)["daytrading"]
|
|
assert h["alive"] is False and h["last_cycle_age"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crash_triggers_alert_and_restart(_clean):
|
|
app = _app()
|
|
runs = []
|
|
|
|
async def boom():
|
|
runs.append(1)
|
|
if len(runs) == 1:
|
|
raise RuntimeError("x")
|
|
await asyncio.sleep(60)
|
|
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom)
|
|
await asyncio.sleep(0.05)
|
|
assert len(runs) == 2
|
|
assert any(c[0] == "SCHEDULER" for c in _clean)
|
|
app.state.swing_trade_task.cancel()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_limit(_clean, monkeypatch):
|
|
app = _app()
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 2)
|
|
runs = []
|
|
|
|
async def boom():
|
|
runs.append(1)
|
|
raise RuntimeError("x")
|
|
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom)
|
|
await asyncio.sleep(0.1)
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 3600.0)
|
|
await asyncio.sleep(0.1)
|
|
assert len(runs) == 3 # 최초 + 재시작 2회
|
|
assert any("한도" in c[1] for c in _clean)
|
|
for h in m._restart_handles.values():
|
|
h.cancel()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancel_does_not_restart(_clean):
|
|
app = _app()
|
|
runs = []
|
|
|
|
async def ok():
|
|
runs.append(1)
|
|
await asyncio.sleep(60)
|
|
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", ok)
|
|
await asyncio.sleep(0.01)
|
|
app.state.swing_trade_task.cancel()
|
|
await asyncio.sleep(0.02)
|
|
assert len(runs) == 1 and not _clean
|
|
|
|
|
|
# ── 하드닝 R2 ──────────────────────────────────────────────────────────
|
|
def test_reset_daily_loss_does_not_touch_mdd(monkeypatch):
|
|
"""MDD reset 계열은 예외 발생형 목 — 호출되면 즉시 실패. DT daily_pnl 만 0 이 된다."""
|
|
from app.services import mdd_manager as mm
|
|
from app.services.daytrading.position_manager import day_trading_state
|
|
|
|
def boom(*a, **k):
|
|
raise AssertionError("MDD reset must not be called")
|
|
for n in ("reset", "reset_v2"):
|
|
monkeypatch.setattr(mm.mdd_manager, n, boom, raising=False)
|
|
day_trading_state.daily_pnl = -12345.0
|
|
out = asyncio.run(m.reset_daily_loss(api_key="x"))
|
|
assert day_trading_state.daily_pnl == 0.0
|
|
assert out["previous_daily_pnl"] == -12345.0
|
|
|
|
|
|
def test_failed_trade_retention_is_long():
|
|
assert m._FAILED_TRADE_RETENTION_DAYS >= 30
|
|
|
|
|
|
def test_dt_scheduler_beat_makes_stale_detectable():
|
|
from app.services.daytrading import scheduler as dts
|
|
app = _app()
|
|
|
|
class T:
|
|
def done(self):
|
|
return False
|
|
app.state.day_trade_task = T()
|
|
m._scheduler_tasks["daytrading"] = "day_trade_task"
|
|
dts._beat_v2("loop")
|
|
far = hb._beats["daytrading"]["at"] + m._HB_SCHEDULER_STALE_SECONDS + 5
|
|
assert m._scheduler_health_v2(app, now=far, market_open=True)["daytrading"]["stale"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dt_exit_consecutive_failures_alert(monkeypatch):
|
|
from app.services.daytrading import scheduler as dts
|
|
sent = []
|
|
monkeypatch.setattr(dts, "_notify_dt_warning_bg_v2", lambda msg: sent.append(msg))
|
|
dts._record_dt_exit_result_v2(True)
|
|
for _ in range(dts.DT_EXIT_FAILURE_ALERT_THRESHOLD - 1):
|
|
dts._record_dt_exit_result_v2(False, "timeout")
|
|
assert sent == []
|
|
dts._record_dt_exit_result_v2(False, "timeout")
|
|
assert len(sent) == 1
|
|
dts._record_dt_exit_result_v2(True)
|
|
assert dts._dt_exit_health["consecutive_failures"] == 0
|
|
|
|
|
|
# ── 하드닝 라운드1: 백오프 / 한도 후 복구 / dead+age None / KIS 스트림 ──────
|
|
@pytest.mark.asyncio
|
|
async def test_restart_uses_backoff_delay(monkeypatch):
|
|
app = _app()
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_RESTART_BACKOFF_SEC", (0.2, 0.4))
|
|
runs = []
|
|
|
|
async def boom():
|
|
runs.append(1)
|
|
raise RuntimeError("x")
|
|
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom)
|
|
await asyncio.sleep(0.05)
|
|
assert len(runs) == 1 # 지연 중 즉시 재호출 없음
|
|
await asyncio.sleep(0.25)
|
|
assert len(runs) == 2
|
|
for h in m._restart_handles.values():
|
|
h.cancel()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_resumes_after_limit_cooldown(monkeypatch):
|
|
import time as _real_time
|
|
app = _app()
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 1)
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 0.05)
|
|
offset = {"v": 0.0}
|
|
monkeypatch.setattr(m, "time", types.SimpleNamespace(
|
|
monotonic=lambda: _real_time.monotonic() + offset["v"]))
|
|
runs = []
|
|
|
|
async def boom():
|
|
runs.append(1)
|
|
raise RuntimeError("x")
|
|
|
|
# 한도(1회) 소진 상태에서 시작: 창이 지나기 전엔 재시도해도 재시작하지 않는다
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", boom)
|
|
await asyncio.sleep(0.03)
|
|
assert len(runs) == 2
|
|
assert "swing" in m._restart_handles # 영구 포기가 아니라 재시도 예약됨
|
|
n_before = len(runs)
|
|
offset["v"] = 4000.0 # 1시간 창 경과 가정 -> 다음 쿨다운 재시도가 통과
|
|
# 이미 예약된 핸들은 ~3600s 뒤라 직접 재예약: 핸들 취소 후 새 크래시 유발 대신 상태 검증
|
|
h = m._restart_handles["swing"]
|
|
assert h.when() - asyncio.get_running_loop().time() > 100 # 창이 풀릴 때까지 대기(백오프 아님)
|
|
h.cancel()
|
|
m._restart_handles.clear()
|
|
m._shutting_down = True
|
|
assert len(runs) == n_before
|
|
|
|
|
|
def test_dead_watched_scheduler_with_no_heartbeat_is_flagged_in_market():
|
|
app = _app()
|
|
|
|
class D:
|
|
def done(self):
|
|
return True
|
|
app.state.swing_trade_task = D()
|
|
m._scheduler_tasks["swing"] = "swing_trade_task"
|
|
h = m._scheduler_health_v2(app, market_open=True)["swing"]
|
|
assert h["alive"] is False and h["last_cycle_age"] is None and h["stale"] is True
|
|
assert m._scheduler_health_v2(app, market_open=False)["swing"]["stale"] is False
|
|
|
|
|
|
def test_kis_stream_silent_and_probe(monkeypatch):
|
|
monkeypatch.setattr(m, "_kis_stream_last_msg_mono", None)
|
|
|
|
class T:
|
|
def __init__(self, d):
|
|
self._d = d
|
|
|
|
def done(self):
|
|
return self._d
|
|
app = _app()
|
|
app.state.kis_stream_task = None
|
|
assert m._kis_stream_health_v2(app, market_open=True) is None
|
|
app.state.kis_stream_task = T(False)
|
|
monkeypatch.setattr(m, "_kis_stream_last_msg_mono", 100.0)
|
|
limit = m._HB_KIS_STREAM_SILENT_SECONDS
|
|
assert m._kis_stream_health_v2(app, now=100.0 + limit + 1, market_open=True)["silent"] is True
|
|
assert m._kis_stream_health_v2(app, now=100.0 + limit + 1, market_open=False)["silent"] is False
|
|
assert m._kis_stream_health_v2(app, now=101.0, market_open=True)["silent"] is False
|
|
app.state.kis_stream_task = T(True) # 태스크 사망 + 장중
|
|
assert m._kis_stream_health_v2(app, now=101.0, market_open=True)["silent"] is True
|
|
|
|
seen = []
|
|
|
|
class S:
|
|
def _enqueue_realtime(self, t, c, p):
|
|
seen.append((t, c))
|
|
s = S()
|
|
assert m._install_kis_stream_probe_v2(s) is True
|
|
assert m._install_kis_stream_probe_v2(s) is False # 멱등
|
|
monkeypatch.setattr(m, "_kis_stream_last_msg_mono", None)
|
|
s._enqueue_realtime("tick", "005930", {})
|
|
assert seen == [("tick", "005930")] and m._kis_stream_last_msg_mono is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_kis_stream_done_callback_alerts(_clean):
|
|
async def boom():
|
|
raise RuntimeError("ws")
|
|
t = asyncio.create_task(boom())
|
|
t.add_done_callback(m._on_kis_stream_done_v2)
|
|
await asyncio.sleep(0.02)
|
|
assert any(c[0] == "KIS_STREAM" for c in _clean)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_limit_exhausted_scheduler_restarts_when_window_frees(monkeypatch):
|
|
import time as _t
|
|
app = _app()
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_MAX_RESTARTS_PER_HOUR", 1)
|
|
monkeypatch.setattr(m, "_HB_SCHEDULER_LIMIT_COOLDOWN_SEC", 0.05)
|
|
m._scheduler_restarts["swing"] = [_t.monotonic() - 3599.5] # 0.5초 뒤 창에서 빠짐
|
|
runs = []
|
|
|
|
async def work():
|
|
runs.append(1)
|
|
if len(runs) == 1:
|
|
raise RuntimeError("x")
|
|
await asyncio.sleep(60)
|
|
|
|
m._spawn_scheduler_v2(app, "swing_trade_task", "swing", work)
|
|
await asyncio.sleep(0.1)
|
|
assert len(runs) == 1 # 한도 소진 — 아직 대기
|
|
await asyncio.sleep(1.8)
|
|
assert len(runs) == 2 # 창이 풀리자 자동 복구
|
|
app.state.swing_trade_task.cancel()
|