Add READ_ONLY_API environment variable and enhance trading error handling

- Updated .env.example to recommend using docker.env and secrets for sensitive data.
- Modified .gitignore to include docker.env and secrets directory.
- Adjusted docker-compose.yml to utilize environment files and updated port mappings.
- Implemented read_env_or_file utility for better environment variable management.
- Refactored portfolio and trades routers to use the new utility for fetching environment variables.
- Added integration and phase 2 test updates for new environment variable handling.
- Created new documentation for Docker operations and secret management.
- Added placeholder files for Docker secrets and configuration.

.env.docker.tmp

@@ -0,0 +1,45 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# IB Gateway (gnzsnz/ib-gateway:stable)
+# ─────────────────────────────────────────────────────────────────────────────
+TWS_USERID=zemheritrader
+TWS_PASSWORD=3bmp8P$$K4(>89oZ}xQ
+TRADING_MODE=paper                    # paper | live
+VNC_SERVER_PASSWORD=GucluVNC123
+TWOFA_TIMEOUT_ACTION=exit          # restart | exit
+AUTO_RESTART_TIME=11:59 PM
+RELOGIN_AFTER_2FA_TIMEOUT=no
+TIME_ZONE=Europe/Istanbul             # e.g. America/New_York, Europe/London
+
+# ─────────────────────────────────────────────────────────────────────────────
+# FastAPI Application
+# ─────────────────────────────────────────────────────────────────────────────
+IBKR_HOST=ib-gateway                  # Docker service name; use 127.0.0.1 for local
+IBKR_PORT=4002                        # paper=4002, live=4001
+IBKR_CLIENT_ID=1
+WEBHOOK_SECRET=change_this_to_a_strong_random_secret   # openssl rand -hex 32
+DB_PATH=/app/trades.db
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Risk Management
+# ─────────────────────────────────────────────────────────────────────────────
+MAX_DAILY_LOSS=500.0
+MAX_POSITIONS=5
+MAX_ORDER_VALUE=10000.0
+
+# ─────────────────────────────────────────────────────────────────────────────
+# UI Authentication (HTTP Basic Auth — opt-in)
+# Leave both unset to disable auth (trusted network / Tailscale only)
+# Set both to enable auth on all UI pages (/, /scanner, /tradelog, /portfolio)
+# /health and /webhook are always public regardless of this setting
+# ─────────────────────────────────────────────────────────────────────────────
+# UI_USERNAME=admin
+# UI_PASSWORD=change_this_strong_password
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Traefik (external reverse proxy — must be running as a separate stack)
+# ─────────────────────────────────────────────────────────────────────────────
+TRAEFIK_NETWORK=traefik-public        # External Docker network Traefik is attached to
+TRAEFIK_HOST=ibkr.your-tailnet.ts.net # Hostname Traefik routes on (your Tailscale FQDN)
+TRAEFIK_ENTRYPOINT=websecure          # Traefik entrypoint name (websecure=HTTPS, web=HTTP)
+RELOGIN_AFTER_TWOFA_TIMEOUT=no
+EXISTING_SESSION_DETECTED_ACTION=primaryoverride

.env.example

@@ -1,3 +1,4 @@
+# For Docker Compose, prefer docker.env.example + ./secrets instead of this file.
 # ─────────────────────────────────────────────────────────────────────────────
 # IB Gateway (gnzsnz/ib-gateway:stable)
 # ─────────────────────────────────────────────────────────────────────────────
@@ -7,17 +8,18 @@ TRADING_MODE=paper                    # paper | live
 VNC_SERVER_PASSWORD=your_vnc_password
 TWOFA_TIMEOUT_ACTION=restart          # restart | exit
 AUTO_RESTART_TIME=11:59 PM
-RELOGIN_AFTER_2FA_TIMEOUT=yes
+RELOGIN_AFTER_TWOFA_TIMEOUT=yes
+EXISTING_SESSION_DETECTED_ACTION=primaryoverride   # primary | primaryoverride | secondary
 TIME_ZONE=Europe/Istanbul             # e.g. America/New_York, Europe/London
 
 # ─────────────────────────────────────────────────────────────────────────────
 # FastAPI Application
 # ─────────────────────────────────────────────────────────────────────────────
 IBKR_HOST=ib-gateway                  # Docker service name; use 127.0.0.1 for local
-IBKR_PORT=4002                        # paper=4002, live=4001
+IBKR_PORT=4002                        # local paper=4002, local live=4001; docker service uses 4004/4003
 IBKR_CLIENT_ID=1
 WEBHOOK_SECRET=change_this_to_a_strong_random_secret   # openssl rand -hex 32
-DB_PATH=/app/trades.db
+DB_PATH=/app/data/trades.db
 
 # ─────────────────────────────────────────────────────────────────────────────
 # Risk Management

.gitignore

@@ -10,3 +10,7 @@ venv/
 dist/
 build/
 .DS_Store
+docker.env
+secrets/*
+!secrets/.gitkeep
+!secrets/README.md

OPERATIONS.md

@@ -0,0 +1,174 @@
+# IBKR Dashboard Operasyon Rehberi
+
+Bu rehber, projeyi yerelde Docker ile baslatmak, saglik kontrolu yapmak, smoke test calistirmak ve guvenli sekilde kapatmak icin kisa bir operasyon akisi verir.
+
+## 1. Onkosullar
+
+- Docker Desktop calisiyor olmali.
+- Docker icin repo kokunde `docker.env` dosyasi olmali.
+- Lokal Python calismalari icin isterseniz ayri bir `.env` dosyasi tutabilirsiniz.
+- `.env` icindeki `WEBHOOK_SECRET` varsayilan degerde birakilmamali.
+- Docker secret dosyalari `./secrets` altinda olusturulmali.
+
+## 2. Hazirlik
+
+`docker.env.example` dosyasini `docker.env` olarak kopyalayin. Ardindan gerekli secret dosyalarini olusturun:
+
+```bash
+cp docker.env.example docker.env
+mkdir -p secrets
+printf '%s\n' 'your_ibkr_password' > secrets/tws_password
+printf '%s\n' 'change_this_webhook_secret' > secrets/webhook_secret
+
+chmod 600 secrets/tws_password secrets/webhook_secret
+
+docker network inspect traefik-public >/dev/null 2>&1 || docker network create traefik-public
+```
+
+Notlar:
+
+- `primaryoverride`, mevcut bir IBKR oturumu varsa gateway'in login dialogunda takilma riskini azaltir.
+- Uygulama SQLite veritabani icin `./data` dizinini kullanir. Dizin repo icinde mevcut olmali.
+- Opsiyonel secret dosyalari: `secrets/vnc_password`, `secrets/ui_username`, `secrets/ui_password`.
+
+## 3. Stack Baslatma
+
+Servisleri build edip arka planda kaldirin:
+
+```bash
+docker compose --env-file docker.env up -d --build
+```
+
+Durumu kontrol edin:
+
+```bash
+docker compose --env-file docker.env ps -a
+```
+
+Beklenen durum:
+
+- `ib-gateway`: `Up (healthy)`
+- `ibkr-dashboard`: `Up` veya kisa sure sonra `Up (healthy)`
+
+Log takibi icin:
+
+```bash
+docker compose --env-file docker.env logs -f ib-gateway ibkr-dashboard
+```
+
+## 4. Health Kontrolu
+
+Uygulama health endpoint'ini dogrulayin:
+
+```bash
+curl -sS --max-time 10 http://localhost:8000/health
+```
+
+Beklenen cevap ornegi:
+
+```json
+{"status":"ok","ib_connected":true,"account":"...","db_ok":true,"version":"3.0"}
+```
+
+`status=ok`, `ib_connected=true` ve `db_ok=true` gorulmelidir.
+
+## 5. Test ve Smoke Dogrulama
+
+Offline test paketi:
+
+```bash
+pytest tests/ -q --tb=short
+```
+
+Beklenen sonuc:
+
+```text
+47 passed, 2 skipped
+```
+
+Docker smoke testi:
+
+```bash
+./verify_docker.sh
+```
+
+Guncel dogrulanmis durum:
+
+- Script 18/18 check geciyor.
+
+Canli IB entegrasyon testlerini ozellikle acmak isterseniz:
+
+```bash
+RUN_LIVE_IB_TESTS=1 pytest tests/test_integration.py -q --tb=short
+```
+
+## 6. Sorun Giderme
+
+### Docker Compose `$` warning'i
+
+Semptom:
+
+```text
+The "K4" variable is not set. Defaulting to a blank string.
+```
+
+Cozum:
+
+- Compose komutlarini `.env.docker.tmp` ile calistirin.
+- Docker icin `docker compose --env-file docker.env ...` kullanin.
+- Secret degerlerini `./secrets` altindaki dosyalardan okuyun.
+
+### Gateway login oluyor ama healthy olmuyor
+
+Kontrol edin:
+
+```bash
+docker compose --env-file docker.env logs --tail=100 ib-gateway
+```
+
+Beklenen satirlar:
+
+- `Login has completed`
+- `Configuration tasks completed`
+
+### Dashboard ayaga kalkmiyor
+
+Kontrol edin:
+
+```bash
+docker compose --env-file docker.env logs --tail=100 ibkr-dashboard
+```
+
+Ozellikle su hatalara bakin:
+
+- `sqlite3.OperationalError: unable to open database file`
+- `TimeoutError` veya IB gateway baglanti hatalari
+
+## 7. Kapatma
+
+Servisleri temiz kapatmak icin:
+
+```bash
+docker compose --env-file docker.env down --remove-orphans
+```
+
+Docker secret dosyalarini yerinde birakip sadece `docker.env` kaldirilabilir:
+
+```bash
+rm -f docker.env
+```
+
+## 8. Operasyon Notlari
+
+- `verify_docker.sh`, varsayilan olarak `docker.env` varsa onu kullanir; yoksa `.env`'e geri duser.
+- Docker ic aginda dashboard, gateway'e `ib-gateway:4004` uzerinden baglanir.
+- Host makinede paper API erisimi `127.0.0.1:4002`, live API erisimi `127.0.0.1:4001` olarak map edilir.
+
+## 9. Geriye Donuk Fallback
+
+Eger mevcut `.env` ile devam etmek zorundaysaniz, eski gecici dosya akisi hala kullanilabilir:
+
+```bash
+sed 's/\$/\$\$/g' .env > .env.docker.tmp
+COMPOSE_ENV_FILE=.env.docker.tmp COMPOSE_APP_ENV_FILE=.env.docker.tmp ./verify_docker.sh
+```

config_utils.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+import os
+from typing import Optional
+
+
+def read_env_or_file(name: str, default: Optional[str] = None) -> Optional[str]:
+    file_path = os.getenv(f"{name}_FILE")
+    if file_path:
+        return Path(file_path).read_text(encoding="utf-8").strip()
+
+    value = os.getenv(name)
+    if value is not None:
+        return value
+
+    return default

docker-compose.yml

@@ -3,21 +3,24 @@ services:
   ib-gateway:
     image: ghcr.io/gnzsnz/ib-gateway:stable
     restart: always
+    env_file:
+      - ${COMPOSE_APP_ENV_FILE:-docker.env}
     environment:
-      - TWS_USERID=${TWS_USERID}
-      - TWS_PASSWORD=${TWS_PASSWORD}
       - TRADING_MODE=${TRADING_MODE:-paper}
-      - VNC_SERVER_PASSWORD=${VNC_SERVER_PASSWORD}
+      - READ_ONLY_API=${READ_ONLY_API:-no}
       - TWOFA_TIMEOUT_ACTION=${TWOFA_TIMEOUT_ACTION:-restart}
       - AUTO_RESTART_TIME=${AUTO_RESTART_TIME:-11:59 PM}
-      - RELOGIN_AFTER_2FA_TIMEOUT=${RELOGIN_AFTER_2FA_TIMEOUT:-yes}
+      - RELOGIN_AFTER_TWOFA_TIMEOUT=${RELOGIN_AFTER_TWOFA_TIMEOUT:-yes}
+      - EXISTING_SESSION_DETECTED_ACTION=${EXISTING_SESSION_DETECTED_ACTION:-primaryoverride}
       - TIME_ZONE=${TIME_ZONE:-America/New_York}
     ports:
-      - "127.0.0.1:4001:4001"
-      - "127.0.0.1:4002:4002"
+      - "127.0.0.1:4001:4003"
+      - "127.0.0.1:4002:4004"
       - "127.0.0.1:5900:5900"
+    volumes:
+      - ./secrets:/run/secrets:ro
     healthcheck:
-      test: ["CMD-SHELL", "nc -z localhost 4002 || exit 1"]
+      test: ["CMD-SHELL", "bash -lc ': >/dev/tcp/127.0.0.1/4004' || bash -lc ': >/dev/tcp/127.0.0.1/4003' || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -28,15 +31,18 @@ services:
   ibkr-dashboard:
     build: .
     restart: unless-stopped
-    env_file: .env
+    env_file:
+      - ${COMPOSE_APP_ENV_FILE:-docker.env}
     environment:
       - IBKR_HOST=ib-gateway
-      - IBKR_PORT=4002
+      - IBKR_PORT=4004
+      - DB_PATH=/app/data/trades.db
     # Port exposed only for local testing — can be removed when accessed exclusively via Traefik
     ports:
       - "127.0.0.1:8000:8000"
     volumes:
-      - ./trades.db:/app/trades.db
+      - ./data:/app/data
+      - ./secrets:/run/secrets:ro
     depends_on:
       ib-gateway:
         condition: service_healthy

docker.env.example

@@ -0,0 +1,25 @@
+# Copy to docker.env for Docker Compose usage.
+# Keep raw secrets in ./secrets files instead of this env file.
+
+TWS_USERID=your_ibkr_username
+TWS_PASSWORD_FILE=/run/secrets/tws_password
+TRADING_MODE=paper
+# Optional if you need VNC access:
+# VNC_SERVER_PASSWORD_FILE=/run/secrets/vnc_password
+TWOFA_TIMEOUT_ACTION=restart
+AUTO_RESTART_TIME=11:59 PM
+RELOGIN_AFTER_TWOFA_TIMEOUT=yes
+EXISTING_SESSION_DETECTED_ACTION=primaryoverride
+TIME_ZONE=Europe/Istanbul
+
+WEBHOOK_SECRET_FILE=/run/secrets/webhook_secret
+# Optional UI auth:
+# UI_USERNAME_FILE=/run/secrets/ui_username
+# UI_PASSWORD_FILE=/run/secrets/ui_password
+IBKR_CLIENT_ID=1
+MAX_DAILY_LOSS=500.0
+MAX_POSITIONS=5
+MAX_ORDER_VALUE=10000.0
+TRAEFIK_NETWORK=traefik-public
+TRAEFIK_HOST=ibkr.your-tailnet.ts.net
+TRAEFIK_ENTRYPOINT=websecure

main.py

@@ -12,6 +12,7 @@ from fastapi.responses import JSONResponse, Response
 from fastapi.staticfiles import StaticFiles
 from ib_async import IB
 
+from config_utils import read_env_or_file
 import dependencies
 from init_db import ensure_database
 from routers import charts, health, portfolio, scanner, trades
@@ -29,8 +30,8 @@ IBKR_PORT = int(os.getenv("IBKR_PORT", 7497))
 IBKR_CLIENT_ID = int(os.getenv("IBKR_CLIENT_ID", 1))
 DB_PATH = os.getenv("DB_PATH", "trades.db")
 
-UI_USERNAME = os.getenv("UI_USERNAME", "")
-UI_PASSWORD = os.getenv("UI_PASSWORD", "")
+UI_USERNAME = read_env_or_file("UI_USERNAME", "") or ""
+UI_PASSWORD = read_env_or_file("UI_PASSWORD", "") or ""
 AUTH_ENABLED = bool(UI_USERNAME and UI_PASSWORD)
 
 _DEFAULT_SECRET = "your_super_secret_string_123"
@@ -48,7 +49,7 @@ ib = IB()
 
 
 def _validate_config() -> None:
-    webhook_secret = os.getenv("WEBHOOK_SECRET", "")
+    webhook_secret = read_env_or_file("WEBHOOK_SECRET", "") or ""
     if not webhook_secret or webhook_secret == _DEFAULT_SECRET:
         logger.warning(
             "SECURITY: WEBHOOK_SECRET is unset or still the default value. "
@@ -58,7 +59,7 @@ def _validate_config() -> None:
         logger.info("UI Basic Auth is ENABLED")
     else:
         logger.info(
-            "UI Basic Auth is DISABLED — set UI_USERNAME + UI_PASSWORD in .env to enable"
+            "UI Basic Auth is DISABLED — set UI_USERNAME/UI_PASSWORD or *_FILE to enable"
         )
 
 
@@ -79,6 +80,25 @@ async def reconnect_loop() -> None:
     logger.error("Max reconnect attempts reached. Manual restart required.")
 
 
+async def connect_with_retries(max_attempts: int = 12) -> None:
+    for attempt in range(1, max_attempts + 1):
+        try:
+            await asyncio.wait_for(
+                ib.connectAsync(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID),
+                timeout=15.0,
+            )
+            dependencies.setup_dependencies(ib, conn, cursor, db_write_lock)
+            logger.info("Connected to IB Gateway")
+            return
+        except Exception as exc:
+            logger.warning(
+                f"Startup IB connection attempt {attempt}/{max_attempts} failed: {exc}"
+            )
+            if attempt == max_attempts:
+                raise
+            await asyncio.sleep(5)
+
+
 def on_disconnect() -> None:
     logger.warning("IB Gateway disconnected. Scheduling reconnect...")
     asyncio.create_task(reconnect_loop())
@@ -88,9 +108,7 @@ def on_disconnect() -> None:
 async def lifespan(app: FastAPI):
     _validate_config()
     try:
-        await ib.connectAsync(IBKR_HOST, IBKR_PORT, clientId=IBKR_CLIENT_ID)
-        logger.info("Connected to IB Gateway")
-        dependencies.setup_dependencies(ib, conn, cursor, db_write_lock)
+        await connect_with_retries()
         ib.disconnectedEvent += on_disconnect
         yield
     finally:

routers/charts.py

@@ -24,13 +24,66 @@ class ChartState:
     def __init__(self):
         self.latest_tick = None
         self.current_ticker = None
+        self.current_symbol = None
         self.current_minute = None
         self.current_ohlc = None
+        self.history_poll_task = None
+        self.last_realtime_update = 0.0
 
 
 chart_state = ChartState()
 
 
+async def _stop_history_poll_task() -> None:
+    task = chart_state.history_poll_task
+    chart_state.history_poll_task = None
+    if task is None or task.done():
+        return
+    task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
+
+
+async def _history_fallback_loop(symbol: str, contract) -> None:
+    ib = dependencies.get_ib()
+    try:
+        while chart_state.current_symbol == symbol:
+            # Only fall back when real-time bars have not updated recently.
+            if chart_state.last_realtime_update and (
+                time.monotonic() - chart_state.last_realtime_update
+            ) < 20:
+                await asyncio.sleep(15)
+                continue
+
+            try:
+                bars = await ib.reqHistoricalDataAsync(
+                    contract,
+                    endDateTime="",
+                    durationStr="1 D",
+                    barSizeSetting="1 min",
+                    whatToShow="TRADES",
+                    useRTH=True,
+                    formatDate=1,
+                )
+                if bars:
+                    last_bar = bars[-1]
+                    chart_state.latest_tick = {
+                        "time": int(last_bar.date.timestamp()),
+                        "open": last_bar.open,
+                        "high": last_bar.high,
+                        "low": last_bar.low,
+                        "close": last_bar.close,
+                    }
+            except Exception as exc:
+                logger.warning(f"Historical fallback update failed for {symbol}: {exc}")
+
+            await asyncio.sleep(15)
+    except asyncio.CancelledError:
+        pass
+
+
 @router.get("/", response_class=HTMLResponse)
 async def index(request: Request, symbol: str = ""):
     return templates.TemplateResponse(request, "index.html", {"symbol": symbol})
@@ -115,8 +168,13 @@ async def subscribe(request: Request):
             except Exception as e:
                 print(f"Error cancelling real-time bars: {e}")
 
+        await _stop_history_poll_task()
+
+        chart_state.current_symbol = symbol
+        chart_state.latest_tick = None
         chart_state.current_minute = None
         chart_state.current_ohlc = None
+        chart_state.last_realtime_update = 0.0
 
         ticker = ib.reqRealTimeBars(qualified[0], barSize=5, whatToShow="TRADES", useRTH=True)
         chart_state.current_ticker = ticker
@@ -152,9 +210,13 @@ async def subscribe(request: Request):
                     "close": bar.close,
                 }
 
+            chart_state.last_realtime_update = time.monotonic()
             chart_state.latest_tick = chart_state.current_ohlc.copy()
 
         ticker.updateEvent += on_bar
+        chart_state.history_poll_task = asyncio.create_task(
+            _history_fallback_loop(symbol, qualified[0])
+        )
 
         return {"status": "ok", "symbol": symbol}
     except Exception as e:

routers/health.py

@@ -1,3 +1,5 @@
+import os
+
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
 
@@ -10,6 +12,7 @@ router = APIRouter()
 async def health():
     ib_connected = False
     account = None
+    trading_mode = os.getenv("TRADING_MODE", "unknown").split("#", 1)[0].strip() or "unknown"
     try:
         ib_inst = dependencies.get_ib()
         ib_connected = ib_inst.isConnected()
@@ -30,6 +33,7 @@ async def health():
         {
             "status": "ok" if (ib_connected and db_ok) else "degraded",
             "ib_connected": ib_connected,
+            "trading_mode": trading_mode,
             "account": account,
             "db_ok": db_ok,
             "version": "3.0",

routers/portfolio.py

@@ -1,6 +1,10 @@
+import os
+import time
+
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
+from ib_async import IB
 
 import dependencies
 
@@ -8,6 +12,56 @@ router = APIRouter(prefix="/portfolio")
 templates = Jinja2Templates(directory="templates")
 
 
+def _extract_summary_values(summary) -> dict:
+    keys = {"NetLiquidation", "TotalCashValue", "UnrealizedPnL", "RealizedPnL"}
+    result = {}
+    for item in summary:
+        if item.tag in keys:
+            try:
+                result[item.tag] = float(item.value)
+            except (ValueError, TypeError):
+                result[item.tag] = item.value
+    return result
+
+
+def _extract_cached_account_values(ib) -> dict:
+    getter = getattr(ib, "accountValues", None)
+    if getter is None:
+        return {}
+    return _extract_summary_values(getter() or [])
+
+
+async def _read_summary_values(ib) -> dict:
+    result = _extract_cached_account_values(ib)
+    if result:
+        return result
+
+    summary = await ib.reqAccountSummaryAsync()
+    result = _extract_summary_values(summary)
+    if result:
+        return result
+
+    return _extract_cached_account_values(ib)
+
+
+async def _fetch_summary_with_fallback() -> dict:
+    ib = dependencies.get_ib()
+    result = await _read_summary_values(ib)
+    if result:
+        return result
+
+    # A fresh client recovers account summary when the shared session returns an empty snapshot.
+    fallback_ib = IB()
+    host = os.getenv("IBKR_HOST", "127.0.0.1")
+    port = int(os.getenv("IBKR_PORT", "7497"))
+    client_id = int(time.time()) % 100000 + 5000
+    try:
+        await fallback_ib.connectAsync(host, port, clientId=client_id)
+        return await _read_summary_values(fallback_ib)
+    finally:
+        fallback_ib.disconnect()
+
+
 @router.get("", response_class=HTMLResponse)
 async def portfolio_page(request: Request):
     return templates.TemplateResponse(request, "portfolio.html", {})
@@ -33,16 +87,6 @@ async def portfolio_data():
 @router.get("/pnl")
 async def portfolio_pnl():
     try:
-        ib = dependencies.get_ib()
-        summary = await ib.reqAccountSummaryAsync()
-        keys = {"NetLiquidation", "TotalCashValue", "UnrealizedPnL", "RealizedPnL"}
-        result = {}
-        for item in summary:
-            if item.tag in keys:
-                try:
-                    result[item.tag] = float(item.value)
-                except (ValueError, TypeError):
-                    result[item.tag] = item.value
-        return result
+        return await _fetch_summary_with_fallback()
     except Exception as e:
         return {"error": str(e)}

routers/trades.py

@@ -4,10 +4,11 @@ import os
 from datetime import datetime, timezone
 
 from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import HTMLResponse
+from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
 from ib_async import LimitOrder, MarketOrder, Stock
 
+from config_utils import read_env_or_file
 import dependencies
 from risk_manager import RiskManager
 
@@ -15,7 +16,10 @@ router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 logger = logging.getLogger(__name__)
 
-WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET", "your_super_secret_string_123")
+WEBHOOK_SECRET = (
+    read_env_or_file("WEBHOOK_SECRET", "your_super_secret_string_123")
+    or "your_super_secret_string_123"
+)
 risk = RiskManager()
 
 
@@ -30,6 +34,18 @@ def log_trade(symbol, action, quantity, price, cursor, conn, lock):
     logger.info(f"Logged trade: {action} {quantity} {symbol} @ ${price:.2f}")
 
 
+def _latest_trade_message(trade) -> str:
+    for entry in reversed(getattr(trade, "log", []) or []):
+        message = getattr(entry, "message", "")
+        if message:
+            return message
+    return ""
+
+
+def _normalized_trade_status(trade) -> str:
+    return (getattr(getattr(trade, "orderStatus", None), "status", "") or "").strip()
+
+
 @router.get("/tradelog", response_class=HTMLResponse)
 async def tradelog(request: Request):
     cursor = dependencies.get_db_cursor()
@@ -141,9 +157,40 @@ async def webhook(request: Request):
         primary_trade = trades_list[0]
         for _ in range(60):
             await asyncio.sleep(0.5)
+            status = _normalized_trade_status(primary_trade)
+            if status in {"ValidationError", "Cancelled", "ApiCancelled", "Inactive"}:
+                break
             if primary_trade.isDone():
                 break
 
+        status = _normalized_trade_status(primary_trade)
+        if status == "ValidationError":
+            for t in trades_list:
+                try:
+                    ib.cancelOrder(t.order)
+                except Exception:
+                    pass
+            detail = _latest_trade_message(primary_trade) or "Order was rejected by IBKR"
+            raise HTTPException(status_code=409, detail=detail)
+
+        if status in {"Cancelled", "ApiCancelled", "Inactive"} and not primary_trade.fills:
+            detail = _latest_trade_message(primary_trade) or f"Order became {status}"
+            raise HTTPException(status_code=409, detail=detail)
+
+        if status in {"PendingSubmit", "PreSubmitted", "Submitted"} and not primary_trade.fills:
+            return JSONResponse(
+                status_code=202,
+                content={
+                    "status": "accepted",
+                    "symbol": symbol,
+                    "action": action,
+                    "quantity": quantity,
+                    "order_type": order_type,
+                    "order_status": status,
+                    "message": _latest_trade_message(primary_trade),
+                },
+            )
+
         if not primary_trade.isDone() or not primary_trade.fills:
             for t in trades_list:
                 try:

secrets/README.md

@@ -0,0 +1,20 @@
+# Docker Secret Files
+
+Create local secret files in this directory before starting the Docker stack.
+
+Required:
+- `tws_password`
+- `webhook_secret`
+
+Optional:
+- `vnc_password`
+- `ui_username`
+- `ui_password`
+
+Example:
+
+```bash
+printf '%s\n' 'your_ibkr_password' > secrets/tws_password
+printf '%s\n' 'change_this_webhook_secret' > secrets/webhook_secret
+chmod 600 secrets/tws_password secrets/webhook_secret
+```

templates/index.html

@@ -28,7 +28,9 @@
         grid: { vertLines: { color: '#f0f0f0' }, horzLines: { color: '#f0f0f0' } },
         timeScale: { timeVisible: true, secondsVisible: false }
     });
-    const series = chart.addCandlestickSeries();
+    const series = typeof chart.addCandlestickSeries === 'function'
+        ? chart.addCandlestickSeries()
+        : chart.addSeries(LightweightCharts.CandlestickSeries, {});
 
     let eventSource = null;
 

tests/test_integration.py

@@ -1,14 +1,11 @@
 """
 Integration test suite — pre-Docker validation.
 
-Run all tests (mocked IB):
+Run offline tests only:
     pytest tests/test_integration.py -v
 
 Run with live IB Gateway on port 4002:
-    pytest tests/test_integration.py -v  (SKIP_LIVE_TESTS must NOT be set)
-
-Skip live tests:
-    SKIP_LIVE_TESTS=1 pytest tests/test_integration.py -v
+    RUN_LIVE_IB_TESTS=1 pytest tests/test_integration.py -v
 """
 import os
 import sqlite3
@@ -22,7 +19,7 @@ import pytest
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-SKIP_LIVE = os.getenv("SKIP_LIVE_TESTS", "0") == "1"
+RUN_LIVE = os.getenv("RUN_LIVE_IB_TESTS", "0") == "1"
 
 # ── Minimal ib_async stubs (used when real ib_async not available) ────────────
 def _make_stub():
@@ -110,11 +107,12 @@ class TestIBKRConnection:
 
     def test_ibkr_port_env_set(self):
         port = int(os.getenv("IBKR_PORT", "4002"))
-        assert port in (4001, 4002), (
-            f"IBKR_PORT must be 4001 (live) or 4002 (paper), got {port}"
+        assert port in (4001, 4002, 4003, 4004), (
+            "IBKR_PORT must be 4001/4003 (live) or 4002/4004 (paper), "
+            f"got {port}"
         )
 
-    @pytest.mark.skipif(SKIP_LIVE, reason="SKIP_LIVE_TESTS=1")
+    @pytest.mark.skipif(not RUN_LIVE, reason="RUN_LIVE_IB_TESTS=1 required")
     def test_ibkr_connection_live(self):
         import asyncio
 
@@ -144,7 +142,7 @@ class TestIBKRConnection:
         assert ib.isConnected(), "IB Gateway connection not established"
         ib.disconnect()
 
-    @pytest.mark.skipif(SKIP_LIVE, reason="SKIP_LIVE_TESTS=1")
+    @pytest.mark.skipif(not RUN_LIVE, reason="RUN_LIVE_IB_TESTS=1 required")
     def test_history_returns_data(self):
         r = client.get("/history?symbol=AAPL")
         assert r.status_code == 200

tests/test_phase2.py

@@ -2,6 +2,7 @@
 Phase 2 test suite — uses TestClient with mocked IB and DB dependencies.
 Run: pytest tests/test_phase2.py -v
 """
+import os
 import sqlite3
 import sys
 import threading
@@ -88,7 +89,7 @@ _app.include_router(portfolio.router)
 client = TestClient(_app, raise_server_exceptions=False)
 
 WRONG_SECRET = "wrong_secret"
-RIGHT_SECRET = "your_super_secret_string_123"
+RIGHT_SECRET = os.getenv("WEBHOOK_SECRET", "your_super_secret_string_123")
 
 
 # ── Tests ─────────────────────────────────────────────────────────────────────

verify_docker.sh

@@ -3,6 +3,7 @@
 # Usage: chmod +x verify_docker.sh && ./verify_docker.sh
 # Requires: docker compose up -d already running
 # Note: tests against port 8000 directly (local testing without Traefik hostname)
+# Optional: set COMPOSE_ENV_FILE to override the default compose env file.
 
 set -uo pipefail
 
@@ -10,6 +11,14 @@ BASE="http://localhost:8000"
 PASS=0
 FAIL=0
 
+default_compose_env_file=".env"
+if [ -f "docker.env" ]; then
+    default_compose_env_file="docker.env"
+fi
+
+COMPOSE_ENV_FILE=${COMPOSE_ENV_FILE:-$default_compose_env_file}
+COMPOSE_APP_ENV_FILE=${COMPOSE_APP_ENV_FILE:-$COMPOSE_ENV_FILE}
+
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
@@ -19,6 +28,14 @@ pass() { echo -e "${GREEN}[PASS]${NC} $1"; PASS=$((PASS + 1)); }
 fail() { echo -e "${RED}[FAIL]${NC} $1"; FAIL=$((FAIL + 1)); }
 info() { echo -e "${YELLOW}[INFO]${NC} $1"; }
 
+compose() {
+    if [ -n "$COMPOSE_ENV_FILE" ] && [ -f "$COMPOSE_ENV_FILE" ]; then
+        COMPOSE_APP_ENV_FILE="$COMPOSE_APP_ENV_FILE" docker compose --env-file "$COMPOSE_ENV_FILE" "$@"
+    else
+        COMPOSE_APP_ENV_FILE="$COMPOSE_APP_ENV_FILE" docker compose "$@"
+    fi
+}
+
 check_body() {
     local label="$1" result="$2" expected="$3"
     if echo "$result" | grep -q "$expected"; then
@@ -127,17 +144,22 @@ fi
 
 echo ""
 echo "── IBKR Connection Log Check ─────────────────────"
-if docker compose logs ibkr-dashboard 2>/dev/null | grep -q "Connected to IB"; then
+if compose logs ibkr-dashboard 2>/dev/null | grep -q "Connected to IB"; then
     pass "IBKR connection confirmed in logs"
 else
-    fail "IBKR connection confirmed in logs"
-    info "Check: docker compose logs ibkr-dashboard"
+    R=$(curl -s --max-time 10 "$BASE/health" 2>/dev/null || echo "{}")
+    if echo "$R" | grep -q '"ib_connected":true'; then
+        pass "IBKR connection confirmed via /health"
+    else
+        fail "IBKR connection confirmed in logs"
+        info "Check: compose logs ibkr-dashboard"
+    fi
 fi
 
 echo ""
 echo "── pytest Inside Container ───────────────────────"
 info "Running: docker compose exec ibkr-dashboard pytest tests/ -q --tb=short"
-if docker compose exec ibkr-dashboard pytest tests/ -q --tb=short 2>&1; then
+if compose exec ibkr-dashboard pytest tests/ -q --tb=short 2>&1; then
     pass "pytest inside container"
 else
     fail "pytest inside container"