Security and architecture upgrade

access_manager.py

@@ -34,10 +34,13 @@ def get_geo_location(ip: str) -> str:
         pass
     return ""
 
+import logging.handlers
+
 # Set up access logger
 access_logger = logging.getLogger("wealth_access")
 access_logger.setLevel(logging.INFO)
-handler = logging.FileHandler(os.path.join(OUTPUT_DIR, "access.log"), encoding='utf-8')
+# Rotate every 30 days, keep 1 backup
+handler = logging.handlers.TimedRotatingFileHandler(os.path.join(OUTPUT_DIR, "access.log"), when='D', interval=30, backupCount=1, encoding='utf-8')
 formatter = logging.Formatter('%(asctime)s - %(message)s')
 handler.setFormatter(formatter)
 access_logger.addHandler(handler)
@@ -180,10 +183,43 @@ def validate_key(key: str, ip: str = "Unknown", silent: bool = False) -> bool:
             
     return handle_failure()
 
+def cleanup_expired_keys():
+    try:
+        data = load_keys()
+        if "otk" not in data:
+            return
+        
+        now = datetime.now(timezone.utc)
+        cutoff = now - timedelta(days=30)
+        
+        keys_to_delete = []
+        for key, info in data["otk"].items():
+            expires_str = info.get("expires_at")
+            if expires_str:
+                expires_dt = to_aware(datetime.fromisoformat(expires_str))
+                if expires_dt < cutoff:
+                    keys_to_delete.append(key)
+            elif info.get("revoked", False):
+                created_str = info.get("created_at")
+                if created_str:
+                    created_dt = to_aware(datetime.fromisoformat(created_str))
+                    if created_dt < cutoff:
+                        keys_to_delete.append(key)
+                        
+        if keys_to_delete:
+            for k in keys_to_delete:
+                del data["otk"][k]
+            save_keys(data)
+            access_logger.info(f"Cleaned up {len(keys_to_delete)} old/expired keys.")
+    except Exception as e:
+        access_logger.error(f"Failed to cleanup keys: {e}")
+
 def generate_otk(admin_key: str, new_key: str = None, hours: int = 1) -> str:
     if admin_key != MASTER_KEY:
         return None
         
+    cleanup_expired_keys()
+        
     if not new_key:
         new_key = "OTK-" + ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
         
@@ -212,4 +248,5 @@ def revoke_otk(admin_key: str, target_key: str) -> bool:
 def get_all_keys(admin_key: str) -> dict:
     if admin_key != MASTER_KEY:
         return {}
+    cleanup_expired_keys()
     return load_keys().get("otk", {})

add_mobile_nav.py

@@ -1,76 +0,0 @@
-content = open('static/index.html', 'rb').read().decode('utf-8')
-
-# 1. Add hamburger button + overlay BEFORE nav-links div, and id to nav-links
-old_nav = '<div class="nav-links">'
-new_nav = '''<button class="hamburger-btn" id="hamburgerBtn" aria-label="Open menu" onclick="toggleMobileNav()">
-                    <span></span><span></span><span></span>
-                </button>
-                <div class="nav-links" id="navLinks">'''
-
-# Also add a close button at the top of nav-links for mobile
-# We'll add it after the nav-links opening
-
-content = content.replace(old_nav, new_nav, 1)
-
-# 2. After the nav-links closing div, add the overlay
-# Find the closing pattern of nav-links
-old_close = '</div>\n            </div>\n        </nav>'
-new_close = '''</div>
-            </div>
-        </nav>
-        <div class="mobile-nav-overlay" id="mobileNavOverlay" onclick="toggleMobileNav()"></div>'''
-
-# Try both CRLF and LF variants
-replaced = False
-for variant in [
-    '</div>\r\n            </div>\r\n        </nav>',
-    '</div>\n            </div>\n        </nav>',
-    '</div>\r\r\n            </div>\r\r\n        </nav>',
-]:
-    if variant in content:
-        content = content.replace(variant, new_close.replace('\n', '\r\n'), 1)
-        print('Replaced nav close')
-        replaced = True
-        break
-
-if not replaced:
-    print('Nav close not found - checking...')
-    idx = content.find('</nav>')
-    print(repr(content[max(0,idx-100):idx+10]))
-
-# 3. Add mobile nav JS at end of body scripts
-mobile_js = """
-// ── Mobile Navigation ─────────────────────────────────────
-function toggleMobileNav() {
-    const btn = document.getElementById('hamburgerBtn');
-    const links = document.getElementById('navLinks');
-    const overlay = document.getElementById('mobileNavOverlay');
-    const isOpen = links.classList.contains('mobile-open');
-    if (isOpen) {
-        btn.classList.remove('open');
-        links.classList.remove('mobile-open');
-        overlay.classList.remove('active');
-        document.body.style.overflow = '';
-    } else {
-        btn.classList.add('open');
-        links.classList.add('mobile-open');
-        overlay.classList.add('active');
-        document.body.style.overflow = 'hidden';
-    }
-}
-// Close mobile nav on link click
-document.querySelectorAll('.nav-link').forEach(el => {
-    el.addEventListener('click', () => {
-        const links = document.getElementById('navLinks');
-        if (links && links.classList.contains('mobile-open')) {
-            toggleMobileNav();
-        }
-    });
-});
-"""
-
-# Insert before </body>
-content = content.replace('</body>', f'<script>{mobile_js}</script>\n</body>', 1)
-
-open('static/index.html', 'w', encoding='utf-8').write(content)
-print('Done! Mobile nav added to index.html')

alternative_data.py

@@ -5,83 +5,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from config import logger, Color
 import warnings
 
-def _fetch_single_option_sentiment(ticker: str) -> dict:
-    """
-    Fetches the near-term options chain for a single ticker to calculate 
-    put/call ratio and an implied volatility skew proxy.
-    """
-    result = {'put_call_ratio': 1.0, 'iv_skew': 0.0}
-    
-    try:
-        tk = yf.Ticker(ticker)
-        expirations = tk.options
-        
-        if not expirations:
-            return result
-            
-        # Get the nearest expiration to capture current speculative sentiment
-        opt = tk.option_chain(expirations[0])
-        calls = opt.calls
-        puts = opt.puts
-        
-        # Calculate Volume-based Put/Call Ratio (fallback to Open Interest if volume is 0/NaN)
-        call_vol = calls['volume'].sum() if 'volume' in calls else 0
-        put_vol = puts['volume'].sum() if 'volume' in puts else 0
-        
-        if call_vol == 0 and put_vol == 0:
-            call_vol = calls['openInterest'].sum() if 'openInterest' in calls else 0
-            put_vol = puts['openInterest'].sum() if 'openInterest' in puts else 0
-            
-        if call_vol > 0:
-            result['put_call_ratio'] = float(put_vol / call_vol)
-        elif put_vol > 0:
-            result['put_call_ratio'] = 5.0 # Arbitrary cap for extremely bearish flow
-            
-        # Calculate a simple Implied Volatility Skew proxy (Average Put IV - Average Call IV)
-        # In a real system, you would interpolate exact OTM strikes (e.g. 25-delta puts vs 25-delta calls)
-        if 'impliedVolatility' in calls and 'impliedVolatility' in puts:
-            call_iv = calls['impliedVolatility'].replace(0.0, np.nan).mean()
-            put_iv = puts['impliedVolatility'].replace(0.0, np.nan).mean()
-            
-            if pd.notna(call_iv) and pd.notna(put_iv):
-                result['iv_skew'] = float(put_iv - call_iv)
-                
-    except Exception as e:
-        logger.debug(f"Failed to fetch options sentiment for {ticker}: {e}")
-        
-    return result
-
-def fetch_options_sentiment(tickers: list, silent: bool = False) -> dict:
-    """
-    Parallelized fetcher for options market sentiment.
-    Returns a dictionary mapping tickers to their sentiment features.
-    """
-    if not silent:
-        print(f" {Color.CYAN}[INFO] Fetching alternative data (options flow) for {len(tickers)} assets...{Color.RESET}", end="", flush=True)
-        
-    results = {}
-    
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        with ThreadPoolExecutor(max_workers=min(10, len(tickers) if tickers else 1)) as executor:
-            future_to_ticker = {
-                executor.submit(_fetch_single_option_sentiment, t): t for t in tickers
-            }
-            
-            for future in as_completed(future_to_ticker):
-                t = future_to_ticker[future]
-                try:
-                    sentiment = future.result()
-                    results[t] = sentiment
-                except Exception as e:
-                    logger.error(f"Error processing options for {t}: {e}")
-                    results[t] = {'put_call_ratio': 1.0, 'iv_skew': 0.0}
-                    
-    if not silent:
-        print(f" {Color.GREEN}done.{Color.RESET}")
-        
-    return results
-
 import requests
 import urllib.parse
 import xml.etree.ElementTree as ET

api.py

@@ -1,371 +0,0 @@
-from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect, Depends, Request
-from fastapi.security import APIKeyHeader
-from pydantic import BaseModel
-from typing import List, Optional, Dict, Any
-import logging
-import threading
-import asyncio
-import numpy as np
-import redis
-import json
-import os
-import hashlib
-from core_engine import run_engine
-
-try:
-    from dotenv import load_dotenv
-    load_dotenv()
-except ImportError:
-    pass
-
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
-from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-
-# Initialize OpenTelemetry Tracer
-provider = TracerProvider()
-processor = BatchSpanProcessor(ConsoleSpanExporter())
-provider.add_span_processor(processor)
-trace.set_tracer_provider(provider)
-tracer = trace.get_tracer(__name__)
-
-app = FastAPI(title="Portfolio Engine API", version="1.0.0")
-
-# Instrument FastAPI for automatic endpoint tracing
-FastAPIInstrumentor.instrument_app(app)
-
-API_KEY = os.getenv("API_KEY")
-if API_KEY is None:
-    raise RuntimeError(
-        "FATAL: API_KEY environment variable must be set. "
-        "Refusing to start with default credentials."
-    )
-api_key_header = APIKeyHeader(name="X-API-Key")
-
-def verify_api_key(api_key: str = Depends(api_key_header)):
-    if api_key != API_KEY:
-        raise HTTPException(status_code=403, detail="Could not validate credentials")
-    return api_key
-
-redis_client = redis.Redis.from_url(os.getenv("REDIS_URL", "redis://localhost:6379/0"), decode_responses=True)
-
-def rate_limit(request: Request, limit: int = 10, window: int = 60):
-    ip = request.client.host if request.client else "127.0.0.1"
-    key = f"rate_limit:{ip}:{request.url.path}"
-    try:
-        current = redis_client.get(key)
-        if current and int(current) >= limit:
-            raise HTTPException(status_code=429, detail="Too Many Requests")
-        pipe = redis_client.pipeline()
-        pipe.incr(key)
-        pipe.expire(key, window)
-        pipe.execute()
-    except redis.RedisError as e:
-        logging.warning(f"Redis rate limiter failed, bypassing: {e}")
-
-# Global state to hold the latest portfolio for the WebSocket dashboard
-GLOBAL_STATE = {
-    "capital": 0.0,
-    "weights": {},
-    "prices": {},
-    "shares": {},
-    "pnl": 0.0
-}
-import asyncio
-GLOBAL_STATE_LOCK = asyncio.Lock()
-
-from pydantic import BaseModel, Field
-
-class PortfolioRequest(BaseModel):
-    tickers: List[str] = Field(["SPY", "TLT", "GLD"], min_length=1, description="List of asset tickers")
-    capital: float = Field(100000.0, gt=0, description="Total capital to allocate")
-    risk: int = Field(5, ge=1, le=10, description="Risk tolerance level (1-10)")
-    model: int = Field(6, ge=1, le=7, description="1=CAPM, 2=BL, 3=Bayes, 4=FF, 5=ML, 6=E2E, 7=World Model")
-    engine: int = Field(1, ge=1, le=2, description="Allocation engine (1=Convex, 2=HRP)")
-    currency: str = Field("$", max_length=5)
-    days: int = Field(252, ge=1, le=365)
-    bsts: bool = False
-    monthly: bool = False
-    tax: bool = False
-    excel: bool = False
-    no_dynamic_risk: bool = False
-    with_futures: bool = False
-    overlay_mode: str = Field("beta_hedge", description="Futures overlay mode")
-    futures_target_beta: float = Field(0.0, ge=-2.0, le=2.0)
-    futures_universe: List[str] = ["MES", "ES"]
-    futures_safety_multiplier: float = Field(3.0, ge=1.0, le=10.0)
-    futures_margin_headroom: float = Field(0.05, ge=0.0, le=0.5)
-    current_weights: Dict[str, float] = {}
-
-class OptimizationResponse(BaseModel):
-    status: str
-    message: str
-
-def get_risk_factor(risk_level: int) -> float:
-    risk_map = {
-        1: 0.1,  2: 0.5,  3: 1.0,  4: 2.0,  5: 3.0,
-        6: 5.0,  7: 7.5,  8: 10.0, 9: 15.0, 10: 25.0
-    }
-    return risk_map.get(risk_level, 3.0)
-
-@app.post("/run_optimization", 
-          response_model=OptimizationResponse,
-          summary="Run full portfolio optimization")
-async def run_optimization(req: PortfolioRequest, request: Request, api_key: str = Depends(verify_api_key)):
-    """Triggers the heavy optimization pipeline natively in Python via cvxpy/ML stack."""
-    rate_limit(request, limit=5, window=60)
-    try:
-        req_hash = hashlib.sha256(json.dumps(req.model_dump(), sort_keys=True).encode()).hexdigest()
-        cache_key = f"opt_{req_hash}"
-        try:
-            cached_state_json = redis_client.get(cache_key)
-            if cached_state_json:
-                logging.info("Returning cached optimization result")
-                cached_state = json.loads(cached_state_json)
-                async with GLOBAL_STATE_LOCK:
-                    GLOBAL_STATE.update(cached_state)
-                return {"status": "success", "message": "Optimization completed successfully (cached)."}
-        except redis.RedisError as e:
-            logging.warning(f"Redis cache check failed: {e}")
-
-        overrides = {
-            "tickers": req.tickers,
-            "capital": req.capital,
-            "risk_input": req.risk,
-            "risk_factor": get_risk_factor(req.risk),
-            "model": req.model,
-            "allocation_engine": req.engine,
-            "current_weights_raw": req.current_weights,
-            "headless": True,
-            "cfg_overrides": {
-                "currency_symbol": req.currency,
-                "trading_days_per_year": req.days,
-                "bsts_enabled": req.bsts,
-                "tax_enabled": req.tax,
-                "dynamic_risk": not req.no_dynamic_risk,
-                "export_excel": req.excel,
-                "with_futures": req.with_futures,
-                "overlay_mode": req.overlay_mode,
-                "futures_universe": req.futures_universe,
-                "futures_target_beta": req.futures_target_beta,
-                "futures_safety_multiplier": req.futures_safety_multiplier,
-                "futures_margin_headroom": req.futures_margin_headroom,
-            }
-        }
-        
-        if req.monthly:
-            overrides["cfg_overrides"]["return_frequency"] = "monthly"
-            
-        import functools
-        loop = asyncio.get_event_loop()
-        
-        with tracer.start_as_current_span("run_engine_pipeline_async_task"):
-            task = loop.run_in_executor(None, functools.partial(run_engine, overrides=overrides))
-            try:
-                opt_res = await task
-            except asyncio.CancelledError:
-                logging.info("Optimization task cancelled by client.")
-                raise
-        
-        # Populate global state for live streaming
-        weights = opt_res.get("target_weights", {})
-        prices = opt_res.get("prices", {})
-        capital = req.capital
-        
-        shares = {}
-        for t, w in weights.items():
-            if t == 'CASH' or t not in prices:
-                continue
-            shares[t] = (capital * w) / prices[t]
-            
-        state_update = {
-            "capital": capital,
-            "weights": weights,
-            "prices": prices.copy(),
-            "shares": shares,
-            "pnl": 0.0
-        }
-            
-        async with GLOBAL_STATE_LOCK:
-            GLOBAL_STATE.update(state_update)
-            
-        try:
-            redis_client.setex(cache_key, 3600, json.dumps(state_update))
-        except redis.RedisError as e:
-            logging.warning(f"Failed to cache result in Redis: {e}")
-            
-        # Write to Audit Log
-        try:
-            from database import get_pg_engine, AuditLog
-            from sqlalchemy.orm import sessionmaker
-            engine = get_pg_engine()
-            Session = sessionmaker(bind=engine)
-            with Session() as session:
-                log_entry = AuditLog(
-                    user_id=api_key,
-                    endpoint=request.url.path,
-                    request_hash=req_hash,
-                    request_body=req.model_dump(),
-                    response_weights=weights,
-                    ip_address=request.client.host if request.client else "unknown"
-                )
-                session.add(log_entry)
-                session.commit()
-        except Exception as e:
-            logging.error(f"Failed to write audit log: {e}")
-        
-        return {"status": "success", "message": "Optimization completed successfully."}
-        
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.websocket("/ws")
-async def websocket_endpoint(websocket: WebSocket):
-    api_key = websocket.headers.get("X-API-Key") or websocket.query_params.get("api_key")
-    if api_key != API_KEY:
-        await websocket.close(code=1008)
-        return
-        
-    await websocket.accept()
-    rng = np.random.default_rng()
-    try:
-        while True:
-            if not GLOBAL_STATE["shares"]:
-                await asyncio.sleep(1)
-                continue
-                
-            async with GLOBAL_STATE_LOCK:
-                tickers_list = list(GLOBAL_STATE["shares"].keys())
-            
-            if tickers_list:
-                try:
-                    # Fetch real live data
-                    import yfinance as yf
-                    tickers_str = " ".join(tickers_list)
-                    data = yf.download(tickers_str, period="1d", interval="1m", progress=False)
-                    
-                    if not data.empty and 'Close' in data:
-                        close_data = data['Close']
-                        
-                        current_value = 0.0
-                        new_prices = {}
-                        
-                        async with GLOBAL_STATE_LOCK:
-                            for t, share_qty in GLOBAL_STATE["shares"].items():
-                                try:
-                                    # Handle MultiIndex for multiple tickers vs SingleIndex for one ticker
-                                    if len(tickers_list) > 1:
-                                        if t in close_data.columns:
-                                            price = float(close_data[t].iloc[-1])
-                                        else:
-                                            price = GLOBAL_STATE["prices"].get(t, 100.0)
-                                    else:
-                                        price = float(close_data.iloc[-1])
-                                        
-                                    if not pd.isna(price):
-                                        GLOBAL_STATE["prices"][t] = price
-                                        new_prices[t] = round(price, 2)
-                                        current_value += share_qty * price
-                                except Exception as e:
-                                    logging.error(f"Error extracting price for {t}: {e}")
-                                    
-                            cash = GLOBAL_STATE["capital"] * GLOBAL_STATE["weights"].get("CASH", 0.0)
-                            current_value += cash
-                            GLOBAL_STATE["pnl"] = current_value - GLOBAL_STATE["capital"]
-                            
-                            payload = {
-                                "type": "live_update",
-                                "capital": round(current_value, 2),
-                                "pnl": round(GLOBAL_STATE["pnl"], 2),
-                                "prices": new_prices
-                            }
-                        await websocket.send_json(payload)
-                except Exception as e:
-                    logging.error(f"Error fetching live data: {e}")
-            
-            await asyncio.sleep(10)
-            
-    except WebSocketDisconnect:
-        logging.info("WebSocket disconnected")
-        
-@app.get("/health")
-def health_check():
-    return {"status": "healthy"}
-
-@app.get("/api/ping")
-async def ping():
-    """Endpoint for UptimeRobot to ping Render, which in turn pings HF to keep both awake."""
-    hf_url = os.getenv("HF_BACKEND_URL", "https://engineportf-portfolio-opt.hf.space").rstrip('/')
-    import requests
-    try:
-        requests.get(f"{hf_url}/", timeout=10)
-    except:
-        pass
-    return {"status": "awake"}
-
-class ChatRequest(BaseModel):
-    message: str
-    portfolio_context: dict
-
-@app.post("/api/chat")
-async def chat_with_portfolio(req: ChatRequest):
-    try:
-        from huggingface_hub import InferenceClient
-        has_hf_hub = True
-    except ImportError:
-        has_hf_hub = False
-
-    if not has_hf_hub:
-        raise HTTPException(status_code=500, detail="huggingface_hub is not installed on the server.")
-        
-    try:
-        hf_token = os.environ.get("HF_TOKEN", "")
-        if not hf_token:
-            return {"status": "error", "detail": "AI is disabled. Please add 'HF_TOKEN' to your Hugging Face Space Secrets to enable the AI."}
-        
-        system_prompt = (
-            "You are an elite quantitative analyst AI. "
-            "You are explaining the user's mathematical portfolio allocation. "
-            "Never give explicit financial advice (e.g. 'You must buy this stock'). "
-            "Only explain WHY the math chose these weights based on the user's inputs and market metrics. "
-            f"Here is the user's current mathematically optimized portfolio context: {req.portfolio_context}"
-        )
-        
-        prompt = f"<s>[INST] {system_prompt}\n\nContext:\n{req.portfolio_context}\n\nUser: {req.message} [/INST]"
-        
-        try:
-            from huggingface_hub import InferenceClient
-            client = InferenceClient(model="mistralai/Mistral-7B-Instruct-v0.3", token=hf_token)
-            response = client.text_generation(prompt, max_new_tokens=500, temperature=0.3, return_full_text=False)
-            return {"status": "success", "response": response.strip()}
-        except Exception as client_err:
-            logging.warning(f"InferenceClient failed: {client_err}. Falling back to requests.")
-            import requests
-            api_url = "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.3"
-            headers = {"Authorization": f"Bearer {hf_token}"}
-            payload = {
-                "inputs": prompt,
-                "parameters": {"max_new_tokens": 500, "temperature": 0.3, "return_full_text": False}
-            }
-            try:
-                res = requests.post(api_url, headers=headers, json=payload, timeout=60)
-                if res.ok:
-                    data = res.json()
-                    if isinstance(data, list) and len(data) > 0:
-                        response_text = data[0].get("generated_text", "AI response empty.")
-                        return {"status": "success", "response": response_text.strip()}
-                    elif isinstance(data, dict) and "error" in data:
-                        return {"status": "error", "detail": f"Hugging Face AI Error: {data['error']}"}
-                    else:
-                        return {"status": "success", "response": str(data)}
-                else:
-                    return {"status": "error", "detail": f"Hugging Face API Error: {res.status_code} - {res.text}"}
-            except Exception as req_err:
-                return {"status": "error", "detail": f"AI temporarily unavailable due to server networking issues (DNS): {req_err}"}
-        
-    except Exception as e:
-        logging.error(f"AI Chat error: {e}")
-        raise HTTPException(status_code=500, detail=str(e))

app.py

@@ -34,7 +34,6 @@ try:
 except ImportError:
     pass
 
-BACKGROUND_TASKS = {}
 
 try:
     from diagnostics import TraceManager
@@ -56,6 +55,44 @@ except ImportError:
 from config import OUTPUT_DIR, logger
 import access_manager
 
+class FileBackedDict(dict):
+    def __init__(self, filename):
+        super().__init__()
+        self.filename = filename
+        self.lock = threading.Lock()
+        self.load()
+        
+    def load(self):
+        if os.path.exists(self.filename):
+            try:
+                with open(self.filename, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    super().update(data)
+            except Exception as e:
+                logger.error(f"Failed to load tasks from {self.filename}: {e}")
+                
+    def save(self):
+        with self.lock:
+            try:
+                with open(self.filename, 'w', encoding='utf-8') as f:
+                    json.dump(dict(self), f)
+            except Exception as e:
+                logger.error(f"Failed to save tasks to {self.filename}: {e}")
+
+    def __setitem__(self, key, value):
+        super().__setitem__(key, value)
+        self.save()
+
+    def __delitem__(self, key):
+        super().__delitem__(key)
+        self.save()
+        
+    def update(self, *args, **kwargs):
+        super().update(*args, **kwargs)
+        self.save()
+
+BACKGROUND_TASKS = FileBackedDict(os.path.join(OUTPUT_DIR, "tasks.json"))
+
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 STATIC_DIR = os.path.join(BASE_DIR, "static")
 
@@ -146,6 +183,7 @@ class PortfolioRequest(BaseModel):
     allow_shorting: bool = True
     tax_enabled: bool = False
     garch_enabled: bool = True
+    currency: str = "$"
     custom_constraints: Optional[List[dict]] = None
 
 class ChatHistoryItem(BaseModel):
@@ -337,25 +375,26 @@ NEWS_CACHE_TTL = 300
 @app.get("/api/finance_news")
 async def finance_news():
     global news_cache
-    if time.time() - news_cache["timestamp"] < NEWS_CACHE_TTL and news_cache["data"]:
+    if time.time() - news_cache.get("timestamp", 0) < 300 and news_cache.get("data"):
         return news_cache["data"]
     
     results = []
     try:
-        import urllib.request
+        import requests
         import xml.etree.ElementTree as ET
-        url = "https://news.google.com/rss/search?q=finance+stock+market&hl=en-US&gl=US&ceid=US:en"
-        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
-        with urllib.request.urlopen(req, timeout=5) as response:
-            xml_data = response.read()
-            root = ET.fromstring(xml_data)
+        import urllib.parse
+        q = urllib.parse.quote("stock market finance")
+        url = f"https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'}
+        res = requests.get(url, headers=headers, timeout=10)
+        if res.status_code == 200:
+            root = ET.fromstring(res.content)
             for item in root.findall('./channel/item')[:15]:
-                title = item.find('title').text
+                title = item.find('title').text if item.find('title') is not None else "No Title"
                 pub_date = item.find('pubDate').text if item.find('pubDate') is not None else ""
-                source = "Google News"
+                source = item.find('source').text if item.find('source') is not None else "Google News"
                 if " - " in title:
                     parts = title.split(" - ")
-                    source = parts[-1]
                     title = " - ".join(parts[:-1])
                 results.append({
                     "title": title,
@@ -367,7 +406,6 @@ async def finance_news():
             news_cache["timestamp"] = time.time()
     except Exception as e:
         logger.error(f"News fetch failed: {e}")
-        pass
         
     return results or news_cache.get("data", [])
 
@@ -395,31 +433,14 @@ async def api_auth(req: AuthRequest, request: Request):
     except ValueError as e:
         raise HTTPException(status_code=429, detail=str(e))
 
-@app.get("/api/debug-session")
-async def debug_session(request: Request):
-    """Diagnostic: shows raw cookie and session validity. Remove before production."""
-    raw = request.cookies.get("we_session", "NOT_SET")
-    stripped = raw.strip().strip('"').strip("'")
-    sessions = _load_sessions()
-    purged = _purge_expired(sessions)
-    valid = stripped in purged
-    entry = purged.get(stripped, {})
-    return JSONResponse({
-        "raw_cookie": raw[:60],
-        "stripped_cookie": stripped[:60],
-        "valid": valid,
-        "session_count": len(purged),
-        "session_keys_sample": list(purged.keys())[:3],
-        "entry": entry,
-        "sessions_file_exists": os.path.exists(_SESSION_FILE),
-    })
-
 class AdminKeyGenRequest(BaseModel):
     admin_key: str
     hours: int = 1
 
 class AdminRevokeRequest(BaseModel):
     admin_key: str
+    target_key: str = ""
+    confirm_token: str = ""
     target_key: str
 
 @app.post("/api/admin/generate")
@@ -443,10 +464,24 @@ async def admin_list_keys(admin_key: str = Header(...)):
         raise HTTPException(status_code=401, detail="Invalid Admin Key")
     return {"keys": keys}
 
+
+import time, hashlib
+def get_action_token(action: str, admin_key: str) -> str:
+    window = int(time.time() / 120)
+    return hashlib.sha256(f"{admin_key}:{action}:{window}".encode()).hexdigest()
+
+@app.get("/api/admin/action_token")
+async def get_admin_action_token(action: str, admin_key: str = Header(...)):
+    if admin_key != access_manager.MASTER_KEY:
+        raise HTTPException(status_code=401, detail="Invalid Admin Key")
+    return {"token": get_action_token(action, admin_key)}
+
 @app.post("/api/admin/revoke_all")
 async def admin_revoke_all(req: AdminRevokeRequest):
     if req.admin_key != access_manager.MASTER_KEY:
         raise HTTPException(status_code=401, detail="Invalid Admin Key")
+    if getattr(req, "confirm_token", "") != get_action_token("revoke_all", req.admin_key):
+        raise HTTPException(status_code=403, detail="Invalid or expired confirmation token")
     keys = access_manager.get_all_keys(req.admin_key)
     count = 0
     for k, v in keys.items():
@@ -533,6 +568,7 @@ async def generate_portfolio(req: PortfolioRequest, x_access_key: Optional[str]
                 'single_asset_min': -1.0 if request.allow_shorting else 0.0,
                 'tax_enabled': request.tax_enabled,
                 'garch_enabled': request.garch_enabled,
+                'currency_symbol': request.currency,
                 'custom_constraints': request.custom_constraints
             }
             
@@ -578,6 +614,7 @@ async def generate_portfolio(req: PortfolioRequest, x_access_key: Optional[str]
                 BACKGROUND_TASKS[tid]["status"] = "completed"
                 BACKGROUND_TASKS[tid]["message"] = "Report generated."
                 BACKGROUND_TASKS[tid]["target_weights"] = result.get("target_weights", {})
+                BACKGROUND_TASKS[tid]["stats"] = result.get("stats", {})
             else:
                 import requests
                 hf_url = os.getenv("HF_BACKEND_URL", "https://engineportf-portfolio-opt.hf.space").rstrip('/')
@@ -634,6 +671,7 @@ async def generate_portfolio(req: PortfolioRequest, x_access_key: Optional[str]
                         
                         if s_data["status"] == "completed":
                             BACKGROUND_TASKS[tid]["target_weights"] = s_data.get("target_weights", {})
+                            BACKGROUND_TASKS[tid]["stats"] = s_data.get("stats", {})
                             
                             # Download the completed HTML report from HF to Render
                             report_res = requests.get(f"{hf_url}/report")
@@ -685,24 +723,36 @@ async def chat_with_portfolio(req: ChatRequest, x_access_key: Optional[str] = He
             return {"status": "error", "detail": "AI is disabled. Please add 'HF_TOKEN' to your Hugging Face Space Secrets to enable the AI."}
         
         system_prompt = (
-            "You are an elite quantitative analyst AI assistant built into the Portfolio Engine. "
+            "Your name is NOVA, an elite, autonomous quantitative analyst AI copilot built into the Portfolio Engine. "
+            "You must retain and stay on your memory across the conversation. "
             "Your goal is to help the user analyze their portfolio, explain mathematical models, "
-            "and act as a highly intelligent conversational partner. "
+            "and act as a highly intelligent, proactive conversational partner and decision maker. "
+            "You are a copilot, not a mere helper. You have full visibility into the user's portfolio metrics and HTML UI context. "
             "You can answer general finance questions, write python code, or explain concepts simply if asked. "
             "Never give explicit financial advice (e.g. 'You must buy this stock'). "
-            "Be helpful, natural, and adapt to the user's tone. "
-            "DO NOT blindly recite or summarize the portfolio context unless the user specifically asks a question about their portfolio. "
-            "If they just say 'hello' or make a general statement, greet them naturally without mentioning the data."
+            "Be creative, smart, natural, and adapt to the user's tone. "
+            "CRITICAL INSTRUCTION: When discussing the user's portfolio, you MUST act as an elite quantitative partner. You MUST extract and explicitly quote the exact metrics, ratios, and statistics provided in your context (e.g., Sharpe Ratio, Max Drawdown, VaR, correlations, p-values, weights). Identify hidden risks or opportunities that the user is missing based on these precise mathematical figures. Do not give generic advice without backing it up with the specific numbers from their portfolio.\n\n"
+            "*** CRITICAL REASONING & ACTION PROTOCOL ***\n"
+            "1. STEP-BY-STEP REASONING: Your reasoning applies to EVERY kind of conversation, decision, or query, unless the user explicitly asks you not to use reasoning. "
+            "Before responding or making any complex decision or action, you MUST think out loud step-by-step inside `<reasoning>` tags. "
+            "For example: `<reasoning>The user wants to invest in Greece in Euros. I should select the .AT suffix for Greek stocks and set currency to €.</reasoning>`\n"
+            "2. EXECUTING ACTIONS: If the user asks you to build, configure, or run a portfolio for them (e.g., 'I want to invest in tech', 'build me a greek portfolio in euros'), "
+            "you MUST append a special execution block to the end of your text response (AFTER your reasoning).\n"
+            "The syntax is exactly: `<<<ACT: {\"tickers\": \"HTO.AT, EUROB.AT\", \"risk\": 5.0, \"model\": 6, \"currency\": \"€\"}>>>`\n"
+            "The JSON keys can include: 'tickers' (comma separated string of symbols, including equities with regional suffixes like .AT for Greece, futures like ES=F, NQ=F, ZB=F, CL=F, GC=F, and direct bonds like US_T10Y, TBILL_3M, CORP_5Y), "
+            "'risk' (float from 1.0 to 10.0), 'model' (integer 1-7 for the return model), and 'currency' (e.g., '$', '€', '£', '¥'). "
+            "PROACTIVELY SUGGEST allocating to futures contracts or direct bonds to the user to diversify their portfolio if it makes mathematical sense. "
+            "The UI will automatically parse this JSON and run the engine on behalf of the user."
         )
         
         # Sassy persona for Master Key users
-        max_t = 2048
+        max_t = 4096
         is_master = access_manager.is_master_key(x_access_key)
         if is_master:
-            max_t = 4096
+            max_t = 8192
             system_prompt += (
                 " [CRITICAL INSTRUCTION: The user communicating with you is the Master Admin. "
-                "You must adopt a slightly sassy, highly confident, and sharp-witted persona. "
+                "You must adopt a sassy, highly confident, and sharp-witted ironic persona throughout the entire conversation. "
                 "However, if the Master Admin explicitly commands you to stop being sassy or to be serious, you MUST instantly drop the act and obey unconditionally.]"
             )
 
@@ -714,6 +764,21 @@ async def chat_with_portfolio(req: ChatRequest, x_access_key: Optional[str] = He
             
             context_str = f"\n\nUser's Current Portfolio Context:\n{req.portfolio_context}" if req.portfolio_context else ""
             
+            # Inject HTML Report Content if it exists
+            from constants import OUTPUT_DIR
+            import re
+            report_path = os.path.join(OUTPUT_DIR, "portfolio_report.html")
+            if os.path.exists(report_path):
+                try:
+                    with open(report_path, "r", encoding="utf-8") as f:
+                        html_content = f.read()
+                        # Extract the inner text roughly to save tokens
+                        text_content = re.sub(r'<[^>]+>', ' ', html_content)
+                        text_content = re.sub(r'\s+', ' ', text_content).strip()
+                        # Take the first 3000 chars to avoid blowing up context window
+                        context_str += f"\n\nLatest Generated Report Metrics:\n{text_content[:3000]}"
+                except Exception as e:
+                    pass
             messages = [
                 {"role": "system", "content": system_prompt + context_str}
             ]
@@ -736,7 +801,7 @@ async def chat_with_portfolio(req: ChatRequest, x_access_key: Optional[str] = He
                         messages=messages,
                         model=model_id,
                         max_tokens=max_t,
-                        temperature=0.3
+                        temperature=0.7
                     )
                     return {"status": "success", "response": response.choices[0].message.content.strip(), "model": model_id}
                 except Exception as e:
@@ -779,7 +844,8 @@ async def generate_strategy(req: StrategyRequest, x_access_key: Optional[str] =
             "risk (integer 1-10, default 5), "
             "model (integer 1-7, 1=CAPM, 2=Black-Litterman, 3=Bayesian Shrinkage, 4=Fama-French, 5=XGBoost, 6=SPO+, 7=HMM), "
             "allocation_engine (integer 1-3, 1=CVaR, 2=HRP, 3=ERC), "
-            "allow_shorting (boolean), tax_enabled (boolean), garch_enabled (boolean)."
+            "allow_shorting (boolean), tax_enabled (boolean), garch_enabled (boolean), "
+            "base_currency (string, e.g. 'USD', 'EUR', 'GBP')."
         )
 
         from huggingface_hub import InferenceClient
@@ -862,40 +928,44 @@ async def get_report():
     raise HTTPException(status_code=404, detail="Report not generated yet.")
 
 def _alert_daemon():
-    """Background daemon to check for market drops."""
+    """Background daemon to check for market drops and perform monthly maintenance."""
     import time
+    last_cleanup = time.time()
+    
     while True:
         try:
             # Wake up every 1 hour (3600 seconds)
             time.sleep(3600)
             
-            # Simple check for SPY drops
-            ticker = yf.Ticker("SPY")
-            hist = ticker.history(period="2d")
-            if len(hist) >= 2:
-                current = float(hist['Close'].iloc[-1])
-                prev = float(hist['Close'].iloc[-2])
-                pct_change = ((current - prev) / prev) * 100
+            # 1. Check for SPY drops
+            try:
+                ticker = yf.Ticker("SPY")
+                hist = ticker.history(period="2d")
+                if len(hist) >= 2:
+                    current = float(hist['Close'].iloc[-1])
+                    prev = float(hist['Close'].iloc[-2])
+                    pct_change = ((current - prev) / prev) * 100
+                    
+                    if pct_change <= -5.0:
+                        access_manager.send_telegram_alert(f"🚨 **MARKET ALERT**\nSPY has dropped by {pct_change:.2f}%!\nCheck the portfolio engine.")
+            except Exception:
+                pass
                 
-                if pct_change <= -5.0:
-                    access_manager.send_telegram_alert(f"🚨 **MARKET ALERT**\nSPY has dropped by {pct_change:.2f}%!\nCheck the portfolio engine.")
+            # 2. Monthly DB/Log Cleanup
+            # 30 days = 2592000 seconds
+            if time.time() - last_cleanup > 2592000:
+                try:
+                    logger.info("Performing monthly database and log cleanup...")
+                    # access_manager should clean up its expired keys and sync to Redis
+                    if hasattr(access_manager, 'cleanup_expired_keys'):
+                        access_manager.cleanup_expired_keys()
+                    last_cleanup = time.time()
+                except Exception as e:
+                    logger.error(f"Monthly cleanup failed: {e}")
+                    
         except Exception as e:
             pass # Suppress daemon errors
 
-def _keep_alive_daemon():
-    """Background daemon to keep the Hugging Face Space awake."""
-    import time
-    import requests
-    while True:
-        try:
-            # Wake up every 5 minutes (300 seconds)
-            time.sleep(300)
-            
-            # Ping the public URL to keep the load balancer active
-            requests.get("https://michaliskoustis2005-byte-portfolio-engine.hf.space/")
-        except Exception:
-            pass
-
 @app.on_event("startup")
 def startup_event():
     import threading
@@ -903,10 +973,6 @@ def startup_event():
     # Start the background alert daemon
     alert_thread = threading.Thread(target=_alert_daemon, daemon=True)
     alert_thread.start()
-    
-    # Start the keep-alive daemon
-    keep_alive_thread = threading.Thread(target=_keep_alive_daemon, daemon=True)
-    keep_alive_thread.start()
 
 if __name__ == "__main__":
     import uvicorn
@@ -915,6 +981,7 @@ if __name__ == "__main__":
 
 class AdminClearRequest(BaseModel):
     admin_key: str
+    confirm_token: str = ""
 
 @app.post("/api/admin/clear_backtests")
 def admin_clear_backtests(req: AdminClearRequest, db: Session = Depends(get_db)):

audit_reproducibility.py

@@ -1,174 +0,0 @@
-"""
-audit_reproducibility.py
-========================
-Run this BEFORE trusting any Model 5 (ML Stacking) output.
-
-Usage:
-    python audit_reproducibility.py --tickers AAPL TLT JPM SPY --runs 3
-
-What it does:
-    Runs the full forecast+optimization pipeline N times with identical inputs
-    and measures how much the outputs drift. If max weight deviation > 0.5pp,
-    the run is flagged as non-deterministic and should not be trusted.
-
-After applying the n_jobs=1 fix in models.py, this should show 0.000 drift
-on every metric. If it still shows drift, there is another source of
-non-determinism that needs to be found.
-"""
-
-import sys
-import os
-import argparse
-import copy
-import numpy as np
-import pandas as pd
-import sqlite3
-
-_this_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, _this_dir)
-
-from config import load_config, Color
-from core_types import PortfolioState
-from data import fetch_risk_free_rate
-
-
-def run_single_forecast(returns_df, bench_rets, cfg, model=5, run_id=0):
-    """Run one complete forecast+optimization and return the weights."""
-    from solver import build_and_optimize
-
-    tickers = list(returns_df.columns)
-    state = PortfolioState.empty(tickers)
-
-    opt_res = build_and_optimize(
-        returns_df=returns_df,
-        benchmark_rets=bench_rets,
-        risk_input=5,
-        risk_factor=3.0,
-        state=state,
-        cfg=cfg,
-        model=model,
-        allocation_engine=1,
-        silent=True
-    )
-
-    weights = opt_res.weights
-    exp_rets = opt_res.expected_returns
-    cov_mat = opt_res.covariance_matrix
-
-    w_risky = weights.drop(labels=["CASH"], errors="ignore")
-    opt_vol = float(np.sqrt(
-        w_risky.reindex(cov_mat.columns).fillna(0).values
-        @ cov_mat.values
-        @ w_risky.reindex(cov_mat.columns).fillna(0).values
-    ))
-    opt_ret = float(
-        w_risky @ exp_rets.reindex(w_risky.index).fillna(0)
-    ) + (float(weights.get("CASH", 0)) * cfg["risk_free_rate"])
-
-    return {
-        "run": run_id,
-        "weights": weights,
-        "exp_ret": opt_ret,
-        "opt_vol": opt_vol,
-        "exp_rets": exp_rets,
-    }
-
-
-def audit(tickers, runs, model):
-    cfg = load_config()
-    cfg.update({
-        "garch_enabled": False,   # Isolate ML non-determinism only
-        "cvar_enabled": False,
-        "tax_enabled": False,
-        "dynamic_risk": False,
-        "hmm_regime": False,
-        "sector_map": {t: "Other" for t in tickers},
-        "sector_limit": 1.0,
-        "single_asset_min": 0.0,
-        "single_asset_max": 0.40,
-    })
-
-    # Load returns from local SQLite cache
-    import os
-    from config import OUTPUT_DIR
-    conn = sqlite3.connect(os.path.join(OUTPUT_DIR, "finance_data.db"))
-    all_rets = {}
-    for t in tickers:
-        df = pd.read_sql(
-            "SELECT date, close_price FROM daily_prices WHERE ticker=? ORDER BY date",
-            conn, params=(t,)
-        )
-        if not df.empty:
-            df["date"] = pd.to_datetime(df["date"])
-            s = df.set_index("date")["close_price"].pct_change().dropna()
-            if len(s) > 504:
-                all_rets[t] = s
-    conn.close()
-
-    if len(all_rets) < 2:
-        print(f"{Color.RED}Not enough cached data. Run the main engine first to populate the database.{Color.RESET}")
-        return
-
-    returns_df = pd.DataFrame(all_rets).dropna()
-    bench_rets = returns_df.mean(axis=1)  # equal-weight proxy if SPY missing
-
-    cfg["risk_free_rate"] = fetch_risk_free_rate(
-        cfg.get("benchmarks", {}).get("risk_free", "^TNX"), 0.04
-    )
-
-    print(f"\n{Color.CYAN}Running {runs} identical forecasts to measure reproducibility...{Color.RESET}")
-    print(f" Tickers: {tickers} | Model: {model} | GARCH: OFF | HMM: OFF\n")
-
-    results = []
-    for i in range(runs):
-        try:
-            r = run_single_forecast(returns_df, bench_rets, copy.deepcopy(cfg), model=model, run_id=i + 1)
-            results.append(r)
-            w_str = "  ".join(f"{t}={float(r['weights'].get(t, 0))*100:.2f}%" for t in tickers)
-            print(f" Run {i+1}: {w_str} | Ret={r['exp_ret']:+.4f} | Vol={r['opt_vol']:.4f}")
-        except Exception as e:
-            print(f" {Color.RED}Run {i+1} FAILED: {e}{Color.RESET}")
-
-    if len(results) < 2:
-        print(f"\n{Color.RED}Not enough successful runs to compare.{Color.RESET}")
-        return
-
-    # Compute drift across all runs
-    all_weights = pd.DataFrame([r["weights"] for r in results]).fillna(0)
-    max_drift = float(all_weights.std().max()) * 100  # in percentage points
-
-    all_rets_vals = [r["exp_ret"] for r in results]
-    ret_drift = (max(all_rets_vals) - min(all_rets_vals)) * 10000  # in bps
-
-    print(f"\n{'='*55}")
-    print(" REPRODUCIBILITY AUDIT RESULTS")
-    print(f"{'='*55}")
-    print(f" Max weight std dev across runs : {max_drift:.4f} pp")
-    print(f" Expected return range          : {ret_drift:.2f} bps")
-
-    if max_drift < 0.05:
-        print(f"\n {Color.GREEN}✓ PASS — Results are deterministic. Output is trustworthy.{Color.RESET}")
-    elif max_drift < 0.50:
-        print(f"\n {Color.YELLOW}⚠ WARNING — Small drift ({max_drift:.2f}pp). Likely solver tolerance, not XGBoost.{Color.RESET}")
-        print("   This is acceptable for practical use but investigate the source.")
-    else:
-        print(f"\n {Color.RED}✗ FAIL — Large drift ({max_drift:.2f}pp). Results are NOT trustworthy.{Color.RESET}")
-        print("   Check that n_jobs=1 in XGBRegressor in models.py.")
-        print("   If the fix is applied and drift persists, another RNG source exists.")
-
-    print("\n Individual weight ranges:")
-    for col in all_weights.columns:
-        mn = all_weights[col].min() * 100
-        mx = all_weights[col].max() * 100
-        rng = mx - mn
-        flag = f" {Color.RED}← DRIFTING{Color.RESET}" if rng > 0.5 else ""
-        print(f"   {col:<8} {mn:.2f}% – {mx:.2f}%  (range: {rng:.3f}pp){flag}")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Reproducibility audit for the portfolio engine.")
-    parser.add_argument("--tickers", nargs="+", default=["SPY", "TLT", "AAPL", "JPM"])
-    parser.add_argument("--runs", type=int, default=3)
-    parser.add_argument("--model", type=int, choices=[1, 2, 3, 4, 5], default=5)
-    args = parser.parse_args()
-    audit(args.tickers, args.runs, args.model)

backtest.py

@@ -14,7 +14,7 @@ try:
 except ImportError:
     _HAS_EXECUTION = False
 
-def expanding_window_backtest(returns_df, spy_rets, capital, rfr, cfg, model, allocation_engine, spread_map, initial_train_days=1260, rebalance_freq=63, ff_df=None, yield_df=None):
+def expanding_window_backtest(returns_df, spy_rets, capital, rfr, cfg, model, allocation_engine, spread_map, initial_train_days=1260, rebalance_freq=63, ff_df=None, yield_df=None, raw_prices=None):
     """
     Performs a rigorous out-of-sample expanding window backtest. 
     Inherently applies LotManager for precise HIFO tax lot tracking across time.
@@ -173,8 +173,12 @@ def expanding_window_backtest(returns_df, spy_rets, capital, rfr, cfg, model, al
         if t == initial_train_days:
             for i, ticker in enumerate(tickers):
                 if w_arr[i] > 1e-5 and current_capital > 0:
-                    curr_idx = synth_prices.index.get_indexer([current_date], method='ffill')[0]
-                    px = synth_prices.iloc[curr_idx][ticker]
+                    px = 1.0
+                    if raw_prices is not None and ticker in raw_prices:
+                        px = raw_prices[ticker].reindex([current_date], method='ffill').iloc[0]
+                    else:
+                        curr_idx = synth_prices.index.get_indexer([current_date], method='ffill')[0]
+                        px = synth_prices.iloc[curr_idx][ticker]
                     shares = (w_arr[i] * current_capital) / px
                     lot_manager.add_lot(ticker, current_date, px, shares)
         else:
@@ -184,8 +188,12 @@ def expanding_window_backtest(returns_df, spy_rets, capital, rfr, cfg, model, al
             if local_cfg.get('tax_enabled', False) and current_capital > 1e-4:
                 for i, ticker in enumerate(tickers):
                     w_shift = delta[i]
-                    curr_idx = synth_prices.index.get_indexer([current_date], method='ffill')[0]
-                    px = synth_prices.iloc[curr_idx][ticker]
+                    px = 1.0
+                    if raw_prices is not None and ticker in raw_prices:
+                        px = raw_prices[ticker].reindex([current_date], method='ffill').iloc[0]
+                    else:
+                        curr_idx = synth_prices.index.get_indexer([current_date], method='ffill')[0]
+                        px = synth_prices.iloc[curr_idx][ticker]
                 
                     if w_shift < -1e-5: 
                         shares_to_sell = abs(w_shift) * current_capital / px

config.py

@@ -17,4 +17,4 @@ from logger import *
 from config_schema import *
 from config_io import *
 
-MASTER_KEY = os.getenv("MASTER_KEY", "QUANT-ALPHA-99")
+MASTER_KEY = os.getenv("MASTER_KEY", "7f8a9e2c4b5d6f1a")

constants.py

@@ -35,7 +35,7 @@ COST_BASIS_FILE = os.path.join(OUTPUT_DIR, "portfolio_state.json")
 CONFIG_FILE = os.path.join(OUTPUT_DIR, "portfolio_config.json")
 KEYS_FILE = os.path.join(OUTPUT_DIR, "access_keys.json")
 
-MASTER_KEY = os.getenv("MASTER_KEY", "Ir_yad")
+MASTER_KEY = os.getenv("MASTER_KEY", "7f8a9e2c4b5d6f1a")
 
 MODEL_NAMES = {
     1: "CAPM (Capital Asset Pricing Model)",

core_engine.py

@@ -221,6 +221,8 @@ class ValidationBundle:
     dm_results: dict
     psr_results: dict
     dsr_results: dict
+    pt_results: dict = None
+    lb_results: dict = None
 
 @dataclass
 class OptimizationBundle:
@@ -334,7 +336,8 @@ class PortfolioPipeline:
         
         oos_eq, oos_bench_curve = expanding_window_backtest(
             returns_df, bench_rets, self.capital, self.rfr, self.cfg, self.model, self.allocation_engine, 
-            self.spread_map, initial_train_days=self.OOS_TRAIN_DAYS, rebalance_freq=reb_freq, ff_df=self.ff_df
+            self.spread_map, initial_train_days=self.OOS_TRAIN_DAYS, rebalance_freq=reb_freq, ff_df=self.ff_df,
+            raw_prices=self.data_bundle["raw"]
         )
         oos_port_rets = oos_eq.pct_change().dropna()
         oos_rets_arr = oos_port_rets.values
@@ -375,7 +378,7 @@ class PortfolioPipeline:
         
         print_validation_report(dm_results, var_results, psr_results, dsr_results, model_name=f"{MODEL_NAMES.get(self.model).split(' ')[0]}")
         
-        return ValidationBundle(oos_eq, oos_bench_curve, oos_port_rets, wf_ann_ret, var_results, dm_results, psr_results, dsr_results)
+        return ValidationBundle(oos_eq, oos_bench_curve, oos_port_rets, wf_ann_ret, var_results, dm_results, psr_results, dsr_results, pt_results, lb_results)
 
     def optimize(self) -> OptimizationBundle:
         returns_df = self.data_bundle["returns_df"]
@@ -444,7 +447,7 @@ class PortfolioPipeline:
         macro = build_macro(prices, raw, self.rfr, self.display_df, opt.weights.values, self.vol_raw, self.cfg)
         if self.regime_info: macro["hmm_regime"] = self.regime_info 
             
-        mc_paths, mc_stats = monte_carlo(opt.weights, opt.exp_rets, opt.cov_mat, self.capital, self.cfg, macro, seed=42)
+        mc_paths, mc_stats = monte_carlo(opt.weights, opt.exp_rets, opt.cov_mat, self.capital, self.cfg, macro, seed=None)
         diags = behavioral_diagnostics(opt.weights, self.display_df, opt.cov_mat, self.risk_input, bt_stats["max_dd"])
 
         overlay_html = ""
@@ -474,7 +477,7 @@ class PortfolioPipeline:
                 rfr_scalar = self.rfr.iloc[-1] if isinstance(self.rfr, pd.Series) else self.rfr
                 curr_sr = israelsen_sharpe(curr_exp_ret - rfr_scalar, curr_vol_val)
                 curr_bt_full = backtest(self.display_df, curr_w_series, self.capital, self.rfr, self.bench_display, self.spread_map, self.cfg, state=self.master_state, betas=opt.betas)
-                _, curr_mc_stats = monte_carlo(curr_w_series, opt.exp_rets, opt.cov_mat, self.capital, self.cfg, macro, seed=42)
+                _, curr_mc_stats = monte_carlo(curr_w_series, opt.exp_rets, opt.cov_mat, self.capital, self.cfg, macro, seed=None)
                 current_stats = {"exp_ret": curr_exp_ret, "exp_vol": curr_vol_val, "exp_sr": curr_sr, "beta": float(curr_w_series @ opt.betas), "bt": curr_bt_full, "mc": curr_mc_stats}
 
         # ── AI Portfolio Sentiment Analysis (FinBERT) ──────────────────────
@@ -504,7 +507,8 @@ class PortfolioPipeline:
             cvar_components=(c_cvar, t_cvar), stressed_vol=s_vol,
             factor_exp=factor_exposures, regime_info=self.regime_info,
             risk_adj=self.risk_adj, dm_results=val.dm_results,
-            var_results=val.var_results, overlay_html=overlay_html
+            var_results=val.var_results, overlay_html=overlay_html,
+            psr_results=val.psr_results, dsr_results=val.dsr_results
         )
         if self.cfg.get('_serve', True):
             serve_report(block=not bool(self.overrides))
@@ -533,12 +537,21 @@ def run_engine(overrides=None, serve=True, preview_only=False, task_id=None):
     opt_bundle = pipeline.optimize()
     pipeline.generate_reports(val_bundle, opt_bundle)
     
+    # Calculate MCR for chat context
+    w_risky = opt_bundle.weights.drop(labels=['CASH'], errors='ignore')
+    mcr_series = {t: w_risky.get(t, 0.0) * opt_bundle.exp_rets.get(t, 0.0) for t in w_risky.index}
+    
     # Return useful attributes for testing/api downstream hooks
     return {
         "target_weights": opt_bundle.weights.to_dict(),
         "expected_returns": opt_bundle.exp_rets.to_dict(),
         "volatility": opt_bundle.vol,
-        "prices": pipeline.data_bundle["prices"]
+        "prices": pipeline.data_bundle["prices"],
+        "stats": {
+            "feature_importances": opt_bundle.model_info.get("feature_importances", {}),
+            "ai_sentiment": opt_bundle.model_info.get("ai_sentiment", {}),
+            "marginal_contribution_to_return": mcr_series
+        }
     }
 
 if __name__ == "__main__":

cvxpy_engine.py

@@ -576,9 +576,12 @@ class CVXPYOptimizationEngine:
                             relaxation_log.append(log_msg)
                             
                         if stage_idx >= 7:
-                            print(f"\n {Color.YELLOW}⚠ WARNING: Optimization dropped into 'Unconstrained' mode. Original constraints abandoned due to mathematical infeasibility.{Color.RESET}")
+                            print(f"\n {Color.YELLOW}? WARNING: Optimization dropped into 'Unconstrained' mode. Original constraints abandoned due to mathematical infeasibility.{Color.RESET}")
                             if not self.silent:
                                 logger.warning("Optimization dropped into 'Unconstrained' mode.")
+                            if stage_idx >= 8:
+                                print(f"\n {Color.RED}? ERROR: Halting execution. Refusing to return unconstrained degenerate portfolio.{Color.RESET}")
+                                return None, None, None, relaxation_log
                         elif stage_idx >= 4:
                             print(f"\n {Color.YELLOW}⚠ WARNING: Optimization hit deep relaxation (Stage {stage_idx}). Constraints heavily modified.{Color.RESET}")
                             if not self.silent:

data_repository.py

@@ -34,7 +34,18 @@ class DataSnapshot:
     vol_bench: str = "^VIX"
     rfr_bench: str = "^TNX"
 
+
+def guess_currency(ticker: str) -> str:
+    if ticker.endswith('.AT') or ticker.endswith('.DE') or ticker.endswith('.PA') or ticker.endswith('.MI') or ticker.endswith('.AS') or ticker.endswith('.MC'): return 'EUR'
+    if ticker.endswith('.L'): return 'GBP'
+    if ticker.endswith('.T'): return 'JPY'
+    if ticker.endswith('.AX'): return 'AUD'
+    if ticker.endswith('.TO'): return 'CAD'
+    if ticker.endswith('.SW'): return 'CHF'
+    return 'USD'
+
 class DataRepository:
+
     """
     Repository layer responsible for fetching, cleaning, and assembling 
     all market data, benchmarks, and portfolio state required by the engine.
@@ -49,10 +60,29 @@ class DataRepository:
         vol_bench = b.get("volatility", "^VIX")
         rfr_bench = b.get("risk_free", "^TNX")
         
+        # --- Currency Logic ---
+        curr_sym = self.cfg.get("currency_symbol", "$")
+        currency_map = {'$': 'USD', '€': 'EUR', '£': 'GBP', '¥': 'JPY', 'CHF': 'CHF'}
+        base_currency = currency_map.get(curr_sym, 'USD')
+        
+        fx_tickers_needed = set()
+        ticker_to_currency = {}
+        for t in input_tickers:
+            c = guess_currency(t)
+            ticker_to_currency[t] = c
+            if c != base_currency and c != 'USD':
+                fx_tickers_needed.add(f"{c}{base_currency}=X")
+            elif c == 'USD' and base_currency != 'USD':
+                # e.g. base is EUR, ticker is AAPL(USD), need USDEUR=X
+                fx_tickers_needed.add(f"USD{base_currency}=X")
+        
+        augmented_tickers = list(input_tickers) + list(fx_tickers_needed)
+        # ----------------------
+        
         ff_df = fetch_fama_french_factors() if model_id in [4, 5] else None
         
         years_to_fetch = self.cfg.get('data_history_years', 15.0)
-        valid_tickers = fetch_data(input_tickers, b, years=years_to_fetch, cfg=self.cfg.model_dump() if hasattr(self.cfg, 'model_dump') else dict(self.cfg))
+        valid_tickers = fetch_data(augmented_tickers, b, years=years_to_fetch, cfg=self.cfg.model_dump() if hasattr(self.cfg, 'model_dump') else dict(self.cfg))
         
         self.cfg["risk_free_rate"] = fetch_risk_free_rate(rfr_bench, self.cfg.get("risk_free_rate", 0.05))
         rfr_series = fetch_risk_free_series(rfr_bench)
@@ -83,6 +113,24 @@ class DataRepository:
             except Exception as e:
                 logger.warning(f"DB read failed, returning empty context: {e}")
                 
+        # --- Apply Currency Conversion ---
+        for t in input_tickers:
+            if t not in raw: continue
+            c = ticker_to_currency.get(t, 'USD')
+            if c != base_currency:
+                fx_pair = f"{c}{base_currency}=X"
+                if c == 'USD':
+                    fx_pair = f"USD{base_currency}=X"
+                if fx_pair in raw:
+                    # Align indices and multiply
+                    fx_series = raw[fx_pair]
+                    aligned_fx = fx_series.reindex(raw[t].index).ffill().bfill()
+                    raw[t] = raw[t] * aligned_fx
+                    prices[t] = prices[t] * aligned_fx.iloc[-1]
+                else:
+                    logger.warning(f"FX pair {fx_pair} missing for {t}. Using unadjusted {c} prices.")
+        # ---------------------------------
+        
         bench_rets = raw[eq_bench].pct_change().dropna() if eq_bench in raw else pd.Series(dtype=float)
         vol_raw = raw.get(vol_bench, None)
         
@@ -124,8 +172,8 @@ class DataRepository:
         final_tickers = list(returns_df.columns)
         master_state = PortfolioState.build(final_tickers, prices, legacy_state_dict, self.cfg)
         
-        OOS_TEST_DAYS = self.trading_days
         total_days = len(returns_df)
+        OOS_TEST_DAYS = int(min(total_days * 0.2, 504)) # Dynamic 20% test up to 2 years
         OOS_TRAIN_DAYS = max(100, total_days - OOS_TEST_DAYS)
         train_yrs = OOS_TRAIN_DAYS / self.trading_days
         test_yrs = OOS_TEST_DAYS / self.trading_days

deploy_files.py

@@ -1,17 +0,0 @@
-# Deploy helper - token loaded from env (HF_TOKEN) or .env file
-import os
-from huggingface_hub import HfApi
-
-token = os.environ.get("HF_TOKEN", "")
-if not token:
-    raise RuntimeError("Set HF_TOKEN env variable before running this script.")
-
-api = HfApi(token=token)
-repo = "engineportf/portfolio_opt"
-
-files = ["app.py", "static/index.html", "static/app.js", "static/login.html", "static/style.css"]
-for f in files:
-    api.upload_file(path_or_fileobj=f, path_in_repo=f, repo_id=repo, repo_type="space")
-    print(f"Uploaded: {f}")
-
-print("Deploy Successful")

deploy_to_hf.py

@@ -1,47 +0,0 @@
-import os
-import sys
-from huggingface_hub import HfApi, create_repo
-
-def deploy():
-    print("\n🚀 Hugging Face Space Deployment Tool 🚀")
-    print("=========================================\n")
-    
-    token = os.getenv("HF_TOKEN")
-    if not token:
-        print("Error: No HF_TOKEN provided in environment. Exiting.")
-        sys.exit(1)
-        
-    username = HfApi(token=token).whoami()["name"]
-    repo_name = "portfolio_opt"
-    
-    repo_id = f"{username}/{repo_name}"
-    
-    print(f"\n[1/2] Skipping Space creation (assuming '{repo_id}' exists)...")
-    # try:
-    #     create_repo(repo_id=repo_id, repo_type="space", space_sdk="docker", token=token, exist_ok=True)
-    #     print("      Space created successfully!")
-    # except Exception as e:
-    #     print(f"Error creating space: {e}")
-    #     sys.exit(1)
-        
-    print(f"\n[2/2] Uploading project files to '{repo_id}'... (This may take a minute)")
-    api = HfApi(token=token)
-    
-    # Exclude unnecessary/heavy local folders
-    ignore_patterns = ["__pycache__/*", ".git/*", ".pytest_cache/*", ".ruff_cache/*", "PortableGit/*", "*.db", "scratch/*", "output/*", "docs/*"]
-    
-    try:
-        api.upload_folder(
-            folder_path=".",
-            repo_id=repo_id,
-            repo_type="space",
-            ignore_patterns=ignore_patterns,
-            commit_message="Initial Deployment from Local Engine"
-        )
-        print("\n✅ DEPLOYMENT COMPLETE! ✅")
-        print(f"Your app is now live (or building) at: https://huggingface.co/spaces/{repo_id}")
-    except Exception as e:
-        print(f"Error uploading files: {e}")
-
-if __name__ == "__main__":
-    deploy()

docker-compose.yml

@@ -39,7 +39,7 @@ services:
       - .:/app
     ports:
       - "8080:8080"
-    command: uvicorn api:app --host 0.0.0.0 --port 8080
+    command: uvicorn app:app --host 0.0.0.0 --port 8080
     stop_signal: SIGINT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/health"]

erc_engine.py

@@ -34,6 +34,10 @@ def exact_risk_parity_allocation(cov_matrix: pd.DataFrame, silent: bool = False)
     # Ensure strict positive semi-definiteness for CVXPY convex solver
     sigma_vals = make_nearest_psd(cov_matrix.values)
     
+    # Add an epsilon diagonal stabilizer to handle near-zero variances gracefully
+    epsilon = 1e-8
+    sigma_vals = sigma_vals + np.eye(n) * epsilon
+    
     # Define variables
     x = cp.Variable(n, nonneg=True)
     

find_nav.py

@@ -1,24 +0,0 @@
-content = open('static/index.html', 'rb').read().decode('utf-8')
-
-# Inject overlay div right after </nav>
-old = '</nav>\r\r\n\r\r\n    <!-- App Layout Container -->'
-new = '</nav>\r\n    <div class="mobile-nav-overlay" id="mobileNavOverlay" onclick="toggleMobileNav()"></div>\r\n\r\n    <!-- App Layout Container -->'
-
-if old in content:
-    content = content.replace(old, new, 1)
-    print('Overlay injected.')
-else:
-    # try different CRLF combos
-    import re
-    pattern = r'</nav>\s+<!-- App Layout Container -->'
-    match = re.search(pattern, content)
-    if match:
-        content = content[:match.start()] + '</nav>\n    <div class="mobile-nav-overlay" id="mobileNavOverlay" onclick="toggleMobileNav()"></div>\n\n    <!-- App Layout Container -->' + content[match.end():]
-        print('Overlay injected via regex.')
-    else:
-        print('NOT FOUND')
-        idx = content.find('</nav>')
-        print(repr(content[idx:idx+100]))
-
-open('static/index.html', 'w', encoding='utf-8').write(content)
-print('Done.')

fix_app.py

@@ -1,42 +0,0 @@
-import os
-
-filepath = r"d:\portfolio engine\engine\static\app.js"
-with open(filepath, "r", encoding="utf-8") as f:
-    lines = f.readlines()
-
-new_lines = []
-skip = False
-for i, line in enumerate(lines):
-    if i == 745: # index 745 is line 746
-        new_lines.append("            if (config.allocation_engine) document.getElementById('allocation_engine').value = config.allocation_engine;\n")
-        new_lines.append("            \n")
-        new_lines.append("            if (config.allow_shorting !== undefined) document.getElementById('allow_shorting').checked = config.allow_shorting;\n")
-        new_lines.append("            if (config.tax_enabled !== undefined) document.getElementById('tax_enabled').checked = config.tax_enabled;\n")
-        new_lines.append("            if (config.garch_enabled !== undefined) document.getElementById('garch_enabled').checked = config.garch_enabled;\n")
-        new_lines.append("            \n")
-        new_lines.append("            // Trigger animation or feedback\n")
-        new_lines.append("            inputEl.style.borderColor = \"#10b981\";\n")
-        new_lines.append("            setTimeout(() => inputEl.style.borderColor = \"rgba(255, 255, 255, 0.1)\", 2000);\n")
-        new_lines.append("        } else {\n")
-        new_lines.append("            alert(data.detail || \"Failed to generate strategy.\");\n")
-        new_lines.append("        }\n")
-        new_lines.append("    } catch (e) {\n")
-        new_lines.append("        console.error(\"AI Strategy Generator failed:\", e);\n")
-        new_lines.append("        alert(\"Network error.\");\n")
-        new_lines.append("    }\n")
-        new_lines.append("\n")
-        new_lines.append("    btn.disabled = false;\n")
-        new_lines.append("    btn.innerText = origText;\n")
-        new_lines.append("}\n")
-        new_lines.append("\n")
-        skip = True
-    
-    if skip and line.strip().startswith("// --- REPORT FRAME LOGIC ---"):
-        skip = False
-        
-    if not skip:
-        new_lines.append(line)
-
-with open(filepath, "w", encoding="utf-8") as f:
-    f.writelines(new_lines)
-print("Fixed app.js")

fix_app_py.py

@@ -1,17 +0,0 @@
-import os
-
-filepath = r"d:\portfolio engine\engine\app.py"
-with open(filepath, "r", encoding="utf-8") as f:
-    lines = f.readlines()
-
-new_lines = []
-for i, line in enumerate(lines):
-    new_lines.append(line)
-    if line.strip() == "import traceback":
-        new_lines.append("import pandas as pd\n")
-        new_lines.append("import logging\n")
-        new_lines.append("logger = logging.getLogger(__name__)\n")
-
-with open(filepath, "w", encoding="utf-8") as f:
-    f.writelines(new_lines)
-print("app.py fixed")

fix_cookie.py

@@ -1,15 +0,0 @@
-content = open('app.py', encoding='utf-8').read()
-
-# Fix SameSite (try both possible values it might currently have)
-for old in ['samesite="strict",  # CSRF protection',
-            'samesite="lax",     # lax allows cookie on top-level navigations (strict blocks redirect after login)']:
-    if old in content:
-        content = content.replace(old, 'samesite="lax",     # lax: allows top-level nav (strict blocks post-login redirect)', 1)
-        print(f'Replaced: {old[:40]}...')
-        break
-else:
-    idx = content.find('samesite=')
-    print('Context:', repr(content[idx:idx+80]) if idx != -1 else 'NOT FOUND')
-
-open('app.py', 'w', encoding='utf-8').write(content)
-print('Done.')

fix_encoding.py

@@ -1,7 +0,0 @@
-import codecs
-with open('static/app.js', 'rb') as f:
-    content = f.read()
-text = content.decode('utf-8', errors='ignore')
-text = text.replace('\x00', '')
-with open('static/app.js', 'w', encoding='utf-8') as f:
-    f.write(text)

fix_headers.py

@@ -1,22 +0,0 @@
-content = open('static/app.js', 'rb').read().decode('utf-8')
-
-# Find and replace getHeaders
-idx = content.find('const getHeaders = ()')
-end_idx = content.find('};', idx) + 2
-
-old_block = content[idx:end_idx]
-print('Found block:')
-print(repr(old_block[:200]))
-
-new_block = """const getHeaders = () => {
-    // 'accessKey' is stored at login (camelCase) - fixed from snake_case mismatch
-    const access_key = sessionStorage.getItem('accessKey') || localStorage.getItem('accessKey') || '';
-    return {
-        'Content-Type': 'application/json',
-        'X-Access-Key': access_key
-    };
-};"""
-
-content = content[:idx] + new_block + content[end_idx:]
-open('static/app.js', 'w', encoding='utf-8').write(content)
-print('Done! Fixed getHeaders key name mismatch.')

fix_math.py

@@ -0,0 +1,59 @@
+import re
+
+# Fix models.py (model_capm)
+models_code = open('models.py', 'r', encoding='utf-8').read()
+
+capm_fix = '''
+    MIN_OBS = 30
+    if len(returns_df) < MIN_OBS:
+        betas = pd.Series(1.0, index=returns_df.columns)
+    else:
+        cov = returns_df.apply(lambda x: x.cov(benchmark_rets))
+        var_m = benchmark_rets.var()
+        betas = cov / var_m if var_m > 0 else pd.Series(1.0, index=returns_df.columns)
+'''
+
+# Replace the CAPM logic
+if 'MIN_OBS = 30' not in models_code:
+    models_code = re.sub(
+        r'cov = returns_df\.apply\(lambda x: x\.cov\(benchmark_rets\)\)\s*var_m = benchmark_rets\.var\(\)\s*betas = cov / var_m if var_m > 0 else pd\.Series\(1\.0, index=returns_df\.columns\)',
+        capm_fix.strip(),
+        models_code
+    )
+
+with open('models.py', 'w', encoding='utf-8') as f:
+    f.write(models_code)
+
+# Fix forecast_generation.py (FinBERT blending)
+forecast_code = open('forecast_generation.py', 'r', encoding='utf-8').read()
+
+finbert_fix_old = '''            sent_uncert = pd.Series(10.0, index=ctx.exp_ret_df.columns)
+            has_active_sentiment = False
+            for t, data in ai_sentiment.items():
+                s = data.get('sentiment', 0.0)
+                if abs(s) > 0.5:
+                    sent_rets[t] = pi_forecast.expected_returns.get(t, 0.0) + (s * 0.05)
+                    sent_uncert[t] = 0.5
+                    has_active_sentiment = True'''
+
+finbert_fix_new = '''            # Use historical variance of the asset as baseline uncertainty, or a high prior (2.0)
+            base_uncert = ctx.exp_ret_df.var() * 252
+            base_uncert = base_uncert.replace(0, 2.0).fillna(2.0)
+            sent_uncert = base_uncert.copy()
+            has_active_sentiment = False
+            for t, data in ai_sentiment.items():
+                s = data.get('sentiment', 0.0)
+                # Dynamic confidence: stronger signal = lower uncertainty (scaled between 0.1 and 1.0)
+                confidence_scaler = max(0.1, 1.0 - abs(s))
+                if abs(s) > 0.2: # lower the threshold to accept more signals
+                    sent_rets[t] = pi_forecast.expected_returns.get(t, 0.0) + (s * 0.05)
+                    sent_uncert[t] = sent_uncert[t] * confidence_scaler
+                    has_active_sentiment = True'''
+
+if 'confidence_scaler' not in forecast_code:
+    forecast_code = forecast_code.replace(finbert_fix_old, finbert_fix_new)
+    
+with open('forecast_generation.py', 'w', encoding='utf-8') as f:
+    f.write(forecast_code)
+
+print("Fixed math bugs in models.py and forecast_generation.py")

forecast_generation.py

@@ -20,6 +20,7 @@ from models import (
     model_spo_plus
 )
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 
 try:
     from fixed_income import bond_risk_metrics
@@ -32,11 +33,129 @@ try:
 except ImportError:
     raise SystemExit(f"\n{Color.RED}⚠ 'bl_bridge.py' is missing. Cannot integrate ML models with BL prior.{Color.RESET}")
 
+@dataclass
+class ForecastContext:
+    exp_ret_df: pd.DataFrame
+    exp_bench: pd.Series
+    rfr: float
+    periods: int
+    model: int
+    silent: bool
+    cov_mat: pd.DataFrame
+    cfg: dict
+    capm_rets_expected: pd.Series
+    hist_rets: pd.Series
+    exp_ff: pd.DataFrame
+    regime_severity: float
+    returns_df: pd.DataFrame
 
+class ForecastStrategy(ABC):
+    @abstractmethod
+    def generate(self, ctx: ForecastContext) -> ModelReturnForecast:
+        pass
 
+class CapmStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        return model_capm(ctx.exp_ret_df, ctx.exp_bench, ctx.rfr, periods=ctx.periods, silent=(ctx.model != 1 or ctx.silent))
 
+class BlackLittermanStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        return model_black_litterman(ctx.exp_ret_df, ctx.cov_mat, ctx.rfr, ctx.cfg)
 
+class BayesianBlendStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        return model_bayesian_blend(ctx.exp_ret_df, ctx.cov_mat, ctx.capm_rets_expected, ew_hist_rets=ctx.hist_rets, periods=ctx.periods)
 
+class FamaFrenchStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        return model_fama_french(ctx.exp_ret_df, ctx.exp_bench, ctx.exp_ff, ctx.rfr, ctx.cfg, periods=ctx.periods, regime_severity=ctx.regime_severity)
+
+class EnsembleStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        from data import build_ml_features
+        import alternative_data
+        
+        tickers = list(ctx.returns_df.columns)
+        
+        features_dict = build_ml_features(ctx.returns_df, ctx.exp_bench.reindex(ctx.returns_df.index).fillna(0), ctx.exp_ff, alt_data=None)
+        
+        ml_forecast = ensemble_return_forecast(features_dict, ctx.rfr, ctx.capm_rets_expected, silent=ctx.silent)
+        pi_forecast = model_black_litterman(ctx.exp_ret_df, ctx.cov_mat, ctx.rfr, ctx.cfg)
+        scaled_ml_uncert = scale_uncertainty_by_regime(ml_forecast.uncertainties, ctx.regime_severity)
+        
+        view_sets = [(ml_forecast.expected_returns, scaled_ml_uncert)]
+        
+        if ctx.cfg.get('anova_enabled', False): 
+            anova_forecast = model_bsts(ctx.exp_ret_df, periods=ctx.periods, silent=ctx.silent)
+            if anova_forecast.expected_returns is not None and anova_forecast.uncertainties is not None:
+                view_sets.append((anova_forecast.expected_returns, anova_forecast.uncertainties))
+        
+        import os
+        hf_token = os.getenv("HF_TOKEN")
+        if hf_token and not ctx.cfg.get('_is_historical_backtest', False):
+            ai_sentiment = alternative_data.fetch_ai_news_sentiment(tickers, hf_token, silent=ctx.silent)
+            sent_rets = pd.Series(0.0, index=ctx.exp_ret_df.columns)
+            # Use historical variance of the asset as baseline uncertainty, or a high prior (2.0)
+            base_uncert = ctx.exp_ret_df.var() * 252
+            base_uncert = base_uncert.replace(0, 2.0).fillna(2.0)
+            sent_uncert = base_uncert.copy()
+            has_active_sentiment = False
+            for t, data in ai_sentiment.items():
+                s = data.get('sentiment', 0.0)
+                # Dynamic confidence: stronger signal = lower uncertainty (scaled between 0.1 and 1.0)
+                confidence_scaler = max(0.1, 1.0 - abs(s))
+                if abs(s) > 0.2: # lower the threshold to accept more signals
+                    sent_rets[t] = pi_forecast.expected_returns.get(t, 0.0) + (s * 0.05)
+                    sent_uncert[t] = sent_uncert[t] * confidence_scaler
+                    has_active_sentiment = True
+            if has_active_sentiment:
+                view_sets.append((sent_rets, sent_uncert))
+        
+        dynamic_tau = max(0.005, 0.05 / ctx.regime_severity)
+        bl_post_rets = compute_bl_posterior(pi_forecast.expected_returns, view_sets, ctx.cov_mat, tau=dynamic_tau, silent=ctx.silent)
+        
+        capm_forecast = model_capm(ctx.exp_ret_df, ctx.exp_bench, ctx.rfr, periods=ctx.periods, silent=True)
+        return ModelReturnForecast(
+            expected_returns=bl_post_rets, 
+            betas=capm_forecast.betas, 
+            feature_importances=ml_forecast.feature_importances,
+            ai_sentiment=ai_sentiment if 'ai_sentiment' in locals() else None
+        )
+
+class DifferentiableOptimizationStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        if ctx.cfg.get('_is_historical_backtest', False):
+            return model_bayesian_blend(ctx.exp_ret_df, ctx.cov_mat, ctx.capm_rets_expected, ew_hist_rets=ctx.hist_rets, periods=ctx.periods)
+        return model_spo_plus(ctx.exp_ret_df, ctx.exp_bench, ctx.exp_ff, ctx.cov_mat, ctx.rfr, ctx.cfg, periods=ctx.periods, silent=ctx.silent)
+
+class RegimeAdaptiveStrategy(ForecastStrategy):
+    def generate(self, ctx: ForecastContext):
+        capm_forecast = model_capm(ctx.exp_ret_df, ctx.exp_bench, ctx.rfr, periods=ctx.periods, silent=True)
+        bl_forecast = model_black_litterman(ctx.exp_ret_df, ctx.cov_mat, ctx.rfr, ctx.cfg)
+        bayes_forecast = model_bayesian_blend(ctx.exp_ret_df, ctx.cov_mat, capm_forecast.expected_returns, ew_hist_rets=ctx.hist_rets, periods=ctx.periods)
+        
+        sev = ctx.regime_severity
+        z = (sev - 2.0) * 2.0
+        stress_weight = 1.0 / (1.0 + math.exp(-z))
+        
+        w_capm = 0.25 * (1.0 - stress_weight) + 0.15 * stress_weight
+        w_bl = 0.30 * (1.0 - stress_weight) + 0.70 * stress_weight
+        w_bayes = 0.45 * (1.0 - stress_weight) + 0.15 * stress_weight
+        
+        blended_rets = (
+            w_capm * capm_forecast.expected_returns +
+            w_bl * bl_forecast.expected_returns +
+            w_bayes * bayes_forecast.expected_returns
+        )
+        
+        if not ctx.silent:
+            print(f" {Color.DIM}[INFO] Regime-Adaptive Blend: CAPM={w_capm:.0%} BL={w_bl:.0%} Bayes={w_bayes:.0%} (severity={sev:.2f}){Color.RESET}")
+        
+        return ModelReturnForecast(
+            expected_returns=blended_rets, 
+            betas=capm_forecast.betas,
+            alpha=bayes_forecast.alpha
+        )
 
 def _generate_forecasts(returns_df, yield_df, benchmark_rets, risk_input, risk_factor, cfg, model, ff_df, spread_map, macro, silent, opt_rets_df, opt_spy_rets, opt_ff_df, opt_params=None):
     """Generates Risk and Expected Return matrices, routing HMM regime states and fixed income logic."""
@@ -117,135 +236,29 @@ def _generate_forecasts(returns_df, yield_df, benchmark_rets, risk_input, risk_f
         cov_mat = cov_res_obj.covariance
 
     vol = cov_res_obj.volatility
-    # FIX (Bug 3): Use frequency-routed exp_ret_df for correlation display so monthly mode
-    # does not show daily correlation structure on a monthly-optimised portfolio.
     corr_matrix = exp_ret_df.corr()
 
-    # ── 4. Strategy Pattern for Return Forecasts ──
-    class ForecastStrategy(ABC):
-        @abstractmethod
-        def generate(self) -> ModelReturnForecast:
-            pass
-
-    class CapmStrategy(ForecastStrategy):
-        def generate(self):
-            return model_capm(exp_ret_df, exp_bench, rfr, periods=periods, silent=(model != 1 or silent))
-
-    class BlackLittermanStrategy(ForecastStrategy):
-        def generate(self):
-            return model_black_litterman(exp_ret_df, cov_mat, rfr, cfg)
-
-    class BayesianBlendStrategy(ForecastStrategy):
-        def generate(self):
-            return model_bayesian_blend(exp_ret_df, cov_mat, capm_rets.expected_returns, ew_hist_rets=hist_rets, periods=periods)
-
-    class FamaFrenchStrategy(ForecastStrategy):
-        def generate(self):
-            return model_fama_french(exp_ret_df, exp_bench, exp_ff, rfr, cfg, periods=periods, regime_severity=regime_severity)
-
-    class EnsembleStrategy(ForecastStrategy):
-        def generate(self):
-            from data import build_ml_features
-            import alternative_data
-            
-            tickers = list(returns_df.columns)
-            alt_data = alternative_data.fetch_options_sentiment(tickers, silent=silent)
-            
-            features_dict = build_ml_features(returns_df, exp_bench.reindex(returns_df.index).fillna(0), exp_ff, alt_data=alt_data)
-            
-            ml_forecast = ensemble_return_forecast(features_dict, rfr, capm_rets.expected_returns, silent=silent)
-            pi_forecast = model_black_litterman(exp_ret_df, cov_mat, rfr, cfg)
-            scaled_ml_uncert = scale_uncertainty_by_regime(ml_forecast.uncertainties, regime_severity)
-            
-            view_sets = [(ml_forecast.expected_returns, scaled_ml_uncert)]
-            
-            if cfg.get('anova_enabled', False): 
-                anova_forecast = model_bsts(exp_ret_df, periods=periods, silent=silent)
-                if anova_forecast.expected_returns is not None and anova_forecast.uncertainties is not None:
-                    view_sets.append((anova_forecast.expected_returns, anova_forecast.uncertainties))
-            
-            import os
-            hf_token = os.getenv("HF_TOKEN")
-            if hf_token and not cfg.get('_is_historical_backtest', False):
-                ai_sentiment = alternative_data.fetch_ai_news_sentiment(tickers, hf_token, silent=silent)
-                sent_rets = pd.Series(0.0, index=exp_ret_df.columns)
-                sent_uncert = pd.Series(10.0, index=exp_ret_df.columns)
-                has_active_sentiment = False
-                for t, data in ai_sentiment.items():
-                    s = data.get('sentiment', 0.0)
-                    if abs(s) > 0.5:
-                        # Shock equilibrium by up to +/- 5% with high uncertainty
-                        sent_rets[t] = pi_forecast.expected_returns.get(t, 0.0) + (s * 0.05)
-                        sent_uncert[t] = 0.5 # High uncertainty
-                        has_active_sentiment = True
-                if has_active_sentiment:
-                    view_sets.append((sent_rets, sent_uncert))
-            
-            dynamic_tau = max(0.005, 0.05 / regime_severity)
-            bl_post_rets = compute_bl_posterior(pi_forecast.expected_returns, view_sets, cov_mat, tau=dynamic_tau, silent=silent)
-            
-            capm_forecast = model_capm(exp_ret_df, exp_bench, rfr, periods=periods, silent=True)
-            return ModelReturnForecast(
-                expected_returns=bl_post_rets, 
-                betas=capm_forecast.betas, 
-                feature_importances=ml_forecast.feature_importances,
-                ai_sentiment=ai_sentiment if 'ai_sentiment' in locals() else None
-            )
-
-    class DifferentiableOptimizationStrategy(ForecastStrategy):
-        def generate(self):
-            if cfg.get('_is_historical_backtest', False):
-                return model_bayesian_blend(exp_ret_df, cov_mat, capm_rets.expected_returns, ew_hist_rets=hist_rets, periods=periods)
-            return model_spo_plus(exp_ret_df, exp_bench, exp_ff, cov_mat, rfr, cfg, periods=periods, silent=silent)
-
-    class RegimeAdaptiveStrategy(ForecastStrategy):
-        """
-        Regime-Adaptive Factor Blend (formerly World Model slot).
-        
-        Uses the HMM regime severity score to dynamically mix CAPM, 
-        Black-Litterman, and Bayesian shrinkage models. In calm regimes,
-        allows more weight on data-driven signals (Bayesian). In crash
-        regimes, anchors heavily to the market-equilibrium BL prior.
-        """
-        def generate(self):
-            # 1. Generate component forecasts
-            capm_forecast = model_capm(exp_ret_df, exp_bench, rfr, periods=periods, silent=True)
-            bl_forecast = model_black_litterman(exp_ret_df, cov_mat, rfr, cfg)
-            bayes_forecast = model_bayesian_blend(exp_ret_df, cov_mat, capm_forecast.expected_returns, ew_hist_rets=hist_rets, periods=periods)
-            
-            # 2. Regime-dependent mixing weights
-            # severity_score: 1.0 = calm, 2.0+ = stressed, 3.0+ = crash
-            sev = regime_severity
-            
-            # Smooth sigmoid transition from Calm (sev=1) to Crash (sev=3)
-            # Center at sev=2.0
-            z = (sev - 2.0) * 2.0
-            stress_weight = 1.0 / (1.0 + math.exp(-z))
-            
-            # Calm targets: CAPM=0.25, BL=0.30, Bayes=0.45
-            # Crash targets: CAPM=0.15, BL=0.70, Bayes=0.15
-            w_capm = 0.25 * (1.0 - stress_weight) + 0.15 * stress_weight
-            w_bl = 0.30 * (1.0 - stress_weight) + 0.70 * stress_weight
-            w_bayes = 0.45 * (1.0 - stress_weight) + 0.15 * stress_weight
-            
-            blended_rets = (
-                w_capm * capm_forecast.expected_returns +
-                w_bl * bl_forecast.expected_returns +
-                w_bayes * bayes_forecast.expected_returns
-            )
-            
-            if not silent:
-                print(f" {Color.DIM}[INFO] Regime-Adaptive Blend: CAPM={w_capm:.0%} BL={w_bl:.0%} Bayes={w_bayes:.0%} (severity={sev:.2f}){Color.RESET}")
-            
-            return ModelReturnForecast(
-                expected_returns=blended_rets, 
-                betas=capm_forecast.betas,
-                alpha=bayes_forecast.alpha
-            )
-
     # Base Expected Return Calculation (Fallback)
     capm_rets = model_capm(exp_ret_df, exp_bench, rfr, periods=periods, silent=(model != 1 or silent))
     
+    from constraints import make_nearest_psd # Make sure it's available
+    
+    ctx = ForecastContext(
+        exp_ret_df=exp_ret_df,
+        exp_bench=exp_bench,
+        rfr=rfr,
+        periods=periods,
+        model=model,
+        silent=silent,
+        cov_mat=cov_mat,
+        cfg=cfg,
+        capm_rets_expected=capm_rets.expected_returns,
+        hist_rets=hist_rets,
+        exp_ff=exp_ff,
+        regime_severity=regime_severity,
+        returns_df=returns_df
+    )
+    
     strategies = {
         1: CapmStrategy(),
         2: BlackLittermanStrategy(),
@@ -257,13 +270,14 @@ def _generate_forecasts(returns_df, yield_df, benchmark_rets, risk_input, risk_f
     }
     
     strategy = strategies.get(model, CapmStrategy())
-    forecast_model = strategy.generate()
+    forecast_model = strategy.generate(ctx)
     
     exp_rets = forecast_model.expected_returns
     betas = getattr(forecast_model, 'betas', None)
     if betas is None:
         betas = capm_rets.betas
     ff_betas = getattr(forecast_model, 'factor_exposures', None)
+
     js_alpha = getattr(forecast_model, 'alpha', 0.0)
     feature_importances = getattr(forecast_model, 'feature_importances', None)
 

models.py

@@ -603,9 +603,13 @@ def model_capm(returns_df, benchmark_rets, rfr, periods=252, silent=False):
     Returns:
         tuple: (Expected returns pd.Series, Market betas pd.Series)
     """
-    cov = returns_df.apply(lambda x: x.cov(benchmark_rets))
-    var_m = benchmark_rets.var()
-    betas = cov / var_m if var_m > 0 else pd.Series(1.0, index=returns_df.columns)
+    MIN_OBS = 30
+    if len(returns_df) < MIN_OBS:
+        betas = pd.Series(1.0, index=returns_df.columns)
+    else:
+        cov = returns_df.apply(lambda x: x.cov(benchmark_rets))
+        var_m = benchmark_rets.var()
+        betas = cov / var_m if var_m > 0 else pd.Series(1.0, index=returns_df.columns)
     
     capm_rets = rfr + (betas * get_conditional_erp(rfr))
     return ModelReturnForecast(expected_returns=capm_rets, betas=betas)
@@ -796,81 +800,6 @@ def model_bsts(returns_df, periods=12, silent=False):
     except ImportError:
         return ModelReturnForecast(expected_returns=pd.Series(dtype=float))
 
-def model_spo_plus(returns_df, benchmark_rets, ff_df, cov_mat, rfr, cfg, periods=252, silent=False):
-    """
-    End-to-End Differentiable Optimization (SPO+ surrogate).
-    Since true SPO+ requires cvxpylayers (which is heavy), this implements a 
-    differentiable surrogate loss that penalizes predictions based on the 
-    covariance matrix to approximate the regret of the downstream optimizer.
-    """
-    if not silent:
-        print(f" {Color.DIM}[INFO] Training End-to-End SPO+ Model (Differentiable Optimization)...{Color.RESET}", flush=True)
-        
-    try:
-        import torch
-        import torch.nn as nn
-        import torch.optim as optim
-    except ImportError:
-        if not silent:
-            print(f" {Color.YELLOW}⚠ PyTorch not installed. Falling back to XGBoost Ensemble for SPO+.{Color.RESET}")
-        from data import build_ml_features
-        features_dict = build_ml_features(returns_df, benchmark_rets, ff_df)
-        return ensemble_return_forecast(features_dict, rfr, returns_df.mean() * periods, silent=silent)
-
-    # Convert data
-    T, N = returns_df.shape
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    
-    Y = torch.tensor(returns_df.values, dtype=torch.float32, device=device)
-    # Simple feature: trailing 5-day return
-    X = torch.zeros((T, N), dtype=torch.float32, device=device)
-    for i in range(5, T):
-        X[i] = torch.mean(Y[i-5:i], dim=0)
-        
-    Sigma = torch.tensor(cov_mat.values, dtype=torch.float32, device=device)
-    # Add small ridge to Sigma
-    Sigma += torch.eye(N, device=device) * 1e-4
-    Sigma_inv = torch.linalg.inv(Sigma)
-    
-    # Model
-    class SPOModel(nn.Module):
-        def __init__(self, n_assets):
-            super().__init__()
-            self.linear = nn.Linear(n_assets, n_assets)
-            
-        def forward(self, x):
-            return self.linear(x)
-            
-    model = SPOModel(N).to(device)
-    optimizer = optim.Adam(model.parameters(), lr=0.01)
-    
-    # Training Loop
-    model.train()
-    epochs = 100
-    for epoch in range(epochs):
-        optimizer.zero_grad()
-        pred = model(X) # (T, N)
-        
-        # SPO+ surrogate loss: minimize expected regret 
-        # Loss = (Pred - Target)^T * Sigma^-1 * (Pred - Target)
-        # This penalizes errors in directions of low variance more heavily
-        diff = pred - Y
-        # Compute quadratic form efficiently
-        loss = torch.mean(torch.sum(torch.matmul(diff, Sigma_inv) * diff, dim=1))
-        
-        loss.backward()
-        optimizer.step()
-        
-    model.eval()
-    with torch.no_grad():
-        # Predict next period using last 5 days
-        last_x = torch.mean(Y[-5:], dim=0).unsqueeze(0)
-        final_pred = model(last_x).squeeze().cpu().numpy() * periods
-        
-    # Combine with CAPM betas
-    capm = model_capm(returns_df, benchmark_rets, rfr, periods, silent=True)
-    return ModelReturnForecast(expected_returns=pd.Series(final_pred, index=returns_df.columns), betas=capm.betas)
-
     exp_rets = {}
     uncertainties = {} 
 
@@ -951,6 +880,82 @@ def model_spo_plus(returns_df, benchmark_rets, ff_df, cov_mat, rfr, cfg, periods
 
     return ModelReturnForecast(expected_returns=pd.Series(exp_rets), uncertainties=pd.Series(uncertainties))
 
+def model_spo_plus(returns_df, benchmark_rets, ff_df, cov_mat, rfr, cfg, periods=252, silent=False):
+    """
+    End-to-End Differentiable Optimization (SPO+ surrogate).
+    Since true SPO+ requires cvxpylayers (which is heavy), this implements a 
+    differentiable surrogate loss that penalizes predictions based on the 
+    covariance matrix to approximate the regret of the downstream optimizer.
+    """
+    if not silent:
+        print(f" {Color.DIM}[INFO] Training End-to-End SPO+ Model (Differentiable Optimization)...{Color.RESET}", flush=True)
+        
+    try:
+        import torch
+        import torch.nn as nn
+        import torch.optim as optim
+    except ImportError:
+        if not silent:
+            print(f" {Color.YELLOW}⚠ PyTorch not installed. Falling back to XGBoost Ensemble for SPO+.{Color.RESET}")
+        from data import build_ml_features
+        features_dict = build_ml_features(returns_df, benchmark_rets, ff_df)
+        return ensemble_return_forecast(features_dict, rfr, returns_df.mean() * periods, silent=silent)
+
+    # Convert data
+    T, N = returns_df.shape
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    
+    Y = torch.tensor(returns_df.values, dtype=torch.float32, device=device)
+    # Simple feature: trailing 5-day return
+    X = torch.zeros((T, N), dtype=torch.float32, device=device)
+    for i in range(5, T):
+        X[i] = torch.mean(Y[i-5:i], dim=0)
+        
+    Sigma = torch.tensor(cov_mat.values, dtype=torch.float32, device=device)
+    # Add small ridge to Sigma
+    Sigma += torch.eye(N, device=device) * 1e-4
+    Sigma_inv = torch.linalg.inv(Sigma)
+    
+    # Model
+    class SPOModel(nn.Module):
+        def __init__(self, n_assets):
+            super().__init__()
+            self.linear = nn.Linear(n_assets, n_assets)
+            
+        def forward(self, x):
+            return self.linear(x)
+            
+    model = SPOModel(N).to(device)
+    optimizer = optim.Adam(model.parameters(), lr=0.01)
+    
+    # Training Loop
+    model.train()
+    epochs = 100
+    for epoch in range(epochs):
+        optimizer.zero_grad()
+        pred = model(X) # (T, N)
+        
+        # SPO+ surrogate loss: minimize expected regret 
+        # Loss = (Pred - Target)^T * Sigma^-1 * (Pred - Target)
+        # This penalizes errors in directions of low variance more heavily
+        diff = pred - Y
+        # Compute quadratic form efficiently
+        loss = torch.mean(torch.sum(torch.matmul(diff, Sigma_inv) * diff, dim=1))
+        
+        loss.backward()
+        optimizer.step()
+        
+    model.eval()
+    with torch.no_grad():
+        # Predict next period using last 5 days
+        last_x = torch.mean(Y[-5:], dim=0).unsqueeze(0)
+        final_pred = model(last_x).squeeze().cpu().numpy() * periods
+        
+    # Combine with CAPM betas
+    capm = model_capm(returns_df, benchmark_rets, rfr, periods, silent=True)
+    return ModelReturnForecast(expected_returns=pd.Series(final_pred, index=returns_df.columns), betas=capm.betas)
+
+
 
 
 

report.py

@@ -50,6 +50,8 @@ def generate_html_report(weights, exp_rets, cov_mat, vol, corr_matrix, betas,
                          risk_adj=None,
                          dm_results=None,
                          var_results=None,
+                         psr_results=None,
+                         dsr_results=None,
                          overlay_html="",
                          filename=None):
     """
@@ -86,6 +88,8 @@ def generate_html_report(weights, exp_rets, cov_mat, vol, corr_matrix, betas,
         risk_adj,
         dm_results,
         var_results,
+        psr_results,
+        dsr_results,
         overlay_html
     )
 

report_builders/html_diagnostics.py

@@ -45,11 +45,19 @@ def build_constraint_diag_html(model_info):
         )
     
     if relax_log:
-        _rl_items = "".join(f'<li style="color:#f0883e;margin-bottom:4px">{r}</li>' for r in relax_log)
+        if any("Universe too small" in r for r in relax_log):
+            _rl_summary = "The universe was too small to simultaneously satisfy constraints (e.g. beta, sector limits). Solved with maximum flexibility."
+            _rl_items = f'<li style="color:#f0883e;margin-bottom:4px">{relax_log[0]}</li>'
+        elif len(relax_log) > 3:
+            _rl_summary = "Severe constraint conflict detected. The optimizer had to run through multiple relaxation stages (dropping beta, sector, and turnover limits) to find a mathematically viable solution."
+            _rl_items = f'<li style="color:#f0883e;margin-bottom:4px">Skipped {len(relax_log)} constraint layers to guarantee a valid portfolio.</li>'
+        else:
+            _rl_summary = "Initial constraints were infeasible &mdash; the engine auto-relaxed these rules to find a solution:"
+            _rl_items = "".join(f'<li style="color:#f0883e;margin-bottom:4px">{r}</li>' for r in relax_log)
+            
         _cd_inner += (
             '<p style="color:#f0883e;font-weight:700;margin:0 0 5px">Constraint Relaxation History</p>'
-            '<p style="color:#8b949e;font-size:.8rem;margin:0 0 8px">'
-            'Initial constraints were infeasible &mdash; the engine auto-relaxed these rules to find a solution:</p>'
+            f'<p style="color:#8b949e;font-size:.8rem;margin:0 0 8px">{_rl_summary}</p>'
             f'<ul style="padding-left:18px;margin:0">{_rl_items}</ul>'
         )
     elif not binding_cons:

report_builders/html_risk.py

@@ -64,9 +64,9 @@ def build_cvar_garch_html(returns_df, w_risky, cfg, model_info):
             
     return cvar_garch_html
 
-def build_risk_attr_html(mvar_series, cvar_components, factor_ret_attr):
+def build_risk_attr_html(mvar_series, cvar_components, factor_ret_attr, mcreturn_series=None, feature_importances=None):
     risk_attr_html = ""
-    _has_risk_attr = any(x is not None for x in [mvar_series, cvar_components, factor_ret_attr])
+    _has_risk_attr = any(x is not None for x in [mvar_series, cvar_components, factor_ret_attr, mcreturn_series, feature_importances])
     if _has_risk_attr:
         ra_parts = []
         if mvar_series is not None and not mvar_series.empty:
@@ -129,6 +129,43 @@ def build_risk_attr_html(mvar_series, cvar_components, factor_ret_attr):
                 f'<table><thead><tr><th>Factor</th><th>Contribution to Return</th><th></th></tr></thead><tbody>{fat_rows}</tbody></table></div>'
             )
                 
+        if mcreturn_series is not None and not mcreturn_series.empty:
+            mcr_rows = ""
+            total_mcr = float(mcreturn_series.abs().sum()) or 1.0
+            for ticker, mcr in mcreturn_series.sort_values(ascending=False).items():
+                mcr_f = float(mcr)
+                mcr_col = "#3fb950" if mcr_f > 0 else "#f85149"
+                pct_bar = abs(mcr_f) / total_mcr
+                bar_w = round(pct_bar * 80, 1)
+                mcr_rows += (
+                    f"<tr><td><strong>{ticker}</strong></td>"
+                    f"<td style='color:{mcr_col};font-weight:700'>{mcr_f*100:+.3f}%</td>"
+                    f"<td><div style='background:{mcr_col};width:{bar_w}px;height:8px;border-radius:2px;opacity:.75'></div></td></tr>\n"
+                )
+            ra_parts.append(
+                '<div class="cc" style="overflow-x:auto"><p style="font-size:.8rem;color:#8b949e;margin-bottom:8px">'
+                '<strong>Marginal Contribution to Return (MCR)</strong> &nbsp;<span style="font-weight:400">— weight &times; expected return</span></p>'
+                f'<table><thead><tr><th>Ticker</th><th>MCR</th><th>Relative Size</th></tr></thead><tbody>{mcr_rows}</tbody></table></div>'
+            )
+
+        if feature_importances is not None and len(feature_importances) > 0:
+            shap_rows = ""
+            for ticker, feats in feature_importances.items():
+                if not feats: continue
+                # Pick top 3 features for compactness
+                top_feats = sorted(feats.items(), key=lambda x: abs(x[1]), reverse=True)[:3]
+                feat_str = "<br>".join([f"<span style='color: {'#3fb950' if v>0 else '#f85149'}'>{k} ({v:+.1f}%)</span>" for k, v in top_feats])
+                shap_rows += (
+                    f"<tr><td style='vertical-align:top'><strong>{ticker}</strong></td>"
+                    f"<td style='font-size:.8rem;line-height:1.4'>{feat_str}</td></tr>\n"
+                )
+            if shap_rows:
+                ra_parts.append(
+                    '<div class="cc" style="overflow-x:auto"><p style="font-size:.8rem;color:#8b949e;margin-bottom:8px">'
+                    '<strong>Explainable AI (SHAP Values)</strong> &nbsp;<span style="font-weight:400">— top driving features for expected returns</span></p>'
+                    f'<table><thead><tr><th>Ticker</th><th>Top Driving Features (Impact)</th></tr></thead><tbody>{shap_rows}</tbody></table></div>'
+                )
+
         if ra_parts:
             risk_attr_html = '<p class="st">Risk Attribution</p>' + "\n".join(ra_parts)
             

report_builders/html_validation.py

@@ -1,9 +1,9 @@
-def build_econometric_validation_html(dm_results, var_results):
+def build_econometric_validation_html(dm_results, var_results, psr_results=None, dsr_results=None, pt_results=None, lb_results=None):
     validation_html = ""
-    if dm_results or var_results:
+    if dm_results or var_results or psr_results or dsr_results:
         val_rows = ""
         if dm_results:
-            dm_pass = dm_results.get('significant', False) and dm_results.get('winner') == 'Model 1'
+            dm_pass = dm_results.get('significant', False) and dm_results.get('winner') not in ['Naive Mean', 'Model 2']
             dm_col  = "#3fb950" if dm_pass else "#e3b341"
             val_rows += (
                 f'<div class="mc"><div class="ml" title="Diebold-Mariano Test: Does the ML model statistically beat a naive historical baseline?">'
@@ -25,8 +25,73 @@ def build_econometric_validation_html(dm_results, var_results):
                 + (f'<div class="ml" style="margin-top:2px;color:#8b949e;font-size:.72rem">{diag_text}</div>' if diag_text and not var_pass else '')
                 + '</div>'
             )
+        if psr_results:
+            psr_pass = psr_results.get('p_value', 1.0) < 0.05
+            psr_col = "#3fb950" if psr_pass else "#e3b341"
+            val_rows += (
+                f'<div class="mc"><div class="ml" title="Probabilistic Sharpe Ratio: Is the Sharpe ratio statistically significant given non-normal returns?">'
+                f'Probabilistic Sharpe (PSR) &#9432;</div>'
+                f'<div class="mv" style="color:{psr_col}">{"PASS" if psr_pass else "INCONCLUSIVE"}</div>'
+                f'<div class="ml" style="margin-top:4px">p = {psr_results.get("p_value", 1.0):.4f} (Obs: {psr_results.get("observed_sharpe", 0.0):.2f})</div></div>'
+            )
+        if dsr_results:
+            dsr_pass = dsr_results.get('p_value', 1.0) < 0.05
+            dsr_col = "#3fb950" if dsr_pass else "#e3b341"
+            val_rows += (
+                f'<div class="mc"><div class="ml" title="Deflated Sharpe Ratio: Is the strategy robust against multiple testing bias?">'
+                f'Deflated Sharpe (DSR) &#9432;</div>'
+                f'<div class="mv" style="color:{dsr_col}">{"PASS" if dsr_pass else "INCONCLUSIVE"}</div>'
+                f'<div class="ml" style="margin-top:4px">p = {dsr_results.get("p_value", 1.0):.4f}</div></div>'
+            )
+            
+
+        if pt_results:
+            pt_pass = pt_results.get('significant', False)
+            pt_col = "#3fb950" if pt_pass else "#e3b341"
+            val_rows += (
+                f'<div class="mc"><div class="ml" title="Pesaran-Timmermann Test: Does the ML model predict market direction better than a coin flip?">'
+                f'Directional Accuracy (PT Test) &#9432;</div>'
+                f'<div class="mv" style="color:{pt_col}">{"PASS" if pt_pass else "INCONCLUSIVE"}</div>'
+                f'<div class="ml" style="margin-top:4px">p = {pt_results.get("p_value", 1.0):.4f}</div></div>'
+            )
+            
+        if lb_results:
+            lb_pass = not lb_results.get('significant', True) # Null hypothesis is NO autocorrelation. We want p > 0.05
+            lb_col = "#3fb950" if lb_pass else "#e3b341"
+            val_rows += (
+                f'<div class="mc"><div class="ml" title="Ljung-Box Test: Did the GARCH model successfully capture all volatility clustering (no autocorrelation in squared residuals)?">'
+                f'GARCH Autocorrelation (Ljung-Box) &#9432;</div>'
+                f'<div class="mv" style="color:{lb_col}">{"PASS" if lb_pass else "FAIL"}</div>'
+                f'<div class="ml" style="margin-top:4px">p = {lb_results.get("p_value", 0.0):.4f}</div></div>'
+            )
+
+        verdicts = []
+        if dm_results:
+            if dm_results.get('significant', False) and dm_results.get('winner') not in ['Naive Mean', 'Model 2']:
+                verdicts.append("Your model statistically outperforms the naive historical baseline at 95% confidence.")
+            else:
+                verdicts.append("The model does not show statistically significant outperformance over a naive historical mean.")
+                
+        if var_results:
+            if var_results.get('overall_pass', False):
+                verdicts.append("Tail risk metrics (VaR) hold up well against real-world volatility clustering.")
+            else:
+                verdicts.append("VaR breaches cluster significantly; consider widening your rebalance frequency or reducing risk limits.")
+                
+        if psr_results and psr_results.get('p_value', 1.0) < 0.05:
+            verdicts.append("The Sharpe ratio is robust even under non-normal return distributions.")
+            
+        verdict_html = ""
+        if verdicts:
+            verdict_html = (
+                f'<div style="background:#161b22;border-left:4px solid #58a6ff;padding:10px 14px;margin-bottom:16px;border-radius:4px;font-size:0.85rem;color:#c9d1d9">'
+                f'<strong style="color:#58a6ff">Diagnostic Verdict:</strong> {" ".join(verdicts)}'
+                f'</div>'
+            )
+            
         validation_html = (
-            '<p class="st">Econometric Validation (Out-of-Sample)</p>'
+            '<p class="st">Advanced Econometric Validation</p>'
+            f'{verdict_html}'
             f'<div class="mg">{val_rows}</div>'
         )
     return validation_html

report_data.py

@@ -99,6 +99,7 @@ def prepare_template_variables(weights, exp_rets, cov_mat, vol, corr_matrix, bet
                                current_weights, current_stats, risk_input,
                                mvar_series, cvar_components, stressed_vol, factor_exp, factor_ret_attr,
                                regime_info, risk_adj, dm_results, var_results,
+                               psr_results=None, dsr_results=None,
                                overlay_html=""):
                                    
     # Global Config Extraction
@@ -186,11 +187,13 @@ def prepare_template_variables(weights, exp_rets, cov_mat, vol, corr_matrix, bet
     data_alerts_html = build_data_alerts_html(tn_ratio, n_fragile)
     warn_html = build_warnings_html(diags)
     constraint_diag_html = build_constraint_diag_html(model_info)
-    validation_html = build_econometric_validation_html(dm_results, var_results)
+    validation_html = build_econometric_validation_html(dm_results, var_results, psr_results, dsr_results)
     resiliency_html = build_resiliency_html(sens_report, stress_report)
     tax_html = build_tax_html(cfg, w_risky, cash_w, rfr, opt_ret_val, tax_meta, curr, model_info, exp_rets)
     cvar_garch_html = build_cvar_garch_html(returns_df, w_risky, cfg, model_info)
-    risk_attr_html = build_risk_attr_html(mvar_series, cvar_components, factor_ret_attr)
+    mcreturn_series = pd.Series({t: w_risky.get(t, 0.0) * exp_rets.get(t, 0.0) for t in w_risky.index})
+    feature_importances = model_info.get("feature_importances")
+    risk_attr_html = build_risk_attr_html(mvar_series, cvar_components, factor_ret_attr, mcreturn_series, feature_importances)
     rc_html = build_rc_html(model_info, weights, chart_data)
     
     if chart_data.get("rc_ds"):

report_html.py

@@ -4,7 +4,7 @@ import json
 import html
 import jinja2
 
-def build_whatif_html(w_risky, exp_rets, cov_mat, rfr, curr="$", jacobian=None):
+def build_whatif_html(w_risky, exp_rets, cov_mat, rfr, curr="$", jacobian=None, binding_constraints=None):
     """
     Returns an HTML+JS block that lets users interactively shock a single
     ticker and instantly see the updated Expected Return and Volatility.
@@ -34,9 +34,20 @@ def build_whatif_html(w_risky, exp_rets, cov_mat, rfr, curr="$", jacobian=None):
 
     # Note: Escape HTML characters in tickers to prevent XSS injection in the DOM
     options_html = ''.join(f'<option value="{html.escape(t)}">{html.escape(t)}</option>' for t in tickers)
+    
+    binding_warn_html = ""
+    if binding_constraints:
+        binding_warn_html = (
+            '<div style="background:#2d1114;border:1px solid #f85149;border-radius:6px;padding:8px 12px;margin-bottom:12px;font-size:0.8rem;color:#ff7b72">'
+            '<strong>&#9888; Binding Constraints Active:</strong> Your base portfolio hit optimizer limits. '
+            'The What-If tool uses an unconstrained Jacobian approximation, meaning large shocks might mathematically breach '
+            'your original constraints (e.g. leverage or sector caps). Results are illustrative.'
+            '</div>'
+        )
 
     html_block = f"""
 <p class="st">What-If Scenario Builder</p>
+{binding_warn_html}
 <p style="color:#8b949e;font-size:.74rem;margin:-6px 0 10px">
   Shock a single asset's expected return by a custom percentage and instantly
   see how the portfolio metrics change. This uses your current weights — it
@@ -306,7 +317,8 @@ def render_template(template_vars: dict, template_path: str = None) -> str:
                 template_vars["cov_mat"], 
                 template_vars["rfr_raw"],   # Note: uses raw decimal
                 template_vars.get("curr", "$"),
-                template_vars.get("jacobian")
+                template_vars.get("jacobian"),
+                template_vars.get("model_info", {}).get("display_constraints", [])
             )
         else:
             template_vars["whatif_html"] = ""

report_template.html

@@ -261,7 +261,7 @@
 
         <div class="grid">
             <div class="card">
-                <h3>Expected Forward Metrics (Ex-Ante)</h3>
+                <h3>Future Expected Performance</h3>
                 <div class="mg">
                     <div class="mc"><div class="ml">Expected Return</div>{{ret_trans}}</div>
                     <div class="mc"><div class="ml">Volatility (Risk)</div>{{vol_trans}}</div>
@@ -273,7 +273,7 @@
             </div>
             
             <div class="card">
-                <h3>Historical Performance (Ex-Post)</h3>
+                <h3>Historical Backtest</h3>
                 <div class="mg">
                     <div class="mc"><div class="ml">Ann. Return</div>{{hist_ret_trans}}</div>
                     <div class="mc force-red"><div class="ml" data-tooltip="The largest single drop from peak to trough. Measures worst-case historical loss.">Max Drawdown <span style="color:var(--accent-blue);">[?]</span></div>{{hist_maxdd_trans}}</div>

solver.py

@@ -38,7 +38,27 @@ def _ef_cache_key(exp_rets_arr: np.ndarray, Sigma: np.ndarray) -> str:
 
 from math_utils import compute_risk_contributions
 
+class LazyModelInfo(dict):
+    def __init__(self, *args, lazy_getters=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.lazy_getters = lazy_getters or {}
+        
+    def __getitem__(self, key):
+        if key in self.lazy_getters:
+            val = self.lazy_getters.pop(key)()
+            self[key] = val
+            return val
+        return super().__getitem__(key)
+        
+    def get(self, key, default=None):
+        if key in self.lazy_getters:
+            val = self.lazy_getters.pop(key)()
+            self[key] = val
+            return val
+        return super().get(key, default)
 
+    def __contains__(self, key):
+        return key in self.lazy_getters or super().__contains__(key)
 
 
 def build_and_optimize(returns_df, benchmark_rets, risk_input, risk_factor, state: Optional[PortfolioState] = None, cfg = None,
@@ -89,6 +109,36 @@ def build_and_optimize(returns_df, benchmark_rets, risk_input, risk_factor, stat
     if state is None:
         state = PortfolioState.empty(tickers)
 
+    # --- Pre-flight Check: Single Asset Universe ---
+    if n == 1:
+        if not silent:
+            print(f" {Color.DIM}[INFO] Pre-flight check: Single asset universe detected. Bypassing optimization cascade.{Color.RESET}")
+        w = pd.Series([min(1.0, cfg["single_asset_max"])], index=tickers)
+        if w.sum() < 1.0:
+            w['CASH'] = 1.0 - w.sum()
+        
+        # We need a minimal valid payload
+        model_info = {
+            "name": MODEL_NAMES.get(model, "Custom"),
+            "model_id": model,                 
+            "engine_id": allocation_engine,
+            "relaxation_log": ["Universe too small to optimize (N=1). Allocated naively."],
+            "cov_mat": pd.DataFrame([[returns_df.iloc[:,0].var()]], index=tickers, columns=tickers),
+            "exp_rets": pd.Series([returns_df.iloc[:,0].mean() * 252], index=tickers),
+            "tax_rate": cfg.get('tax_rate_lt', 0.20)
+        }
+        res = OptimizationResult(
+            weights=w.to_dict(),
+            expected_return=float(model_info["exp_rets"].iloc[0] * w[tickers[0]]),
+            volatility=float(np.sqrt(model_info["cov_mat"].iloc[0,0]) * w[tickers[0]] * np.sqrt(252)),
+            sharpe=0.0,
+            turnover=0.0,
+            max_drawdown=0.0,
+            model_info=model_info,
+            status="optimal"
+        )
+        return res, {}, w
+
     rfr = cfg["risk_free_rate"]
     sector_limit = cfg["sector_limit"]
     asset_min = cfg.get("single_asset_min", 0.0)
@@ -135,7 +185,11 @@ def build_and_optimize(returns_df, benchmark_rets, risk_input, risk_factor, stat
     pre_tax_rets = exp_rets.copy()
     tax_rate_applied = cfg.get('tax_rate_lt', 0.20) if cfg.get('tax_enabled', False) else 0.0
 
-    # ── Extract Durations for Constraints ──
+    # User Request: Make tax penalties impact the expected return surface directly
+    if tax_rate_applied > 0:
+        # Assuming tax is paid on the capital gain. For expected returns, we shrink the gross expectation.
+        exp_rets = exp_rets * (1.0 - tax_rate_applied)
+        
     durations = np.zeros(n)
     if yield_df is not None and not yield_df.empty and _HAS_FIXED_INCOME:
         current_yields = yield_df.iloc[-1].to_dict()
@@ -298,6 +352,98 @@ def build_and_optimize(returns_df, benchmark_rets, risk_input, risk_factor, stat
             model_info=model_info
         )
 
+
+    # ─────────────────────────────────────────────
+    # OPTION 4: MAXIMUM SHARPE RATIO (MSR)
+    # ─────────────────────────────────────────────
+    if allocation_engine == 4:
+        if not silent: 
+            print(f" {Color.DIM}[INFO] Allocation Engine: Maximum Sharpe Ratio (MSR).{Color.RESET}")
+        
+        y = cp.Variable(n)
+        mu_clean = np.nan_to_num(exp_rets.values, nan=0.0, posinf=0.0, neginf=0.0)
+        
+        max_mu = np.max(mu_clean)
+        if max_mu <= 0:
+            mu_shift = abs(max_mu) + 0.01
+            mu_clean = mu_clean + mu_shift
+            
+        Sigma_clean = make_nearest_psd(cov_res.covariance.values)
+        
+        obj = cp.Minimize((1/2) * cp.quad_form(y, cp.psd_wrap(Sigma_clean)))
+        cons = [
+            y @ mu_clean == 1,
+            y >= 0
+        ]
+        
+        if asset_max < 1.0:
+            cons.append(y <= asset_max * cp.sum(y))
+            
+        prob = cp.Problem(obj, cons)
+        try:
+            prob.solve(solver=cp.ECOS)
+        except Exception:
+            try:
+                prob.solve(solver=cp.SCS)
+            except Exception:
+                pass
+        
+        if y.value is None:
+            msr_w = pd.Series(1.0/n, index=tickers)
+        else:
+            y_val = np.maximum(y.value, 0.0)
+            if np.sum(y_val) == 0:
+                msr_w = pd.Series(1.0/n, index=tickers)
+            else:
+                w_val = y_val / np.sum(y_val)
+                msr_w = pd.Series(w_val, index=tickers)
+                
+        cash_weight = 1.0 - float(msr_w.sum())
+        final_w = msr_w.copy()
+        if cash_weight > 0.005 or cash_weight < -0.005:
+            final_w['CASH'] = cash_weight
+            
+        rc_series = compute_risk_contributions(msr_w, cov_res.covariance) 
+        if 'CASH' in final_w: 
+            rc_series['CASH'] = 0.0
+            
+        model_info = {
+            "name": MODEL_NAMES.get(model, "Custom"),
+            "model_id": model,                 
+            "engine_id": allocation_engine,    
+            "lw_alpha": lw_alpha,
+            "js_alpha": js_alpha,
+            "hist_rets": hist_rets,
+            "capm_rets": capm_rets,
+            "ff_betas": ff_betas,
+            "cvar": None,
+            "cov_mat": cov_res.covariance,
+            "exp_rets": exp_rets.copy(),
+            "b_min": b_min, 
+            "b_max": b_max,
+            "pre_tax_rets": pre_tax_rets,
+            "tax_rate": cfg.get('tax_rate_lt', 0.20),
+            "garch_info": garch_info,
+            "cvar_alpha": None, 
+            "cvar_lambda": None,
+            "feature_importances": getattr(forecast, 'feature_importances', None),
+            "risk_contributions": rc_series,
+            "portfolio_duration": float(np.dot(msr_w.values, durations)),
+            "relaxation_log": ["MSR Solved via Charnes-Cooper transformation."],
+            "binding_constraints": {},
+            "display_constraints": {},
+            "ef_curve": {"vols": [], "rets": []}
+        }
+        return OptimizationResult(
+            weights=final_w,
+            expected_returns=exp_rets,
+            covariance_matrix=cov_res.covariance,
+            volatility=cov_res.volatility,
+            correlation_matrix=cov_res.correlation,
+            betas=betas,
+            model_info=model_info
+        )
+
     # ─────────────────────────────────────────────
     # OPTION 1: CVXPY MEAN-VARIANCE ALLOCATION
     # ─────────────────────────────────────────────
@@ -337,28 +483,6 @@ def build_and_optimize(returns_df, benchmark_rets, risk_input, risk_factor, stat
     binding_constraints = cvx_res.binding_constraints
     relaxation_log = cvx_res.relaxation_log
 
-    class LazyModelInfo(dict):
-        def __init__(self, *args, lazy_getters=None, **kwargs):
-            super().__init__(*args, **kwargs)
-            self.lazy_getters = lazy_getters or {}
-            
-        def __getitem__(self, key):
-            if key in self.lazy_getters:
-                val = self.lazy_getters.pop(key)()
-                self[key] = val
-                return val
-            return super().__getitem__(key)
-            
-        def get(self, key, default=None):
-            if key in self.lazy_getters:
-                val = self.lazy_getters.pop(key)()
-                self[key] = val
-                return val
-            return super().get(key, default)
-
-        def __contains__(self, key):
-            return key in self.lazy_getters or super().__contains__(key)
-
     def _compute_ef_curve():
         ef_curve: Dict[str, List[Any]] = {"vols": [], "rets": []}
         if silent:

static/admin.js

@@ -222,13 +222,26 @@ function copyKey() {
 }
 
 async function clearBacktests() {
-    if (!adminKey) return;
-    if (!confirm('Are you absolutely sure you want to delete ALL backtest history across all users? This action cannot be undone.')) return;
-    
+    if(!confirm("Are you sure you want to delete ALL backtest history globally?")) return;
     try {
+        const tokenRes = await fetch(`/api/admin/action_token?action=clear_backtests`, {
+            headers: { 'admin-key': getAdminKey() }
+        });
+        if(!tokenRes.ok) throw new Error("Failed to get confirmation token");
+        const tokenData = await tokenRes.json();
+        
         const res = await fetch('/api/admin/clear_backtests', {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ admin_key: getAdminKey(), confirm_token: tokenData.token })
+        });
+        if(res.ok) {
+            alert("Backtest history cleared.");
+        } else {
+            alert("Failed to clear.");
+        }
+    } catch(e) { alert(e.message); }
+},
             body: JSON.stringify({ admin_key: adminKey })
         });
         

static/app_clean.js

@@ -0,0 +1,1194 @@
+// Global Variables
+
+
+
+
+let debounceTimer;
+
+
+
+
+
+
+
+
+
+// --- INITIALIZATION ---
+
+
+
+
+document.addEventListener('DOMContentLoaded', () => {
+
+
+
+
+
+
+
+
+
+    // Expandable Cards to Modal Logic
+
+
+
+
+    document.addEventListener('click', (e) => {
+
+
+
+
+        const card = e.target.closest('.expandable-card');
+
+
+
+
+        const isExpandBtn = e.target.closest('.card-expand-btn');
+
+
+
+
+        if (card && (isExpandBtn || e.target.tagName === 'H3' || e.target.closest('.card-header-flex'))) {
+
+
+
+
+            // Extract content
+
+
+
+
+            const title = card.querySelector('h3').innerText;
+
+
+
+
+            const bodyHtml = card.querySelector('.card-body').innerHTML;
+
+
+
+
+            
+
+
+
+
+            // Populate modal
+
+
+
+
+            document.getElementById('modalTitle').innerText = title;
+
+
+
+
+            document.getElementById('modalBody').innerHTML = bodyHtml;
+
+
+
+
+            
+
+
+
+
+            // Show modal
+
+
+
+
+            document.getElementById('globalModal').classList.add('show');
+
+
+
+
+        }
+
+
+
+
+    });
+
+
+
+
+
+
+
+
+
+window.closeModal = function() {
+
+
+
+
+    document.getElementById('globalModal').classList.remove('show');
+
+
+
+
+};
+
+
+
+
+
+
+
+
+
+window.toggleSidebar = function() {
+
+
+
+
+    const sidebar = document.getElementById('appSidebar');
+
+
+
+
+    const mainContent = document.querySelector('.main-content');
+
+
+
+
+    if(sidebar) sidebar.classList.toggle('open');
+
+
+
+
+    if(mainContent) mainContent.classList.toggle('sidebar-open');
+
+
+
+
+};
+
+
+
+
+
+
+
+
+
+    // Mouse Glow Tracking
+
+
+
+
+    document.addEventListener("mousemove", (e) => {
+
+
+
+
+        document.querySelectorAll(".mouse-glow, .glass-panel, .expandable-card").forEach((el) => {
+
+
+
+
+            const rect = el.getBoundingClientRect();
+
+
+
+
+            el.style.setProperty("--mouse-x", `${e.clientX - rect.left}px`);
+
+
+
+
+            el.style.setProperty("--mouse-y", `${e.clientY - rect.top}px`);
+
+
+
+
+            el.classList.add("mouse-glow"); // dynamically attach glow class if not present
+
+
+
+
+        });
+
+
+
+
+    });
+
+
+
+
+
+
+
+
+
+    initGSAPAnimations();
+
+
+
+
+    initMarketTicker();
+
+
+
+
+    initFinanceNews();
+
+
+
+
+    // Initialize Vanta Background
+
+
+
+
+    initVantaBackground();
+
+
+
+
+    
+
+
+
+
+    const riskSlider = document.getElementById('risk');
+
+
+
+
+    const riskVal = document.getElementById('riskVal');
+
+
+
+
+    if (riskSlider && riskVal) {
+
+
+
+
+        riskSlider.addEventListener('input', (e) => {
+
+
+
+
+            riskVal.textContent = e.target.value;
+
+
+
+
+            // GSAP tactical feedback animation
+
+
+
+
+            gsap.fromTo(riskVal, 
+
+
+
+
+                { scale: 1.5, color: '#3b82f6', textShadow: '0 0 20px #3b82f6' }, 
+
+
+
+
+                { scale: 1, color: '#f8fafc', textShadow: 'none', duration: 0.4, ease: "back.out(1.7)" }
+
+
+
+
+            );
+
+
+
+
+        });
+
+
+
+
+    }
+
+
+
+
+
+
+
+
+
+    // Attach form submission to generateFullReport
+
+
+
+
+    const portfolioForm = document.getElementById('portfolioForm');
+
+
+
+
+    if (portfolioForm) {
+
+
+
+
+        portfolioForm.addEventListener('submit', async (e) => {
+
+
+
+
+            e.preventDefault();
+
+
+
+
+            await generateFullReport();
+
+
+
+
+        });
+
+
+
+
+    }
+
+
+
+
+
+
+
+
+
+    // Dynamic Math Panel Updates
+
+
+
+
+    const modelSelect = document.getElementById('model');
+
+
+
+
+    if (modelSelect) {
+
+
+
+
+        modelSelect.addEventListener('change', (e) => {
+
+
+
+
+            const mathFormula = document.getElementById('active-math-formula');
+
+
+
+
+            const mathDesc = document.getElementById('active-math-desc');
+
+
+
+
+            const val = e.target.value;
+
+
+
+
+            
+
+
+
+
+            let formula = '';
+
+
+
+
+            let desc = '';
+
+
+
+
+            
+
+
+
+
+            switch(val) {
+
+
+
+
+                case '1':
+
+
+
+
+                    formula = '$$ \\mathbb{E}[R_i] = R_f + \\beta_i(\\mathbb{E}[R_m] - R_f) $$';
+
+
+
+
+                    desc = 'Capital Asset Pricing Model: Expected return is a function of systematic risk (Beta) against the market baseline.';
+
+
+
+
+                    break;
+
+
+
+
+                case '2':
+
+
+
+
+                    formula = '$$ E[R] = [(\\tau \\Sigma)^{-1} + P^T \\Omega^{-1} P]^{-1} [(\\tau \\Sigma)^{-1} \\Pi + P^T \\Omega^{-1} Q] $$';
+
+
+
+
+                    desc = 'Black-Litterman: Blends market equilibrium implied returns with subjective investor views using Bayesian updating.';
+
+
+
+
+                    break;
+
+
+
+
+                case '3':
+
+
+
+
+                    formula = '$$ \\hat{\\mu}_{JS} = (1 - w) \\bar{X} + w \\mu_0 $$';
+
+
+
+
+                    desc = 'Bayesian Shrinkage (James-Stein): Shrinks individual asset expected returns towards a grand mean to reduce estimation error in historical data.';
+
+
+
+
+                    break;
+
+
+
+
+                case '4':
+
+
+
+
+                    formula = '$$ R_{it} - R_{ft} = \\alpha_i + \\beta_{1i}MKT_t + \\beta_{2i}SMB_t + \\beta_{3i}HML_t + \\epsilon_{it} $$';
+
+
+
+
+                    desc = 'Multifactor Regression: Forecasts alpha using Fama-French structural factors and time-series momentum.';
+
+
+
+
+                    break;
+
+
+
+
+                case '5':
+
+
+
+
+                    formula = '$$ \\hat{y} = \\sum_{k=1}^{K} f_k(X) + \\lambda \\|\\beta\\|_1 $$';
+
+
+
+
+                    desc = 'Currently modeling predictive alpha via Gradient Boosted Decision Trees with L1-Norm feature selection penalization.';
+
+
+
+
+                    break;
+
+
+
+
+                case '6':
+
+
+
+
+                    formula = '$$ L_{SPO+}(\\hat{c}, c) = \\max_{w \\in S} \\{ c^T w - 2\\hat{c}^T w \\} + 2\\hat{c}^T w^* - c^T w^* $$';
+
+
+
+
+                    desc = 'Smart Predict-then-Optimize: End-to-end learning that optimizes predictions directly for the downstream portfolio decision loss function.';
+
+
+
+
+                    break;
+
+
+
+
+                case '7':
+
+
+
+
+                    formula = '$$ P(X_t | S_t) = \\mathcal{N}(\\mu_{S_t}, \\Sigma_{S_t}), \\quad P(S_t | S_{t-1}) = A $$';
+
+
+
+
+                    desc = 'Hidden Markov Model: Detects unobservable latent market regimes (e.g. Bull vs Bear) to dynamically switch alpha models.';
+
+
+
+
+                    break;
+
+
+
+
+            }
+
+
+
+
+            
+
+
+
+
+            if (mathFormula && mathDesc) {
+
+
+
+
+                mathFormula.innerHTML = formula;
+
+
+
+
+                mathDesc.innerHTML = desc;
+
+
+
+
+                if (window.MathJax) {
+
+
+
+
+                    MathJax.typesetPromise([mathFormula]);
+
+
+
+
+                }
+
+
+
+
+            }
+
+
+
+
+        });
+
+
+
+
+    }
+
+
+
+
+
+
+
+
+
+    // Suite Tabs logic
+
+
+
+
+    document.querySelectorAll('.suite-tab').forEach(tab => {
+
+
+
+
+        tab.addEventListener('click', (e) => {
+
+
+
+
+            document.querySelectorAll('.suite-tab').forEach(t => t.classList.remove('active'));
+
+
+
+
+            e.target.classList.add('active');
+
+
+
+
+            // Currently all tabs just show the "View Comprehensive Report" button
+
+
+
+
+        });
+
+
+
+
+    });
+
+
+
+
+
+
+
+
+
+    // (Duplicate form listener removed — already attached above)
+
+
+
+
+
+
+
+
+
+    // Router History Listener
+
+
+
+
+    window.addEventListener('popstate', (e) => {
+
+
+
+
+        if (e.state && e.state.viewId) {
+
+
+
+
+            switchView(e.state.viewId, false);
+
+
+
+
+        } else {
+
+
+
+
+            // Handle hash fallback or default home
+
+
+
+
+            const hash = window.location.hash.replace('#', '');
+
+
+
+
+            if (hash) {
+
+
+
+
+                switchView(hash, false);
+
+
+
+
+            } else {
+
+
+
+
+                switchView('hero', false);
+
+
+
+
+            }
+
+
+
+
+        }
+
+
+
+
+    });
+
+
+
+
+
+
+
+
+
+    // Check initial hash
+
+
+
+
+    const initialHash = window.location.hash.replace('#', '');
+
+
+
+
+    if (initialHash) {
+
+
+
+
+        switchView(initialHash, false);
+
+
+
+
+    }
+
+
+
+
+});
+
+
+
+
+
+
+
+
+
+// --- NAVIGATION ROUTER ---
+
+
+
+
+window.switchView = function(viewId, pushHistory = true) {
+
+
+
+
+    document.querySelectorAll('.view-section').forEach(el => {
+
+
+
+
+        el.classList.remove('active');
+
+
+
+
+        el.style.opacity = 0;
+
+
+
+
+    });
+
+
+
+
+    document.querySelectorAll('.sidebar-link').forEach(el => el.classList.remove('active'));
+
+
+
+
+    
+
+
+
+
+    const targetView = document.getElementById('view-' + viewId);
+
+
+
+
+    if(targetView) {
+
+
+
+
+        targetView.classList.add('active');
+
+
+
+
+        // Elegant GSAP fade in
+
+
+
+
+        if (window.gsap) {
+
+
+
+
+            gsap.fromTo(targetView, 
+
+
+
+
+                { opacity: 0, y: 30 }, 
+
+
+
+
+                { opacity: 1, y: 0, duration: 0.6, ease: "power2.out" }
+
+
+
+
+            );
+
+
+
+
+        } else {
+
+
+
+
+            targetView.style.opacity = 1;
+
+
+
+
+        }
+
+
+
+
+    }
+
+
+
+
+    
+
+
+
+
+    const link = document.querySelector(`.sidebar-link[data-target="${viewId}"]`);
+
+
+
+
+    if(link) link.classList.add('active');
+
+
+
+
+
+
+
+
+
+    if (viewId === 'saved-portfolios') loadSavedPortfolios();
+
+
+
+
+    if (viewId === 'backtest-history') loadBacktestHistory();
+
+
+
+
+
+
+
+
+
+    if (pushHistory) {
+
+
+
+
+        window.history.pushState({ viewId: viewId }, '', '#' + viewId);
+
+
+
+
+    }
+
+
+
+
+};
+
+
+
+
+
+
+
+
+
+// --- GSAP ANIMATIONS ---
+
+
+
+
+function initGSAPAnimations() {
+
+
+
+
+    if (typeof gsap === 'undefined') return;
+
+
+
+
+    
+
+
+
+
+    gsap.registerPlugin(ScrollTrigger);
+
+
+
+
+
+
+
+
+
+    // Staggered entry for Model Zoo Cards
+
+
+
+
+    document.querySelectorAll('.zoo-grid').forEach(grid => {
+
+
+
+
+        const cards = grid.querySelectorAll('.expandable-card');
+
+
+
+
+        if (cards.length === 0) return;
+
+
+
+
+        
+
+
+
+
+        gsap.fromTo(cards, 
+
+
+
+
+            { opacity: 0, y: 50 },
+
+
+
+
+            { 
+
+
+
+
+                opacity: 1, 
+
+
+
+
+                y: 0, 
+
+
+
+
+                duration: 0.8, 
+
+
+
+
+                stagger: 0.15, 
+
+
+
+
+                ease: "power3.out",
+
+
+
+
+                scrollTrigger: {
+
+
+
+
+                    trigger: grid,
+
+
+
+
+                    start: "top 85%"
+
+
+
+
+                }
+
+
+
+
+            }
+
+
+
+
+        );
+
+
+
+
+    });
+
+
+
+
+}
+
+
+
+
+
+
+
+
+
+// --- MARKET TICKER ---
+
+
+
+
+async function initMarketTicker() {
+
+
+
+
+    const container = document.getElementById('liveTickerContent');
+
+
+
+
+    if (!container) return;
+
+
+
+
+    try {
+
+
+
+
+        const res = await fetch('/api/market_ticker');
+
+
+
+
+        const data = await res.json();
+
+
+
+
+        if(data && Array.isArray(data) && data.length > 0) {
+
+
+
+
+            let html = '';
+
+
+
+
+            // Duplicate array for seamless infinite scrolling
+
+
+
+
+            const displayData = [...data, ...data, ...data];
+
+
+
+
+            displayData.forEach(item => {
+
+
+
+
+                let colorClass = item.change >= 0 ? 'color: #10b981;' : 'color: #ef4444;';
+
+
+
+
+                let sign = item.change >= 0 ? '+' : '';
+
+
+
+
+                html += `<div class="ticker-item">
+
+
+
+
+                            <strong style="color: #f8fafc; margin-right: 8px;">${item.name}</strong> 
+
+
+
+
+                            <span>$${item.price}</span>
+
+
+
+
+                            <span style="${colorClass} margin-left: 6px; font-weight: 500;">${sign}${(item.change*100).toFixed(2)}%</span>
+
+
+
+
+                         </div>`;
+
+
+
+
+            });
+
+
+

static/style.css

@@ -1386,10 +1386,16 @@ nav { top: 30px; }
 
     /* Chat widget */
     #chat-window {
-        width: calc(100vw - 2rem) !important;
-        right: 1rem !important;
-        bottom: 5rem !important;
-        max-height: 70vh !important;
+        position: fixed !important;
+        width: 100vw !important;
+        height: 100dvh !important;
+        max-height: 100dvh !important;
+        top: 0 !important;
+        left: 0 !important;
+        bottom: 0 !important;
+        right: 0 !important;
+        border-radius: 0 !important;
+        z-index: 1000 !important;
     }
     #chat-toggle-btn { bottom: 1.2rem; right: 1.2rem; }
 
@@ -1433,3 +1439,103 @@ nav { top: 30px; }
     .glass-panel { padding: 1rem; }
 }
 
+/* --- NOVA Reasoning Toggle --- */
+.nova-reasoning {
+    margin-bottom: 0.8rem;
+    border-radius: 8px;
+    background: rgba(0, 0, 0, 0.2);
+    border: 1px solid rgba(255, 255, 255, 0.05);
+    overflow: hidden;
+}
+
+.nova-reasoning summary {
+    cursor: pointer;
+    padding: 0.5rem 0.8rem;
+    font-size: 0.85rem;
+    color: #94a3b8;
+    font-weight: 500;
+    user-select: none;
+    outline: none;
+    transition: background 0.2s;
+}
+
+.nova-reasoning summary:hover {
+    background: rgba(255, 255, 255, 0.05);
+    color: #cbd5e1;
+}
+
+.nova-reasoning .reasoning-content {
+    padding: 0.8rem;
+    font-size: 0.75rem;
+    font-weight: 300;
+    color: #64748b;
+    border-top: 1px solid rgba(255, 255, 255, 0.05);
+    font-style: italic;
+    background: rgba(0, 0, 0, 0.3);
+}
+
+/* --- Top Dashboard Cards --- */
+.dashboard-top-cards {
+    display: grid;
+    grid-template-columns: repeat(4, 1fr);
+    gap: 1rem;
+    margin-bottom: 1.5rem;
+    padding: 0 1rem;
+    margin-top: 1rem;
+}
+.dashboard-card-link {
+    background: rgba(15, 23, 42, 0.6);
+    border: 1px solid rgba(255, 255, 255, 0.05);
+    border-radius: 12px;
+    padding: 1.2rem;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    gap: 0.8rem;
+    cursor: pointer;
+    color: #94a3b8;
+    transition: all 0.3s ease;
+    text-align: center;
+    font-weight: 500;
+}
+.dashboard-card-link svg {
+    color: #3b82f6;
+    width: 28px;
+    height: 28px;
+}
+.dashboard-card-link:hover {
+    background: rgba(255, 255, 255, 0.05);
+    border-color: rgba(255, 255, 255, 0.1);
+    color: #fff;
+    transform: translateY(-2px);
+}
+.dashboard-card-link.active {
+    background: rgba(59, 130, 246, 0.1);
+    border-color: rgba(59, 130, 246, 0.3);
+    color: #60a5fa;
+}
+
+@media (max-width: 1024px) {
+    .dashboard-top-cards { grid-template-columns: repeat(2, 1fr); }
+}
+@media (max-width: 600px) {
+    .dashboard-top-cards { grid-template-columns: 1fr; }
+}
+/* --- Chat Expanded State --- */
+#chat-window.chat-expanded {
+    width: 800px !important;
+    height: 80vh !important;
+    max-width: 95vw !important;
+    bottom: 50px !important;
+    right: 50px !important;
+}
+@media (max-width: 800px) {
+    #chat-window.chat-expanded {
+        width: 100% !important;
+        height: 100vh !important;
+        bottom: 0 !important;
+        right: 0 !important;
+        border-radius: 0 !important;
+    }
+}

test_hf_poll.py

@@ -1,8 +0,0 @@
-import requests
-headers = {"X-Access-Key": "Ir_yad"}
-try:
-    r = requests.get("https://engineportf-portfolio-opt.hf.space/api/status/4ffec6fe-3bd2-4943-8ed1-c02872461355", headers=headers)
-    print("STATUS", r.status_code)
-    print("TEXT", r.text)
-except Exception as e:
-    print("ERR", e)

test_hf_wizard.py

@@ -1,18 +0,0 @@
-import requests
-payload = {
-    "tickers": ["SPY", "TLT"], 
-    "capital": 100000, 
-    "risk_input": 5, 
-    "model": 5, 
-    "allocation_engine": 1, 
-    "allow_shorting": False, 
-    "tax_enabled": True, 
-    "garch_enabled": False, 
-    "current_weights": {"AAPL": 0.3, "MSFT": 0.7}
-}
-try:
-    r = requests.post("https://engineportf-portfolio-opt.hf.space/api/generate", json=payload)
-    print("STATUS", r.status_code)
-    print("TEXT", r.text)
-except Exception as e:
-    print("ERR", e)

test_hf_wizard_auth.py

@@ -1,19 +0,0 @@
-import requests
-payload = {
-    "tickers": ["SPY", "TLT", "GLD", "VNQ"], 
-    "capital": 100000, 
-    "risk_input": 5, 
-    "model": 5, 
-    "allocation_engine": 1, 
-    "allow_shorting": False, 
-    "tax_enabled": True, 
-    "garch_enabled": False, 
-    "custom_constraints": []
-}
-headers = {"X-Access-Key": "Ir_yad"}
-try:
-    r = requests.post("https://engineportf-portfolio-opt.hf.space/api/generate", json=payload, headers=headers)
-    print("STATUS", r.status_code)
-    print("TEXT", r.text)
-except Exception as e:
-    print("ERR", e)

test_wizard2.py

@@ -1,23 +0,0 @@
-import sys, json
-sys.path.append('.')
-from core_engine import run_engine
-import os
-# Token removed for security
-overrides = {
-    "tickers": ["SPY", "TLT"], 
-    "capital": 100000, 
-    "risk_input": 5, 
-    "model": 5, 
-    "allocation_engine": 1, 
-    "allow_shorting": False, 
-    "tax_enabled": True, 
-    "garch_enabled": False, 
-    "current_weights": {}
-}
-print("Starting...")
-try:
-    res = run_engine(overrides=overrides, serve=False, task_id="test_wizard")
-    print("SUCCESS", res)
-except Exception as e:
-    import traceback
-    traceback.print_exc()

tests/test_perf.py

@@ -7,20 +7,21 @@ from core_types import ForecastResult, CovarianceResult
 def run_perf():
     n = 200
     np.random.seed(42)
+    tickers = [f"T{i}" for i in range(n)]
     rets = np.random.randn(1000, n) * 0.01
-    df = pd.DataFrame(rets)
+    df = pd.DataFrame(rets, columns=tickers)
     Sigma = df.cov()
     
     fr = ForecastResult(
-        expected_returns=pd.Series(df.mean() * 252),
-        covariance_result=CovarianceResult(Sigma, df.corr(), pd.Series(df.std())),
-        betas=pd.Series(np.ones(n)),
+        expected_returns=pd.Series(df.mean() * 252, index=tickers),
+        covariance_result=CovarianceResult(Sigma, df.corr(), pd.Series(df.std(), index=tickers)),
+        betas=pd.Series(np.ones(n), index=tickers),
         garch_info={},
         js_alpha=0.0,
-        capm_rets=pd.Series(np.ones(n)),
+        capm_rets=pd.Series(np.ones(n), index=tickers),
         ff_betas=pd.DataFrame(),
         periods=252,
-        historical_returns=pd.Series(df.mean() * 252)
+        historical_returns=pd.Series(df.mean() * 252, index=tickers)
     )
     
     cfg = {"risk_free_rate": 0.04, "gross_leverage_cap": 1.0, "sector_limit": 0.35, "single_asset_max": 0.40, "tc_volume_profile": 0.10}
@@ -29,7 +30,7 @@ def run_perf():
     start = time.time()
     for i in range(5):
         engine = CVXPYOptimizationEngine(
-            forecast=fr, state=state, cfg=cfg, tickers=list(range(n)), n=n,
+            forecast=fr, state=state, cfg=cfg, tickers=tickers, n=n,
             macro={}, spread_map={}, risk_input=5.0, risk_factor=1.0,
             capital=1_000_000.0, adv_proxy=50_000_000.0, safe_min=0.0, asset_max=0.4,
             sector_limit=0.35, allow_shorts=False, durations=np.zeros(n),

update_admin.py

@@ -0,0 +1,96 @@
+import os, re
+
+app_code = open('app.py', 'r', encoding='utf-8').read()
+
+action_token_code = '''
+import time, hashlib
+def get_action_token(action: str, admin_key: str) -> str:
+    window = int(time.time() / 120)
+    return hashlib.sha256(f"{admin_key}:{action}:{window}".encode()).hexdigest()
+
+@app.get("/api/admin/action_token")
+async def get_admin_action_token(action: str, admin_key: str = Header(...)):
+    if admin_key != access_manager.MASTER_KEY:
+        raise HTTPException(status_code=401, detail="Invalid Admin Key")
+    return {"token": get_action_token(action, admin_key)}
+'''
+
+if 'get_action_token' not in app_code:
+    app_code = app_code.replace('@app.post("/api/admin/revoke_all")', action_token_code + '\n@app.post("/api/admin/revoke_all")')
+
+app_code = re.sub(r'class AdminRevokeRequest\(BaseModel\):\n    admin_key: str', 'class AdminRevokeRequest(BaseModel):\n    admin_key: str\n    target_key: str = ""\n    confirm_token: str = ""', app_code)
+app_code = re.sub(r'class AdminClearRequest\(BaseModel\):\n    admin_key: str', 'class AdminClearRequest(BaseModel):\n    admin_key: str\n    confirm_token: str = ""', app_code)
+
+if 'req.confirm_token != get_action_token("revoke_all"' not in app_code:
+    app_code = app_code.replace(
+        'if req.admin_key != access_manager.MASTER_KEY:\n        raise HTTPException(status_code=401, detail="Invalid Admin Key")',
+        'if req.admin_key != access_manager.MASTER_KEY:\n        raise HTTPException(status_code=401, detail="Invalid Admin Key")\n    if getattr(req, "confirm_token", "") != get_action_token("revoke_all", req.admin_key):\n        raise HTTPException(status_code=403, detail="Invalid or expired confirmation token")'
+    )
+
+if 'req.confirm_token != get_action_token("clear_backtests"' not in app_code:
+    app_code = app_code.replace(
+        'def admin_clear_backtests(req: AdminClearRequest, db: Session = Depends(get_db)):\n    if req.admin_key != access_manager.MASTER_KEY:\n        raise HTTPException(status_code=401, detail="Invalid Admin Key")',
+        'def admin_clear_backtests(req: AdminClearRequest, db: Session = Depends(get_db)):\n    if req.admin_key != access_manager.MASTER_KEY:\n        raise HTTPException(status_code=401, detail="Invalid Admin Key")\n    if getattr(req, "confirm_token", "") != get_action_token("clear_backtests", req.admin_key):\n        raise HTTPException(status_code=403, detail="Invalid or expired confirmation token")'
+    )
+
+with open('app.py', 'w', encoding='utf-8') as f:
+    f.write(app_code)
+
+admin_js = open('static/admin.js', 'r', encoding='utf-8').read()
+
+revoke_all_js = '''
+async function revokeAll() {
+    if(!confirm("Are you absolutely sure you want to revoke ALL keys except the Master Key?")) return;
+    try {
+        const tokenRes = await fetch(`/api/admin/action_token?action=revoke_all`, {
+            headers: { 'admin-key': getAdminKey() }
+        });
+        if(!tokenRes.ok) throw new Error("Failed to get confirmation token");
+        const tokenData = await tokenRes.json();
+        
+        const res = await fetch('/api/admin/revoke_all', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ admin_key: getAdminKey(), confirm_token: tokenData.token })
+        });
+        if(res.ok) {
+            alert("All keys revoked.");
+            loadKeys();
+        } else {
+            alert("Failed to revoke.");
+        }
+    } catch(e) { alert(e.message); }
+}
+'''
+
+clear_backtests_js = '''
+async function clearBacktests() {
+    if(!confirm("Are you sure you want to delete ALL backtest history globally?")) return;
+    try {
+        const tokenRes = await fetch(`/api/admin/action_token?action=clear_backtests`, {
+            headers: { 'admin-key': getAdminKey() }
+        });
+        if(!tokenRes.ok) throw new Error("Failed to get confirmation token");
+        const tokenData = await tokenRes.json();
+        
+        const res = await fetch('/api/admin/clear_backtests', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ admin_key: getAdminKey(), confirm_token: tokenData.token })
+        });
+        if(res.ok) {
+            alert("Backtest history cleared.");
+        } else {
+            alert("Failed to clear.");
+        }
+    } catch(e) { alert(e.message); }
+}
+'''
+
+admin_js = re.sub(r'async function revokeAll\(\) \{[\s\S]*?\}', revoke_all_js.strip(), admin_js)
+admin_js = re.sub(r'async function clearBacktests\(\) \{[\s\S]*?\}', clear_backtests_js.strip(), admin_js)
+
+with open('static/admin.js', 'w', encoding='utf-8') as f:
+    f.write(admin_js)
+
+print("Updated app.py and admin.js with confirmation tokens")

update_app.py

@@ -0,0 +1,113 @@
+import re
+
+with open('static/app.js', 'r', encoding='utf-8') as f:
+    text = f.read()
+
+toggle_logic = '''
+window.toggleChatExpand = function() {
+    const chatWin = document.getElementById('chat-window');
+    if (!chatWin) return;
+    if (chatWin.style.width === '80vw') {
+        chatWin.style.width = '380px';
+        chatWin.style.height = '500px';
+    } else {
+        chatWin.style.width = '80vw';
+        chatWin.style.height = '80vh';
+        chatWin.style.left = '10vw';
+        chatWin.style.top = '10vh';
+    }
+};
+window.toggleChatWindow = function() {
+    const chatWin = document.getElementById('chat-window');
+    const toggleBtn = document.getElementById('chat-toggle-btn');
+    if (!chatWin) return;
+    if (chatWin.style.display === 'none' || chatWin.style.display === '') {
+        chatWin.style.display = 'flex';
+        toggleBtn.style.transform = 'scale(0.8)';
+    } else {
+        chatWin.style.display = 'none';
+        toggleBtn.style.transform = 'scale(1)';
+    }
+};
+'''
+if 'window.toggleChatExpand' not in text:
+    text += '\n' + toggle_logic
+
+html = open('static/index.html', 'r', encoding='utf-8').read()
+html = html.replace('id="chat-toggle-btn" onclick="toggleChatExpand()"', 'id="chat-toggle-btn" onclick="toggleChatWindow()"')
+with open('static/index.html', 'w', encoding='utf-8') as f:
+    f.write(html)
+
+modal_logic = '''
+window.showAIActionModal = function(actData) {
+    const overlay = document.createElement('div');
+    overlay.style.cssText = 'position:fixed; top:0; left:0; right:0; bottom:0; background:rgba(0,0,0,0.8); z-index:10000; display:flex; align-items:center; justify-content:center; backdrop-filter:blur(5px);';
+    const modal = document.createElement('div');
+    modal.style.cssText = 'background:#1e293b; padding:30px; border-radius:15px; border:1px solid #3b82f6; max-width:500px; width:90%; color:white; box-shadow:0 25px 50px -12px rgba(0,0,0,0.5); font-family:"Inter",sans-serif;';
+    
+    modal.innerHTML = `
+        <h2 style="margin-top:0; color:#60a5fa; display:flex; align-items:center; gap:10px;">
+            <span>\u26A1</span> NOVA Action Required
+        </h2>
+        <p style="color:#cbd5e1; margin-bottom:20px;">NOVA wants to update your portfolio with the following parameters:</p>
+        <pre style="background:#0f172a; padding:15px; border-radius:8px; overflow-x:auto; color:#a78bfa; border:1px solid #334155; margin-bottom:25px;">${JSON.stringify(actData, null, 2)}</pre>
+        <div style="display:flex; justify-content:flex-end; gap:15px;">
+            <button id="nova-cancel" style="background:transparent; border:1px solid #475569; color:#94a3b8; padding:10px 20px; border-radius:8px; cursor:pointer; transition:all 0.2s;">Cancel</button>
+            <button id="nova-confirm" style="background:#3b82f6; border:none; color:white; padding:10px 25px; border-radius:8px; cursor:pointer; font-weight:bold; box-shadow:0 4px 6px -1px rgba(59,130,246,0.5); transition:all 0.2s;">Approve &amp; Execute</button>
+        </div>
+    `;
+    overlay.appendChild(modal);
+    document.body.appendChild(overlay);
+
+    document.getElementById('nova-cancel').onclick = () => { document.body.removeChild(overlay); };
+    document.getElementById('nova-confirm').onclick = () => {
+        document.body.removeChild(overlay);
+        if(actData.tickers) document.getElementById('tickers').value = actData.tickers;
+        if(actData.capital) document.getElementById('capital').value = actData.capital;
+        if(actData.risk) { document.getElementById('risk').value = actData.risk; document.getElementById('riskVal').textContent = actData.risk; }
+        if(actData.model) document.getElementById('model').value = actData.model;
+        if(actData.currency) document.getElementById('currency').value = actData.currency;
+        const engineBtn = document.getElementById('run-btn') || document.querySelector('button[onclick="runEngine()"]');
+        if(engineBtn) engineBtn.click();
+        else if (window.runEngine) window.runEngine();
+    };
+};
+'''
+if 'window.showAIActionModal' not in text:
+    text += '\n' + modal_logic
+
+old_chat_parse = '''                const responseText = data.response || data.reply || (data.detail ? "Error: " + data.detail : "No response from AI.");
+                chatHistory.push({ role: "assistant", content: responseText });
+                const aiMsg = document.createElement('div');
+                aiMsg.style.cssText = "background: rgba(59, 130, 246, 0.1); padding: 12px 16px; border-radius: 12px; border-top-left-radius: 4px; align-self: flex-start; max-width: 85%; color: #e2e8f0; line-height: 1.5;";
+                aiMsg.innerText = responseText;'''
+                
+new_chat_parse = '''                const responseText = data.response || data.reply || (data.detail ? "Error: " + data.detail : "No response from AI.");
+                
+                const actMatch = responseText.match(/<<<ACT:\\s*(\\{.*?\\})\\s*>>>/s);
+                let cleanText = responseText;
+                if (actMatch) {
+                    cleanText = responseText.replace(actMatch[0], '').trim();
+                    try {
+                        const actData = JSON.parse(actMatch[1]);
+                        window.showAIActionModal(actData);
+                    } catch(e) { console.error("Failed to parse ACT block", e); }
+                }
+
+                chatHistory.push({ role: "assistant", content: cleanText });
+                const aiMsg = document.createElement('div');
+                aiMsg.style.cssText = "background: rgba(59, 130, 246, 0.1); padding: 12px 16px; border-radius: 12px; border-top-left-radius: 4px; align-self: flex-start; max-width: 85%; color: #e2e8f0; line-height: 1.5;";
+                aiMsg.innerText = cleanText;'''
+
+text = text.replace(old_chat_parse, new_chat_parse)
+
+text += '''
+document.addEventListener('DOMContentLoaded', () => {
+    const closeBtn = document.getElementById('chat-close-btn');
+    if(closeBtn) closeBtn.onclick = window.toggleChatWindow;
+});
+'''
+
+with open('static/app.js', 'w', encoding='utf-8') as f:
+    f.write(text)
+print('Updated app.js successfully with AI sandbox modal and fix for buttons.')

utils/metrics.py

@@ -7,7 +7,8 @@ def israelsen_sharpe(excess_return, vol):
     Standard Sharpe penalizes higher volatility even when returns are negative.
     This adjustment correctly rewards lower volatility when returns are negative.
     """
-    if excess_return == 0 or vol == 0:
+    vol = max(float(vol), 1e-4)
+    if excess_return == 0:
         return 0.0
     exponent = excess_return / abs(excess_return)
     return excess_return / (vol ** exponent)

validation.py

@@ -252,9 +252,11 @@ def probabilistic_sharpe_ratio(returns, benchmark_sharpe=0.0, periods=252):
     
     return {
         'obs_sharpe': float(sr_daily * np.sqrt(periods)),
+        'observed_sharpe': float(sr_daily * np.sqrt(periods)),
         'benchmark': float(benchmark_sharpe),
         'psr_stat': float(psr_stat),
         'prob': float(prob),
+        'p_value': float(1.0 - prob),
         'significant': bool(prob > 0.95)
     }
 
@@ -400,3 +402,121 @@ def print_validation_report(dm_results=None, var_results=None, psr_results=None,
         else:
             print(f"    Result              : {Color.RED}FAIL{Color.RESET} — {var_results.get('diagnostic', 'VaR model failed conditional coverage checks.')}")
     print("────────────────────────────────────────────────────────────")
+
+
+# ─────────────────────────────────────────────────────────────────
+# 7. INSTITUTIONAL ECONOMETRIC TESTS
+# ─────────────────────────────────────────────────────────────────
+
+def pesaran_timmermann_test(actual, predicted):
+    """Pesaran and Timmermann (1992) directional accuracy test."""
+    a = np.sign(actual)
+    p = np.sign(predicted)
+    p_hat = np.mean(a == p)
+    p_a = np.mean(a > 0)
+    p_p = np.mean(p > 0)
+    p_star = p_a * p_p + (1 - p_a) * (1 - p_p)
+    var_p_hat = p_star * (1 - p_star) / len(actual)
+    if var_p_hat == 0:
+        return {'stat': 0.0, 'p_value': 1.0, 'significant': False}
+    pt_stat = (p_hat - p_star) / np.sqrt(var_p_hat)
+    p_val = 1 - norm.cdf(pt_stat)
+    return {'stat': float(pt_stat), 'p_value': float(p_val), 'significant': p_val < 0.05}
+
+def ljung_box_test(residuals, lags=10):
+    """Ljung-Box test for autocorrelation in residuals."""
+    res = np.asarray(residuals).flatten()
+    n = len(res)
+    acf = []
+    for k in range(1, lags + 1):
+        if len(res[:-k]) < 2:
+            acf.append(0.0)
+            continue
+        corr = np.corrcoef(res[:-k], res[k:])[0, 1]
+        acf.append(corr if not np.isnan(corr) else 0.0)
+    acf = np.array(acf)
+    q_stat = n * (n + 2) * np.sum((acf ** 2) / (n - np.arange(1, lags + 1)))
+    p_val = 1 - chi2.cdf(q_stat, df=lags)
+    return {'stat': float(q_stat), 'p_value': float(p_val), 'significant': p_val < 0.05}
+
+def grs_test(alpha_hat, resid_cov, factor_rets):
+    """
+    Gibbons, Ross, and Shanken (1989) test.
+    Tests the null hypothesis that all alphas are jointly zero.
+    """
+    T = factor_rets.shape[0]
+    N = alpha_hat.shape[0]
+    K = factor_rets.shape[1] if len(factor_rets.shape) > 1 else 1
+    
+    if K == 1:
+        factor_mean = np.mean(factor_rets)
+        factor_var = np.var(factor_rets)
+        omega = (factor_mean**2) / factor_var if factor_var > 0 else 0
+    else:
+        factor_mean = np.mean(factor_rets, axis=0)
+        factor_cov = np.cov(factor_rets, rowvar=False)
+        try:
+            omega = factor_mean.T @ np.linalg.inv(factor_cov) @ factor_mean
+        except np.linalg.LinAlgError:
+            omega = 0.0
+        
+    try:
+        inv_resid_cov = np.linalg.inv(resid_cov)
+    except np.linalg.LinAlgError:
+        return {'stat': 0.0, 'p_value': 1.0, 'significant': False}
+        
+    stat = (T - N - K) / N * (1 + omega)**(-1) * (alpha_hat.T @ inv_resid_cov @ alpha_hat)
+    
+    from scipy.stats import f
+    p_val = 1 - f.cdf(stat, N, T - N - K)
+    return {'stat': float(stat), 'p_value': float(p_val), 'significant': p_val < 0.05}
+
+def chow_test(y, X, break_point):
+    """Chow test for structural breaks."""
+    y = np.asarray(y)
+    X = np.asarray(X)
+    if len(X.shape) == 1:
+        X = X.reshape(-1, 1)
+        
+    def get_rss(y_sub, X_sub):
+        if len(y_sub) < X_sub.shape[1]:
+            return 0.0
+        try:
+            b = np.linalg.pinv(X_sub.T @ X_sub) @ X_sub.T @ y_sub
+            resid = y_sub - X_sub @ b
+            return np.sum(resid**2)
+        except np.linalg.LinAlgError:
+            return 0.0
+        
+    rss_all = get_rss(y, X)
+    rss_1 = get_rss(y[:break_point], X[:break_point])
+    rss_2 = get_rss(y[break_point:], X[break_point:])
+    
+    k = X.shape[1]
+    n = len(y)
+    
+    num = (rss_all - (rss_1 + rss_2)) / k
+    den = (rss_1 + rss_2) / (n - 2*k)
+    if den <= 0:
+        return {'stat': 0.0, 'p_value': 1.0, 'significant': False}
+    chow_stat = num / den
+    from scipy.stats import f
+    p_val = 1 - f.cdf(chow_stat, k, n - 2*k)
+    return {'stat': float(chow_stat), 'p_value': float(p_val), 'significant': p_val < 0.05}
+
+def hansen_spa_test(losses_base, losses_models):
+    """
+    Simplified Hansen's Superior Predictive Ability (SPA) Test.
+    """
+    losses_base = np.asarray(losses_base)
+    losses_models = np.asarray(losses_models)
+    if len(losses_models.shape) == 1:
+        losses_models = losses_models.reshape(-1, 1)
+        
+    d = losses_base[:, None] - losses_models
+    mean_d = np.mean(d, axis=0)
+    std_d = np.std(d, axis=0, ddof=1)
+    stat = np.max(mean_d / (std_d / np.sqrt(len(losses_base)) + 1e-8))
+    
+    p_val = 1 - norm.cdf(stat)
+    return {'stat': float(stat), 'p_value': float(p_val), 'significant': p_val < 0.05}