"""ml_walkforward.py - walk-forward LightGBM forecaster on the alpha_v3 decision panel. DISCOVERY ONLY (panel_discovery.parquet, <= 2020-12-31).

Written by: Claude (2026-10-19). Design fixed before the first run (one model config, three horizons = 3 trials, plus a shuffled-target control):
  target   y_<h> / own_vol30 (vol-scaled mid return), clipped at +/-6
  model    LightGBM regression, 300 trees, lr 0.03, 15 leaves, min_data_in_leaf 400, feature_fraction 0.7, bagging 0.8, L2 1.0, seed 7
  folds    expanding window, train on years < Y, test year Y, Y = 2010..2020 (2005-2009 is warm-up)
  trade    a row is traded when |pred| >= the 80th percentile of |pred| on the TRAINING rows of that fold; side = sign(pred);
           net = executed long_<h> / short_<h> (bid/ask from the file) in bp; one position per (symbol, day, slot) row, rows may overlap in time
Statistics: day-clustered t (all trades of a symbol-day summed), per-year means, per-symbol means. Nothing here is a verdict.
"""
from __future__ import annotations

import os
import sys

import lightgbm as lgb
import numpy as np
import pandas as pd

HERE = os.path.dirname(os.path.abspath(__file__))
PARAMS = dict(objective="regression", learning_rate=0.03, num_leaves=15, min_data_in_leaf=400, feature_fraction=0.7, bagging_fraction=0.8,
              bagging_freq=1, lambda_l2=1.0, seed=7, verbose=-1, num_threads=12)
N_TREES = 300
Q_TRADE = 0.80


def features(P: pd.DataFrame) -> list[str]:
    return [c for c in P.columns if c.startswith("f_") or c.startswith("own_") and c != "own_vol30"] + ["slot", "dow", "gex_rank", "dix_rank", "gex_neg", "sym_code"]


def tstat_day(df: pd.DataFrame) -> float:
    d = df.groupby(["sym", "day"]).net.sum()
    return d.mean() / d.std(ddof=1) * np.sqrt(len(d)) if len(d) > 2 else np.nan


def run(P: pd.DataFrame, h: str, shuffle: bool = False) -> pd.DataFrame:
    P = P.copy()
    P["sym_code"] = P["sym"].astype("category").cat.codes
    feats = features(P)
    y = (P[f"y_{h}"] / P["own_vol30"]).clip(-6, 6)
    ok = np.isfinite(y)
    P, y = P[ok], y[ok]
    yr = P["day"].dt.year
    out = []
    rng = np.random.default_rng(11)
    for Y in range(2010, 2021):
        tr, te = yr < Y, yr == Y
        if te.sum() == 0:
            continue
        ytr = y[tr].to_numpy()
        if shuffle:
            ytr = rng.permutation(ytr)
        m = lgb.train(PARAMS, lgb.Dataset(P.loc[tr, feats], ytr), N_TREES)
        thr = np.quantile(np.abs(m.predict(P.loc[tr, feats])), Q_TRADE)
        p = m.predict(P.loc[te, feats])
        T = P.loc[te, ["sym", "day", "slot", f"y_{h}", f"long_{h}", f"short_{h}"]].copy()
        T["pred"] = p
        T["ic_y"] = y[te].to_numpy()
        T = T[np.abs(p) >= thr]
        T["side"] = np.sign(T["pred"]).astype(int)
        T["net"] = np.where(T.side > 0, T[f"long_{h}"], T[f"short_{h}"])
        T["gross"] = T.side * T[f"y_{h}"]
        T["year"] = Y
        out.append(T)
    return pd.concat(out)


def summary(T: pd.DataFrame, label: str):
    print(f"\n--- {label}: trades {len(T)}  net {T.net.mean():.2f} bp  gross {T.gross.mean():.2f} bp  t_day {tstat_day(T):.2f}  "
          f"IC {T[['pred', 'ic_y']].corr(method='spearman').iloc[0, 1]:.3f}")
    print("  by year :", T.groupby("year").net.agg(["count", "mean"]).round(2).T.to_dict())
    print("  by sym  :", T.groupby("sym").net.agg(["count", "mean"]).round(2).T.to_dict())
    print("  by slot :", T.groupby("slot").net.mean().round(2).to_dict())


def main():
    P = pd.read_parquet(os.path.join(HERE, "panel_discovery.parquet"))
    for h in ("h30", "h60", "close"):
        T = run(P, h)
        summary(T, f"{h}")
        T.to_parquet(os.path.join(HERE, f"ml_wf_trades_{h}.parquet"))
        Ts = run(P, h, shuffle=True)
        summary(Ts, f"{h} SHUFFLED-TARGET CONTROL")


if __name__ == "__main__":
    main()