dualea/Experts/Advisors/DualEA/ML/dataset.py

import os
import pandas as pd
from typing import Tuple

# Default MT5 Common Files path on Windows
DEFAULT_COMMON = os.path.join(os.environ.get("APPDATA", ""), "MetaQuotes", "Terminal", "Common", "Files", "DualEA")
FEATURES_NAME = "features.csv"


def resolve_features_path(common_dir: str = None) -> str:
    base = common_dir or DEFAULT_COMMON
    return os.path.join(base, FEATURES_NAME)


def _read_csv_robust(path: str) -> pd.DataFrame:
    encodings = [
        "utf-8",
        "utf-8-sig",
        "utf-16",
        "utf-16-le",
        "utf-16-be",
        "cp1252",
        "latin1",
    ]
    last_exc = None
    for enc in encodings:
        try:
            return pd.read_csv(path, encoding=enc)
        except UnicodeDecodeError as e:
            last_exc = e
            continue
        except Exception as e:
            # If it's not a decode error, bubble up immediately
            raise
    # Fallback with python engine ignoring bad lines if all decodes failed
    return pd.read_csv(path, encoding="latin1", engine="python")


def load_features(common_dir: str = None) -> pd.DataFrame:
    path = resolve_features_path(common_dir)
    if not os.path.exists(path):
        raise FileNotFoundError(f"features.csv not found at {path}")
    # Comma-delimited, result of EA writes (encoding may vary depending on MT5 settings)
    df = _read_csv_robust(path)
    # Normalize columns to lower snake-case (handles header casing/spacing)
    df.columns = [str(c).strip().lower().replace(" ", "_").replace("-", "_") for c in df.columns]
    # Basic cleaning
    df = df.dropna(how="all")
    # Ensure timestamp is parsed if present
    for col in ("time", "timestamp", "entry_time", "close_time"):
        if col in df.columns:
            try:
                df[col] = pd.to_datetime(df[col])
            except Exception:
                pass
    # If long format (feature/value), pivot to wide
    if {"feature", "value"}.issubset(set(df.columns)):
        # Coerce value to numeric
        df["value"] = pd.to_numeric(df["value"], errors="coerce")
        # Candidate keys to keep identity and slices stable
        key_priority = [
            ["position_id"],
            ["deal_id"],
            ["order_id"],
            ["position_id", "deal_id", "order_id"],
            ["strategy", "symbol", "timeframe", "entry_time", "close_time"],
            ["strategy", "symbol", "entry_time", "close_time"],
            ["symbol", "entry_time", "close_time"],
        ]
        keys = []
        for cand in key_priority:
            kk = [c for c in cand if c in df.columns]
            if len(kk) == len(cand):
                keys = kk
                break
        # Always try to include context columns for downstream use
        for c in ("strategy", "symbol", "timeframe", "entry_time", "close_time"):
            if c in df.columns and c not in keys:
                keys.append(c)
        if not keys:
            # Fallback: use all non-feature columns except 'value' as keys to avoid data loss
            keys = [c for c in df.columns if c not in ("feature", "value")]
        # Pivot
        wide = df.pivot_table(index=keys, columns="feature", values="value", aggfunc="last")
        wide = wide.reset_index()
        # Flatten columns after pivot
        wide.columns = [str(c).strip().lower().replace(" ", "_") for c in wide.columns]
        return wide
    return df


def make_label_from_r_multiple(df: pd.DataFrame, threshold: float = 0.0) -> Tuple[pd.DataFrame, pd.Series]:
    """Return labels only for explicitly closed, finite realized outcomes.

    This diagnostic helper never joins Common Files implicitly, fills missing
    outcomes with zero, or interprets a gate/status flag as an outcome. The
    standard training entry point is the shared real-history MQL5 pipeline.
    """
    import math

    if not math.isfinite(float(threshold)):
        raise ValueError("Outcome threshold must be finite")
    outcome = next((name for name in ("r_multiple", "r", "rmultiple") if name in df.columns), None)
    if outcome is None:
        if "profit" not in df.columns:
            raise ValueError("Realized R multiple or profit is required; no implicit file merge or unknown-label fallback")
        if threshold != 0.0:
            raise ValueError("A nonzero R-multiple threshold cannot be applied to currency profit")
        outcome = "profit"

    if "is_closed" in df.columns:
        closed = df["is_closed"].astype(str).str.strip().str.lower().isin({"true", "1", "1.0", "yes", "closed"})
    elif "close_event" in df.columns:
        closed = df["close_event"].astype(str).str.strip().str.lower().isin({"true", "1", "1.0", "yes", "close", "closed"})
    elif "close_time" in df.columns:
        values = pd.to_numeric(df["close_time"], errors="coerce")
        nonempty = df["close_time"].notna()
        if nonempty.any() and values[nonempty].notna().all():
            times = pd.to_datetime(values, unit="s", errors="coerce", utc=True)
        else:
            times = pd.to_datetime(df["close_time"], errors="coerce", utc=True)
        closed = times.notna() & (times > pd.Timestamp("1970-01-02", tz="UTC"))
    else:
        raise ValueError("Explicit is_closed/close_event/close_time evidence is required; open trades cannot be labeled")

    realized = pd.to_numeric(df[outcome], errors="coerce")
    finite = realized.map(lambda value: bool(pd.notna(value) and math.isfinite(float(value))))
    observed = closed & finite
    selected = df.loc[observed].copy()
    if selected.empty:
        raise ValueError("No explicitly closed trades with finite observed outcomes; collect real outcomes instead of bootstrapping labels")
    labels = (realized.loc[observed] > float(threshold)).astype(int)
    return selected, labels