#property strict
#include "../Include/CMarketModelFeatures.mqh"

input string BuildOutputDirectory="DualEA\\model_build\\universal_h1_logistic_v1";
input string TrainingSymbols="EURUSD,GBPUSD,USDCHF,USDJPY,USDCAD,AUDUSD,NZDUSD,EURGBP,EURJPY,GBPJPY,XAUUSD,XAGUSD,XAUEUR,AUS200,GER40,NAS100,UK100,US30,US500,BRENT,WTI,BTCUSD,ETHUSD";
input int DecisionStrideBars=1;
input int TrainingEpochs=300;
input double LearningRate=0.08;
input double L2Penalty=0.01;
input datetime TrainingStart=D'2024.01.01';
input datetime TrainingEnd=D'2026.07.01';
input datetime ValidationEnd=D'2026.09.01';
input datetime TestEnd=D'2026.10.01';

#define MODEL_FEATURES 12
#define ROW_FEATURES 16
struct STrainingRow
{
   int symbol_index;
   datetime time;
   datetime target_end;
   int direction;
   int split;
   int label;
   double net_return;
   double entry_spread;
   double exit_spread;
   double x[ROW_FEATURES];
};
STrainingRow rows[];
string training_symbols[],rejected_symbols[];
double means[],scales[],weights[],effective_weights[];
double effective_intercept=0;
int symbol_counts[],symbol_positives[];
double intercept=0.0;
int counts[3],positives[3];
bool trained=false;

double Logistic(double score)
{
   if(score>=0.0) return 1.0/(1.0+MathExp(-MathMin(score,700.0)));
   double e=MathExp(MathMax(score,-700.0));
   return e/(1.0+e);
}
double Prediction(const STrainingRow &row)
{
   double score=effective_intercept;
   for(int j=0;j<ROW_FEATURES;j++)
   {
      int feature=j<MODEL_FEATURES ? j : MODEL_FEATURES+4*row.symbol_index+j-MODEL_FEATURES;
      score+=effective_weights[feature]*row.x[j];
   }
   return Logistic(score);
}
void RefreshEffectiveWeights()
{
   effective_intercept=intercept;
   for(int j=0;j<ArraySize(weights);j++)
   {
      effective_weights[j]=weights[j]/scales[j];
      effective_intercept-=weights[j]*means[j]/scales[j];
   }
}
string Number(double value) { return DoubleToString(value,16); }
string VectorJSON(const double &values[])
{
   string result="[";
   for(int j=0;j<ArraySize(values);j++) { if(j) result+=","; result+=Number(values[j]); }
   return result+"]";
}
string SplitName(int split) { return split==0 ? "train" : (split==1 ? "validation" : "test"); }
bool WriteText(string name,string content)
{
   int h=FileOpen(BuildOutputDirectory+"\\"+name,FILE_WRITE|FILE_TXT|FILE_ANSI,0,CP_UTF8);
   if(h==INVALID_HANDLE) { Print("[MODEL-TRAIN FAIL] Cannot write ",name," error=",GetLastError()); return false; }
   FileWriteString(h,content);
   FileFlush(h);
   FileClose(h);
   return true;
}

bool LoadRealRows()
{
   string requested[];
   int requested_count=StringSplit(TrainingSymbols,',',requested);
   int used=0;
   ArrayResize(rows,0);
   ArrayResize(training_symbols,0);
   ArrayResize(rejected_symbols,0);
   ArrayInitialize(counts,0);
   ArrayInitialize(positives,0);
   for(int symbol_slot=0;symbol_slot<requested_count;symbol_slot++)
   {
      string symbol=requested[symbol_slot];
      StringTrimLeft(symbol);StringTrimRight(symbol);
      if(symbol=="") continue;
      bool duplicate=false;
      for(int k=0;k<ArraySize(training_symbols);k++) if(training_symbols[k]==symbol) duplicate=true;
      if(duplicate) continue;
      MqlRates rates[];
      ArraySetAsSeries(rates,false);
      int n=0;
      if(SymbolSelect(symbol,true))
      {
         // The first cross-symbol request can initiate asynchronous history loading.
         for(int retry=0;retry<12;retry++)
         {
            n=CopyRates(symbol,PERIOD_H1,TrainingStart,TestEnd-1,rates);
            if(n>=1500) break;
            Sleep(250);
         }
      }
      double point=SymbolInfoDouble(symbol,SYMBOL_POINT);
      int local_counts[3],local_positive[3];
      ArrayInitialize(local_counts,0);ArrayInitialize(local_positive,0);
      int start=used;
      int accepted_index=ArraySize(training_symbols);
      if(n>=1500 && point>0)
      {
         ArrayResize(rows,used+n*2);
         for(int d=15;d+4<n;d+=DecisionStrideBars)
         {
            datetime decision=rates[d].time,end=rates[d+4].time;
            if(decision<TrainingStart || end>=TestEnd) continue;
            bool continuous=true;
            for(int k=d;k<d+4;k++) if(rates[k+1].time-rates[k].time!=3600) continuous=false;
            if(!continuous || rates[d].open<=0 || rates[d+4].open<=0 || rates[d].spread<0 || rates[d+4].spread<0) continue;
            int split=decision<TrainingEnd ? 0 : (decision<ValidationEnd ? 1 : 2);
            datetime boundary=split==0 ? TrainingEnd : (split==1 ? ValidationEnd : TestEnd);
            if(end>=boundary) continue;
            for(int direction=0;direction<2;direction++)
            {
               double f[];
               if(!MarketModelFeaturesFromRates(rates,d-1,direction,point,decision,f) || ArraySize(f)!=MODEL_FEATURES) continue;
               double entry_bid=rates[d].open,exit_bid=rates[d+4].open;
               double entry_ask=entry_bid+rates[d].spread*point,exit_ask=exit_bid+rates[d+4].spread*point;
               if(entry_ask<=0 || exit_ask<=0) continue;
               double target=direction==0 ? MathLog(exit_bid/entry_ask) : MathLog(entry_bid/exit_ask);
               if(!MathIsValidNumber(target)) continue;
               rows[used].symbol_index=accepted_index;
               rows[used].time=decision;rows[used].target_end=end;
               rows[used].direction=direction;rows[used].split=split;
               rows[used].net_return=target;rows[used].label=target>0 ? 1 : 0;
               rows[used].entry_spread=rates[d].spread;rows[used].exit_spread=rates[d+4].spread;
               for(int j=0;j<MODEL_FEATURES;j++) rows[used].x[j]=f[j];
               rows[used].x[12]=1;
               rows[used].x[13]=f[0]/MathMax(f[4],1e-8);
               rows[used].x[14]=f[1]/MathMax(f[4],1e-8);
               rows[used].x[15]=f[3];
               local_counts[split]++;local_positive[split]+=rows[used].label;
               used++;
            }
         }
      }
      // Eligibility must not select the universe using held-out labels or scores.
      bool eligible=local_counts[0]>=1000 && local_positive[0]>=20 && local_counts[0]-local_positive[0]>=20;
      if(!eligible)
      {
         used=start;ArrayResize(rows,used);
         int rejected=ArraySize(rejected_symbols);ArrayResize(rejected_symbols,rejected+1);rejected_symbols[rejected]=symbol;
         PrintFormat("[MODEL-TRAIN SKIP] %s insufficient real history/class counts: bars=%d train=%d val=%d test=%d",symbol,n,local_counts[0],local_counts[1],local_counts[2]);
         continue;
      }
      ArrayResize(training_symbols,accepted_index+1);training_symbols[accepted_index]=symbol;
      ArrayResize(symbol_counts,(accepted_index+1)*3);ArrayResize(symbol_positives,(accepted_index+1)*3);
      for(int split=0;split<3;split++)
      {
         counts[split]+=local_counts[split];positives[split]+=local_positive[split];
         symbol_counts[accepted_index*3+split]=local_counts[split];symbol_positives[accepted_index*3+split]=local_positive[split];
         PrintFormat("[MODEL-TRAIN DATA] %s %s rows=%d positive=%d negative=%d",symbol,SplitName(split),local_counts[split],local_positive[split],local_counts[split]-local_positive[split]);
      }
      ArrayResize(rows,used);
   }
   string feature_names[];
   if(ArraySize(training_symbols)<2 || !UniversalModelFeatureNames(training_symbols,feature_names)) return false;
   int width=ArraySize(feature_names);
   ArrayResize(means,width);ArrayResize(scales,width);ArrayResize(weights,width);ArrayResize(effective_weights,width);
   return true;
}
void FitTrainingOnly()
{
   int width=ArraySize(weights);
   ArrayInitialize(means,0.0);ArrayInitialize(scales,0.0);ArrayInitialize(weights,0.0);
   for(int i=0;i<ArraySize(rows);i++)
      if(rows[i].split==0)
         for(int j=0;j<ROW_FEATURES;j++)
         {
            int f=j<MODEL_FEATURES ? j : MODEL_FEATURES+4*rows[i].symbol_index+j-MODEL_FEATURES;
            means[f]+=rows[i].x[j]/counts[0];
            scales[f]+=rows[i].x[j]*rows[i].x[j]/counts[0];
         }
   // E[x^2]-E[x]^2 includes the inactive per-symbol zeros exactly.
   for(int j=0;j<width;j++) {scales[j]=MathSqrt(MathMax(0.0,scales[j]-means[j]*means[j]));if(scales[j]<1e-12)scales[j]=1.0;}
   double prior=(double)positives[0]/counts[0];
   intercept=MathLog(prior/(1.0-prior));
   double gradient[];ArrayResize(gradient,width);
   for(int epoch=0;epoch<TrainingEpochs;epoch++)
   {
      RefreshEffectiveWeights();
      ArrayInitialize(gradient,0.0);
      double bias_gradient=0.0;
      for(int i=0;i<ArraySize(rows);i++)
      {
         if(rows[i].split!=0) continue;
         double error=Prediction(rows[i])-rows[i].label;
         bias_gradient+=error/counts[0];
         for(int j=0;j<ROW_FEATURES;j++)
         {
            int f=j<MODEL_FEATURES ? j : MODEL_FEATURES+4*rows[i].symbol_index+j-MODEL_FEATURES;
            gradient[f]+=error*(rows[i].x[j]/scales[f])/counts[0];
         }
      }
      for(int j=0;j<width;j++)
      {
         gradient[j]-=(means[j]/scales[j])*bias_gradient;
         weights[j]-=LearningRate*(gradient[j]+L2Penalty*weights[j]);
      }
      intercept-=LearningRate*bias_gradient;
      if(epoch%50==0) PrintFormat("[MODEL-TRAIN FIT] epoch=%d/%d real_train_rows=%d width=%d",epoch,TrainingEpochs,counts[0],width);
   }
   RefreshEffectiveWeights();
}
string MetricsJSON(int split,int symbol_index=-1)
{
   double ranked[][2];
   int expected_count=symbol_index<0 ? counts[split] : symbol_counts[symbol_index*3+split];
   int expected_positive=symbol_index<0 ? positives[split] : symbol_positives[symbol_index*3+split];
   if(expected_count==0)
      return "{\"rows\":0,\"positive\":0,\"negative\":0,\"auc\":null,\"log_loss\":null,\"constant_train_prior_log_loss\":null,\"brier_score\":null,\"accuracy\":null}";
   ArrayResize(ranked,expected_count);
   double loss=0,brier=0,baseline_loss=0,baseline_brier=0;
   int correct=0,slot=0;
   double prior=(double)positives[0]/counts[0];
   for(int i=0;i<ArraySize(rows);i++)
   {
      if(rows[i].split!=split || (symbol_index>=0 && rows[i].symbol_index!=symbol_index)) continue;
      double p=Prediction(rows[i]);
      p=MathMax(1e-12,MathMin(1.0-1e-12,p));
      int y=rows[i].label;
      loss-=y*MathLog(p)+(1-y)*MathLog(1-p);
      brier+=MathPow(p-y,2);
      baseline_loss-=y*MathLog(prior)+(1-y)*MathLog(1-prior);
      baseline_brier+=MathPow(prior-y,2);
      if((p>=0.5 ? 1 : 0)==y) correct++;
      ranked[slot][0]=p;
      ranked[slot][1]=y;
      slot++;
   }
   ArraySort(ranked);
   double positive_rank_sum=0;
   int begin=0;
   while(begin<slot)
   {
      int end=begin+1;
      while(end<slot && ranked[end][0]==ranked[begin][0]) end++;
      double average_rank=((begin+1)+end)*0.5;
      for(int k=begin;k<end;k++) if(ranked[k][1]>0.5) positive_rank_sum+=average_rank;
      begin=end;
   }
   double pos=expected_positive,neg=expected_count-expected_positive;
   double auc=pos>0 && neg>0 ? (positive_rank_sum-pos*(pos+1)/2)/(pos*neg) : 0;
   string auc_json=pos>0 && neg>0 ? Number(auc) : "null";
   PrintFormat("[MODEL-TRAIN METRICS] %s AUC=%s logloss=%.6f baseline=%.6f accuracy=%.6f",SplitName(split),auc_json,loss/slot,baseline_loss/slot,(double)correct/slot);
   return "{\"rows\":"+IntegerToString(slot)+",\"positive\":"+IntegerToString(expected_positive)+",\"negative\":"+IntegerToString(expected_count-expected_positive)+",\"auc\":"+auc_json+",\"log_loss\":"+Number(loss/slot)+",\"constant_train_prior_log_loss\":"+Number(baseline_loss/slot)+",\"constant_train_prior_brier\":"+Number(baseline_brier/slot)+",\"constant_train_prior_accuracy\":"+Number(prior>=0.5 ? pos/slot : neg/slot)+",\"brier_score\":"+Number(brier/slot)+",\"accuracy\":"+Number((double)correct/slot)+"}";
}

string StringArrayJSON(const string &values[])
{
   string output="[";
   for(int j=0;j<ArraySize(values);j++){if(j)output+=",";output+="\""+values[j]+"\"";}
   return output+"]";
}
bool SaveArtifacts()
{
   FolderCreate("DualEA");FolderCreate("DualEA\\model_build");FolderCreate(BuildOutputDirectory);
   string names[],base_names[];
   if(!UniversalModelFeatureNames(training_symbols,names)) return false;
   MarketModelFeatureNames(base_names);
   string joined_names="",joined_symbols="";
   for(int j=0;j<ArraySize(names);j++){if(j)joined_names+="|";joined_names+=names[j];}
   for(int j=0;j<ArraySize(training_symbols);j++){if(j)joined_symbols+="|";joined_symbols+=training_symbols[j];}
   string model="{\"schema\":\"market_universal_v1\",\"symbol\":\"*\",\"supported_symbols\":"+StringArrayJSON(training_symbols)+",\"rejected_symbols\":"+StringArrayJSON(rejected_symbols)+",\"timeframe\":16385,\"horizon_bars\":4,\"feature_names\":"+StringArrayJSON(names)+",\"weights\":"+VectorJSON(weights)+",\"intercept\":"+Number(intercept)+",\"scaler_mean\":"+VectorJSON(means)+",\"scaler_scale\":"+VectorJSON(scales)+",\"training_start\":\""+TimeToString(TrainingStart,TIME_DATE)+"\",\"training_end\":\""+TimeToString(TrainingEnd,TIME_DATE)+"\",\"validation_end\":\""+TimeToString(ValidationEnd,TIME_DATE)+"\",\"test_end\":\""+TimeToString(TestEnd,TIME_DATE)+"\",\"target\":\"positive_side_aware_four_bar_log_return_after_recorded_spread\",\"commission\":0,\"slippage\":0,\"decision_stride_bars\":"+IntegerToString(DecisionStrideBars)+",\"epochs\":"+IntegerToString(TrainingEpochs)+",\"learning_rate\":"+Number(LearningRate)+",\"l2\":"+Number(L2Penalty)+"}";
   if(!WriteText("model_data.json",model)) return false;
   string metrics="{\"train\":"+MetricsJSON(0)+",\"validation\":"+MetricsJSON(1)+",\"test\":"+MetricsJSON(2)+",\"per_symbol\":{";
   for(int s=0;s<ArraySize(training_symbols);s++)
   {
      if(s)metrics+=",";
      metrics+="\""+training_symbols[s]+"\":{\"train\":"+MetricsJSON(0,s)+",\"validation\":"+MetricsJSON(1,s)+",\"test\":"+MetricsJSON(2,s)+"}";
   }
   metrics+="},\"selection\":\"fixed_hyperparameters_no_validation_or_test_selection\",\"training_data\":\"broker_H1_OHLC_and_recorded_spread\",\"purge\":\"target_end_before_split_boundary\",\"supported_symbols\":"+StringArrayJSON(training_symbols)+",\"rejected_symbols\":"+StringArrayJSON(rejected_symbols)+"}";
   if(!WriteText("metrics.json",metrics)) return false;
   string ini="version=1\nfeature_contract=market_universal_v1\ntrained_symbol=*\nsupported_symbols="+joined_symbols+"\ntrained_timeframe=16385\nprediction_horizon_bars=4\ntraining_start="+TimeToString(TrainingStart,TIME_DATE)+"\ntraining_end="+TimeToString(TrainingEnd,TIME_DATE)+"\nvalidation_end="+TimeToString(ValidationEnd,TIME_DATE)+"\ntest_end="+TimeToString(TestEnd,TIME_DATE)+"\nfeature_count="+IntegerToString(ArraySize(names))+"\nfeature_names="+joined_names+"\nscaler_mean=";
   for(int j=0;j<ArraySize(means);j++){if(j)ini+="|";ini+=Number(means[j]);}
   ini+="\nscaler_scale=";
   for(int j=0;j<ArraySize(scales);j++){if(j)ini+="|";ini+=Number(scales[j]);}
   ini+="\ninput_name=model_input\noutput_name=probability\nlabel_mode=future_side_aware_return\n";
   if(!WriteText("onnx_config.ini",ini)) return false;
   int dataset=FileOpen(BuildOutputDirectory+"\\dataset.csv",FILE_WRITE|FILE_CSV|FILE_ANSI,',',CP_UTF8);
   int parity=FileOpen(BuildOutputDirectory+"\\parity_samples.csv",FILE_WRITE|FILE_CSV|FILE_ANSI,',',CP_UTF8);
   if(dataset==INVALID_HANDLE || parity==INVALID_HANDLE){if(dataset!=INVALID_HANDLE)FileClose(dataset);if(parity!=INVALID_HANDLE)FileClose(parity);return false;}
   string header="decision_time,target_end,split,symbol,direction,label,net_log_return,entry_spread_points,exit_spread_points";
   for(int j=0;j<MODEL_FEATURES;j++)header+=","+base_names[j];
   header+=",active_symbol_flag,active_ret1,active_ret3,active_zscore";
   FileWriteString(dataset,header+"\r\n");
   string parity_header="split,decision_time,symbol,direction,expected_probability";
   for(int j=0;j<ArraySize(names);j++)parity_header+=","+names[j];
   FileWriteString(parity,parity_header+"\r\n");
   int selected[];ArrayResize(selected,ArraySize(training_symbols)*3);ArrayInitialize(selected,0);
   for(int i=0;i<ArraySize(rows);i++)
   {
      string symbol=training_symbols[rows[i].symbol_index];
      string line=TimeToString(rows[i].time,TIME_DATE|TIME_MINUTES)+","+TimeToString(rows[i].target_end,TIME_DATE|TIME_MINUTES)+","+SplitName(rows[i].split)+","+symbol+","+IntegerToString(rows[i].direction)+","+IntegerToString(rows[i].label)+","+Number(rows[i].net_return)+","+Number(rows[i].entry_spread)+","+Number(rows[i].exit_spread);
      for(int j=0;j<ROW_FEATURES;j++)line+=","+Number(rows[i].x[j]);
      FileWriteString(dataset,line+"\r\n");
      int group=rows[i].symbol_index*3+rows[i].split;
      if(selected[group]<8)
      {
         string sample=SplitName(rows[i].split)+","+TimeToString(rows[i].time,TIME_DATE|TIME_MINUTES)+","+symbol+","+IntegerToString(rows[i].direction)+","+Number(Prediction(rows[i]));
         for(int j=0;j<ArraySize(names);j++)
         {
            double value=0;
            if(j<MODEL_FEATURES)value=rows[i].x[j];
            else if((j-MODEL_FEATURES)/4==rows[i].symbol_index)value=rows[i].x[MODEL_FEATURES+(j-MODEL_FEATURES)%4];
            sample+=","+Number(value);
         }
         FileWriteString(parity,sample+"\r\n");selected[group]++;
      }
   }
   FileFlush(dataset);FileClose(dataset);FileFlush(parity);FileClose(parity);
   return true;
}
int OnInit()
{
   if(!MQLInfoInteger(MQL_TESTER) || _Period!=PERIOD_H1) {Print("[MODEL-TRAIN FAIL] Run only in H1 Strategy Tester.");return INIT_FAILED;}
   if(!(TrainingStart<TrainingEnd && TrainingEnd<ValidationEnd && ValidationEnd<TestEnd && TestEnd<=TimeCurrent()) ||
      TrainingStart%86400!=0 || TrainingEnd%86400!=0 || ValidationEnd%86400!=0 || TestEnd%86400!=0)
   {
      Print("[MODEL-TRAIN FAIL] Date boundaries must be ordered midnight dates, with TestEnd no later than tester time.");
      return INIT_PARAMETERS_INCORRECT;
   }
   if(DecisionStrideBars<1 || DecisionStrideBars>24 || TrainingEpochs<1 || TrainingEpochs>2000 ||
      !MathIsValidNumber(LearningRate) || !MathIsValidNumber(L2Penalty) ||
      LearningRate<=0 || LearningRate>1 || L2Penalty<0) return INIT_PARAMETERS_INCORRECT;
   if(!LoadRealRows()) {Print("[MODEL-TRAIN FAIL] Real history/class-count requirements were not met.");return INIT_FAILED;}
   FitTrainingOnly();
   if(!MathIsValidNumber(intercept)) return INIT_FAILED;
   for(int j=0;j<ArraySize(weights);j++)
      if(!MathIsValidNumber(weights[j]) || !MathIsValidNumber(means[j]) || !MathIsValidNumber(scales[j]) || scales[j]<=0)
      {
         Print("[MODEL-TRAIN FAIL] Invalid fitted coefficient/scaler at ",j);
         return INIT_FAILED;
      }
   if(!SaveArtifacts()) return INIT_FAILED;
   trained=true;
   PrintFormat("[MODEL-TRAIN RESULT] rows=%d train=%d validation=%d test=%d output=%s",ArraySize(rows),counts[0],counts[1],counts[2],BuildOutputDirectory);
   return INIT_SUCCEEDED;
}
void OnTick() {}
double OnTester() { return trained ? ArraySize(rows) : -1; }