140 lines
6 KiB
MQL5
140 lines
6 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| TestCSCVEngine.mq5 |
|
|
//| Validation of the CSCV engine - Astralys LLC |
|
|
//| |
|
|
//| Before running the engine on real data, it has to be checked |
|
|
//| against cases whose answer is known in advance. Otherwise there |
|
|
//| is no way to tell a bug from a discovery, which is precisely the |
|
|
//| failure mode the article is about. |
|
|
//| |
|
|
//| Two brackets, both stated by Bailey et al. (2015): |
|
|
//| |
|
|
//| CASE A - pure noise, no skill anywhere. |
|
|
//| "if the relative ranks are distributed close to uniformly, |
|
|
//| the distribution of the logits will approximate the standard |
|
|
//| Normal distribution with a 0 mean and a standard deviation |
|
|
//| of 1. This is the case when the backtest lacks information" |
|
|
//| Expected: PBO close to 50%, logit mean close to 0. |
|
|
//| |
|
|
//| CASE B - one genuinely superior strategy, present in every |
|
|
//| partition. The in-sample winner is also the out-of-sample |
|
|
//| winner every time. |
|
|
//| Expected: PBO close to 0%, logits large and positive. |
|
|
//| |
|
|
//| If the engine passes both, its arithmetic is sound. |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Astralys LLC"
|
|
#property link "https://pulsar-terminal.com"
|
|
#property version "1.00"
|
|
#property script_show_inputs
|
|
|
|
#include <PBO/CSCVEngine.mqh>
|
|
|
|
input int InpRows = 1600; // bars
|
|
input int InpCols = 200; // parameter combinations
|
|
input int InpPartitions = 16; // S
|
|
input int InpSeed = 20260815;
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Standard normal deviate, Box-Muller. MathRand is seeded by the |
|
|
//| caller so the whole test is reproducible. |
|
|
//+------------------------------------------------------------------+
|
|
double Gauss(void)
|
|
{
|
|
double u1 = (MathRand() + 1.0) / 32768.0; // (0,1], never exactly 0
|
|
double u2 = (MathRand() + 0.5) / 32768.0;
|
|
return(MathSqrt(-2.0 * MathLog(u1)) * MathCos(2.0 * M_PI * u2));
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Mean and standard deviation of an array. |
|
|
//+------------------------------------------------------------------+
|
|
void MeanSd(const double &v[], double &mean, double &sd)
|
|
{
|
|
const int n = ArraySize(v);
|
|
mean = 0.0; sd = 0.0;
|
|
if(n < 2) return;
|
|
|
|
for(int i = 0; i < n; i++) mean += v[i];
|
|
mean /= n;
|
|
|
|
for(int i = 0; i < n; i++) { const double d = v[i] - mean; sd += d * d; }
|
|
sd = MathSqrt(sd / (n - 1));
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Fill a T x N matrix of log returns. |
|
|
//| |
|
|
//| edge > 0 gives column `star` a constant per-bar drift, which |
|
|
//| makes it dominate in every partition. |
|
|
//+------------------------------------------------------------------+
|
|
void BuildMatrix(double &m[], const int rows, const int cols,
|
|
const double edge, const int star)
|
|
{
|
|
const double vol = 0.01;
|
|
|
|
for(int t = 0; t < rows; t++)
|
|
{
|
|
const int base = t * cols;
|
|
for(int n = 0; n < cols; n++)
|
|
m[base + n] = vol * Gauss() + ((n == star) ? edge : 0.0);
|
|
}
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Run one case and report. |
|
|
//+------------------------------------------------------------------+
|
|
void RunCase(const string label, const double edge, const int star,
|
|
const double pboLow, const double pboHigh)
|
|
{
|
|
double m[];
|
|
if(ArrayResize(m, InpRows * InpCols) != InpRows * InpCols)
|
|
{
|
|
Print("cannot allocate matrix");
|
|
return;
|
|
}
|
|
|
|
MathSrand(InpSeed);
|
|
BuildMatrix(m, InpRows, InpCols, edge, star);
|
|
|
|
CCSCVEngine engine;
|
|
if(!engine.SetPartitions(InpPartitions)) return;
|
|
if(!engine.SetReturns(m, InpRows, InpCols)) return;
|
|
if(!engine.Run()) return;
|
|
|
|
double logits[];
|
|
engine.GetLogits(logits);
|
|
|
|
double mean, sd;
|
|
MeanSd(logits, mean, sd);
|
|
|
|
const double pbo = engine.PBO();
|
|
const bool passed = (pbo >= pboLow && pbo <= pboHigh);
|
|
|
|
PrintFormat("--- %s", label);
|
|
PrintFormat(" PBO = %6.2f%% (expected %.0f%% to %.0f%%) %s",
|
|
pbo, pboLow, pboHigh, passed ? "PASS" : "FAIL");
|
|
PrintFormat(" probability of loss = %6.2f%%", engine.ProbabilityOfLoss());
|
|
PrintFormat(" logits: mean %+.3f, sd %.3f, count %d",
|
|
mean, sd, engine.CombinationCount());
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Script entry point |
|
|
//+------------------------------------------------------------------+
|
|
void OnStart(void)
|
|
{
|
|
PrintFormat("CSCV validation: %d bars, %d trials, S = %d, seed %d",
|
|
InpRows, InpCols, InpPartitions, InpSeed);
|
|
|
|
// No skill anywhere. The in-sample winner is a coin toss out of sample,
|
|
// so roughly half the combinations should land below the median.
|
|
RunCase("CASE A - pure noise", 0.0, -1, 40.0, 60.0);
|
|
|
|
// One strategy is genuinely better, in every partition.
|
|
// The engine must recognise it and report almost no overfitting.
|
|
RunCase("CASE B - one persistent winner", 0.004, 7, 0.0, 2.0);
|
|
|
|
Print("--- done. Case A near 50% and case B near 0% means the arithmetic holds.");
|
|
}
|
|
//+------------------------------------------------------------------+
|