dng/NN_in_Trading
Vigiar
2
4
Derivar
Você já tinha feito uma derivação do repositório NN_in_Trading, anteriormente
 4
Código Questões Pedidos de integração Projetos Lançamentos Pacotes Wiki Trabalho Operações
NN_in_Trading/Experts/OMPB/Study.mq5

735 linhas
34 KiB
MQL5
Em brutoLigação permanenteVista normalHistórico

OMPB
2026-09-04 17:41:04 +03:00
//+------------------------------------------------------------------+
//| Study.mq5 |
//+------------------------------------------------------------------+
#property copyright "Copyright DNG®"
#property link "https://www.mql5.com/ru/users/dng"
#property version "1.00"
#property strict
OMPB
2026-09-14 09:56:18 +03:00
#define D2SKILL
OMPB
2026-09-04 17:41:04 +03:00
#include "Trajectory.mqh"
//---
input group "---- D2Skill training ----"
input ENUM_D2SKILL_STAGE InpD2SkillStage = D2Skill_D2_STAGE_FORMATION; //Lifecycle stage
input ENUM_D2SKILL_MODE InpD2SkillExecutionMode = D2_COLLECT; //Bank execution mode
input ED2SkillRepresentation InpD2SkillRepresentation = D2SkillDirectionMagnitude; //Bank representation
OMPB
2026-09-14 09:56:18 +03:00
input bool InpD2SkillResetBanksOnRepresentationMismatch = false; //Reset incompatible banks
input bool InpD2SkillRecreateIncompatibleCheckpoint = false; //Recreate incompatible policy checkpoint
OMPB
2026-09-04 17:41:04 +03:00
//---
input group "---- OMPB production checkpoint ----"
input ENUM_OMPB_STAGE InpOMPBStage = OMPB_STAGE_BASE_POLICY; //Must be Stage 03
//---
input group "---- Actor-Critic training ----"
input datetime Start = D'2024.01.01'; //Training period start
input datetime End = D'2026.01.01'; //Training period end
input int Iterations = 1000000; //Training iterations
input int EpisodeBars = 2 * StackSize; //Bars per episode
input double MinBalance = 50.0; //Minimum account balance
input int UpdatePolicy = 5; //Actor update interval
OMPB
2026-09-14 09:56:18 +03:00
//---
OMPB
2026-09-04 17:41:04 +03:00
int Epochs = 0;
OMPB
2026-09-14 09:56:18 +03:00
CNet Actor;
CNet Q1;
CNet Q2;
CBufferFloat State;
CBufferFloat TimeState;
CBufferFloat Account;
CBufferFloat NextAccount;
CBufferFloat CurrentAction;
CBufferFloat BaselineAction;
CBufferFloat BaselineAccount;
CBufferFloat BaselineNextAccount;
CBufferFloat TeacherAction;
CBufferFloat RandomAction;
CBufferFloat CriticInput;
CBufferFloat ScalarTarget;
CBufferFloat ActorObjective;
ulong TeacherTransitions = 0;
ulong TeacherCriticTransitions = 0;
ulong RandomCriticTransitions = 0;
double TeacherRewardSum = 0;
ulong PolicyTransitions = 0;
ulong PolicySkippedInvalid = 0;
ulong PairedTransitions = 0;
double PairedDeltaSum = 0;
SD2SkillActorForwardState ActorForwardState;
SD2SkillActorForwardState BaselineActorForwardState;
SD2SkillEpisodeOutcome BaselineEpisodeOutcome;
SD2SkillEpisodeOutcome SkillEpisodeOutcome;
OMPB
2026-09-04 17:41:04 +03:00
OMPB
2026-09-14 09:56:18 +03:00
bool PairedEpisodeActive = false;
bool Stage03StopRequestedLogged = false;
OMPB
2026-09-04 17:41:04 +03:00
//+------------------------------------------------------------------+
//| Logs one user-requested Stage 03 stop without misclassifying it. |
//+------------------------------------------------------------------+
void D2SkillLogStage03StopRequested(const string phase)
{
if(Stage03StopRequestedLogged)
return;
PrintFormat("OMPB_STAGE03_STOP_REQUESTED phase=%s", phase);
Stage03StopRequestedLogged = true;
}
//+------------------------------------------------------------------+
//| Resets only D2Skill episode influence, preserving bank params. |
//+------------------------------------------------------------------+
bool ResetD2SkillEpisodeInfluence(CNet &actor)
{
CNeuronBaseOCL *layer = actor.Layer(1);
if(!layer || layer.Type() != defNeuronD2Skill)
ReturnFalse;
CD2Skill *skill = (CD2Skill*)layer;
if(!skill.Ready() || !skill.ResetEpisodeInfluence())
ReturnFalse;
return(true);
}
//+------------------------------------------------------------------+
//| Adds one simulated account transition to its own episode result.|
//+------------------------------------------------------------------+
bool AccumulateEpisodeOutcome(SD2SkillEpisodeOutcome &outcome,
CBufferFloat *account,
const double reward)
{
if(!account || account.Total() != AccountDescr || !MathIsValidNumber(reward) ||
(account.GetIndex() >= 0 && !account.BufferRead()))
ReturnFalse;
const double balance = MathMax(0.0, double(account[0]) * EtalonBalance);
const double equity = MathMax(0.0, double(account[2]) * MathMax(balance, 1.0));
const double drawdown = MathMin(0.0, double(account[1]));
const double cost = MathMax(0.0, -reward);
const double risk = MathMax(0.0, double(account[4]) + double(account[5]));
return(outcome.Accumulate(reward, balance, equity, drawdown, cost, 1, risk));
}
//+------------------------------------------------------------------+
//| Loads OrCreateD2SkillPolicies. |
//+------------------------------------------------------------------+
bool LoadOrCreateD2SkillPolicies(void)
{
return(D2SkillLoadOrCreatePolicySet(Actor, Q1, Q2, true));
}
//+------------------------------------------------------------------+
//| Saves D2SkillPolicies. |
//+------------------------------------------------------------------+
bool SaveD2SkillPolicies(void)
{
//--- Forecast artifacts are immutable throughout Actor-Critic study. This
//--- bounded lifecycle check avoids hashing three large files every batch.
return(D2SkillSaveStage03StopCheckpoint(Actor, Q1, Q2));
}
//+------------------------------------------------------------------+
//| Implements PrepareHistory. |
//+------------------------------------------------------------------+
bool PrepareHistory(int &first_position, int &last_position)
{
int start = iBarShift(Symb.Name(), TimeFrame, Start);
int end = iBarShift(Symb.Name(), TimeFrame, End);
int bars = CopyRates(Symb.Name(), TimeFrame, 0, start, Rates);
if(bars <= 0 || !RSI.BufferResize(bars) || !CCI.BufferResize(bars) ||
!ATR.BufferResize(bars) || !MACD.BufferResize(bars))
ReturnFalse;
int wait = -1;
bool calculated = false;
do
{
calculated = (RSI.BarsCalculated() >= bars && CCI.BarsCalculated() >= bars &&
ATR.BarsCalculated() >= bars && MACD.BarsCalculated() >= bars);
Sleep(100);
wait++;
}
while(!calculated && wait < 100);
if(!calculated)
ReturnFalse;
RSI.Refresh();
CCI.Refresh();
ATR.Refresh();
MACD.Refresh();
if(!ArraySetAsSeries(Rates, true))
ReturnFalse;
first_position = end + 1;
//--- CreateBuffers(position,...,forecast) forms its state from
//--- position+NForecast and needs HistoryBars bars behind it. position is
//--- therefore the end of the realized future window, not its first bar.
last_position = start - HistoryBars - NForecast;
return(last_position > first_position);
}
//+------------------------------------------------------------------+
//| Implements PolicyBackward. |
//+------------------------------------------------------------------+
bool PolicyBackward(void)
{
//--- PolicyBackward builds Q+1 in this buffer. ScalarTarget must remain the
//--- immutable TD target consumed by both Critic backward passes.
return(PolicyBackward(Actor, Q1, GetPointer(ActorObjective), GetPointer(D2SkillMarket), -1));
}
//+------------------------------------------------------------------+
//| Builds one causal baseline/skill pair on the same state. |
//+------------------------------------------------------------------+
bool BuildPairedTransition(const int position, const bool trace_td, bool &terminal,
double &reward, double &delta_j, bool &paired,
bool &baseline_terminal)
{
terminal = false;
reward = 0;
delta_j = 0;
paired = false;
baseline_terminal = false;
CNeuronBaseOCL *layer = Actor.Layer(1);
CD2Skill *skill = (layer && layer.Type() == defNeuronD2Skill ? (CD2Skill*)layer : NULL);
paired = D2SkillUsePairedHindsight(skill, ActorForwardState);
bool task_enabled = false;
bool step_enabled = false;
D2SkillD2BankFlags(task_enabled, step_enabled);
static bool pair_setup_logged = false;
if(!pair_setup_logged)
{
PrintFormat("D2Skill paired setup stage=%d layer_type=%d skill_ready=%s paired=%s",
D2SkillD2Stage, (layer ? layer.Type() : -1),
(skill != NULL && skill.Ready() ? "true" : "false"),
(paired ? "true" : "false"));
pair_setup_logged = true;
}
double baseline_reward = 0;
if(paired)
{
if(!D2SkillActorForwardStateReady(BaselineActorForwardState) ||
!D2SkillRestorePairedActorState(BaselineActorForwardState, skill,
false, false, "baseline_start"))
{
Print("TrainTransition stage=baseline_restore");
ReturnFalse;
}
if(!Actor.feedForward(GetPointer(BaselineAccount), 1, false, GetPointer(D2SkillMarket), -1) ||
!ReadAction(Actor, GetPointer(BaselineAction)) ||
!AdvanceAccount(GetPointer(BaselineAccount), GetPointer(BaselineAction), position, MinBalance,
GetPointer(BaselineNextAccount), baseline_reward, baseline_terminal) ||
!D2SkillCaptureActorForwardState(BaselineActorForwardState) ||
!AccumulateEpisodeOutcome(BaselineEpisodeOutcome, GetPointer(BaselineNextAccount),
baseline_reward))
{
D2SkillRestorePairedActorState(ActorForwardState, skill,
task_enabled, step_enabled,
"baseline_failure");
Print("TrainTransition stage=baseline_transition");
ReturnFalse;
}
if(!D2SkillRestorePairedActorState(ActorForwardState, skill,
task_enabled, step_enabled,
"baseline_to_skill"))
{ Print("TrainTransition stage=restore_skill_state"); ReturnFalse; }
}
if(!Actor.feedForward(GetPointer(Account), 1, false, GetPointer(D2SkillMarket), -1) ||
!ReadAction(Actor, GetPointer(CurrentAction)) ||
!AdvanceAccount(GetPointer(Account), GetPointer(CurrentAction), position, MinBalance,
GetPointer(NextAccount), reward, terminal) ||
(paired && (!D2SkillCaptureActorForwardState(ActorForwardState) ||
!AccumulateEpisodeOutcome(SkillEpisodeOutcome, GetPointer(NextAccount), reward))))
{
if(paired)
D2SkillRestorePairedActorState(ActorForwardState, skill,
task_enabled, step_enabled,
"skill_failure");
Print("TrainTransition stage=skill_transition");
ReturnFalse;
}
if(paired)
{
delta_j = reward - baseline_reward;
if(!MathIsValidNumber(delta_j))
{
D2SkillRestorePairedActorState(ActorForwardState, skill,
task_enabled, step_enabled,
"nonfinite_delta");
Print("TrainTransition stage=paired_delta_nonfinite");
ReturnFalse;
}
if(trace_td)
PrintFormat("D2Skill paired iteration=%d JBase=%.8f JSkill=%.8f DeltaJ=%.8f baseline_terminal=%s " +
"skill_terminal=%s",
position, baseline_reward, reward, delta_j,
(baseline_terminal ? "true" : "false"), (terminal ? "true" : "false"));
}
return(true);
}
//+------------------------------------------------------------------+
//| Implements TrainTransition. |
//+------------------------------------------------------------------+
bool TrainTransition(const int position, const int iteration, bool &skill_terminal,
bool &pair_terminal)
{
skill_terminal = false;
pair_terminal = false;
const bool trace_td = (iteration < 8 || iteration % 10000 == 0);
if(!D2SkillForwardForecast(position, GetPointer(State), GetPointer(TimeState)))
{ Print("TrainTransition stage=current_forecast"); ReturnFalse; }
double reward = 0;
double delta_j = 0;
bool paired = false;
bool baseline_terminal = false;
if(!BuildPairedTransition(position, trace_td, skill_terminal, reward, delta_j, paired,
baseline_terminal))
ReturnFalse;
pair_terminal = (paired && D2SkillPairReachedTerminal(baseline_terminal,
skill_terminal));
const double balance = MathMax(0.0, double(Account[0]) * EtalonBalance);
const double buy_lot = MathMax(0.0, double(CurrentAction[0] - CurrentAction[3]));
const double sell_lot = MathMax(0.0, double(CurrentAction[3] - CurrentAction[0]));
if(trace_td)
PrintFormat("D2Skill action iteration=%d balance=%.2f buy_lot=%.8f buy_tp=%.8f buy_sl=%.8f " +
"sell_lot=%.8f sell_tp=%.8f sell_sl=%.8f",
iteration, balance, buy_lot, double(CurrentAction[1]), double(CurrentAction[2]),
sell_lot, double(CurrentAction[4]), double(CurrentAction[5]));
//--- Utility is emitted only at the common episode boundary. The two branch
//--- outcomes are accumulated independently, never inferred from this reward.
if(paired && trace_td)
PrintFormat("D2Skill paired transition base_reward=%.8f skill_reward=%.8f",
reward - delta_j, reward);
const double target_value = reward;
if(trace_td)
PrintFormat("D2Skill return iteration=%d reward=%.8f target=%.8f balance=%.2f equity=%.2f "
"floating_buy=%.8f floating_sell=%.8f next_buy=%.8f next_sell=%.8f terminal=%s",
iteration, reward, target_value, double(NextAccount[0])*EtalonBalance,
double(NextAccount[2]) * balance,
double(NextAccount[6]) * MathMax(double(NextAccount[0]) * EtalonBalance, 1.0),
double(NextAccount[7]) * MathMax(double(NextAccount[0]) * EtalonBalance, 1.0),
double(NextAccount[4]), double(NextAccount[5]),
(skill_terminal ? "true" : "false"));
if(!MathIsValidNumber(target_value) || !ScalarTarget.BufferInit(1, 0) ||
!ScalarTarget.Update(0, float(target_value)))
{ Print("TrainTransition stage=scalar_target"); ReturnFalse; }
CNeuronBaseOCL *context_layer = Actor.Layer(0);
CNeuronBaseOCL *actor_layer = Actor.Layer(3);
if(!context_layer || !actor_layer)
{ Print("TrainTransition stage=actor_layer"); ReturnFalse; }
//--- Use the live device activation, not the CPU account object.
if(!BuildCriticInput(context_layer.getOutput(), actor_layer.getOutput(), GetPointer(CriticInput)))
{ Print("TrainTransition stage=critic_input"); ReturnFalse; }
if(!Q1.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1))
{ Print("TrainTransition stage=q1_forward"); ReturnFalse; }
if(!Q2.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1))
{ Print("TrainTransition stage=q2_forward"); ReturnFalse; }
//--- Emit current critic values only for the configured trace interval.
if(trace_td)
{
CNeuronBaseOCL *q1_layer = Q1.Layer(3);
CNeuronBaseOCL *q2_layer = Q2.Layer(3);
CBufferFloat *q1_output = (q1_layer ? q1_layer.getOutput() : NULL);
CBufferFloat *q2_output = (q2_layer ? q2_layer.getOutput() : NULL);
if(!q1_output || !q2_output || q1_output.GetIndex() < 0 || q2_output.GetIndex() < 0 ||
q1_output.Total() != 1 || q2_output.Total() != 1 ||
!q1_output.BufferRead() || !q2_output.BufferRead())
{ Print("TrainTransition stage=current_q_read"); ReturnFalse; }
PrintFormat("D2Skill Q iteration=%d q1=%.8f q2=%.8f td_q1=%.8f td_q2=%.8f",
iteration, double(q1_output[0]), double(q2_output[0]),
target_value - double(q1_output[0]), target_value - double(q2_output[0]));
}
//--- A critic gradient has no valid direction on the discontinuous no-trade
//--- plateau. Keep learning Q there, but let only the realized-future teacher
//--- move Actor back into the executable action manifold.
const bool executable_action = (IsExecutableOrder(buy_lot, CurrentAction[1], CurrentAction[2]) ||
IsExecutableOrder(sell_lot, CurrentAction[4], CurrentAction[5]));
//--- Apply the Actor policy update at its configured cadence.
if(iteration > 0 && UpdatePolicy > 0 && iteration % UpdatePolicy == 0)
{
//--- Skip policy backpropagation for a discontinuous no-trade action.
if(executable_action)
{
if(!PolicyBackward())
{ Print("TrainTransition stage=policy_backward"); ReturnFalse; }
PolicyTransitions++;
}
else
PolicySkippedInvalid++;
}
const bool q1_backward = Q1.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!q1_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("primary_critic_backward");
return(false);
}
const int error = GetLastError();
PrintFormat("OMPB_STAGE03_CRITIC_BACKWARD_FAIL critic=Q1 iteration=%u position=%u error=%d",
(uint)iteration, (uint)position, error);
ReturnFalse;
}
const bool q2_backward = Q2.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!q2_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("primary_critic_backward");
return(false);
}
const int error = GetLastError();
PrintFormat("OMPB_STAGE03_CRITIC_BACKWARD_FAIL critic=Q2 iteration=%u position=%u error=%d",
(uint)iteration, (uint)position, error);
ReturnFalse;
}
//--- The teacher uses realized future bars, not one selected Forecast scenario.
//--- It is deliberately a second Critic-only sample: Q1/Q2 are stateless,
//--- whereas Actor has already performed its only forward for this state.
double teacher_reward = 0;
if(!BuildTeacherAction(position, GetPointer(Account), GetPointer(TeacherAction), teacher_reward))
{ Print("TrainTransition stage=teacher_action"); ReturnFalse; }
//--- Supervise Actor only with a profitable realized-future teacher action.
if(teacher_reward > 0)
{
if(trace_td)
PrintFormat("D2Skill teacher iteration=%d reward=%.8f buy_lot=%.8f sell_lot=%.8f",
iteration, teacher_reward,
MathMax(0.0, double(TeacherAction[0] - TeacherAction[3])),
MathMax(0.0, double(TeacherAction[3] - TeacherAction[0])));
//--- Supervised Actor update reuses the current activation; it never advances
//--- the history stack or invokes a second Actor forward.
if(!Actor.backProp(GetPointer(TeacherAction), GetPointer(D2SkillMarket), -1))
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("teacher_actor_backward");
return(false);
}
Print("TrainTransition stage=teacher_actor_backward");
ReturnFalse;
}
TeacherTransitions++;
TeacherRewardSum += teacher_reward;
}
//--- Critic receives every oracle action, including a losing one. Only Actor
//--- is restricted to profitable oracle supervision above.
if(!ScalarTarget.BufferInit(1, 0) || !ScalarTarget.Update(0, float(teacher_reward)) ||
!BuildCriticInput(context_layer.getOutput(), GetPointer(TeacherAction), GetPointer(CriticInput)))
{ Print("TrainTransition stage=teacher_critic_input"); ReturnFalse; }
if(!Q1.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1) ||
!Q2.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1))
{ Print("TrainTransition stage=teacher_critic_forward"); ReturnFalse; }
const bool teacher_q1_backward = Q1.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!teacher_q1_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("teacher_critic_backward");
return(false);
}
Print("TrainTransition stage=teacher_critic_backward");
ReturnFalse;
}
const bool teacher_q2_backward = Q2.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!teacher_q2_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("teacher_critic_backward");
return(false);
}
Print("TrainTransition stage=teacher_critic_backward");
ReturnFalse;
}
TeacherCriticTransitions++;
//--- A second critic-only sample is uniformly random over executable lots and
//--- stop distances. It never runs Actor backward or advances policy history.
double random_reward = 0;
if(!BuildRandomAction(position, GetPointer(Account), GetPointer(RandomAction), random_reward))
{ Print("TrainTransition stage=random_action"); ReturnFalse; }
//--- Supervise Actor only with a profitable randomized teacher action.
if(random_reward > 0)
{
if(!Actor.backProp(GetPointer(RandomAction), GetPointer(D2SkillMarket), -1))
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("random_actor_backward");
return(false);
}
Print("TrainTransition stage=random_actor_backward");
ReturnFalse;
}
}
if(!ScalarTarget.BufferInit(1, 0) || !ScalarTarget.Update(0, float(random_reward)) ||
!BuildCriticInput(context_layer.getOutput(), GetPointer(RandomAction), GetPointer(CriticInput)))
{ Print("TrainTransition stage=random_critic_input"); ReturnFalse; }
if(!Q1.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1) ||
!Q2.feedForward(GetPointer(CriticInput), 1, false, GetPointer(D2SkillMarket), -1))
{ Print("TrainTransition stage=random_critic_forward"); ReturnFalse; }
const bool random_q1_backward = Q1.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!random_q1_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("random_critic_backward");
return(false);
}
Print("TrainTransition stage=random_critic_backward");
ReturnFalse;
}
const bool random_q2_backward = Q2.backProp(GetPointer(ScalarTarget), GetPointer(D2SkillMarket), -1);
if(!random_q2_backward)
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("random_critic_backward");
return(false);
}
Print("TrainTransition stage=random_critic_backward");
ReturnFalse;
}
RandomCriticTransitions++;
//--- Only the three declared Forecast trainable buffers are read here. The
//--- complete persistent Codebook is checked at checkpoint/lifecycle bounds.
if(!D2SkillVerifyFrozenWeightsExact())
{ Print("TrainTransition stage=frozen_forecast_check"); ReturnFalse; }
return(true);
}
//+------------------------------------------------------------------+
//| Implements TrainD2SkillActorCritic. |
//+------------------------------------------------------------------+
void TrainD2SkillActorCritic(void)
{
Stage03StopRequestedLogged = false;
int first = 0, last = 0;
if(!PrepareHistory(first, last))
{ PrintFormat("%s -> %d history unavailable", __FUNCTION__, __LINE__); return; }
uint shown = GetTickCount();
int completed_iterations = 0;
int failed_iteration = -1;
int failed_position = -1;
int position = last;
int episode = 0;
bool episode_reset_failed = false;
TeacherTransitions = 0;
TeacherCriticTransitions = 0;
RandomCriticTransitions = 0;
TeacherRewardSum = 0;
PolicyTransitions = 0;
PolicySkippedInvalid = 0;
PairedTransitions = 0;
PairedDeltaSum = 0;
for(int iteration = 0; iteration < Iterations && !IsStopped(); iteration++)
{
//--- Create a fresh paired episode and seed both account states.
if(episode == 0)
{
if(!ResetD2SkillEpisodeInfluence(Actor) ||
!D2SkillForwardForecast(position, GetPointer(State), GetPointer(TimeState)))
break;
const vector<float> sampled = SampleAccount(GetPointer(State), Rates[position].time,
EtalonBalance, MinBalance);
if(sampled.Size() != AccountDescr || !Account.AssignArray(sampled) ||
(Account.GetIndex() >= 0 && !Account.BufferWrite()) ||
!BaselineAccount.AssignArray(GetPointer(Account)) ||
(BaselineAccount.GetIndex() >= 0 && !BaselineAccount.BufferWrite()) ||
!BaselineEpisodeOutcome.Reset() || !SkillEpisodeOutcome.Reset() || !Actor.Clear())
break;
CNeuronBaseOCL *layer = Actor.Layer(1);
CD2Skill *skill = (layer && layer.Type() == defNeuronD2Skill ? (CD2Skill*)layer : NULL);
PairedEpisodeActive = (D2SkillUsePairedHindsight(skill, ActorForwardState) &&
D2SkillActorForwardStateReady(BaselineActorForwardState));
if(PairedEpisodeActive && (!D2SkillCaptureActorForwardState(ActorForwardState) ||
!D2SkillCaptureActorForwardState(BaselineActorForwardState)))
break;
}
bool skill_terminal = false;
bool pair_terminal = false;
//--- Stop the loop cleanly when a transition fails or manual stop arrives.
if(!TrainTransition(position, iteration, skill_terminal, pair_terminal))
{
if(IsStopped())
{
D2SkillLogStage03StopRequested("train_transition");
break;
}
failed_iteration = iteration;
failed_position = position;
if(!ResetD2SkillEpisodeInfluence(Actor))
episode_reset_failed = true;
break;
}
completed_iterations++;
if((NextAccount.GetIndex() >= 0 && !NextAccount.BufferRead()) || !Account.AssignArray(GetPointer(NextAccount)) ||
(Account.GetIndex() >= 0 && !Account.BufferWrite()) ||
(PairedEpisodeActive &&
((BaselineNextAccount.GetIndex() >= 0 && !BaselineNextAccount.BufferRead()) ||
!BaselineAccount.AssignArray(GetPointer(BaselineNextAccount)) ||
(BaselineAccount.GetIndex() >= 0 && !BaselineAccount.BufferWrite()))))
{
failed_iteration = iteration;
failed_position = position;
if(!ResetD2SkillEpisodeInfluence(Actor))
episode_reset_failed = true;
break;
}
position--;
episode++;
//--- Close an episode on range exhaustion or its configured bar limit.
const bool episode_limit = (position < first ||
episode >= MathMax(1, EpisodeBars));
double terminal_delta = 0.0;
bool utility_applied = false;
bool influence_reset = false;
bool episode_closed = false;
if(!D2SkillCloseOfflineEpisode(Actor, PairedEpisodeActive, pair_terminal,
episode_limit, BaselineEpisodeOutcome,
SkillEpisodeOutcome, terminal_delta,
utility_applied, influence_reset,
episode_closed))
{
failed_iteration = iteration;
failed_position = position;
episode_reset_failed = true;
break;
}
if(episode_closed && !influence_reset)
{
failed_iteration = iteration;
failed_position = position;
episode_reset_failed = true;
break;
}
if(episode_closed)
{
if(PairedEpisodeActive)
{
PairedTransitions++;
PairedDeltaSum += terminal_delta;
PrintFormat("D2Skill paired terminal JBase=%.8f JSkill=%.8f DeltaJ=%.8f utility_mutated=%s " +
"base_balance=%.2f skill_balance=%.2f base_duration=%u skill_duration=%u",
BaselineEpisodeOutcome.Outcome(), SkillEpisodeOutcome.Outcome(), terminal_delta,
(utility_applied ? "true" : "false"),
BaselineEpisodeOutcome.Balance(), SkillEpisodeOutcome.Balance(),
BaselineEpisodeOutcome.Duration(), SkillEpisodeOutcome.Duration());
}
if(position < first)
position = last;
episode = 0;
}
//--- Refresh the progress panel without delaying training updates.
if(GetTickCount() - shown > 500)
{
const double q1_rmse = MathSqrt(MathMax(0.0, double(Q1.getRecentAverageError()))) /
MathMax(EtalonBalance, 1.0);
const double q2_rmse = MathSqrt(MathMax(0.0, double(Q2.getRecentAverageError()))) /
MathMax(EtalonBalance, 1.0);
const double teacher_mean = (TeacherTransitions > 0 ? TeacherRewardSum / double(TeacherTransitions) : 0.0);
Comment(StringFormat("D2Skill AC %6.2f%% Q1 rRMSE %.8f Q2 rRMSE %.8f teacherA %I64u " +
"teacherQ %I64u randomQ %I64u R %.2f policy %I64u skip %I64u",
100.0 * iteration / MathMax(Iterations, 1), q1_rmse, q2_rmse,
TeacherTransitions, TeacherCriticTransitions, RandomCriticTransitions,
teacher_mean, PolicyTransitions, PolicySkippedInvalid));
shown = GetTickCount();
}
}
if(IsStopped())
{
D2SkillLogStage03StopRequested("training_loop");
Comment("");
if(!ResetD2SkillEpisodeInfluence(Actor))
{
Print("OMPB_STAGE03_CHECKPOINT_FAIL reason=manual_stop_reset");
return;
}
ResetLastError();
if(!SaveD2SkillPolicies())
{
const int error = GetLastError();
PrintFormat("OMPB_STAGE03_CHECKPOINT_FAIL reason=manual_stop error=%d", error);
return;
}
Print("OMPB_STAGE03_CHECKPOINT_PASS reason=manual_stop");
return;
}
if(!ResetD2SkillEpisodeInfluence(Actor))
episode_reset_failed = true;
Comment("");
//--- Reject publication unless every requested iteration completed safely.
if(completed_iterations != Iterations || episode_reset_failed)
{
if(episode_reset_failed)
PrintFormat("%s -> %d publication aborted: episode influence reset failed completed=%d/%d",
__FUNCTION__, __LINE__, completed_iterations, Iterations);
else
if(failed_iteration >= 0)
PrintFormat("%s -> %d publication aborted: TrainTransition failed iteration=%d position=%d completed=%d/%d",
__FUNCTION__, __LINE__, failed_iteration, failed_position, completed_iterations, Iterations);
else
PrintFormat("%s -> %d publication aborted: stop requested completed=%d/%d",
__FUNCTION__, __LINE__, completed_iterations, Iterations);
return;
}
if(!SaveD2SkillPolicies())
PrintFormat("%s -> %d save failed", __FUNCTION__, __LINE__);
if(PairedTransitions > 0)
PrintFormat("D2Skill paired summary transitions=%I64u mean_delta_j=%.8f",
PairedTransitions, PairedDeltaSum / double(PairedTransitions));
}
//+------------------------------------------------------------------+
//| Initializes the Stage 03 BasePolicy Expert. |
//+------------------------------------------------------------------+
int OnInit()
{
ResetLastError();
//--- Confirm that Forecast buffers and the OMPB posterior remain frozen.
if(InpOMPBStage != OMPB_STAGE_BASE_POLICY ||
!D2SkillConfigureRuntime(InpD2SkillStage, InpD2SkillMode,
InpD2SkillExecutionMode, InpD2SkillUtilityAware,
InpD2SkillMinUtility, InpD2SkillUtilityScale,
false, false, InpD2SkillRepresentation,
InpD2SkillResetBanksOnRepresentationMismatch,
InpD2SkillRecreateIncompatibleCheckpoint) ||
!D2SkillInitIndicators() || !D2SkillLoadForecastInference() ||
!D2SkillConfigureProductionOMPBCheckpoint() ||
!D2SkillValidateProductionOMPBCheckpoint() ||
!D2SkillCaptureProductionOMPBFingerprints() || !LoadOrCreateD2SkillPolicies() ||
!D2SkillConfigureActorCriticUpdates(Actor, Q1, Q2) ||
!D2SkillVerifyFrozenForecastExact() ||
!D2SkillVerifyProductionOMPBFingerprints())
{
PrintFormat("D2Skill Actor-Critic initialization failed at line %d error=%d", __LINE__, GetLastError());
return(INIT_FAILED);
}
if(!D2SkillConfigureD2UtilityMode(D2Skill_D2_UTILITY_PAIRED_HINDSIGHT) ||
((D2SkillD2ExecutionMode == D2_EVALUATE ||
D2SkillD2ExecutionMode == D2_ONLINE_CALIBRATION) &&
D2SkillD2Mode != D2Skill_D2_MODE_BASE &&
(!D2SkillInitActorForwardState(ActorForwardState, Actor) ||
!D2SkillInitActorForwardState(BaselineActorForwardState, Actor))))
{
PrintFormat("D2Skill paired snapshot initialization failed at line %d error=%d",
__LINE__, GetLastError());
return(INIT_FAILED);
}
if(!EventSetMillisecondTimer(1))
{
PrintFormat("D2Skill Actor-Critic timer initialization failed at line %d error=%d",
__LINE__, GetLastError());
return(INIT_FAILED);
}
//--- Finalize initialization only after the Stage 03 timer is armed.
return(INIT_SUCCEEDED);
}
//+------------------------------------------------------------------+
//| Function OnDeinit. |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
{
EventKillTimer();
//--- Only a completed explicit training loop persists all six artifacts and
//--- writes the hash manifest last. Deinit must not create a mixed generation.
if(D2SkillFrozenBaselineReady && D2SkillForecast != NULL && !D2SkillVerifyFrozenForecastExact())
PrintFormat("%s -> %d forecast mutation", __FUNCTION__, __LINE__);
if(D2SkillProductionSignatureReady && !D2SkillVerifyProductionOMPBFingerprints())
PrintFormat("%s -> %d OMPB production signature mutation", __FUNCTION__, __LINE__);
D2SkillForecast = NULL;
}
//+------------------------------------------------------------------+
//| Runs the one-shot training lifecycle. |
//+------------------------------------------------------------------+
void OnTimer(void)
{
TrainD2SkillActorCritic();
ExpertRemove();
}
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
Criar uma nova questão referindo estaCopiar ligação permanente