forked from mnbvc188199/Warrior_EA
413 lines
24 KiB
MQL5
413 lines
24 KiB
MQL5
//+------------------------------------------------------------------+
| |
//| Trajectory.mqh |
| |
//| Copyright DNG® |
| |
//| https://www.mql5.com/en/users/dng |
| |
//+------------------------------------------------------------------+
| |
#property copyright "Copyright DNG®"
| |
#property link "https://www.mql5.com/en/users/dng"
| |
#property version "1.00"
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
#include "..\RL\FQF.mqh"
| |
//---
| |
#define HistoryBars 20 //Depth of history
| |
#define BarDescr 12 //Elements for 1 bar description
| |
#define AccountDescr 8 //Account description
| |
#define NActions 4 //Number of possible Actions
| |
#define NSkills 20 //Number of skills
| |
#define Buffer_Size 6500
| |
#define FileName "GCRL"
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
struct SState
| |
{
| |
float state[HistoryBars * BarDescr];
| |
float account[AccountDescr-1];
| |
//---
| |
SState(void);
| |
//---
| |
bool Save(int file_handle);
| |
bool Load(int file_handle);
| |
//--- overloading
| |
void operator=(const SState &obj) { ArrayCopy(state, obj.state); ArrayCopy(account, obj.account); }
| |
};
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
SState::SState(void)
| |
{
| |
ArrayInitialize(state, 0);
| |
ArrayInitialize(account, 0);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool SState::Save(int file_handle)
| |
{
| |
if(file_handle == INVALID_HANDLE)
| |
return false;
| |
//---
| |
int total = ArraySize(state);
| |
if(FileWriteInteger(file_handle, total) < sizeof(int))
| |
return false;
| |
for(int i = 0; i < total; i++)
| |
if(FileWriteFloat(file_handle, state[i]) < sizeof(float))
| |
return false;
| |
//---
| |
total = ArraySize(account);
| |
if(FileWriteInteger(file_handle, total) < sizeof(int))
| |
return false;
| |
for(int i = 0; i < total; i++)
| |
if(FileWriteFloat(file_handle, account[i]) < sizeof(float))
| |
return false;
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool SState::Load(int file_handle)
| |
{
| |
if(file_handle == INVALID_HANDLE)
| |
return false;
| |
if(FileIsEnding(file_handle))
| |
return false;
| |
//---
| |
int total = FileReadInteger(file_handle);
| |
if(total != ArraySize(state))
| |
return false;
| |
//---
| |
for(int i = 0; i < total; i++)
| |
{
| |
if(FileIsEnding(file_handle))
| |
return false;
| |
state[i] = FileReadFloat(file_handle);
| |
}
| |
//---
| |
total = FileReadInteger(file_handle);
| |
if(total != ArraySize(account))
| |
return false;
| |
//---
| |
for(int i = 0; i < total; i++)
| |
{
| |
if(FileIsEnding(file_handle))
| |
return false;
| |
account[i] = FileReadFloat(file_handle);
| |
}
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
struct STrajectory
| |
{
| |
SState States[Buffer_Size];
| |
int Actions[Buffer_Size];
| |
float Revards[Buffer_Size];
| |
int Total;
| |
float DiscountFactor;
| |
bool CumCounted;
| |
//---
| |
STrajectory(void);
| |
//---
| |
bool Add(SState &state, int action, float reward);
| |
void CumRevards(void);
| |
//---
| |
bool Save(int file_handle);
| |
bool Load(int file_handle);
| |
};
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
STrajectory::STrajectory(void) : Total(0),
| |
DiscountFactor(0.9f),
| |
CumCounted(false)
| |
{
| |
ArrayInitialize(Actions, -1);
| |
ArrayInitialize(Revards, 0);
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool STrajectory::Save(int file_handle)
| |
{
| |
if(file_handle == INVALID_HANDLE)
| |
return false;
| |
//---
| |
if(!CumCounted)
| |
CumRevards();
| |
if(FileWriteInteger(file_handle, Total) < sizeof(int))
| |
return false;
| |
for(int i = 0; i < Total; i++)
| |
{
| |
if(!States[i].Save(file_handle))
| |
return false;
| |
if(FileWriteInteger(file_handle, Actions[i]) < sizeof(int))
| |
return false;
| |
if(FileWriteFloat(file_handle, Revards[i]) < sizeof(float))
| |
return false;
| |
}
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool STrajectory::Load(int file_handle)
| |
{
| |
if(file_handle == INVALID_HANDLE)
| |
return false;
| |
//---
| |
Total = FileReadInteger(file_handle);
| |
if(Total >= ArraySize(Actions))
| |
return false;
| |
CumCounted = true;
| |
//---
| |
for(int i = 0; i < Total; i++)
| |
{
| |
if(!States[i].Load(file_handle))
| |
return false;
| |
Actions[i] = FileReadInteger(file_handle);
| |
Revards[i] = FileReadFloat(file_handle);
| |
}
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
void STrajectory::CumRevards(void)
| |
{
| |
if(CumCounted)
| |
return;
| |
//---
| |
for(int i = Total - 2; i >= 0; i--)
| |
Revards[i] += Revards[i + 1] * DiscountFactor;
| |
CumCounted = true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool STrajectory::Add(SState &state, int action, float reward)
| |
{
| |
if(Total + 1 >= ArraySize(Actions))
| |
return false;
| |
States[Total] = state;
| |
Actions[Total] = action;
| |
if(Total > 0)
| |
Revards[Total - 1] = reward;
| |
Total++;
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
| |
//| |
| |
//+------------------------------------------------------------------+
| |
bool CreateDescriptions(CArrayObj *actor)
| |
{
| |
//---
| |
CLayerDescription *descr;
| |
//---
| |
if(!actor)
| |
{
| |
actor = new CArrayObj();
| |
if(!actor)
| |
return false;
| |
}
| |
//--- Actor
| |
actor.Clear();
| |
//--- Input layer
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
int prev_count = descr.count = (HistoryBars * BarDescr);
| |
descr.window = 0;
| |
descr.activation = None;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 1
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBatchNormOCL;
| |
descr.count = prev_count;
| |
descr.batch = 1000;
| |
descr.activation = None;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 2
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronConvOCL;
| |
prev_count = descr.count = prev_count - 1;
| |
descr.window = 2;
| |
descr.step = 1;
| |
descr.window_out = 4;
| |
descr.activation = LReLU;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 3
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronProofOCL;
| |
prev_count = descr.count = prev_count;
| |
descr.window = 4;
| |
descr.step = 4;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 4
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronConvOCL;
| |
prev_count = descr.count = prev_count - 1;
| |
descr.window = 2;
| |
descr.step = 1;
| |
descr.window_out = 4;
| |
descr.activation = LReLU;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 5
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronProofOCL;
| |
prev_count = descr.count = prev_count;
| |
descr.window = 4;
| |
descr.step = 4;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 6
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = 256;
| |
descr.optimization = ADAM;
| |
descr.activation = TANH;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 7
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = 128;
| |
descr.activation = LReLU;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 8
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = 2 * NSkills;
| |
descr.activation = None;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 9
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronVAEOCL;
| |
descr.count = NSkills;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 10
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronConcatenate;
| |
descr.count = 256;
| |
descr.window=prev_count;
| |
descr.step=AccountDescr;
| |
descr.optimization = ADAM;
| |
descr.activation = TANH;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 11
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = 256;
| |
descr.activation = LReLU;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 12
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = 256;
| |
descr.activation = LReLU;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 13
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronBaseOCL;
| |
descr.count = NActions;
| |
descr.optimization = ADAM;
| |
descr.activation=None;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//--- layer 14
| |
if(!(descr = new CLayerDescription()))
| |
return false;
| |
descr.type = defNeuronSoftMaxOCL;
| |
descr.count = NActions;
| |
descr.step=1;
| |
descr.optimization = ADAM;
| |
if(!actor.Add(descr))
| |
{
| |
delete descr;
| |
return false;
| |
}
| |
//---
| |
return true;
| |
}
| |
//+------------------------------------------------------------------+
|