//+------------------------------------------------------------------+ //| NeuronGPT.mqh | //| Copyright 2021, MetaQuotes Ltd. | //| https://www.mql5.com | //+------------------------------------------------------------------+ #property copyright "Copyright 2021, MetaQuotes Ltd." #property link "https://www.mql5.com" //+------------------------------------------------------------------+ //| Подключаем библиотеки | //+------------------------------------------------------------------+ #ifndef ArrayLayers #include "arraylayers.mqh" #endif //+------------------------------------------------------------------+ //| Class CNeuronGPT | //| Назначение: Класс оргпнизации GPT блока | //+------------------------------------------------------------------+ class CNeuronGPT : public CNeuronBase { protected: CArrayLayers *m_cQuerys; CArrayLayers *m_cKeys; CArrayLayers *m_cValues; CArrayLayers *m_cScores; CBufferDouble *m_cScoreTemp; CArrayLayers *m_cAttentionOut; CArrayLayers *m_cW0; CArrayLayers *m_cFF1; CArrayLayers *m_cFF2; //--- int m_iLayers; int m_iWindow; int m_iUnits; int m_iKeysSize; int m_iHeads; double m_dStd[][2]; int m_iCurrentPosition; bool CheckArrayLayers(CArrayLayers *&layers); public: CNeuronGPT(void); ~CNeuronGPT(void); //--- virtual bool Init(CLayerDescription *description); virtual bool SetOpenCL(CMyOpenCL *opencl); virtual bool FeedForward(CNeuronBase *prevLayer); virtual bool CalcHiddenGradient(CNeuronBase *prevLayer); virtual bool CalcDeltaWeights(CNeuronBase *prevLayer); virtual bool UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[]); //--- virtual int GetUnits(void) const { return m_iUnits; } virtual int GetLayers(void) const { return m_iLayers; } //--- methods for working with files virtual bool Save(const int file_handle); virtual bool Load(const int file_handle); //--- method of identifying the object virtual int Type(void) const { return(defNeuronGPT); } }; //+------------------------------------------------------------------+ //| Конструктор класса | //+------------------------------------------------------------------+ CNeuronGPT::CNeuronGPT(void) : m_iHeads(8), m_iWindow(0), m_iKeysSize(0), m_iUnits(0), m_iLayers(0), m_iCurrentPosition(0) { m_cQuerys = new CArrayLayers(); m_cKeys = new CArrayLayers(); m_cValues = new CArrayLayers(); m_cScores = new CArrayLayers(); m_cAttentionOut = new CArrayLayers(); m_cW0 = new CArrayLayers(); m_cFF1 = new CArrayLayers(); m_cFF2 = new CArrayLayers(); } //+------------------------------------------------------------------+ //| Деструктор класса | //+------------------------------------------------------------------+ CNeuronGPT::~CNeuronGPT(void) { if(CheckPointer(m_cQuerys) != POINTER_INVALID) delete m_cQuerys; if(CheckPointer(m_cKeys) != POINTER_INVALID) delete m_cKeys; if(CheckPointer(m_cValues) != POINTER_INVALID) delete m_cValues; if(CheckPointer(m_cScores) != POINTER_INVALID) delete m_cScores; if(CheckPointer(m_cScoreTemp) != POINTER_INVALID) delete m_cScoreTemp; if(CheckPointer(m_cAttentionOut) != POINTER_INVALID) delete m_cAttentionOut; if(CheckPointer(m_cW0) != POINTER_INVALID) delete m_cW0; if(CheckPointer(m_cFF1) != POINTER_INVALID) delete m_cFF1; if(CheckPointer(m_cFF2) != POINTER_INVALID) delete m_cFF2; m_iLayers = 0; } //+------------------------------------------------------------------+ //| Мутод инициализации класса | //+------------------------------------------------------------------+ bool CNeuronGPT::Init(CLayerDescription *description) { //--- Проверяем исходные данные if(CheckPointer(description) == POINTER_INVALID || description.type != Type() || description.count <= 0 || description.window <= 0 || description.window_out <= 0 || description.step <= 0 || description.layers <= 0) return false; //--- Сохраняем константы m_iWindow = description.window; m_iUnits = description.count; m_iKeysSize = description.window_out; m_iHeads = description.step; m_iLayers = description.layers; if(!ArrayResize(m_dStd, m_iLayers)) return false; //--- Вызываем метод инициализации родительского класса description.count *= m_iWindow; description.window_out = 1; description.window = 0; if(!CNeuronBase::Init(description)) return false; //--- Создаём динамические массивы для хранения указателей на объекты внутренних слоёв if(!CheckArrayLayers(m_cQuerys)) return false; if(!CheckArrayLayers(m_cKeys)) return false; if(!CheckArrayLayers(m_cValues)) return false; if(!CheckArrayLayers(m_cScores)) return false; if(!CheckArrayLayers(m_cAttentionOut)) return false; if(!CheckArrayLayers(m_cW0)) return false; if(!CheckArrayLayers(m_cFF1)) return false; if(!CheckArrayLayers(m_cFF2)) return false; //--- Запускаем цикл для создания объектов внутренних слоёв for(int layer = 0; layer < m_iLayers; layer++) { //--- Создаём описание для внутренних нейронных слоёв CLayerDescription *temp = new CLayerDescription(); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.type = defNeuronBase; temp.window = m_iWindow; temp.count = (int)(3 * m_iKeysSize * m_iHeads); temp.activation = ACT_None; temp.activation_params[0] = 1; temp.activation_params[1] = 0; temp.optimization = description.optimization; //--- Инициализируем Querys CNeuronBase *Querys = new CNeuronBase(); if(CheckPointer(Querys) == POINTER_INVALID) { delete temp; return false; } if(!Querys.Init(temp)) { delete Querys; delete temp; return false; } if(!m_cQuerys.Add(Querys)) { delete Querys; delete temp; return false; } //--- Инициализируем Keys CNeuronBase *Keys = new CNeuronBase(); if(CheckPointer(Keys) == POINTER_INVALID) { delete temp; return false; } temp.window = 0; temp.count = (int)(m_iUnits * m_iKeysSize * m_iHeads); if(!Keys.Init(temp)) { delete Keys; delete temp; return false; } if(!m_cKeys.Add(Keys)) { delete Keys; delete temp; return false; } //--- Инициализируем Values CNeuronBase *Values = new CNeuronBase(); if(CheckPointer(Values) == POINTER_INVALID) { delete temp; return false; } if(!Values.Init(temp)) { delete Values; delete temp; return false; } if(!m_cValues.Add(Values)) { delete Values; delete temp; return false; } //--- Инициализируем Scores CNeuronBase *Scores = new CNeuronBase(); if(CheckPointer(Scores) == POINTER_INVALID) { delete temp; return false; } temp.count = (int)(m_iUnits * m_iHeads); if(!Scores.Init(temp)) { delete Scores; delete temp; return false; } if(!m_cScores.Add(Scores)) { delete Scores; delete temp; return false; } //--- Инициализируем AttentionOut CNeuronBase *AttentionOut = new CNeuronBase(); if(CheckPointer(AttentionOut) == POINTER_INVALID) { delete temp; return false; } temp.count = (int)(m_iKeysSize * m_iHeads); if(!AttentionOut.Init(temp)) { delete AttentionOut; delete temp; return false; } if(!m_cAttentionOut.Add(AttentionOut)) { delete AttentionOut; delete temp; return false; } //--- Инициализируем W0 CNeuronBase *W0 = new CNeuronBase(); if(CheckPointer(W0) == POINTER_INVALID) { delete temp; return false; } temp.window = temp.count; temp.count = m_iWindow; temp.activation = ACT_None; temp.activation_params[0] = 1; temp.activation_params[1] = 0; if(!W0.Init(temp)) { delete W0; delete temp; return false; } if(!m_cW0.Add(W0)) { delete W0; delete temp; return false; } //--- Инициализируем FF1 CNeuronBase *FF1 = new CNeuronBase(); if(CheckPointer(m_cFF1) == POINTER_INVALID) { delete temp; return false; } temp.window = m_iWindow; temp.count = temp.window * 4; temp.activation = ACT_SWISH; temp.activation_params[0] = 1; temp.activation_params[1] = 0; if(!FF1.Init(temp)) { delete FF1; delete temp; return false; } if(!m_cFF1.Add(FF1)) { delete FF1; delete temp; return false; } //--- Инициализируем FF2 CNeuronBase *FF2 = new CNeuronBase(); if(CheckPointer(FF2) == POINTER_INVALID) { delete temp; return false; } temp.window = temp.count; temp.count = m_iWindow; temp.activation = ACT_None; temp.activation_params[0] = 1; temp.activation_params[1] = 0; if(!FF2.Init(temp)) { delete FF2; delete temp; return false; } if(!m_cFF2.Add(FF2)) { delete FF2; delete temp; return false; } delete temp; } //--- Для исключениия копирования буферов осуществим их подмену if(m_cFF2.Total() < m_iLayers) return false; if(CheckPointer(m_cOutputs) != POINTER_INVALID) delete m_cOutputs; CNeuronBase *temp = m_cFF2.At(m_iLayers - 1); if(CheckPointer(temp) == POINTER_INVALID) return false; m_cOutputs = temp.GetOutputs(); if(CheckPointer(m_cGradients) != POINTER_INVALID) delete m_cGradients; m_cGradients = temp.GetGradients(); //--- SetOpenCL(m_cOpenCL); //--- return true; } //+------------------------------------------------------------------+ //| Метод передачи указателя на объект OpenCL во все внутренние | //| объекты | //+------------------------------------------------------------------+ bool CNeuronGPT::SetOpenCL(CMyOpenCL *opencl) { CNeuronBase::SetOpenCL(opencl); if(CheckPointer(m_cQuerys) != POINTER_INVALID) m_cQuerys.SetOpencl(m_cOpenCL); if(CheckPointer(m_cKeys) != POINTER_INVALID) m_cKeys.SetOpencl(m_cOpenCL); if(CheckPointer(m_cValues) != POINTER_INVALID) m_cValues.SetOpencl(m_cOpenCL); if(CheckPointer(m_cScores) != POINTER_INVALID) m_cScores.SetOpencl(m_cOpenCL); if(CheckPointer(m_cScoreTemp) != POINTER_INVALID) m_cScoreTemp.BufferCreate(m_cOpenCL); if(CheckPointer(m_cAttentionOut) != POINTER_INVALID) m_cAttentionOut.SetOpencl(m_cOpenCL); if(CheckPointer(m_cW0) != POINTER_INVALID) m_cW0.SetOpencl(m_cOpenCL); if(CheckPointer(m_cFF1) != POINTER_INVALID) m_cFF1.SetOpencl(m_cOpenCL); if(CheckPointer(m_cFF2) != POINTER_INVALID) m_cFF2.SetOpencl(m_cOpenCL); //--- return(CheckPointer(m_cOpenCL) != POINTER_INVALID); } //+------------------------------------------------------------------+ //| Метод прямого прохода | //+------------------------------------------------------------------+ bool CNeuronGPT::FeedForward(CNeuronBase *prevLayer) { //--- Проверяем актуальность всех объектов if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID || CheckPointer(m_cQuerys) == POINTER_INVALID || CheckPointer(m_cValues) == POINTER_INVALID || CheckPointer(m_cKeys) == POINTER_INVALID || CheckPointer(m_cScores) == POINTER_INVALID || CheckPointer(m_cAttentionOut) == POINTER_INVALID || CheckPointer(m_cW0) == POINTER_INVALID || CheckPointer(m_cFF1) == POINTER_INVALID || CheckPointer(m_cFF2) == POINTER_INVALID) return false; //--- Увеличиваем указатель на текущий объект в стеке данных m_iCurrentPosition++; if(m_iCurrentPosition >= m_iUnits) m_iCurrentPosition = 0; //--- Запускаем цикл перебора всех внутренних слоёв CNeuronBase *prevL = prevLayer; for(int layer = 0; layer < m_iLayers; layer++) { CNeuronBase *Querys = m_cQuerys.At(layer); if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevL)) return false; CNeuronBase *Keys = m_cKeys.At(layer); if(CheckPointer(Keys) == POINTER_INVALID) return false; CNeuronBase *Values = m_cValues.At(layer); if(CheckPointer(Values) == POINTER_INVALID) return false; int shift_key = m_iCurrentPosition * m_iKeysSize * m_iHeads; if(!Keys.GetOutputs().UpdateArray(shift_key, Querys.GetOutputs(), m_iKeysSize * m_iHeads, m_iKeysSize * m_iHeads)) return false; if(!Values.GetOutputs().UpdateArray(shift_key, Querys.GetOutputs(), 2 * m_iKeysSize * m_iHeads, m_iKeysSize * m_iHeads)) return false; //--- Инициализируем Scores CNeuronBase *Scores = m_cScores.At(layer); if(CheckPointer(Scores) == POINTER_INVALID) return false; //--- Инициализируем AttentionOut CNeuronBase *AttentionOut = m_cAttentionOut.At(layer); if(CheckPointer(AttentionOut) == POINTER_INVALID) return false; //--- Разветвление алгоритма по вычислительному устройству double summs[]; if(CheckPointer(m_cOpenCL) == POINTER_INVALID) { CBufferDouble *querys = Querys.GetOutputs(); CBufferDouble *keys = Keys.GetOutputs(); double scores[]; if(ArrayResize(scores, m_iUnits * m_iHeads) <= 0) return false; //--- Определяем Scores for(int head = 0; head < m_iHeads; head++) { int shift_query = head * m_iKeysSize; double summ = 0; for(int key = 0; key < m_iUnits; key++) { shift_key = key * m_iKeysSize * m_iHeads + head * m_iKeysSize; int shift_score = head * m_iUnits + key; double score = 0; for(int i = 0; i < m_iKeysSize; i++) score += querys.At(shift_query + i) * keys.At(shift_key + i); scores[shift_score] = MathExp(score / MathSqrt(m_iKeysSize)); summ += scores[shift_score]; } //--- Нормализуем Scores if(summ == 0) continue; for(int key = 0; key < m_iUnits; key++) scores[head * m_iUnits + key] /= summ; } if(!Scores.GetOutputs().AssignArray(scores)) return false; //--- Выход блока внимания int total = m_iKeysSize * m_iHeads; if(ArrayResize(summs, total) < total) return false; if(ArrayInitialize(summs, 0) < total) return false; CBufferDouble *values = Values.GetOutputs(); for(int head = 0; head < m_iHeads; head++) { for(int value = 0; value < m_iUnits; value++) { int shift_value = m_iKeysSize * (value * m_iHeads + head); for(int pos = 0; pos < m_iKeysSize; pos++) { double val = values.At(shift_value + pos); summs[m_iKeysSize * head + pos] += val * scores[m_iUnits * head + value]; } } } if(!AttentionOut.GetOutputs().AssignArray(summs)) return false; } else // Блок OpenCL { //--- Создание буферов данных if(Querys.GetOutputs().GetIndex() < 0 && !Querys.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(Keys.GetOutputs().GetIndex() < 0 && !Keys.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(Values.GetOutputs().GetIndex() < 0 && !Values.GetOutputs().BufferCreate(m_cOpenCL)) return false; int scores = (int)MathPow(m_iUnits, 2) * m_iHeads; if(Scores.GetOutputs().Total() != scores && !Scores.GetOutputs().BufferInit(scores, 0)) return false; if(Scores.GetOutputs().GetIndex() < 0 && !Scores.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(AttentionOut.GetOutputs().GetIndex() < 0 && !AttentionOut.GetOutputs().BufferCreate(m_cOpenCL)) return false; //--- Передача параметров кернелу if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_keys, Keys.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_outputs, AttentionOut.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_querys, Querys.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_scores, Scores.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_values, Values.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_key_size, m_iKeysSize)) return false; if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_units, m_iUnits)) return false; //--- Постановка кернела в очередь выполнения int off_set[] = {0}; int NDRange[] = {m_iHeads}; if(!m_cOpenCL.Execute(def_k_GPTFeedForward, 1, off_set, NDRange)) return false; //--- Считываниие результатов операций if(!AttentionOut.GetOutputs().BufferRead()) return false; if(!Scores.GetOutputs().BufferRead()) return false; Querys.GetOutputs().BufferFree(); Keys.GetOutputs().BufferFree(); Values.GetOutputs().BufferFree(); Scores.GetOutputs().BufferFree(); prevL.GetOutputs().BufferFree(); } //--- Взвешенный выхщд всех голов внимания CNeuronBase *W0 = m_cW0.At(layer); if(CheckPointer(W0) == POINTER_INVALID || !W0.FeedForward(AttentionOut)) return false; int total = W0.GetOutputs().GetData(summs, false); if(total <= 0) return false; if(ArrayResize(summs, total) != total) return false; //--- Суммируем с исходными данными и нормализуем double mean = 0; CBufferDouble *prev = prevL.GetOutputs(); for(int i = 0; i < total; i++) { summs[i] += prev.At(i); mean += summs[i]; } mean /= total; m_dStd[layer][0] = MathStandardDeviation(summs); for(int i = 0; i < total; i++) summs[i] = (summs[i] - mean) / m_dStd[layer][0]; if(!W0.GetOutputs().AssignArray(summs)) return false; //--- Прямой проход блока Feed Forward CNeuronBase *FF1 = m_cFF1.At(layer); if(CheckPointer(FF1) == POINTER_INVALID || !FF1.FeedForward(W0)) return false; CNeuronBase *FF2 = m_cFF2.At(layer); if(CheckPointer(FF2) == POINTER_INVALID || !FF2.FeedForward(FF1)) return false; //--- Суммируем с выходом внимания и нормализуем mean = 0; prev = FF2.GetOutputs(); for(int i = 0; i < total; i++) { summs[i] += prev.At(i); mean += summs[i]; } mean /= total; m_dStd[layer][1] = MathStandardDeviation(summs); for(int i = 0; i < total; i++) summs[i] = (summs[i] - mean) / m_dStd[layer][1]; if(!prev.AssignArray(summs)) return false; prevL = FF2; } //--- return true; } //+------------------------------------------------------------------+ //| Метод распределения градиента через скрытый слой | //+------------------------------------------------------------------+ bool CNeuronGPT::CalcHiddenGradient(CNeuronBase *prevLayer) { //--- Проверяем актуальность всех объектов if(CheckPointer(m_cOutputs) == POINTER_INVALID || CheckPointer(m_cGradients) == POINTER_INVALID || CheckPointer(m_cScores) == POINTER_INVALID || CheckPointer(m_cFF2) == POINTER_INVALID || CheckPointer(m_cFF1) == POINTER_INVALID || CheckPointer(m_cW0) == POINTER_INVALID || CheckPointer(m_cAttentionOut) == POINTER_INVALID || CheckPointer(m_cQuerys) == POINTER_INVALID || CheckPointer(m_cKeys) == POINTER_INVALID || CheckPointer(m_cValues) == POINTER_INVALID || m_cOutputs.Total() != m_cGradients.Total()) return false; //--- Запускаем цикл перебора всех внутренних слоёв в обратном порядке for(int layer = m_iLayers - 1; layer >= 0; layer--) { CNeuronBase *FF2 = m_cFF2.At(layer); if(CheckPointer(FF2) == POINTER_INVALID) return false; CBufferDouble *Gradients = FF2.GetGradients(); if(m_dStd[layer][1] != 0 && Gradients.Scaling(1 / m_dStd[layer][1]) <= 0) return false; //--- Проводим градиент через блок Feed Forward CNeuronBase *FF1 = m_cFF1.At(layer); if(!FF2.CalcHiddenGradient(FF1)) return false; CNeuronBase *W0 = m_cW0.At(layer); if(!FF1.CalcHiddenGradient(W0)) return false; CBufferDouble *attention_grad = W0.GetGradients(); if(!attention_grad.SumArray(Gradients)) return false; if(m_dStd[layer][0] != 0 && attention_grad.Scaling(1 / m_dStd[layer][0]) <= 0) return false; //--- Распределеяем градиент ошибки по головам внимания CNeuronBase *AttentionOut = m_cAttentionOut.At(layer); if(!W0.CalcHiddenGradient(AttentionOut)) return false; //--- Получаем указатели на объекты Querys, Keys, Values CNeuronBase *Querys = m_cQuerys.At(layer); if(CheckPointer(Querys) == POINTER_INVALID) return false; CNeuronBase *Keys = m_cKeys.At(layer); if(CheckPointer(Keys) == POINTER_INVALID) return false; CNeuronBase *Values = m_cValues.At(layer); if(CheckPointer(Values) == POINTER_INVALID) return false; //--- Разветвление алгоритма по вычислительному устройству attention_grad = AttentionOut.GetGradients(); if(CheckPointer(m_cOpenCL) == POINTER_INVALID) { int total = attention_grad.Total(); double values[]; double gradients[]; if(ArrayResize(values, total) < total || attention_grad.GetData(gradients, false) < total) return false; if(ArrayInitialize(values, 0) < total) return false; //--- Распределение градиента на Values AttentionOut = m_cScores.At(layer); if(CheckPointer(AttentionOut) == POINTER_INVALID || CheckPointer(AttentionOut.GetOutputs()) == POINTER_INVALID) return false; CBufferDouble *Scores = AttentionOut.GetOutputs(); for(int head = 0; head < m_iHeads; head++) { double score = Scores.At(m_iUnits * head + m_iCurrentPosition); for(int i = 0; i < m_iKeysSize; i++) values[m_iKeysSize * head + i] += gradients[m_iKeysSize * head + i] * score; } if(!Querys.GetGradients().UpdateArray(2 * m_iKeysSize * m_iHeads, values, 0, m_iKeysSize * m_iHeads)) return false; //--- Распределение градиента на Querys и Keys if(Values.GetOutputs().GetData(values, false) <= 0) return false; double querys[], querys_grad[]; double keys[]; int keys_total = m_iKeysSize * m_iHeads; if(Querys.GetOutputs().GetData(querys, false) < 3 * keys_total) return false; if(Keys.GetOutputs().GetData(keys, false) < keys_total * m_iUnits) return false; if(ArrayResize(querys_grad, 2 * keys_total) <= 0) return false; if(ArrayInitialize(querys_grad, 0) <= 0) return false; double score_grad[]; if(ArrayResize(score_grad, m_iUnits) <= 0) return false; for(int head = 0; head < m_iHeads; head++) { if(ArrayInitialize(score_grad, 0) <= 0) return false; int shift_grad = m_iKeysSize * head; for(int k = 0; k < m_iUnits; k++) { for(int i = 0; i < m_iKeysSize; i++) score_grad[k] += gradients[shift_grad + i] * values[m_iKeysSize * (k * m_iHeads + head) + i]; } //--- for(int k = 0; k < m_iUnits; k++) { double score = Scores.At(m_iUnits * head + k); if(score == 0) continue; double grad = 0; for(int i = 0; i < m_iUnits; i++) grad += Scores.At(m_iUnits * head + i) * ((int)(i == k) - score) * score_grad[i]; grad /= MathSqrt(m_iKeysSize); //--- int shift_key = m_iKeysSize * (k * m_iHeads + head); for(int i = 0; i < m_iKeysSize; i++) { querys_grad[shift_grad + i] += grad * keys[shift_key + i]; if(k == m_iCurrentPosition) querys_grad[m_iKeysSize * (m_iHeads + head) + i ] += grad * querys[shift_grad + i]; } } } if(!Querys.GetGradients().UpdateArray(0, querys_grad, 0, 2 * keys_total)) return false; } else // Блок OpenCL { //--- Создание буферов данных if(Values.GetOutputs().GetIndex() < 0 && !Values.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(Querys.GetGradients().GetIndex() < 0 && !Querys.GetGradients().BufferCreate(m_cOpenCL)) return false; CNeuronBase *Scores = m_cScores.At(layer); if(CheckPointer(Scores) == POINTER_INVALID) return false; if(Scores.GetOutputs().GetIndex() < 0 && !Scores.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(attention_grad.GetIndex() < 0 && !attention_grad.BufferCreate(m_cOpenCL)) return false; if(CheckPointer(Scores.GetGradients()) == POINTER_INVALID) { return false; } if(Scores.GetGradients().Total() != Scores.GetOutputs().Total() && !Scores.GetGradients().BufferInit(Scores.GetOutputs().Total(), 0)) return false; if(Scores.GetGradients().GetIndex() < 0 && !Scores.GetGradients().BufferCreate(m_cOpenCL)) return false; //--- if(CheckPointer(m_cScoreTemp) == POINTER_INVALID) { m_cScoreTemp = new CBufferDouble(); if(CheckPointer(m_cScoreTemp) == POINTER_INVALID) return false; } if(m_cScoreTemp.Total() != Scores.GetGradients().Total() && !m_cScoreTemp.BufferInit(Scores.GetGradients().Total(), 0)) return false; if(m_cScoreTemp.GetIndex() < 0 && !m_cScoreTemp.BufferCreate(m_cOpenCL)) return false; //--- Передача параметров кернелу if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_outputs_grad, attention_grad.GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores, Scores.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_grad, Scores.GetGradients().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_temp, m_cScoreTemp.GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values, Values.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values_grad, Querys.GetGradients().GetIndex())) return false; if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_window, m_iKeysSize)) return false; if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_units, m_iUnits)) return false; if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_current, m_iCurrentPosition)) return false; //--- Постановка кернела в очередь выполнения int off_set[] = {0}; int NDRange[] = {m_iHeads}; if(!m_cOpenCL.Execute(def_k_GPTScoreGradients, 1, off_set, NDRange)) return false; //--- Загрузка результатов if(!Querys.GetGradients().BufferRead()) return false; Values.GetOutputs().BufferFree(); Scores.GetOutputs().BufferFree(); m_cScoreTemp.BufferFree(); AttentionOut.GetOutputs().BufferFree(); //--- if(Querys.GetOutputs().GetIndex() < 0 && !Querys.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(Keys.GetOutputs().GetIndex() < 0 && !Keys.GetOutputs().BufferCreate(m_cOpenCL)) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_keys, Keys.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys, Querys.GetOutputs().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys_grad, Querys.GetGradients().GetIndex())) return false; if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_scores_grad, Scores.GetGradients().GetIndex())) return false; if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_key_size, m_iKeysSize)) return false; if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_units, m_iUnits)) return false; if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_current, m_iCurrentPosition)) return false; if(!m_cOpenCL.Execute(def_k_GPTHiddenGradients, 1, off_set, NDRange)) return false; //--- Загрузка результатов if(!Querys.GetGradients().BufferRead()) return false; //--- Scores.GetGradients().BufferFree(); Keys.GetOutputs().BufferFree(); Querys.GetOutputs().BufferFree(); } //--- Перенос градиента ошибки на предыдущий слой CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1)); if(!Querys.CalcHiddenGradient(prevL)) return false; if(!prevL.GetGradients().SumArray(W0.GetGradients())) return false; } //--- return true; } //+------------------------------------------------------------------+ //| Метод распределения градиентов ошибки до матриц весовых | //| коэффициентов | //+------------------------------------------------------------------+ bool CNeuronGPT::CalcDeltaWeights(CNeuronBase *prevLayer) { //--- Проверяем актуальность всех объектов if(CheckPointer(m_cFF2) == POINTER_INVALID) return false; if(CheckPointer(m_cFF1) == POINTER_INVALID) return false; if(CheckPointer(m_cW0) == POINTER_INVALID) return false; if(CheckPointer(m_cAttentionOut) == POINTER_INVALID) return false; if(CheckPointer(m_cQuerys) == POINTER_INVALID) return false; //--- В цикле вызываем аналогичный метод для каждого внутреннего объекта for(int layer = 0; layer < m_iLayers; layer++) { if(CheckPointer(m_cFF2.At(layer)) == POINTER_INVALID) return false; CNeuronBase *temp = m_cFF2.At(layer); if(!temp.CalcDeltaWeights(m_cFF1.At(layer))) return false; temp = m_cFF1.At(layer); if(!temp.CalcDeltaWeights(m_cW0.At(layer))) return false; temp = m_cW0.At(layer); if(!temp.CalcDeltaWeights(m_cAttentionOut.At(layer))) return false; temp = m_cQuerys.At(layer); if(CheckPointer(temp) == POINTER_INVALID) return false; CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1)); if(!temp.CalcDeltaWeights(prevL)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| Метод обновления параметров матрицы весовых коэффициентов | //+------------------------------------------------------------------+ bool CNeuronGPT::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[]) { //--- Блок контролей if(CheckPointer(m_cFF2) == POINTER_INVALID) return false; if(CheckPointer(m_cFF1) == POINTER_INVALID) return false; if(CheckPointer(m_cW0) == POINTER_INVALID) return false; if(CheckPointer(m_cQuerys) == POINTER_INVALID) return false; //--- В цикле вызываем аналогичный метод для каждого внутреннего объекта for(int layer = 0; layer < m_iLayers; layer++) { CNeuronBase *temp = m_cFF2.At(layer); if(CheckPointer(temp) == POINTER_INVALID || !temp.UpdateWeights(batch_size, learningRate, Beta, Lambda)) return false; temp = m_cFF1.At(layer); if(CheckPointer(temp) == POINTER_INVALID || !temp.UpdateWeights(batch_size, learningRate, Beta, Lambda)) return false; temp = m_cW0.At(layer); if(CheckPointer(temp) == POINTER_INVALID || !temp.UpdateWeights(batch_size, learningRate, Beta, Lambda)) return false; temp = m_cQuerys.At(layer); if(CheckPointer(temp) == POINTER_INVALID || !temp.UpdateWeights(batch_size, learningRate, Beta, Lambda)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| Метод сохранения элементов класса в файл | //+------------------------------------------------------------------+ bool CNeuronGPT::Save(const int file_handle) { //--- Вызов метода родительского класса if(!CNeuronBase::Save(file_handle)) return false; //--- Сохраняем константы if(FileWriteInteger(file_handle, m_iLayers) <= 0) return false; if(FileWriteInteger(file_handle, m_iWindow) <= 0) return false; if(FileWriteInteger(file_handle, m_iKeysSize) <= 0) return false; if(FileWriteInteger(file_handle, m_iHeads) <= 0) return false; if(FileWriteInteger(file_handle, m_iUnits) <= 0) return false; if(FileWriteInteger(file_handle, m_iCurrentPosition) <= 0) return false; //--- Вызываем аналогичный метод для всех колекций внутренних слоёв if(CheckPointer(m_cQuerys) == POINTER_INVALID || !m_cQuerys.Save(file_handle)) return false; if(CheckPointer(m_cKeys) == POINTER_INVALID || !m_cKeys.Save(file_handle)) return false; if(CheckPointer(m_cValues) == POINTER_INVALID || !m_cValues.Save(file_handle)) return false; if(CheckPointer(m_cScores) == POINTER_INVALID || !m_cScores.Save(file_handle)) return false; if(CheckPointer(m_cAttentionOut) == POINTER_INVALID || !m_cAttentionOut.Save(file_handle)) return false; if(CheckPointer(m_cW0) == POINTER_INVALID || !m_cW0.Save(file_handle)) return false; if(CheckPointer(m_cFF1) == POINTER_INVALID || !m_cFF1.Save(file_handle)) return false; if(CheckPointer(m_cFF2) == POINTER_INVALID || !m_cFF2.Save(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| Метод восстановления работы класса из файла | //+------------------------------------------------------------------+ bool CNeuronGPT::Load(const int file_handle) { //--- Вызов метода родительского класса if(!CNeuronBase::Load(file_handle)) return false; //--- Считываем константы из файла m_iLayers = FileReadInteger(file_handle); m_iWindow = FileReadInteger(file_handle); m_iKeysSize = FileReadInteger(file_handle); m_iHeads = FileReadInteger(file_handle); m_iUnits = FileReadInteger(file_handle); m_iCurrentPosition = FileReadInteger(file_handle); if(ArrayResize(m_dStd, m_iLayers) <= 0) return false; //--- Вызываем аналогичный метод для всех колекций внутренних слоёв if(!CheckArrayLayers(m_cQuerys) || !m_cQuerys.Load(file_handle)) return false; if(!CheckArrayLayers(m_cKeys) || !m_cKeys.Load(file_handle)) return false; if(!CheckArrayLayers(m_cValues) || !m_cValues.Load(file_handle)) return false; if(!CheckArrayLayers(m_cScores) || !m_cScores.Load(file_handle)) return false; if(!CheckArrayLayers(m_cAttentionOut) || !m_cAttentionOut.Load(file_handle)) return false; if(!CheckArrayLayers(m_cW0) || !m_cW0.Load(file_handle)) return false; if(!CheckArrayLayers(m_cFF1) || !m_cFF1.Load(file_handle)) return false; if(!CheckArrayLayers(m_cFF2) || !m_cFF2.Load(file_handle)) return false; //--- Осуществляем подмену буферов данных для исключения излишнего копирования CNeuronBase *last=m_cFF2.At(m_cFF2.Total()-1); if(CheckPointer(last)==POINTER_INVALID) return false; if(CheckPointer(m_cOutputs)!=POINTER_INVALID) delete m_cOutputs; m_cOutputs=last.GetOutputs(); if(CheckPointer(m_cGradients)!=POINTER_INVALID) delete m_cGradients; m_cGradients=last.GetGradients(); //--- return true; } //+------------------------------------------------------------------+ //| Метод проверки актуальности указателя на объект коллекции | //| нейроных слоёв | //+------------------------------------------------------------------+ bool CNeuronGPT::CheckArrayLayers(CArrayLayers *&layers) { if(CheckPointer(layers) == POINTER_INVALID) layers = new CArrayLayers(); //--- return CheckPointer(layers) != POINTER_INVALID; } //+------------------------------------------------------------------+