//+------------------------------------------------------------------+ //| OMV Model.mqh | //| Copyright 2026, Allan Munene Mutiiria. | //| https://t.me/Forex_Algo_Trader | //+------------------------------------------------------------------+ #property copyright "Copyright 2026, Allan Munene Mutiiria." #property link "https://t.me/Forex_Algo_Trader" #ifndef OMV_MODEL_MQH #define OMV_MODEL_MQH #include "OMV Protobuf.mqh" //+------------------------------------------------------------------+ //| Capacity limits | //+------------------------------------------------------------------+ #define OMV_MAX_NODES 4096 // Layers the graph may hold #define OMV_MAX_TENSORS 4096 // Weight tensors the graph may hold #define OMV_MAX_PORTS 64 // Names one node may carry per side #define OMV_MAX_ATTRS 32 // Attributes one node may carry #define OMV_MAX_META 32 // Key and value pairs the file may carry #define OMV_MAX_FUNCS 64 // Custom operators the file may define #define OMV_MAX_DIMS 16 // Axes one tensor may have #define OMV_MAX_VALUES 4096 // Weights one tensor keeps for display //+------------------------------------------------------------------+ //| Graph layer | //+------------------------------------------------------------------+ struct OmvNode { // Name the operator this layer applies string op; // Name this layer was given string name; // Name the operator set this layer's operator comes from string domain; // Name each attribute the operator was given string attrName[OMV_MAX_ATTRS]; // Print the value each attribute holds string attrValue[OMV_MAX_ATTRS]; // Record which kind each attribute declared itself to be int attrKind[OMV_MAX_ATTRS]; // Count the attributes present int attrCount; // Carry whatever the file says about this layer string docString; // Name the overload the operator was resolved to string overload; // List the names feeding this layer string feeds[OMV_MAX_PORTS]; // Count the names present int feedCount; // List the names this layer produces string emits[OMV_MAX_PORTS]; // Count the names present int emitCount; }; //+------------------------------------------------------------------+ //| Weight tensor | //+------------------------------------------------------------------+ struct OmvTensor { // Name the nodes refer to it by string name; // Measure each axis int dim[OMV_MAX_DIMS]; // Count the axes present int dims; // Record the ONNX type code int dataType; // Multiply every axis to size the block int values; // Track the smallest weight held double minValue; // Track the largest weight held double maxValue; // Track the average weight double meanValue; // Track how far the weights spread from their average double stdValue; // Track what share of the weights are zero double sparsity; // Keep the weights themselves so the inspector can show them double val[]; // Count the weights kept, which stops at the display limit int kept; // Report where the file keeps this tensor's values string location; // Name the category the file gives this tensor string category; // Keep the strings when the tensor holds text rather than numbers string text[]; // Count the strings kept int texts; // Report whether the file stored this tensor in a segment bool segmented; }; //+------------------------------------------------------------------+ //| Exposed graph port | //+------------------------------------------------------------------+ struct OmvPort { // Name the graph exposes string name; // Measure each axis int dim[OMV_MAX_DIMS]; // Count the axes present int dims; // Record the ONNX type code int elemType; }; //+------------------------------------------------------------------+ //| ONNX file reader | //+------------------------------------------------------------------+ class COmvModel { private: // Store the layers in file order OmvNode m_node[]; // Store the weight tensors OmvTensor m_tensor[]; // Store the graph inputs OmvPort m_input[]; // Store the graph outputs OmvPort m_output[]; // Store what the file says about the values passing between layers OmvPort m_inner[]; // Count the layers held int m_nodes; // Count the tensors held int m_tensors; // Count the graph inputs held int m_inputs; // Count the graph outputs held int m_outputs; // Count the inner values held int m_inners; // Record the format revision long m_irVersion; // Record the operator set version int m_opset; // Name the tool that wrote the file string m_producer; // Record the version of the tool that wrote it string m_producerVersion; // Name the domain the model belongs to string m_domain; // Record the revision the model gives itself long m_modelVersion; // Carry whatever the file says about itself string m_docString; // Name the graph inside the file string m_graphName; // Carry whatever the file says about the graph string m_graphDoc; // Name each key the file attached to the model string m_metaKey[OMV_MAX_META]; // Print the value each key carries string m_metaValue[OMV_MAX_META]; // Count the pairs held int m_metas; // Name each operator the file defines for itself string m_funcName[OMV_MAX_FUNCS]; // Describe what each defined operator holds string m_funcDetail[OMV_MAX_FUNCS]; // Count the defined operators held int m_funcs; // Report whether the file carries training state bool m_hasTraining; // Name each key the file attached to the graph string m_gMetaKey[OMV_MAX_META]; // Print the value each graph key carries string m_gMetaValue[OMV_MAX_META]; // Count the graph pairs held int m_gMetas; // Count the quantization mappings the graph carries int m_quants; // Explain why the last load failed string m_error; bool ParseGraph(const uchar &raw[], const int from, const int to); bool ParseNode(const uchar &raw[], const int from, const int to); bool ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer); string ReadList(const uchar &raw[], const int from, const int to, const int field); string ReadSubgraph(const uchar &raw[], const int from, const int to); string ReadEntry(const uchar &raw[], const int from, const int to); void ReadPair(const uchar &raw[], const int from, const int to, string &keys[], string &values[], int &count); void ReadFunction(const uchar &raw[], const int from, const int to); bool ParseTensor(const uchar &raw[], const int from, const int to); bool ParseSparse(const uchar &raw[], const int from, const int to); bool ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port); void Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros, double &total, double &squares, double &lowest, double &highest); void Publish(OmvTensor &tensor, const int seen, const int zeros, const double total, const double squares, const double lowest, const double highest); void RawStats(const uchar &raw[], const int from, const int to, OmvTensor &tensor); void PackedStats(const uchar &raw[], const int from, const int to, const int field, OmvTensor &tensor); public: COmvModel(void); bool Load(const string path); void Clear(void); int Nodes(void) const { return m_nodes; } int Tensors(void) const { return m_tensors; } int Inputs(void) const { return m_inputs; } int Outputs(void) const { return m_outputs; } int Inners(void) const { return m_inners; } long IrVersion(void) const { return m_irVersion; } int Opset(void) const { return m_opset; } string Producer(void) const { return m_producer; } string ProducerVersion(void) const { return m_producerVersion; } string Domain(void) const { return m_domain; } long ModelVersion(void) const { return m_modelVersion; } string DocString(void) const { return m_docString; } string GraphName(void) const { return m_graphName; } string GraphDoc(void) const { return m_graphDoc; } int Metas(void) const { return m_metas; } int GraphMetas(void) const { return m_gMetas; } string GraphMetaKey(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaKey[i] : ""; } string GraphMetaValue(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaValue[i] : ""; } int Quants(void) const { return m_quants; } string MetaKey(const int i) const { return (i >= 0 && i < m_metas) ? m_metaKey[i] : ""; } string MetaValue(const int i) const { return (i >= 0 && i < m_metas) ? m_metaValue[i] : ""; } int Funcs(void) const { return m_funcs; } string FuncName(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcName[i] : ""; } string FuncDetail(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcDetail[i] : ""; } bool HasTraining(void) const { return m_hasTraining; } string Error(void) const { return m_error; } int Parameters(void) const; void NodeAt(const int i, OmvNode &out) const { if(i >= 0 && i < m_nodes) out = m_node[i]; } void TensorAt(const int i, OmvTensor &out) const { if(i >= 0 && i < m_tensors) out = m_tensor[i]; } void InputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inputs) out = m_input[i]; } void OutputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_outputs) out = m_output[i]; } void InnerAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inners) out = m_inner[i]; } string DimsText(const int &dim[], const int dims) const; string DimsList(const int &dim[], const int dims) const; string DimsBare(const int &dim[], const int dims) const; string TypeName(const int code) const; string AttrType(const int kind) const; }; //+------------------------------------------------------------------+ //| Construct an empty model | //+------------------------------------------------------------------+ COmvModel::COmvModel(void) { //--- Clear every field so a query before loading reports empty Clear(); } //+------------------------------------------------------------------+ //| Drop everything the previous file filled | //+------------------------------------------------------------------+ void COmvModel::Clear(void) { //--- Zero the counts that bound every array m_nodes = 0; m_tensors = 0; m_inputs = 0; m_outputs = 0; m_inners = 0; //--- Release the stored structures ArrayResize(m_node, 0); ArrayResize(m_tensor, 0); ArrayResize(m_input, 0); ArrayResize(m_output, 0); ArrayResize(m_inner, 0); //--- Clear the header fields m_irVersion = 0; m_opset = 0; m_producer = ""; m_producerVersion = ""; m_domain = ""; m_modelVersion = 0; m_docString = ""; m_graphName = ""; m_graphDoc = ""; m_metas = 0; m_gMetas = 0; m_quants = 0; m_funcs = 0; m_hasTraining = false; m_error = ""; } //+------------------------------------------------------------------+ //| Walk raw weight bytes for their range and average | //+------------------------------------------------------------------+ void COmvModel::Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros, double &total, double &squares, double &lowest, double &highest) { //--- Seed the range from the first value that arrives if(seen == 0) { lowest = v; highest = v; } //--- Lower the floor when this value is smaller if(v < lowest) lowest = v; //--- Raise the ceiling when this value is larger if(v > highest) highest = v; //--- Add the value to the running totals total += v; squares += v * v; //--- Count a value that is exactly zero if(v == 0.0) zeros++; //--- Keep the value itself while room remains if(tensor.kept < OMV_MAX_VALUES) { ArrayResize(tensor.val, tensor.kept + 1); tensor.val[tensor.kept++] = v; } seen++; } //+------------------------------------------------------------------+ //| Publish what the accumulated values measured | //+------------------------------------------------------------------+ void COmvModel::Publish(OmvTensor &tensor, const int seen, const int zeros, const double total, const double squares, const double lowest, const double highest) { //--- Abort when the read stopped before any value arrived if(seen <= 0) return; //--- Publish the range and the average tensor.minValue = lowest; tensor.maxValue = highest; tensor.meanValue = total / seen; //--- Take the spread as the root of the mean squared deviation const double variance = squares / seen - tensor.meanValue * tensor.meanValue; tensor.stdValue = (variance > 0.0) ? MathSqrt(variance) : 0.0; //--- Take the share of the values that are exactly zero tensor.sparsity = (double)zeros / seen; } //+------------------------------------------------------------------+ //| Walk a raw byte payload at the width its type implies | //+------------------------------------------------------------------+ void COmvModel::RawStats(const uchar &raw[], const int from, const int to, OmvTensor &tensor) { //--- Take the width one value occupies for this element type int width = 4; if(tensor.dataType == 11 || tensor.dataType == 7 || tensor.dataType == 13) width = 8; else if(tensor.dataType == 2 || tensor.dataType == 3 || tensor.dataType == 9) width = 1; else if(tensor.dataType == 4 || tensor.dataType == 5 || tensor.dataType == 10) width = 2; //--- Count the whole values the payload holds const int count = (to - from) / width; if(count <= 0) return; //--- Point a cursor at the payload CPbReader bytes; if(!bytes.Attach(raw, from, to)) return; //--- Start the running measures double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0; int zeros = 0, seen = 0; //--- Visit every stored value for(int i = 0; i < count; i++) { //--- Read the value the way its type is written double v = 0.0; if(tensor.dataType == 1) v = (double)bytes.ReadFloat(); else if(tensor.dataType == 11) v = bytes.ReadDouble(); else if(width == 8) v = (double)(long)bytes.ReadFixed64(); else if(width == 4) v = (double)(int)bytes.ReadFixed32(); else if(width == 2) v = (double)(int)(bytes.ReadVarint() & 0xFFFF); else v = (double)bytes.ReadVarint(); //--- Stop on a broken read if(bytes.Failed()) break; //--- Fold the value into the running measures Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest); } //--- Publish what the walk measured Publish(tensor, seen, zeros, total, squares, lowest, highest); } //+------------------------------------------------------------------+ //| Walk a typed payload the file used instead of raw bytes | //+------------------------------------------------------------------+ void COmvModel::PackedStats(const uchar &raw[], const int from, const int to, const int field, OmvTensor &tensor) { //--- Point a cursor at the payload CPbReader bytes; if(!bytes.Attach(raw, from, to)) return; //--- Start the running measures double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0; int zeros = 0, seen = 0; //--- Read the payload the way the field it arrived in is written while(!bytes.Eof()) { //--- Take the next value in the packed run double v = 0.0; if(field == 4) v = (double)bytes.ReadFloat(); else if(field == 10) v = bytes.ReadDouble(); else if(field == 6) break; else v = (double)(long)bytes.ReadVarint(); //--- Stop on a broken read if(bytes.Failed()) break; //--- Fold the value into the running measures Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest); } //--- Publish what the walk measured Publish(tensor, seen, zeros, total, squares, lowest, highest); } //+------------------------------------------------------------------+ //| Join a list an attribute carried into one line | //+------------------------------------------------------------------+ string COmvModel::ReadList(const uchar &raw[], const int from, const int to, const int field) { //--- Point a cursor at the list CPbReader items; if(!items.Attach(raw, from, to)) return ""; //--- Join up to a handful of entries, then say how many remain string out = ""; int shown = 0, total = 0; while(!items.Eof()) { //--- Read the next entry the way its field is written string one = ""; if(field == 7) one = DoubleToString((double)items.ReadFloat(), 4); else if(field == 8) one = IntegerToString((long)items.ReadVarint()); else items.ReadString(one); //--- Stop on a broken read if(items.Failed()) break; total++; //--- Keep the first few entries only, a row cannot hold more if(shown >= 8) continue; out += (shown > 0 ? ", " : "") + one; shown++; } //--- Say how many entries the line left out if(total > shown) out += ", ... " + IntegerToString(total - shown) + " more"; return out; } //+------------------------------------------------------------------+ //| Describe a nested graph by what it actually holds | //+------------------------------------------------------------------+ string COmvModel::ReadSubgraph(const uchar &raw[], const int from, const int to) { //--- Point a cursor at the nested graph CPbReader reader; if(!reader.Attach(raw, from, to)) return ""; uchar inner[]; reader.Bytes(inner); //--- Count what the nested graph carries and name its operators string title = ""; string ops = ""; int nodes = 0, tensors = 0, shown = 0; //--- Read every field of the nested graph while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Step over anything that is not a nested message if(wire != PB_LENGTH) { if(!reader.SkipField(wire)) break; continue; } //--- Locate the nested payload int f2, t2; if(!reader.ReadBlock(f2, t2)) break; //--- Count a node and name the operator it applies if(field == 1) { nodes++; //--- Name only the first few, a line cannot hold more if(shown < 6) { //--- Read the operator out of the node CPbReader entry; entry.Attach(inner, f2, t2); while(!entry.Eof()) { //--- Take the next field of the entry int f3, w3; if(!entry.ReadTag(f3, w3)) break; //--- Keep the operator and stop reading this node if(f3 == 4 && w3 == PB_LENGTH) { string op = ""; entry.ReadString(op); ops += (shown > 0 ? ", " : "") + op; shown++; break; } if(!entry.SkipField(w3)) break; } } } //--- Count an initializer the nested graph carries else if(field == 5 || field == 15) tensors++; //--- Take the name the nested graph was given else if(field == 2) { //--- Decode the name from its own payload uchar tmp[]; ArrayResize(tmp, t2 - f2); for(int i = f2; i < t2; i++) tmp[i - f2] = inner[i]; title = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8); } } //--- Report the nested graph by its name, its size and what it runs string out = " 0) out += " " + title; out += ": " + IntegerToString(nodes) + " nodes, " + IntegerToString(tensors) + " tensors"; if(StringLen(ops) > 0) out += " (" + ops + (nodes > shown ? ", ..." : "") + ")"; return out + ">"; } //+------------------------------------------------------------------+ //| Name the type an attribute declared itself to be | //+------------------------------------------------------------------+ string COmvModel::AttrType(const int kind) const { //--- Match the declared kind to the type the app reports for it switch(kind) { case 1: return "float32"; case 2: return "int64"; case 3: return "string"; case 4: return "tensor"; case 5: return "graph"; case 6: return "float32[]"; case 7: return "int64[]"; case 8: return "string[]"; } //--- Report nothing recognizable as undefined return "undefined"; } //+------------------------------------------------------------------+ //| Read one attribute an operator was given | //+------------------------------------------------------------------+ bool COmvModel::ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer) { //--- Point a cursor at this attribute CPbReader reader; if(!reader.Attach(raw, from, to)) return false; uchar inner[]; reader.Bytes(inner); //--- Refuse an attribute beyond what the layer may hold if(layer.attrCount >= OMV_MAX_ATTRS) return false; //--- Start with nothing read const int slot = layer.attrCount; layer.attrName[slot] = ""; layer.attrValue[slot] = ""; //--- Remember which kind of value the attribute declares int kind = 0; double asFloat = 0.0; long asInt = 0; string asText = ""; bool haveInt = false, haveFloat = false; //--- Count the entries a repeated field delivers one at a time int repeats = 0; //--- Read every field of the attribute while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Store the attribute name if(field == 1 && wire == PB_LENGTH) reader.ReadString(layer.attrName[slot]); //--- Store a floating point value else if(field == 2 && wire == PB_FIXED32) { asFloat = (double)reader.ReadFloat(); haveFloat = true; } //--- Store a whole number value else if(field == 3 && wire == PB_VARINT) { asInt = (long)reader.ReadVarint(); haveInt = true; } //--- Store a text value else if(field == 4 && wire == PB_LENGTH) reader.ReadString(asText); //--- Name a nested tensor or graph rather than unfolding it else if((field == 5 || field == 6) && wire == PB_LENGTH) { //--- Step past the payload and record what it was int n1, n2; if(!reader.ReadBlock(n1, n2)) break; asText = (field == 5) ? "" : ReadSubgraph(inner, n1, n2); } //--- Append one string, of which this field delivers many else if(field == 9 && wire == PB_LENGTH) { //--- Read this entry and join it to the ones before string one = ""; if(!reader.ReadString(one)) break; repeats++; if(repeats <= 8) asText += (repeats > 1 ? ", " : "") + one; } //--- Join a list of floats, whole numbers or strings else if((field == 7 || field == 8) && wire == PB_LENGTH) { //--- Locate the list and read it in the form it arrived int l1, l2; if(!reader.ReadBlock(l1, l2)) break; asText = ReadList(inner, l1, l2, field); } //--- Name a list of tensors or graphs rather than unfolding it else if((field == 10 || field == 11) && wire == PB_LENGTH) { //--- Step past this entry and count it among its siblings int n1, n2; if(!reader.ReadBlock(n1, n2)) break; repeats++; //--- Describe the first entry, then say how many followed if(repeats == 1) asText = (field == 10) ? "" : ReadSubgraph(inner, n1, n2); } //--- Store which kind the attribute declares itself to be else if(field == 20 && wire == PB_VARINT) kind = (int)reader.ReadVarint(); //--- Step over anything else else if(!reader.SkipField(wire)) break; } //--- Say how many entries a repeated field delivered beyond those shown if(repeats > 8) asText += ", ... " + IntegerToString(repeats - 8) + " more"; else if(repeats > 1 && (kind == 5 || kind == 6 || kind == 9 || kind == 10)) asText += " and " + IntegerToString(repeats - 1) + " more"; //--- Reject an attribute that never gave a name if(StringLen(layer.attrName[slot]) == 0) return false; //--- Print the value the attribute carried, by the kind it declared if(kind == 3 || kind >= 4 || StringLen(asText) > 0) layer.attrValue[slot] = asText; else if(kind == 2 || (kind == 0 && haveInt)) layer.attrValue[slot] = IntegerToString(asInt); else if(kind == 1 || (kind == 0 && haveFloat)) layer.attrValue[slot] = DoubleToString(asFloat, 6); //--- Record the kind so the row can report the type it opens to layer.attrKind[slot] = kind; //--- Report the attribute as read return true; } //+------------------------------------------------------------------+ //| Read one node into a layer | //+------------------------------------------------------------------+ bool COmvModel::ParseNode(const uchar &raw[], const int from, const int to) { //--- Refuse a file that exceeds the node limit if(m_nodes >= OMV_MAX_NODES) return false; //--- Point a cursor at this node CPbReader reader; if(!reader.Attach(raw, from, to)) return false; //--- Start an empty layer OmvNode layer; layer.op = ""; layer.name = ""; layer.domain = ""; layer.docString = ""; layer.overload = ""; layer.attrCount = 0; layer.feedCount = 0; layer.emitCount = 0; //--- Read every field of the node while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Skip anything that is not text if(wire != PB_LENGTH) { if(!reader.SkipField(wire)) break; continue; } //--- Read an attribute out of its own nested message if(field == 5) { //--- Locate the attribute and decode it while room remains int a1, a2; if(!reader.ReadBlock(a1, a2)) break; if(layer.attrCount < OMV_MAX_ATTRS) if(ParseAttribute(raw, from + a1, from + a2, layer)) layer.attrCount++; continue; } //--- Read the field as text string text; if(!reader.ReadString(text)) break; //--- Store an input name if(field == 1 && layer.feedCount < OMV_MAX_PORTS) layer.feeds[layer.feedCount++] = text; //--- Store an output name else if(field == 2 && layer.emitCount < OMV_MAX_PORTS) layer.emits[layer.emitCount++] = text; //--- Store the layer name else if(field == 3) layer.name = text; //--- Store the operator else if(field == 4) layer.op = text; //--- Store the operator set the operator belongs to else if(field == 7) layer.domain = text; //--- Store whatever the file says about this layer else if(field == 6) layer.docString = text; //--- Store the overload the operator was resolved to else if(field == 8) layer.overload = text; } //--- Reject a node carrying no operator if(StringLen(layer.op) == 0) return false; //--- Append the finished layer ArrayResize(m_node, m_nodes + 1); m_node[m_nodes++] = layer; //--- Report success return true; } //+------------------------------------------------------------------+ //| Read one initializer into a weight tensor | //+------------------------------------------------------------------+ bool COmvModel::ParseTensor(const uchar &raw[], const int from, const int to) { //--- Refuse a file that exceeds the tensor limit if(m_tensors >= OMV_MAX_TENSORS) return false; //--- Point a cursor at this tensor CPbReader reader; if(!reader.Attach(raw, from, to)) return false; //--- Keep the slice so nested payloads can be located uchar inner[]; reader.Bytes(inner); //--- Start an empty tensor OmvTensor weights; weights.name = ""; weights.dims = 0; weights.dataType = 0; weights.values = 0; weights.minValue = 0.0; weights.maxValue = 0.0; weights.meanValue = 0.0; weights.stdValue = 0.0; weights.sparsity = 0.0; weights.kept = 0; weights.texts = 0; weights.segmented = false; weights.location = "default"; weights.category = "Initializer"; //--- Mark every payload the tensor might carry as not yet found int rawFrom = -1, rawTo = -1; int packFrom = -1, packTo = -1, packField = 0; //--- Read every field of the tensor while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Store one axis length if(field == 1 && wire == PB_VARINT) { //--- Keep the axis while room remains if(weights.dims < OMV_MAX_DIMS) weights.dim[weights.dims++] = (int)reader.ReadVarint(); else reader.ReadVarint(); } //--- Store several axis lengths packed together else if(field == 1 && wire == PB_LENGTH) { //--- Locate the packed block int f2, t2; if(!reader.ReadBlock(f2, t2)) break; //--- Read every length the block holds CPbReader packed; packed.Attach(inner, f2, t2); while(!packed.Eof() && weights.dims < OMV_MAX_DIMS) weights.dim[weights.dims++] = (int)packed.ReadVarint(); } //--- Store the element type else if(field == 2 && wire == PB_VARINT) weights.dataType = (int)reader.ReadVarint(); //--- Handle a tensor the file stored in segments else if(field == 3 && wire == PB_LENGTH) { //--- Step past the segment, only its presence is reported int s1, s2; if(!reader.ReadBlock(s1, s2)) break; weights.segmented = true; } //--- Store the tensor name else if(field == 8 && wire == PB_LENGTH) reader.ReadString(weights.name); //--- Locate the raw byte payload else if(field == 9 && wire == PB_LENGTH) { if(!reader.ReadBlock(rawFrom, rawTo)) break; } //--- Collect one string, of which the field may carry many else if(field == 6 && wire == PB_LENGTH) { //--- Append it while room remains string one = ""; if(!reader.ReadString(one)) break; if(weights.texts < OMV_MAX_VALUES) { ArrayResize(weights.text, weights.texts + 1); weights.text[weights.texts++] = one; } } //--- Locate a typed payload, which the file may use instead else if(wire == PB_LENGTH && (field == 4 || field == 5 || field == 7 || field == 10 || field == 11)) { //--- Remember which typed field carried the values if(!reader.ReadBlock(packFrom, packTo)) break; packField = field; } //--- Report values the file keeps somewhere else entirely else if(field == 14 && wire == PB_VARINT) { //--- A location of one means the weights sit outside the file if((int)reader.ReadVarint() == 1) weights.location = "external"; } //--- Name the file the weights were moved out to else if(field == 13 && wire == PB_LENGTH) { //--- Read the entry and keep whatever names a location int e1, e2; if(!reader.ReadBlock(e1, e2)) break; const string named = ReadEntry(inner, e1, e2); if(StringLen(named) > 0) weights.location = named; } //--- Step over anything else else if(!reader.SkipField(wire)) break; } //--- Multiply every axis to size the block weights.values = (weights.dims > 0) ? 1 : 0; for(int i = 0; i < weights.dims; i++) weights.values *= weights.dim[i]; //--- Summarize whichever payload the file actually used if(rawFrom >= 0) RawStats(inner, rawFrom, rawTo, weights); else if(packFrom >= 0) PackedStats(inner, packFrom, packTo, packField, weights); //--- Reject a tensor carrying no name if(StringLen(weights.name) == 0) return false; //--- Append the finished tensor ArrayResize(m_tensor, m_tensors + 1); m_tensor[m_tensors++] = weights; //--- Report success return true; } //+------------------------------------------------------------------+ //| Read a key and value pair into a pair of lists | //+------------------------------------------------------------------+ void COmvModel::ReadPair(const uchar &raw[], const int from, const int to, string &keys[], string &values[], int &count) { //--- Refuse a pair beyond what the lists may hold if(count >= OMV_MAX_META) return; //--- Point a cursor at the pair CPbReader reader; if(!reader.Attach(raw, from, to)) return; //--- Read the key and the value it carries string key = "", value = ""; while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Store the key if(field == 1 && wire == PB_LENGTH) reader.ReadString(key); //--- Store the value else if(field == 2 && wire == PB_LENGTH) reader.ReadString(value); //--- Step over anything else else if(!reader.SkipField(wire)) break; } //--- Keep only a pair that actually named something if(StringLen(key) == 0) return; keys[count] = key; values[count] = value; count++; } //+------------------------------------------------------------------+ //| Read an operator the file defines for itself | //+------------------------------------------------------------------+ void COmvModel::ReadFunction(const uchar &raw[], const int from, const int to) { //--- Refuse a definition beyond what the list may hold if(m_funcs >= OMV_MAX_FUNCS) return; //--- Point a cursor at the definition CPbReader reader; if(!reader.Attach(raw, from, to)) return; //--- Count what the definition carries and name it string named = "", domain = ""; int nodes = 0, ins = 0, outs = 0; while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Step over anything that is not a nested payload if(wire != PB_LENGTH) { if(!reader.SkipField(wire)) break; continue; } //--- Take the name the definition was given if(field == 1) reader.ReadString(named); //--- Take the domain it belongs to else if(field == 9) reader.ReadString(domain); //--- Count the parts the definition is built from else { //--- Locate the payload and count it by which field it arrived in int f2, t2; if(!reader.ReadBlock(f2, t2)) break; if(field == 2) ins++; else if(field == 3) outs++; else if(field == 7) nodes++; } } //--- Keep only a definition that actually named itself if(StringLen(named) == 0) return; m_funcName[m_funcs] = (StringLen(domain) > 0) ? domain + "." + named : named; m_funcDetail[m_funcs] = IntegerToString(nodes) + " nodes, " + IntegerToString(ins) + " in, " + IntegerToString(outs) + " out"; m_funcs++; } //+------------------------------------------------------------------+ //| Read a key and value pair the file uses for metadata | //+------------------------------------------------------------------+ string COmvModel::ReadEntry(const uchar &raw[], const int from, const int to) { //--- Point a cursor at the pair CPbReader reader; if(!reader.Attach(raw, from, to)) return ""; //--- Read the key and the value it carries string key = "", value = ""; while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Store the key if(field == 1 && wire == PB_LENGTH) reader.ReadString(key); //--- Store the value else if(field == 2 && wire == PB_LENGTH) reader.ReadString(value); //--- Step over anything else else if(!reader.SkipField(wire)) break; } //--- Report only the entry that names where the data sits return (key == "location") ? value : ""; } //+------------------------------------------------------------------+ //| Read a sparse initializer through its values tensor | //+------------------------------------------------------------------+ bool COmvModel::ParseSparse(const uchar &raw[], const int from, const int to) { //--- Point a cursor at the sparse tensor CPbReader reader; if(!reader.Attach(raw, from, to)) return false; uchar inner[]; reader.Bytes(inner); //--- Remember where the shape and the values sit int valFrom = -1, valTo = -1; int dim[OMV_MAX_DIMS]; int dims = 0; //--- Read every field of the sparse tensor while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Locate the tensor holding the values themselves if(field == 1 && wire == PB_LENGTH) { if(!reader.ReadBlock(valFrom, valTo)) break; } //--- Store one axis of the shape the sparse tensor stands for else if(field == 3 && wire == PB_VARINT) { //--- Keep the axis while room remains if(dims < OMV_MAX_DIMS) dim[dims++] = (int)reader.ReadVarint(); else reader.ReadVarint(); } //--- Read several axes packed together else if(field == 3 && wire == PB_LENGTH) { //--- Locate the packed block and read every length it holds int p1, p2; if(!reader.ReadBlock(p1, p2)) break; CPbReader packed; packed.Attach(inner, p1, p2); while(!packed.Eof() && dims < OMV_MAX_DIMS) dim[dims++] = (int)packed.ReadVarint(); } //--- Step over the indices and anything else else if(!reader.SkipField(wire)) break; } //--- Nothing to report without the values tensor if(valFrom < 0) return false; //--- Read the values as an ordinary tensor if(!ParseTensor(inner, valFrom, valTo)) return false; //--- Correct the entry to describe the sparse tensor it came from const int slot = m_tensors - 1; m_tensor[slot].category = "Sparse Initializer"; if(dims > 0) { //--- Report the shape the sparse tensor stands for, not the value run m_tensor[slot].dims = dims; for(int i = 0; i < dims; i++) m_tensor[slot].dim[i] = dim[i]; m_tensor[slot].values = 1; for(int i = 0; i < dims; i++) m_tensor[slot].values *= dim[i]; } //--- Report the sparse tensor as read return true; } //+------------------------------------------------------------------+ //| Read one value info into an exposed port | //+------------------------------------------------------------------+ bool COmvModel::ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port) { //--- Start an empty port port.name = ""; port.dims = 0; port.elemType = 0; //--- Point a cursor at this value info CPbReader value; if(!value.Attach(raw, from, to)) return false; //--- Keep the slice so nested payloads can be located uchar level1[]; value.Bytes(level1); //--- Read every field of the value info while(!value.Eof()) { //--- Take the next field tag int field, wire; if(!value.ReadTag(field, wire)) break; //--- Store the port name if(field == 1 && wire == PB_LENGTH) { value.ReadString(port.name); continue; } //--- Step over anything but the type description if(field != 2 || wire != PB_LENGTH) { if(!value.SkipField(wire)) break; continue; } //--- Descend into the type description int t1, t2; if(!value.ReadBlock(t1, t2)) break; CPbReader type; type.Attach(level1, t1, t2); uchar level2[]; type.Bytes(level2); //--- Read every field of the type while(!type.Eof()) { //--- Take the next field tag int f2, w2; if(!type.ReadTag(f2, w2)) break; //--- Step over anything but the tensor description if(f2 != 1 || w2 != PB_LENGTH) { if(!type.SkipField(w2)) break; continue; } //--- Descend into the tensor description int s1, s2; if(!type.ReadBlock(s1, s2)) break; CPbReader tt; tt.Attach(level2, s1, s2); uchar level3[]; tt.Bytes(level3); //--- Read every field of the tensor description while(!tt.Eof()) { //--- Take the next field tag int f3, w3; if(!tt.ReadTag(f3, w3)) break; //--- Store the element type if(f3 == 1 && w3 == PB_VARINT) { port.elemType = (int)tt.ReadVarint(); continue; } //--- Step over anything but the shape if(f3 != 2 || w3 != PB_LENGTH) { if(!tt.SkipField(w3)) break; continue; } //--- Descend into the shape int p1, p2; if(!tt.ReadBlock(p1, p2)) break; CPbReader shape; shape.Attach(level3, p1, p2); uchar level4[]; shape.Bytes(level4); //--- Read every axis the shape holds while(!shape.Eof()) { //--- Take the next field tag int f4, w4; if(!shape.ReadTag(f4, w4)) break; //--- Step over anything but an axis if(f4 != 1 || w4 != PB_LENGTH) { if(!shape.SkipField(w4)) break; continue; } //--- Descend into the axis int d1, d2; if(!shape.ReadBlock(d1, d2)) break; CPbReader dim; dim.Attach(level4, d1, d2); //--- Read every field of the axis while(!dim.Eof()) { //--- Take the next field tag int f5, w5; if(!dim.ReadTag(f5, w5)) break; //--- Store the axis length if(f5 == 1 && w5 == PB_VARINT) { if(port.dims < OMV_MAX_DIMS) port.dim[port.dims++] = (int)dim.ReadVarint(); else dim.ReadVarint(); } //--- Step over a named axis else if(!dim.SkipField(w5)) break; } } } } } //--- Report a port that carries a name return StringLen(port.name) > 0; } //+------------------------------------------------------------------+ //| Read the graph into nodes, weights and ports | //+------------------------------------------------------------------+ bool COmvModel::ParseGraph(const uchar &raw[], const int from, const int to) { //--- Point a cursor at the graph CPbReader graph; if(!graph.Attach(raw, from, to)) { m_error = "graph out of range"; return false; } //--- Keep the slice so nested payloads can be located uchar inner[]; graph.Bytes(inner); //--- Read every field of the graph while(!graph.Eof()) { //--- Take the next field tag int field, wire; if(!graph.ReadTag(field, wire)) break; //--- Step over anything that is not a nested message if(wire != PB_LENGTH && field != 0) { if(!graph.SkipField(wire)) break; continue; } //--- Locate the nested payload int f2, t2; if(!graph.ReadBlock(f2, t2)) break; //--- Read a node if(field == 1) ParseNode(inner, f2, t2); //--- Read an initializer else if(field == 5) ParseTensor(inner, f2, t2); //--- Read a sparse initializer, whose values sit one level deeper else if(field == 15) ParseSparse(inner, f2, t2); //--- Decode the graph name from its payload else if(field == 2) { uchar tmp[]; ArrayResize(tmp, t2 - f2); for(int i = f2; i < t2; i++) tmp[i - f2] = inner[i]; m_graphName = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8); } //--- Read a graph input else if(field == 11) { OmvPort port; if(ParsePort(inner, f2, t2, port)) { ArrayResize(m_input, m_inputs + 1); m_input[m_inputs++] = port; } } //--- Read a key and value pair the file attached to the graph else if(field == 16) ReadPair(inner, f2, t2, m_gMetaKey, m_gMetaValue, m_gMetas); //--- Count a quantization mapping the graph carries else if(field == 14) m_quants++; //--- Read whatever the file says about the graph else if(field == 10) { //--- Decode the text from its own payload uchar tmp[]; ArrayResize(tmp, t2 - f2); for(int i = f2; i < t2; i++) tmp[i - f2] = inner[i]; m_graphDoc = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8); } //--- Read what the file says about a value passing between layers else if(field == 13) { OmvPort port; if(ParsePort(inner, f2, t2, port)) { ArrayResize(m_inner, m_inners + 1); m_inner[m_inners++] = port; } } //--- Read a graph output else if(field == 12) { OmvPort port; if(ParsePort(inner, f2, t2, port)) { ArrayResize(m_output, m_outputs + 1); m_output[m_outputs++] = port; } } } //--- Report success return true; } //+------------------------------------------------------------------+ //| Open a file and read its whole structure | //+------------------------------------------------------------------+ bool COmvModel::Load(const string path) { //--- Drop any model held from a previous call Clear(); //--- Read the whole file into memory CPbReader reader; if(!reader.LoadFile(path)) { m_error = "cannot open " + path; return false; } //--- Keep the bytes so nested payloads can be located uchar raw[]; reader.Bytes(raw); //--- Read every field of the model while(!reader.Eof()) { //--- Take the next field tag int field, wire; if(!reader.ReadTag(field, wire)) break; //--- Store the format revision if(field == 1 && wire == PB_VARINT) m_irVersion = (long)reader.ReadVarint(); //--- Store the producing tool else if(field == 2 && wire == PB_LENGTH) reader.ReadString(m_producer); //--- Store the version of that tool else if(field == 3 && wire == PB_LENGTH) reader.ReadString(m_producerVersion); //--- Store the domain the model belongs to else if(field == 4 && wire == PB_LENGTH) reader.ReadString(m_domain); //--- Store the revision the model gives itself else if(field == 5 && wire == PB_VARINT) m_modelVersion = (long)reader.ReadVarint(); //--- Store whatever the file says about itself else if(field == 6 && wire == PB_LENGTH) reader.ReadString(m_docString); //--- Store a key and value pair the file attached to the model else if(field == 14 && wire == PB_LENGTH) { //--- Read the pair and keep it while room remains int e1, e2; if(!reader.ReadBlock(e1, e2)) break; ReadPair(raw, e1, e2, m_metaKey, m_metaValue, m_metas); } //--- Handle training state the file stores with the model else if(field == 20 && wire == PB_LENGTH) { //--- Step past it, only its presence is reported int g1, g2; if(!reader.ReadBlock(g1, g2)) break; m_hasTraining = true; } //--- Store an operator the file defines for itself else if(field == 25 && wire == PB_LENGTH) { //--- Read enough of it to name it and say what it holds int u1, u2; if(!reader.ReadBlock(u1, u2)) break; ReadFunction(raw, u1, u2); } //--- Read the graph itself else if(field == 7 && wire == PB_LENGTH) { int from, to; if(!reader.ReadBlock(from, to)) break; if(!ParseGraph(raw, from, to)) return false; } //--- Read the operator set version else if(field == 8 && wire == PB_LENGTH) { //--- Locate the operator set message int from, to; if(!reader.ReadBlock(from, to)) break; //--- Read its version field CPbReader ops; ops.Attach(raw, from, to); while(!ops.Eof()) { int f2, w2; if(!ops.ReadTag(f2, w2)) break; if(f2 == 2 && w2 == PB_VARINT) m_opset = (int)ops.ReadVarint(); else if(!ops.SkipField(w2)) break; } } //--- Step over anything else else if(!reader.SkipField(wire)) break; } //--- Report a file that could not be decoded if(reader.Failed() && m_nodes == 0) { m_error = "malformed protobuf in " + path; return false; } //--- Report a file that holds no graph if(m_nodes == 0) { m_error = "no nodes found in " + path; return false; } //--- Report success return true; } //+------------------------------------------------------------------+ //| Total every weight the file carries | //+------------------------------------------------------------------+ int COmvModel::Parameters(void) const { //--- Add the value count of every tensor int total = 0; for(int i = 0; i < m_tensors; i++) total += m_tensor[i].values; //--- Return the running total return total; } //+------------------------------------------------------------------+ //| Render a shape as a bracketed list | //+------------------------------------------------------------------+ string COmvModel::DimsText(const int &dim[], const int dims) const { //--- Print empty brackets for an unshaped tensor if(dims <= 0) return "[]"; //--- Join every axis with a separator string out = "["; for(int i = 0; i < dims; i++) { //--- Separate all but the first axis if(i > 0) out += "x"; //--- Append this axis length out += IntegerToString(dim[i]); } //--- Close the bracket return out + "]"; } //+------------------------------------------------------------------+ //| Name an ONNX element type code | //+------------------------------------------------------------------+ string COmvModel::TypeName(const int code) const { //--- Name each code the way the reader maps the ONNX enum switch(code) { case 0: return "undefined"; case 1: return "float32"; case 2: return "uint8"; case 3: return "int8"; case 4: return "uint16"; case 5: return "int16"; case 6: return "int32"; case 7: return "int64"; case 8: return "string"; case 9: return "boolean"; case 10: return "float16"; case 11: return "float64"; case 12: return "uint32"; case 13: return "uint64"; case 14: return "complex"; case 15: return "complex"; case 16: return "bfloat16"; } //--- Print the raw code for anything unrecognized return "type" + IntegerToString(code); } //+------------------------------------------------------------------+ //| Join the axes the way a shape row reports them | //+------------------------------------------------------------------+ string COmvModel::DimsBare(const int &dim[], const int dims) const { //--- Report nothing for a tensor the file never shaped if(dims <= 0) return ""; //--- Join every axis with a comma and no brackets string out = ""; for(int i = 0; i < dims; i++) { //--- Separate all but the first axis if(i > 0) out += ","; //--- Append this axis length out += IntegerToString(dim[i]); } //--- Report the joined axes return out; } //+------------------------------------------------------------------+ //| Render a shape the way a tensor type is written | //+------------------------------------------------------------------+ string COmvModel::DimsList(const int &dim[], const int dims) const { //--- An unshaped tensor prints as empty brackets rather than nothing if(dims <= 0) return "[]"; //--- Separate the axes with commas the way a tensor type reads string out = "["; for(int i = 0; i < dims; i++) { //--- Separate all but the first axis if(i > 0) out += ","; //--- Append this axis length out += IntegerToString(dim[i]); } //--- Close the bracket return out + "]"; } #endif // OMV_MODEL_MQH //+------------------------------------------------------------------+