| //+------------------------------------------------------------------+
|
| //| OMV Model.mqh |
|
| //| Copyright 2026, Allan Munene Mutiiria. |
|
| //| https://t.me/Forex_Algo_Trader |
|
| //+------------------------------------------------------------------+
|
| #property copyright "Copyright 2026, Allan Munene Mutiiria."
|
| #property link "https://t.me/Forex_Algo_Trader"
|
|
|
| #ifndef OMV_MODEL_MQH
|
| #define OMV_MODEL_MQH
|
|
|
| #include "OMV Protobuf.mqh"
|
|
|
| //+------------------------------------------------------------------+
|
| //| Capacity limits |
|
| //+------------------------------------------------------------------+
|
| #define OMV_MAX_NODES 4096 // Layers the graph may hold
|
| #define OMV_MAX_TENSORS 4096 // Weight tensors the graph may hold
|
| #define OMV_MAX_PORTS 64 // Names one node may carry per side
|
| #define OMV_MAX_ATTRS 32 // Attributes one node may carry
|
| #define OMV_MAX_META 32 // Key and value pairs the file may carry
|
| #define OMV_MAX_FUNCS 64 // Custom operators the file may define
|
| #define OMV_MAX_DIMS 16 // Axes one tensor may have
|
| #define OMV_MAX_VALUES 4096 // Weights one tensor keeps for display
|
|
|
| //+------------------------------------------------------------------+
|
| //| Graph layer |
|
| //+------------------------------------------------------------------+
|
| struct OmvNode
|
| {
|
| // Name the operator this layer applies
|
| string op;
|
| // Name this layer was given
|
| string name;
|
| // Name the operator set this layer's operator comes from
|
| string domain;
|
| // Name each attribute the operator was given
|
| string attrName[OMV_MAX_ATTRS];
|
| // Print the value each attribute holds
|
| string attrValue[OMV_MAX_ATTRS];
|
| // Record which kind each attribute declared itself to be
|
| int attrKind[OMV_MAX_ATTRS];
|
| // Count the attributes present
|
| int attrCount;
|
| // Carry whatever the file says about this layer
|
| string docString;
|
| // Name the overload the operator was resolved to
|
| string overload;
|
| // List the names feeding this layer
|
| string feeds[OMV_MAX_PORTS];
|
| // Count the names present
|
| int feedCount;
|
| // List the names this layer produces
|
| string emits[OMV_MAX_PORTS];
|
| // Count the names present
|
| int emitCount;
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Weight tensor |
|
| //+------------------------------------------------------------------+
|
| struct OmvTensor
|
| {
|
| // Name the nodes refer to it by
|
| string name;
|
| // Measure each axis
|
| int dim[OMV_MAX_DIMS];
|
| // Count the axes present
|
| int dims;
|
| // Record the ONNX type code
|
| int dataType;
|
| // Multiply every axis to size the block
|
| int values;
|
| // Track the smallest weight held
|
| double minValue;
|
| // Track the largest weight held
|
| double maxValue;
|
| // Track the average weight
|
| double meanValue;
|
| // Track how far the weights spread from their average
|
| double stdValue;
|
| // Track what share of the weights are zero
|
| double sparsity;
|
| // Keep the weights themselves so the inspector can show them
|
| double val[];
|
| // Count the weights kept, which stops at the display limit
|
| int kept;
|
| // Report where the file keeps this tensor's values
|
| string location;
|
| // Name the category the file gives this tensor
|
| string category;
|
| // Keep the strings when the tensor holds text rather than numbers
|
| string text[];
|
| // Count the strings kept
|
| int texts;
|
| // Report whether the file stored this tensor in a segment
|
| bool segmented;
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Exposed graph port |
|
| //+------------------------------------------------------------------+
|
| struct OmvPort
|
| {
|
| // Name the graph exposes
|
| string name;
|
| // Measure each axis
|
| int dim[OMV_MAX_DIMS];
|
| // Count the axes present
|
| int dims;
|
| // Record the ONNX type code
|
| int elemType;
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| ONNX file reader |
|
| //+------------------------------------------------------------------+
|
| class COmvModel
|
| {
|
| private:
|
| // Store the layers in file order
|
| OmvNode m_node[];
|
| // Store the weight tensors
|
| OmvTensor m_tensor[];
|
| // Store the graph inputs
|
| OmvPort m_input[];
|
| // Store the graph outputs
|
| OmvPort m_output[];
|
| // Store what the file says about the values passing between layers
|
| OmvPort m_inner[];
|
| // Count the layers held
|
| int m_nodes;
|
| // Count the tensors held
|
| int m_tensors;
|
| // Count the graph inputs held
|
| int m_inputs;
|
| // Count the graph outputs held
|
| int m_outputs;
|
| // Count the inner values held
|
| int m_inners;
|
| // Record the format revision
|
| long m_irVersion;
|
| // Record the operator set version
|
| int m_opset;
|
| // Name the tool that wrote the file
|
| string m_producer;
|
| // Record the version of the tool that wrote it
|
| string m_producerVersion;
|
| // Name the domain the model belongs to
|
| string m_domain;
|
| // Record the revision the model gives itself
|
| long m_modelVersion;
|
| // Carry whatever the file says about itself
|
| string m_docString;
|
| // Name the graph inside the file
|
| string m_graphName;
|
| // Carry whatever the file says about the graph
|
| string m_graphDoc;
|
| // Name each key the file attached to the model
|
| string m_metaKey[OMV_MAX_META];
|
| // Print the value each key carries
|
| string m_metaValue[OMV_MAX_META];
|
| // Count the pairs held
|
| int m_metas;
|
| // Name each operator the file defines for itself
|
| string m_funcName[OMV_MAX_FUNCS];
|
| // Describe what each defined operator holds
|
| string m_funcDetail[OMV_MAX_FUNCS];
|
| // Count the defined operators held
|
| int m_funcs;
|
| // Report whether the file carries training state
|
| bool m_hasTraining;
|
| // Name each key the file attached to the graph
|
| string m_gMetaKey[OMV_MAX_META];
|
| // Print the value each graph key carries
|
| string m_gMetaValue[OMV_MAX_META];
|
| // Count the graph pairs held
|
| int m_gMetas;
|
| // Count the quantization mappings the graph carries
|
| int m_quants;
|
| // Explain why the last load failed
|
| string m_error;
|
| // Hold the file's own bytes, so the terminal can run what we drew
|
| uchar m_raw[];
|
|
|
| bool ParseGraph(const uchar &raw[], const int from, const int to);
|
| bool ParseNode(const uchar &raw[], const int from, const int to);
|
| bool ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer);
|
| string ReadList(const uchar &raw[], const int from, const int to, const int field);
|
| string ReadSubgraph(const uchar &raw[], const int from, const int to);
|
| string ReadEntry(const uchar &raw[], const int from, const int to);
|
| void ReadPair(const uchar &raw[], const int from, const int to,
|
| string &keys[], string &values[], int &count);
|
| void ReadFunction(const uchar &raw[], const int from, const int to);
|
| bool ParseTensor(const uchar &raw[], const int from, const int to);
|
| bool ParseSparse(const uchar &raw[], const int from, const int to);
|
| bool ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port);
|
| void Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
|
| double &total, double &squares,
|
| double &lowest, double &highest);
|
| void Publish(OmvTensor &tensor, const int seen, const int zeros,
|
| const double total, const double squares,
|
| const double lowest, const double highest);
|
| void RawStats(const uchar &raw[], const int from, const int to,
|
| OmvTensor &tensor);
|
| void PackedStats(const uchar &raw[], const int from, const int to,
|
| const int field, OmvTensor &tensor);
|
|
|
| public:
|
| COmvModel(void);
|
| bool Load(const string path);
|
| void Clear(void);
|
|
|
| int Nodes(void) const { return m_nodes; }
|
| int Tensors(void) const { return m_tensors; }
|
| int Inputs(void) const { return m_inputs; }
|
| int Outputs(void) const { return m_outputs; }
|
| int Inners(void) const { return m_inners; }
|
| long IrVersion(void) const { return m_irVersion; }
|
| int Opset(void) const { return m_opset; }
|
| string Producer(void) const { return m_producer; }
|
| string ProducerVersion(void) const { return m_producerVersion; }
|
| string Domain(void) const { return m_domain; }
|
| long ModelVersion(void) const { return m_modelVersion; }
|
| string DocString(void) const { return m_docString; }
|
| string GraphName(void) const { return m_graphName; }
|
| string GraphDoc(void) const { return m_graphDoc; }
|
| int Metas(void) const { return m_metas; }
|
| int GraphMetas(void) const { return m_gMetas; }
|
| string GraphMetaKey(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaKey[i] : ""; }
|
| string GraphMetaValue(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaValue[i] : ""; }
|
| int Quants(void) const { return m_quants; }
|
| string MetaKey(const int i) const { return (i >= 0 && i < m_metas) ? m_metaKey[i] : ""; }
|
| string MetaValue(const int i) const { return (i >= 0 && i < m_metas) ? m_metaValue[i] : ""; }
|
| int Funcs(void) const { return m_funcs; }
|
| string FuncName(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcName[i] : ""; }
|
| string FuncDetail(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcDetail[i] : ""; }
|
| bool HasTraining(void) const { return m_hasTraining; }
|
| string Error(void) const { return m_error; }
|
| void RawTo(uchar &out[]) const { ArrayCopy(out, m_raw); }
|
| int Parameters(void) const;
|
|
|
| void NodeAt(const int i, OmvNode &out) const { if(i >= 0 && i < m_nodes) out = m_node[i]; }
|
| void TensorAt(const int i, OmvTensor &out) const { if(i >= 0 && i < m_tensors) out = m_tensor[i]; }
|
| void InputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inputs) out = m_input[i]; }
|
| void OutputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_outputs) out = m_output[i]; }
|
| void InnerAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inners) out = m_inner[i]; }
|
| string DimsText(const int &dim[], const int dims) const;
|
| string DimsList(const int &dim[], const int dims) const;
|
| string DimsBare(const int &dim[], const int dims) const;
|
| string TypeName(const int code) const;
|
| string AttrType(const int kind) const;
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Construct an empty model |
|
| //+------------------------------------------------------------------+
|
| COmvModel::COmvModel(void)
|
| {
|
| //--- Clear every field so a query before loading reports empty
|
| Clear();
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Drop everything the previous file filled |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::Clear(void)
|
| {
|
| //--- Zero the counts that bound every array
|
| m_nodes = 0;
|
| m_tensors = 0;
|
| m_inputs = 0;
|
| m_outputs = 0;
|
| m_inners = 0;
|
| //--- Release the stored structures
|
| ArrayResize(m_node, 0);
|
| ArrayResize(m_tensor, 0);
|
| ArrayResize(m_input, 0);
|
| ArrayResize(m_output, 0);
|
| ArrayFree(m_raw);
|
| ArrayResize(m_inner, 0);
|
| //--- Clear the header fields
|
| m_irVersion = 0;
|
| m_opset = 0;
|
| m_producer = "";
|
| m_producerVersion = "";
|
| m_domain = "";
|
| m_modelVersion = 0;
|
| m_docString = "";
|
| m_graphName = "";
|
| m_graphDoc = "";
|
| m_metas = 0;
|
| m_gMetas = 0;
|
| m_quants = 0;
|
| m_funcs = 0;
|
| m_hasTraining = false;
|
| m_error = "";
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Walk raw weight bytes for their range and average |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
|
| double &total, double &squares, double &lowest, double &highest)
|
| {
|
| //--- Seed the range from the first value that arrives
|
| if(seen == 0)
|
| {
|
| lowest = v;
|
| highest = v;
|
| }
|
| //--- Lower the floor when this value is smaller
|
| if(v < lowest)
|
| lowest = v;
|
| //--- Raise the ceiling when this value is larger
|
| if(v > highest)
|
| highest = v;
|
| //--- Add the value to the running totals
|
| total += v;
|
| squares += v * v;
|
| //--- Count a value that is exactly zero
|
| if(v == 0.0)
|
| zeros++;
|
| //--- Keep the value itself while room remains
|
| if(tensor.kept < OMV_MAX_VALUES)
|
| {
|
| ArrayResize(tensor.val, tensor.kept + 1);
|
| tensor.val[tensor.kept++] = v;
|
| }
|
| seen++;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Publish what the accumulated values measured |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::Publish(OmvTensor &tensor, const int seen, const int zeros,
|
| const double total, const double squares,
|
| const double lowest, const double highest)
|
| {
|
| //--- Abort when the read stopped before any value arrived
|
| if(seen <= 0)
|
| return;
|
| //--- Publish the range and the average
|
| tensor.minValue = lowest;
|
| tensor.maxValue = highest;
|
| tensor.meanValue = total / seen;
|
| //--- Take the spread as the root of the mean squared deviation
|
| const double variance = squares / seen - tensor.meanValue * tensor.meanValue;
|
| tensor.stdValue = (variance > 0.0) ? MathSqrt(variance) : 0.0;
|
| //--- Take the share of the values that are exactly zero
|
| tensor.sparsity = (double)zeros / seen;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Walk a raw byte payload at the width its type implies |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::RawStats(const uchar &raw[], const int from, const int to, OmvTensor &tensor)
|
| {
|
| //--- Take the width one value occupies for this element type
|
| int width = 4;
|
| if(tensor.dataType == 11 || tensor.dataType == 7 || tensor.dataType == 13)
|
| width = 8;
|
| else
|
| if(tensor.dataType == 2 || tensor.dataType == 3 || tensor.dataType == 9)
|
| width = 1;
|
| else
|
| if(tensor.dataType == 4 || tensor.dataType == 5 || tensor.dataType == 10)
|
| width = 2;
|
| //--- Count the whole values the payload holds
|
| const int count = (to - from) / width;
|
| if(count <= 0)
|
| return;
|
| //--- Point a cursor at the payload
|
| CPbReader bytes;
|
| if(!bytes.Attach(raw, from, to))
|
| return;
|
| //--- Start the running measures
|
| double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
|
| int zeros = 0, seen = 0;
|
| //--- Visit every stored value
|
| for(int i = 0; i < count; i++)
|
| {
|
| //--- Read the value the way its type is written
|
| double v = 0.0;
|
| if(tensor.dataType == 1)
|
| v = (double)bytes.ReadFloat();
|
| else
|
| if(tensor.dataType == 11)
|
| v = bytes.ReadDouble();
|
| else
|
| if(width == 8)
|
| v = (double)(long)bytes.ReadFixed64();
|
| else
|
| if(width == 4)
|
| v = (double)(int)bytes.ReadFixed32();
|
| else
|
| if(width == 2)
|
| v = (double)(int)(bytes.ReadVarint() & 0xFFFF);
|
| else
|
| v = (double)bytes.ReadVarint();
|
| //--- Stop on a broken read
|
| if(bytes.Failed())
|
| break;
|
| //--- Fold the value into the running measures
|
| Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
|
| }
|
| //--- Publish what the walk measured
|
| Publish(tensor, seen, zeros, total, squares, lowest, highest);
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Walk a typed payload the file used instead of raw bytes |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::PackedStats(const uchar &raw[], const int from, const int to,
|
| const int field, OmvTensor &tensor)
|
| {
|
| //--- Point a cursor at the payload
|
| CPbReader bytes;
|
| if(!bytes.Attach(raw, from, to))
|
| return;
|
| //--- Start the running measures
|
| double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
|
| int zeros = 0, seen = 0;
|
| //--- Read the payload the way the field it arrived in is written
|
| while(!bytes.Eof())
|
| {
|
| //--- Take the next value in the packed run
|
| double v = 0.0;
|
| if(field == 4)
|
| v = (double)bytes.ReadFloat();
|
| else
|
| if(field == 10)
|
| v = bytes.ReadDouble();
|
| else
|
| if(field == 6)
|
| break;
|
| else
|
| v = (double)(long)bytes.ReadVarint();
|
| //--- Stop on a broken read
|
| if(bytes.Failed())
|
| break;
|
| //--- Fold the value into the running measures
|
| Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
|
| }
|
| //--- Publish what the walk measured
|
| Publish(tensor, seen, zeros, total, squares, lowest, highest);
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Join a list an attribute carried into one line |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::ReadList(const uchar &raw[], const int from, const int to, const int field)
|
| {
|
| //--- Point a cursor at the list
|
| CPbReader items;
|
| if(!items.Attach(raw, from, to))
|
| return "";
|
| //--- Join up to a handful of entries, then say how many remain
|
| string out = "";
|
| int shown = 0, total = 0;
|
| while(!items.Eof())
|
| {
|
| //--- Read the next entry the way its field is written
|
| string one = "";
|
| if(field == 7)
|
| one = DoubleToString((double)items.ReadFloat(), 4);
|
| else
|
| if(field == 8)
|
| one = IntegerToString((long)items.ReadVarint());
|
| else
|
| items.ReadString(one);
|
| //--- Stop on a broken read
|
| if(items.Failed())
|
| break;
|
| total++;
|
| //--- Keep the first few entries only, a row cannot hold more
|
| if(shown >= 8)
|
| continue;
|
| out += (shown > 0 ? ", " : "") + one;
|
| shown++;
|
| }
|
| //--- Say how many entries the line left out
|
| if(total > shown)
|
| out += ", ... " + IntegerToString(total - shown) + " more";
|
| return out;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Describe a nested graph by what it actually holds |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::ReadSubgraph(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Point a cursor at the nested graph
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return "<graph>";
|
| uchar inner[];
|
| reader.Bytes(inner);
|
| //--- Count what the nested graph carries and name its operators
|
| string title = "";
|
| string ops = "";
|
| int nodes = 0, tensors = 0, shown = 0;
|
| //--- Read every field of the nested graph
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Step over anything that is not a nested message
|
| if(wire != PB_LENGTH)
|
| {
|
| if(!reader.SkipField(wire))
|
| break;
|
| continue;
|
| }
|
| //--- Locate the nested payload
|
| int f2, t2;
|
| if(!reader.ReadBlock(f2, t2))
|
| break;
|
| //--- Count a node and name the operator it applies
|
| if(field == 1)
|
| {
|
| nodes++;
|
| //--- Name only the first few, a line cannot hold more
|
| if(shown < 6)
|
| {
|
| //--- Read the operator out of the node
|
| CPbReader entry;
|
| entry.Attach(inner, f2, t2);
|
| while(!entry.Eof())
|
| {
|
| //--- Take the next field of the entry
|
| int f3, w3;
|
| if(!entry.ReadTag(f3, w3))
|
| break;
|
| //--- Keep the operator and stop reading this node
|
| if(f3 == 4 && w3 == PB_LENGTH)
|
| {
|
| string op = "";
|
| entry.ReadString(op);
|
| ops += (shown > 0 ? ", " : "") + op;
|
| shown++;
|
| break;
|
| }
|
| if(!entry.SkipField(w3))
|
| break;
|
| }
|
| }
|
| }
|
| //--- Count an initializer the nested graph carries
|
| else
|
| if(field == 5 || field == 15)
|
| tensors++;
|
| //--- Take the name the nested graph was given
|
| else
|
| if(field == 2)
|
| {
|
| //--- Decode the name from its own payload
|
| uchar tmp[];
|
| ArrayResize(tmp, t2 - f2);
|
| for(int i = f2; i < t2; i++)
|
| tmp[i - f2] = inner[i];
|
| title = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
| }
|
| }
|
| //--- Report the nested graph by its name, its size and what it runs
|
| string out = "<graph";
|
| if(StringLen(title) > 0)
|
| out += " " + title;
|
| out += ": " + IntegerToString(nodes) + " nodes, " +
|
| IntegerToString(tensors) + " tensors";
|
| if(StringLen(ops) > 0)
|
| out += " (" + ops + (nodes > shown ? ", ..." : "") + ")";
|
| return out + ">";
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Name the type an attribute declared itself to be |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::AttrType(const int kind) const
|
| {
|
| //--- Match the declared kind to the type the app reports for it
|
| switch(kind)
|
| {
|
| case 1:
|
| return "float32";
|
| case 2:
|
| return "int64";
|
| case 3:
|
| return "string";
|
| case 4:
|
| return "tensor";
|
| case 5:
|
| return "graph";
|
| case 6:
|
| return "float32[]";
|
| case 7:
|
| return "int64[]";
|
| case 8:
|
| return "string[]";
|
| }
|
| //--- Report nothing recognizable as undefined
|
| return "undefined";
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read one attribute an operator was given |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer)
|
| {
|
| //--- Point a cursor at this attribute
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return false;
|
| uchar inner[];
|
| reader.Bytes(inner);
|
| //--- Refuse an attribute beyond what the layer may hold
|
| if(layer.attrCount >= OMV_MAX_ATTRS)
|
| return false;
|
| //--- Start with nothing read
|
| const int slot = layer.attrCount;
|
| layer.attrName[slot] = "";
|
| layer.attrValue[slot] = "";
|
| //--- Remember which kind of value the attribute declares
|
| int kind = 0;
|
| double asFloat = 0.0;
|
| long asInt = 0;
|
| string asText = "";
|
| bool haveInt = false, haveFloat = false;
|
| //--- Count the entries a repeated field delivers one at a time
|
| int repeats = 0;
|
| //--- Read every field of the attribute
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Store the attribute name
|
| if(field == 1 && wire == PB_LENGTH)
|
| reader.ReadString(layer.attrName[slot]);
|
| //--- Store a floating point value
|
| else
|
| if(field == 2 && wire == PB_FIXED32)
|
| {
|
| asFloat = (double)reader.ReadFloat();
|
| haveFloat = true;
|
| }
|
| //--- Store a whole number value
|
| else
|
| if(field == 3 && wire == PB_VARINT)
|
| {
|
| asInt = (long)reader.ReadVarint();
|
| haveInt = true;
|
| }
|
| //--- Store a text value
|
| else
|
| if(field == 4 && wire == PB_LENGTH)
|
| reader.ReadString(asText);
|
| //--- Name a nested tensor or graph rather than unfolding it
|
| else
|
| if((field == 5 || field == 6) && wire == PB_LENGTH)
|
| {
|
| //--- Step past the payload and record what it was
|
| int n1, n2;
|
| if(!reader.ReadBlock(n1, n2))
|
| break;
|
| asText = (field == 5) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
|
| }
|
| //--- Append one string, of which this field delivers many
|
| else
|
| if(field == 9 && wire == PB_LENGTH)
|
| {
|
| //--- Read this entry and join it to the ones before
|
| string one = "";
|
| if(!reader.ReadString(one))
|
| break;
|
| repeats++;
|
| if(repeats <= 8)
|
| asText += (repeats > 1 ? ", " : "") + one;
|
| }
|
| //--- Join a list of floats, whole numbers or strings
|
| else
|
| if((field == 7 || field == 8) && wire == PB_LENGTH)
|
| {
|
| //--- Locate the list and read it in the form it arrived
|
| int l1, l2;
|
| if(!reader.ReadBlock(l1, l2))
|
| break;
|
| asText = ReadList(inner, l1, l2, field);
|
| }
|
| //--- Name a list of tensors or graphs rather than unfolding it
|
| else
|
| if((field == 10 || field == 11) && wire == PB_LENGTH)
|
| {
|
| //--- Step past this entry and count it among its siblings
|
| int n1, n2;
|
| if(!reader.ReadBlock(n1, n2))
|
| break;
|
| repeats++;
|
| //--- Describe the first entry, then say how many followed
|
| if(repeats == 1)
|
| asText = (field == 10) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
|
| }
|
| //--- Store which kind the attribute declares itself to be
|
| else
|
| if(field == 20 && wire == PB_VARINT)
|
| kind = (int)reader.ReadVarint();
|
| //--- Step over anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Say how many entries a repeated field delivered beyond those shown
|
| if(repeats > 8)
|
| asText += ", ... " + IntegerToString(repeats - 8) + " more";
|
| else
|
| if(repeats > 1 && (kind == 5 || kind == 6 || kind == 9 || kind == 10))
|
| asText += " and " + IntegerToString(repeats - 1) + " more";
|
| //--- Reject an attribute that never gave a name
|
| if(StringLen(layer.attrName[slot]) == 0)
|
| return false;
|
| //--- Print the value the attribute carried, by the kind it declared
|
| if(kind == 3 || kind >= 4 || StringLen(asText) > 0)
|
| layer.attrValue[slot] = asText;
|
| else
|
| if(kind == 2 || (kind == 0 && haveInt))
|
| layer.attrValue[slot] = IntegerToString(asInt);
|
| else
|
| if(kind == 1 || (kind == 0 && haveFloat))
|
| layer.attrValue[slot] = DoubleToString(asFloat, 6);
|
| //--- Record the kind so the row can report the type it opens to
|
| layer.attrKind[slot] = kind;
|
| //--- Report the attribute as read
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read one node into a layer |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParseNode(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Refuse a file that exceeds the node limit
|
| if(m_nodes >= OMV_MAX_NODES)
|
| return false;
|
| //--- Point a cursor at this node
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return false;
|
| //--- Start an empty layer
|
| OmvNode layer;
|
| layer.op = "";
|
| layer.name = "";
|
| layer.domain = "";
|
| layer.docString = "";
|
| layer.overload = "";
|
| layer.attrCount = 0;
|
| layer.feedCount = 0;
|
| layer.emitCount = 0;
|
| //--- Read every field of the node
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Skip anything that is not text
|
| if(wire != PB_LENGTH)
|
| {
|
| if(!reader.SkipField(wire))
|
| break;
|
| continue;
|
| }
|
| //--- Read an attribute out of its own nested message
|
| if(field == 5)
|
| {
|
| //--- Locate the attribute and decode it while room remains
|
| int a1, a2;
|
| if(!reader.ReadBlock(a1, a2))
|
| break;
|
| if(layer.attrCount < OMV_MAX_ATTRS)
|
| if(ParseAttribute(raw, from + a1, from + a2, layer))
|
| layer.attrCount++;
|
| continue;
|
| }
|
| //--- Read the field as text
|
| string text;
|
| if(!reader.ReadString(text))
|
| break;
|
| //--- Store an input name
|
| if(field == 1 && layer.feedCount < OMV_MAX_PORTS)
|
| layer.feeds[layer.feedCount++] = text;
|
| //--- Store an output name
|
| else
|
| if(field == 2 && layer.emitCount < OMV_MAX_PORTS)
|
| layer.emits[layer.emitCount++] = text;
|
| //--- Store the layer name
|
| else
|
| if(field == 3)
|
| layer.name = text;
|
| //--- Store the operator
|
| else
|
| if(field == 4)
|
| layer.op = text;
|
| //--- Store the operator set the operator belongs to
|
| else
|
| if(field == 7)
|
| layer.domain = text;
|
| //--- Store whatever the file says about this layer
|
| else
|
| if(field == 6)
|
| layer.docString = text;
|
| //--- Store the overload the operator was resolved to
|
| else
|
| if(field == 8)
|
| layer.overload = text;
|
| }
|
| //--- Reject a node carrying no operator
|
| if(StringLen(layer.op) == 0)
|
| return false;
|
| //--- Append the finished layer
|
| ArrayResize(m_node, m_nodes + 1);
|
| m_node[m_nodes++] = layer;
|
| //--- Report success
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read one initializer into a weight tensor |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParseTensor(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Refuse a file that exceeds the tensor limit
|
| if(m_tensors >= OMV_MAX_TENSORS)
|
| return false;
|
| //--- Point a cursor at this tensor
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return false;
|
| //--- Keep the slice so nested payloads can be located
|
| uchar inner[];
|
| reader.Bytes(inner);
|
| //--- Start an empty tensor
|
| OmvTensor weights;
|
| weights.name = "";
|
| weights.dims = 0;
|
| weights.dataType = 0;
|
| weights.values = 0;
|
| weights.minValue = 0.0;
|
| weights.maxValue = 0.0;
|
| weights.meanValue = 0.0;
|
| weights.stdValue = 0.0;
|
| weights.sparsity = 0.0;
|
| weights.kept = 0;
|
| weights.texts = 0;
|
| weights.segmented = false;
|
| weights.location = "default";
|
| weights.category = "Initializer";
|
| //--- Mark every payload the tensor might carry as not yet found
|
| int rawFrom = -1, rawTo = -1;
|
| int packFrom = -1, packTo = -1, packField = 0;
|
| //--- Read every field of the tensor
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Store one axis length
|
| if(field == 1 && wire == PB_VARINT)
|
| {
|
| //--- Keep the axis while room remains
|
| if(weights.dims < OMV_MAX_DIMS)
|
| weights.dim[weights.dims++] = (int)reader.ReadVarint();
|
| else
|
| reader.ReadVarint();
|
| }
|
| //--- Store several axis lengths packed together
|
| else
|
| if(field == 1 && wire == PB_LENGTH)
|
| {
|
| //--- Locate the packed block
|
| int f2, t2;
|
| if(!reader.ReadBlock(f2, t2))
|
| break;
|
| //--- Read every length the block holds
|
| CPbReader packed;
|
| packed.Attach(inner, f2, t2);
|
| while(!packed.Eof() && weights.dims < OMV_MAX_DIMS)
|
| weights.dim[weights.dims++] = (int)packed.ReadVarint();
|
| }
|
| //--- Store the element type
|
| else
|
| if(field == 2 && wire == PB_VARINT)
|
| weights.dataType = (int)reader.ReadVarint();
|
| //--- Handle a tensor the file stored in segments
|
| else
|
| if(field == 3 && wire == PB_LENGTH)
|
| {
|
| //--- Step past the segment, only its presence is reported
|
| int s1, s2;
|
| if(!reader.ReadBlock(s1, s2))
|
| break;
|
| weights.segmented = true;
|
| }
|
| //--- Store the tensor name
|
| else
|
| if(field == 8 && wire == PB_LENGTH)
|
| reader.ReadString(weights.name);
|
| //--- Locate the raw byte payload
|
| else
|
| if(field == 9 && wire == PB_LENGTH)
|
| {
|
| if(!reader.ReadBlock(rawFrom, rawTo))
|
| break;
|
| }
|
| //--- Collect one string, of which the field may carry many
|
| else
|
| if(field == 6 && wire == PB_LENGTH)
|
| {
|
| //--- Append it while room remains
|
| string one = "";
|
| if(!reader.ReadString(one))
|
| break;
|
| if(weights.texts < OMV_MAX_VALUES)
|
| {
|
| ArrayResize(weights.text, weights.texts + 1);
|
| weights.text[weights.texts++] = one;
|
| }
|
| }
|
| //--- Locate a typed payload, which the file may use instead
|
| else
|
| if(wire == PB_LENGTH &&
|
| (field == 4 || field == 5 ||
|
| field == 7 || field == 10 || field == 11))
|
| {
|
| //--- Remember which typed field carried the values
|
| if(!reader.ReadBlock(packFrom, packTo))
|
| break;
|
| packField = field;
|
| }
|
| //--- Report values the file keeps somewhere else entirely
|
| else
|
| if(field == 14 && wire == PB_VARINT)
|
| {
|
| //--- A location of one means the weights sit outside the file
|
| if((int)reader.ReadVarint() == 1)
|
| weights.location = "external";
|
| }
|
| //--- Name the file the weights were moved out to
|
| else
|
| if(field == 13 && wire == PB_LENGTH)
|
| {
|
| //--- Read the entry and keep whatever names a location
|
| int e1, e2;
|
| if(!reader.ReadBlock(e1, e2))
|
| break;
|
| const string named = ReadEntry(inner, e1, e2);
|
| if(StringLen(named) > 0)
|
| weights.location = named;
|
| }
|
| //--- Step over anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Multiply every axis to size the block
|
| weights.values = (weights.dims > 0) ? 1 : 0;
|
| for(int i = 0; i < weights.dims; i++)
|
| weights.values *= weights.dim[i];
|
| //--- Summarize whichever payload the file actually used
|
| if(rawFrom >= 0)
|
| RawStats(inner, rawFrom, rawTo, weights);
|
| else
|
| if(packFrom >= 0)
|
| PackedStats(inner, packFrom, packTo, packField, weights);
|
| //--- Reject a tensor carrying no name
|
| if(StringLen(weights.name) == 0)
|
| return false;
|
| //--- Append the finished tensor
|
| ArrayResize(m_tensor, m_tensors + 1);
|
| m_tensor[m_tensors++] = weights;
|
| //--- Report success
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read a key and value pair into a pair of lists |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::ReadPair(const uchar &raw[], const int from, const int to,
|
| string &keys[], string &values[], int &count)
|
| {
|
| //--- Refuse a pair beyond what the lists may hold
|
| if(count >= OMV_MAX_META)
|
| return;
|
| //--- Point a cursor at the pair
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return;
|
| //--- Read the key and the value it carries
|
| string key = "", value = "";
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Store the key
|
| if(field == 1 && wire == PB_LENGTH)
|
| reader.ReadString(key);
|
| //--- Store the value
|
| else
|
| if(field == 2 && wire == PB_LENGTH)
|
| reader.ReadString(value);
|
| //--- Step over anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Keep only a pair that actually named something
|
| if(StringLen(key) == 0)
|
| return;
|
| keys[count] = key;
|
| values[count] = value;
|
| count++;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read an operator the file defines for itself |
|
| //+------------------------------------------------------------------+
|
| void COmvModel::ReadFunction(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Refuse a definition beyond what the list may hold
|
| if(m_funcs >= OMV_MAX_FUNCS)
|
| return;
|
| //--- Point a cursor at the definition
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return;
|
| //--- Count what the definition carries and name it
|
| string named = "", domain = "";
|
| int nodes = 0, ins = 0, outs = 0;
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Step over anything that is not a nested payload
|
| if(wire != PB_LENGTH)
|
| {
|
| if(!reader.SkipField(wire))
|
| break;
|
| continue;
|
| }
|
| //--- Take the name the definition was given
|
| if(field == 1)
|
| reader.ReadString(named);
|
| //--- Take the domain it belongs to
|
| else
|
| if(field == 9)
|
| reader.ReadString(domain);
|
| //--- Count the parts the definition is built from
|
| else
|
| {
|
| //--- Locate the payload and count it by which field it arrived in
|
| int f2, t2;
|
| if(!reader.ReadBlock(f2, t2))
|
| break;
|
| if(field == 2)
|
| ins++;
|
| else
|
| if(field == 3)
|
| outs++;
|
| else
|
| if(field == 7)
|
| nodes++;
|
| }
|
| }
|
| //--- Keep only a definition that actually named itself
|
| if(StringLen(named) == 0)
|
| return;
|
| m_funcName[m_funcs] = (StringLen(domain) > 0) ? domain + "." + named : named;
|
| m_funcDetail[m_funcs] = IntegerToString(nodes) + " nodes, " +
|
| IntegerToString(ins) + " in, " +
|
| IntegerToString(outs) + " out";
|
| m_funcs++;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read a key and value pair the file uses for metadata |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::ReadEntry(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Point a cursor at the pair
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return "";
|
| //--- Read the key and the value it carries
|
| string key = "", value = "";
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Store the key
|
| if(field == 1 && wire == PB_LENGTH)
|
| reader.ReadString(key);
|
| //--- Store the value
|
| else
|
| if(field == 2 && wire == PB_LENGTH)
|
| reader.ReadString(value);
|
| //--- Step over anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Report only the entry that names where the data sits
|
| return (key == "location") ? value : "";
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read a sparse initializer through its values tensor |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParseSparse(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Point a cursor at the sparse tensor
|
| CPbReader reader;
|
| if(!reader.Attach(raw, from, to))
|
| return false;
|
| uchar inner[];
|
| reader.Bytes(inner);
|
| //--- Remember where the shape and the values sit
|
| int valFrom = -1, valTo = -1;
|
| int dim[OMV_MAX_DIMS];
|
| int dims = 0;
|
| //--- Read every field of the sparse tensor
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Locate the tensor holding the values themselves
|
| if(field == 1 && wire == PB_LENGTH)
|
| {
|
| if(!reader.ReadBlock(valFrom, valTo))
|
| break;
|
| }
|
| //--- Store one axis of the shape the sparse tensor stands for
|
| else
|
| if(field == 3 && wire == PB_VARINT)
|
| {
|
| //--- Keep the axis while room remains
|
| if(dims < OMV_MAX_DIMS)
|
| dim[dims++] = (int)reader.ReadVarint();
|
| else
|
| reader.ReadVarint();
|
| }
|
| //--- Read several axes packed together
|
| else
|
| if(field == 3 && wire == PB_LENGTH)
|
| {
|
| //--- Locate the packed block and read every length it holds
|
| int p1, p2;
|
| if(!reader.ReadBlock(p1, p2))
|
| break;
|
| CPbReader packed;
|
| packed.Attach(inner, p1, p2);
|
| while(!packed.Eof() && dims < OMV_MAX_DIMS)
|
| dim[dims++] = (int)packed.ReadVarint();
|
| }
|
| //--- Step over the indices and anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Nothing to report without the values tensor
|
| if(valFrom < 0)
|
| return false;
|
| //--- Read the values as an ordinary tensor
|
| if(!ParseTensor(inner, valFrom, valTo))
|
| return false;
|
| //--- Correct the entry to describe the sparse tensor it came from
|
| const int slot = m_tensors - 1;
|
| m_tensor[slot].category = "Sparse Initializer";
|
| if(dims > 0)
|
| {
|
| //--- Report the shape the sparse tensor stands for, not the value run
|
| m_tensor[slot].dims = dims;
|
| for(int i = 0; i < dims; i++)
|
| m_tensor[slot].dim[i] = dim[i];
|
| m_tensor[slot].values = 1;
|
| for(int i = 0; i < dims; i++)
|
| m_tensor[slot].values *= dim[i];
|
| }
|
| //--- Report the sparse tensor as read
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read one value info into an exposed port |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port)
|
| {
|
| //--- Start an empty port
|
| port.name = "";
|
| port.dims = 0;
|
| port.elemType = 0;
|
| //--- Point a cursor at this value info
|
| CPbReader value;
|
| if(!value.Attach(raw, from, to))
|
| return false;
|
| //--- Keep the slice so nested payloads can be located
|
| uchar level1[];
|
| value.Bytes(level1);
|
| //--- Read every field of the value info
|
| while(!value.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!value.ReadTag(field, wire))
|
| break;
|
| //--- Store the port name
|
| if(field == 1 && wire == PB_LENGTH)
|
| {
|
| value.ReadString(port.name);
|
| continue;
|
| }
|
| //--- Step over anything but the type description
|
| if(field != 2 || wire != PB_LENGTH)
|
| {
|
| if(!value.SkipField(wire))
|
| break;
|
| continue;
|
| }
|
| //--- Descend into the type description
|
| int t1, t2;
|
| if(!value.ReadBlock(t1, t2))
|
| break;
|
| CPbReader type;
|
| type.Attach(level1, t1, t2);
|
| uchar level2[];
|
| type.Bytes(level2);
|
| //--- Read every field of the type
|
| while(!type.Eof())
|
| {
|
| //--- Take the next field tag
|
| int f2, w2;
|
| if(!type.ReadTag(f2, w2))
|
| break;
|
| //--- Step over anything but the tensor description
|
| if(f2 != 1 || w2 != PB_LENGTH)
|
| {
|
| if(!type.SkipField(w2))
|
| break;
|
| continue;
|
| }
|
| //--- Descend into the tensor description
|
| int s1, s2;
|
| if(!type.ReadBlock(s1, s2))
|
| break;
|
| CPbReader tt;
|
| tt.Attach(level2, s1, s2);
|
| uchar level3[];
|
| tt.Bytes(level3);
|
| //--- Read every field of the tensor description
|
| while(!tt.Eof())
|
| {
|
| //--- Take the next field tag
|
| int f3, w3;
|
| if(!tt.ReadTag(f3, w3))
|
| break;
|
| //--- Store the element type
|
| if(f3 == 1 && w3 == PB_VARINT)
|
| {
|
| port.elemType = (int)tt.ReadVarint();
|
| continue;
|
| }
|
| //--- Step over anything but the shape
|
| if(f3 != 2 || w3 != PB_LENGTH)
|
| {
|
| if(!tt.SkipField(w3))
|
| break;
|
| continue;
|
| }
|
| //--- Descend into the shape
|
| int p1, p2;
|
| if(!tt.ReadBlock(p1, p2))
|
| break;
|
| CPbReader shape;
|
| shape.Attach(level3, p1, p2);
|
| uchar level4[];
|
| shape.Bytes(level4);
|
| //--- Read every axis the shape holds
|
| while(!shape.Eof())
|
| {
|
| //--- Take the next field tag
|
| int f4, w4;
|
| if(!shape.ReadTag(f4, w4))
|
| break;
|
| //--- Step over anything but an axis
|
| if(f4 != 1 || w4 != PB_LENGTH)
|
| {
|
| if(!shape.SkipField(w4))
|
| break;
|
| continue;
|
| }
|
| //--- Descend into the axis
|
| int d1, d2;
|
| if(!shape.ReadBlock(d1, d2))
|
| break;
|
| CPbReader dim;
|
| dim.Attach(level4, d1, d2);
|
| //--- Read every field of the axis
|
| while(!dim.Eof())
|
| {
|
| //--- Take the next field tag
|
| int f5, w5;
|
| if(!dim.ReadTag(f5, w5))
|
| break;
|
| //--- Store the axis length
|
| if(f5 == 1 && w5 == PB_VARINT)
|
| {
|
| if(port.dims < OMV_MAX_DIMS)
|
| port.dim[port.dims++] = (int)dim.ReadVarint();
|
| else
|
| dim.ReadVarint();
|
| }
|
| //--- Step over a named axis
|
| else
|
| if(!dim.SkipField(w5))
|
| break;
|
| }
|
| }
|
| }
|
| }
|
| }
|
| //--- Report a port that carries a name
|
| return StringLen(port.name) > 0;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Read the graph into nodes, weights and ports |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::ParseGraph(const uchar &raw[], const int from, const int to)
|
| {
|
| //--- Point a cursor at the graph
|
| CPbReader graph;
|
| if(!graph.Attach(raw, from, to))
|
| {
|
| m_error = "graph out of range";
|
| return false;
|
| }
|
| //--- Keep the slice so nested payloads can be located
|
| uchar inner[];
|
| graph.Bytes(inner);
|
| //--- Read every field of the graph
|
| while(!graph.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!graph.ReadTag(field, wire))
|
| break;
|
| //--- Step over anything that is not a nested message
|
| if(wire != PB_LENGTH && field != 0)
|
| {
|
| if(!graph.SkipField(wire))
|
| break;
|
| continue;
|
| }
|
| //--- Locate the nested payload
|
| int f2, t2;
|
| if(!graph.ReadBlock(f2, t2))
|
| break;
|
| //--- Read a node
|
| if(field == 1)
|
| ParseNode(inner, f2, t2);
|
| //--- Read an initializer
|
| else
|
| if(field == 5)
|
| ParseTensor(inner, f2, t2);
|
| //--- Read a sparse initializer, whose values sit one level deeper
|
| else
|
| if(field == 15)
|
| ParseSparse(inner, f2, t2);
|
| //--- Decode the graph name from its payload
|
| else
|
| if(field == 2)
|
| {
|
| uchar tmp[];
|
| ArrayResize(tmp, t2 - f2);
|
| for(int i = f2; i < t2; i++)
|
| tmp[i - f2] = inner[i];
|
| m_graphName = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
| }
|
| //--- Read a graph input
|
| else
|
| if(field == 11)
|
| {
|
| OmvPort port;
|
| if(ParsePort(inner, f2, t2, port))
|
| {
|
| ArrayResize(m_input, m_inputs + 1);
|
| m_input[m_inputs++] = port;
|
| }
|
| }
|
| //--- Read a key and value pair the file attached to the graph
|
| else
|
| if(field == 16)
|
| ReadPair(inner, f2, t2, m_gMetaKey, m_gMetaValue, m_gMetas);
|
| //--- Count a quantization mapping the graph carries
|
| else
|
| if(field == 14)
|
| m_quants++;
|
| //--- Read whatever the file says about the graph
|
| else
|
| if(field == 10)
|
| {
|
| //--- Decode the text from its own payload
|
| uchar tmp[];
|
| ArrayResize(tmp, t2 - f2);
|
| for(int i = f2; i < t2; i++)
|
| tmp[i - f2] = inner[i];
|
| m_graphDoc = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
| }
|
| //--- Read what the file says about a value passing between layers
|
| else
|
| if(field == 13)
|
| {
|
| OmvPort port;
|
| if(ParsePort(inner, f2, t2, port))
|
| {
|
| ArrayResize(m_inner, m_inners + 1);
|
| m_inner[m_inners++] = port;
|
| }
|
| }
|
| //--- Read a graph output
|
| else
|
| if(field == 12)
|
| {
|
| OmvPort port;
|
| if(ParsePort(inner, f2, t2, port))
|
| {
|
| ArrayResize(m_output, m_outputs + 1);
|
| m_output[m_outputs++] = port;
|
| }
|
| }
|
| }
|
| //--- Report success
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Open a file and read its whole structure |
|
| //+------------------------------------------------------------------+
|
| bool COmvModel::Load(const string path)
|
| {
|
| //--- Drop any model held from a previous call
|
| Clear();
|
| //--- Read the whole file into memory
|
| CPbReader reader;
|
| if(!reader.LoadFile(path))
|
| {
|
| m_error = "cannot open " + path;
|
| return false;
|
| }
|
| //--- Keep a copy of the raw bytes for nested payloads
|
| uchar raw[];
|
| reader.Bytes(raw);
|
| ArrayResize(m_raw, ArraySize(raw));
|
| ArrayCopy(m_raw, raw);
|
| //--- Read every field of the model
|
| while(!reader.Eof())
|
| {
|
| //--- Take the next field tag
|
| int field, wire;
|
| if(!reader.ReadTag(field, wire))
|
| break;
|
| //--- Store the format revision
|
| if(field == 1 && wire == PB_VARINT)
|
| m_irVersion = (long)reader.ReadVarint();
|
| //--- Store the producing tool
|
| else
|
| if(field == 2 && wire == PB_LENGTH)
|
| reader.ReadString(m_producer);
|
| //--- Store the version of that tool
|
| else
|
| if(field == 3 && wire == PB_LENGTH)
|
| reader.ReadString(m_producerVersion);
|
| //--- Store the domain the model belongs to
|
| else
|
| if(field == 4 && wire == PB_LENGTH)
|
| reader.ReadString(m_domain);
|
| //--- Store the revision the model gives itself
|
| else
|
| if(field == 5 && wire == PB_VARINT)
|
| m_modelVersion = (long)reader.ReadVarint();
|
| //--- Store whatever the file says about itself
|
| else
|
| if(field == 6 && wire == PB_LENGTH)
|
| reader.ReadString(m_docString);
|
| //--- Store a key and value pair the file attached to the model
|
| else
|
| if(field == 14 && wire == PB_LENGTH)
|
| {
|
| //--- Read the pair and keep it while room remains
|
| int e1, e2;
|
| if(!reader.ReadBlock(e1, e2))
|
| break;
|
| ReadPair(raw, e1, e2, m_metaKey, m_metaValue, m_metas);
|
| }
|
| //--- Handle training state the file stores with the model
|
| else
|
| if(field == 20 && wire == PB_LENGTH)
|
| {
|
| //--- Step past it, only its presence is reported
|
| int g1, g2;
|
| if(!reader.ReadBlock(g1, g2))
|
| break;
|
| m_hasTraining = true;
|
| }
|
| //--- Store an operator the file defines for itself
|
| else
|
| if(field == 25 && wire == PB_LENGTH)
|
| {
|
| //--- Read enough of it to name it and say what it holds
|
| int u1, u2;
|
| if(!reader.ReadBlock(u1, u2))
|
| break;
|
| ReadFunction(raw, u1, u2);
|
| }
|
| //--- Read the graph itself
|
| else
|
| if(field == 7 && wire == PB_LENGTH)
|
| {
|
| int from, to;
|
| if(!reader.ReadBlock(from, to))
|
| break;
|
| if(!ParseGraph(raw, from, to))
|
| return false;
|
| }
|
| //--- Read the operator set version
|
| else
|
| if(field == 8 && wire == PB_LENGTH)
|
| {
|
| //--- Locate the operator set message
|
| int from, to;
|
| if(!reader.ReadBlock(from, to))
|
| break;
|
| //--- Read its version field
|
| CPbReader ops;
|
| ops.Attach(raw, from, to);
|
| while(!ops.Eof())
|
| {
|
| int f2, w2;
|
| if(!ops.ReadTag(f2, w2))
|
| break;
|
| if(f2 == 2 && w2 == PB_VARINT)
|
| m_opset = (int)ops.ReadVarint();
|
| else
|
| if(!ops.SkipField(w2))
|
| break;
|
| }
|
| }
|
| //--- Step over anything else
|
| else
|
| if(!reader.SkipField(wire))
|
| break;
|
| }
|
| //--- Report a file that could not be decoded
|
| if(reader.Failed() && m_nodes == 0)
|
| {
|
| m_error = "malformed protobuf in " + path;
|
| return false;
|
| }
|
| //--- Report a file that holds no graph
|
| if(m_nodes == 0)
|
| {
|
| m_error = "no nodes found in " + path;
|
| return false;
|
| }
|
| //--- Report success
|
| return true;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Total every weight the file carries |
|
| //+------------------------------------------------------------------+
|
| int COmvModel::Parameters(void) const
|
| {
|
| //--- Add the value count of every tensor
|
| int total = 0;
|
| for(int i = 0; i < m_tensors; i++)
|
| total += m_tensor[i].values;
|
| //--- Return the running total
|
| return total;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Render a shape as a bracketed list |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::DimsText(const int &dim[], const int dims) const
|
| {
|
| //--- Print empty brackets for an unshaped tensor
|
| if(dims <= 0)
|
| return "[]";
|
| //--- Join every axis with a separator
|
| string out = "[";
|
| for(int i = 0; i < dims; i++)
|
| {
|
| //--- Separate all but the first axis
|
| if(i > 0)
|
| out += "x";
|
| //--- Append this axis length
|
| out += IntegerToString(dim[i]);
|
| }
|
| //--- Close the bracket
|
| return out + "]";
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Name an ONNX element type code |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::TypeName(const int code) const
|
| {
|
| //--- Name each code the way the reader maps the ONNX enum
|
| switch(code)
|
| {
|
| case 0:
|
| return "undefined";
|
| case 1:
|
| return "float32";
|
| case 2:
|
| return "uint8";
|
| case 3:
|
| return "int8";
|
| case 4:
|
| return "uint16";
|
| case 5:
|
| return "int16";
|
| case 6:
|
| return "int32";
|
| case 7:
|
| return "int64";
|
| case 8:
|
| return "string";
|
| case 9:
|
| return "boolean";
|
| case 10:
|
| return "float16";
|
| case 11:
|
| return "float64";
|
| case 12:
|
| return "uint32";
|
| case 13:
|
| return "uint64";
|
| case 14:
|
| return "complex<float32>";
|
| case 15:
|
| return "complex<float64>";
|
| case 16:
|
| return "bfloat16";
|
| }
|
| //--- Print the raw code for anything unrecognized
|
| return "type" + IntegerToString(code);
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Join the axes the way a shape row reports them |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::DimsBare(const int &dim[], const int dims) const
|
| {
|
| //--- Report nothing for a tensor the file never shaped
|
| if(dims <= 0)
|
| return "";
|
| //--- Join every axis with a comma and no brackets
|
| string out = "";
|
| for(int i = 0; i < dims; i++)
|
| {
|
| //--- Separate all but the first axis
|
| if(i > 0)
|
| out += ",";
|
| //--- Append this axis length
|
| out += IntegerToString(dim[i]);
|
| }
|
| //--- Report the joined axes
|
| return out;
|
| }
|
|
|
| //+------------------------------------------------------------------+
|
| //| Render a shape the way a tensor type is written |
|
| //+------------------------------------------------------------------+
|
| string COmvModel::DimsList(const int &dim[], const int dims) const
|
| {
|
| //--- An unshaped tensor prints as empty brackets rather than nothing
|
| if(dims <= 0)
|
| return "[]";
|
| //--- Separate the axes with commas the way a tensor type reads
|
| string out = "[";
|
| for(int i = 0; i < dims; i++)
|
| {
|
| //--- Separate all but the first axis
|
| if(i > 0)
|
| out += ",";
|
| //--- Append this axis length
|
| out += IntegerToString(dim[i]);
|
| }
|
| //--- Close the bracket
|
| return out + "]";
|
| }
|
|
|
| #endif // OMV_MODEL_MQH
|
| //+------------------------------------------------------------------+ |