1710 lines
64 KiB
MQL5
1710 lines
64 KiB
MQL5
 | //+------------------------------------------------------------------+
| ||
//| OMV Model.mqh |
| |||
//| Copyright 2026, Allan Munene Mutiiria. |
| |||
//| https://t.me/Forex_Algo_Trader |
| |||
//+------------------------------------------------------------------+
| |||
#property copyright "Copyright 2026, Allan Munene Mutiiria."
| |||
#property link "https://t.me/Forex_Algo_Trader"
| |||
| |||
#ifndef OMV_MODEL_MQH
| |||
#define OMV_MODEL_MQH
| |||
| |||
#include "OMV Protobuf.mqh"
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Capacity limits |
| |||
//+------------------------------------------------------------------+
| |||
#define OMV_MAX_NODES 4096 // Layers the graph may hold
| |||
#define OMV_MAX_TENSORS 4096 // Weight tensors the graph may hold
| |||
#define OMV_MAX_PORTS 64 // Names one node may carry per side
| |||
#define OMV_MAX_ATTRS 32 // Attributes one node may carry
| |||
#define OMV_MAX_META 32 // Key and value pairs the file may carry
| |||
#define OMV_MAX_FUNCS 64 // Custom operators the file may define
| |||
#define OMV_MAX_DIMS 16 // Axes one tensor may have
| |||
#define OMV_MAX_VALUES 4096 // Weights one tensor keeps for display
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Graph layer |
| |||
//+------------------------------------------------------------------+
| |||
struct OmvNode
| |||
{
| |||
// Name the operator this layer applies
| |||
string op;
| |||
// Name this layer was given
| |||
string name;
| |||
// Name the operator set this layer's operator comes from
| |||
string domain;
| |||
// Name each attribute the operator was given
| |||
string attrName[OMV_MAX_ATTRS];
| |||
// Print the value each attribute holds
| |||
string attrValue[OMV_MAX_ATTRS];
| |||
// Record which kind each attribute declared itself to be
| |||
int attrKind[OMV_MAX_ATTRS];
| |||
// Count the attributes present
| |||
int attrCount;
| |||
// Carry whatever the file says about this layer
| |||
string docString;
| |||
// Name the overload the operator was resolved to
| |||
string overload;
| |||
// List the names feeding this layer
| |||
string feeds[OMV_MAX_PORTS];
| |||
// Count the names present
| |||
int feedCount;
| |||
// List the names this layer produces
| |||
string emits[OMV_MAX_PORTS];
| |||
// Count the names present
| |||
int emitCount;
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Weight tensor |
| |||
//+------------------------------------------------------------------+
| |||
struct OmvTensor
| |||
{
| |||
// Name the nodes refer to it by
| |||
string name;
| |||
// Measure each axis
| |||
int dim[OMV_MAX_DIMS];
| |||
// Count the axes present
| |||
int dims;
| |||
// Record the ONNX type code
| |||
int dataType;
| |||
// Multiply every axis to size the block
| |||
int values;
| |||
// Track the smallest weight held
| |||
double minValue;
| |||
// Track the largest weight held
| |||
double maxValue;
| |||
// Track the average weight
| |||
double meanValue;
| |||
// Track how far the weights spread from their average
| |||
double stdValue;
| |||
// Track what share of the weights are zero
| |||
double sparsity;
| |||
// Keep the weights themselves so the inspector can show them
| |||
double val[];
| |||
// Count the weights kept, which stops at the display limit
| |||
int kept;
| |||
// Report where the file keeps this tensor's values
| |||
string location;
| |||
// Name the category the file gives this tensor
| |||
string category;
| |||
// Keep the strings when the tensor holds text rather than numbers
| |||
string text[];
| |||
// Count the strings kept
| |||
int texts;
| |||
// Report whether the file stored this tensor in a segment
| |||
bool segmented;
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Exposed graph port |
| |||
//+------------------------------------------------------------------+
| |||
struct OmvPort
| |||
{
| |||
// Name the graph exposes
| |||
string name;
| |||
// Measure each axis
| |||
int dim[OMV_MAX_DIMS];
| |||
// Count the axes present
| |||
int dims;
| |||
// Record the ONNX type code
| |||
int elemType;
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| ONNX file reader |
| |||
//+------------------------------------------------------------------+
| |||
class COmvModel
| |||
{
| |||
private:
| |||
// Store the layers in file order
| |||
OmvNode m_node[];
| |||
// Store the weight tensors
| |||
OmvTensor m_tensor[];
| |||
// Store the graph inputs
| |||
OmvPort m_input[];
| |||
// Store the graph outputs
| |||
OmvPort m_output[];
| |||
// Store what the file says about the values passing between layers
| |||
OmvPort m_inner[];
| |||
// Count the layers held
| |||
int m_nodes;
| |||
// Count the tensors held
| |||
int m_tensors;
| |||
// Count the graph inputs held
| |||
int m_inputs;
| |||
// Count the graph outputs held
| |||
int m_outputs;
| |||
// Count the inner values held
| |||
int m_inners;
| |||
// Record the format revision
| |||
long m_irVersion;
| |||
// Record the operator set version
| |||
int m_opset;
| |||
// Name the tool that wrote the file
| |||
string m_producer;
| |||
// Record the version of the tool that wrote it
| |||
string m_producerVersion;
| |||
// Name the domain the model belongs to
| |||
string m_domain;
| |||
// Record the revision the model gives itself
| |||
long m_modelVersion;
| |||
// Carry whatever the file says about itself
| |||
string m_docString;
| |||
// Name the graph inside the file
| |||
string m_graphName;
| |||
// Carry whatever the file says about the graph
| |||
string m_graphDoc;
| |||
// Name each key the file attached to the model
| |||
string m_metaKey[OMV_MAX_META];
| |||
// Print the value each key carries
| |||
string m_metaValue[OMV_MAX_META];
| |||
// Count the pairs held
| |||
int m_metas;
| |||
// Name each operator the file defines for itself
| |||
string m_funcName[OMV_MAX_FUNCS];
| |||
// Describe what each defined operator holds
| |||
string m_funcDetail[OMV_MAX_FUNCS];
| |||
// Count the defined operators held
| |||
int m_funcs;
| |||
// Report whether the file carries training state
| |||
bool m_hasTraining;
| |||
// Name each key the file attached to the graph
| |||
string m_gMetaKey[OMV_MAX_META];
| |||
// Print the value each graph key carries
| |||
string m_gMetaValue[OMV_MAX_META];
| |||
// Count the graph pairs held
| |||
int m_gMetas;
| |||
// Count the quantization mappings the graph carries
| |||
int m_quants;
| |||
// Explain why the last load failed
| |||
string m_error;
| |||
| |||
bool ParseGraph(const uchar &raw[], const int from, const int to);
| |||
bool ParseNode(const uchar &raw[], const int from, const int to);
| |||
bool ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer);
| |||
string ReadList(const uchar &raw[], const int from, const int to, const int field);
| |||
string ReadSubgraph(const uchar &raw[], const int from, const int to);
| |||
string ReadEntry(const uchar &raw[], const int from, const int to);
| |||
void ReadPair(const uchar &raw[], const int from, const int to,
| |||
string &keys[], string &values[], int &count);
| |||
void ReadFunction(const uchar &raw[], const int from, const int to);
| |||
bool ParseTensor(const uchar &raw[], const int from, const int to);
| |||
bool ParseSparse(const uchar &raw[], const int from, const int to);
| |||
bool ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port);
| |||
void Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
| |||
double &total, double &squares,
| |||
double &lowest, double &highest);
| |||
void Publish(OmvTensor &tensor, const int seen, const int zeros,
| |||
const double total, const double squares,
| |||
const double lowest, const double highest);
| |||
void RawStats(const uchar &raw[], const int from, const int to,
| |||
OmvTensor &tensor);
| |||
void PackedStats(const uchar &raw[], const int from, const int to,
| |||
const int field, OmvTensor &tensor);
| |||
| |||
public:
| |||
COmvModel(void);
| |||
bool Load(const string path);
| |||
void Clear(void);
| |||
| |||
int Nodes(void) const { return m_nodes; }
| |||
int Tensors(void) const { return m_tensors; }
| |||
int Inputs(void) const { return m_inputs; }
| |||
int Outputs(void) const { return m_outputs; }
| |||
int Inners(void) const { return m_inners; }
| |||
long IrVersion(void) const { return m_irVersion; }
| |||
int Opset(void) const { return m_opset; }
| |||
string Producer(void) const { return m_producer; }
| |||
string ProducerVersion(void) const { return m_producerVersion; }
| |||
string Domain(void) const { return m_domain; }
| |||
long ModelVersion(void) const { return m_modelVersion; }
| |||
string DocString(void) const { return m_docString; }
| |||
string GraphName(void) const { return m_graphName; }
| |||
string GraphDoc(void) const { return m_graphDoc; }
| |||
int Metas(void) const { return m_metas; }
| |||
int GraphMetas(void) const { return m_gMetas; }
| |||
string GraphMetaKey(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaKey[i] : ""; }
| |||
string GraphMetaValue(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaValue[i] : ""; }
| |||
int Quants(void) const { return m_quants; }
| |||
string MetaKey(const int i) const { return (i >= 0 && i < m_metas) ? m_metaKey[i] : ""; }
| |||
string MetaValue(const int i) const { return (i >= 0 && i < m_metas) ? m_metaValue[i] : ""; }
| |||
int Funcs(void) const { return m_funcs; }
| |||
string FuncName(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcName[i] : ""; }
| |||
string FuncDetail(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcDetail[i] : ""; }
| |||
bool HasTraining(void) const { return m_hasTraining; }
| |||
string Error(void) const { return m_error; }
| |||
int Parameters(void) const;
| |||
| |||
void NodeAt(const int i, OmvNode &out) const { if(i >= 0 && i < m_nodes) out = m_node[i]; }
| |||
void TensorAt(const int i, OmvTensor &out) const { if(i >= 0 && i < m_tensors) out = m_tensor[i]; }
| |||
void InputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inputs) out = m_input[i]; }
| |||
void OutputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_outputs) out = m_output[i]; }
| |||
void InnerAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inners) out = m_inner[i]; }
| |||
string DimsText(const int &dim[], const int dims) const;
| |||
string DimsList(const int &dim[], const int dims) const;
| |||
string DimsBare(const int &dim[], const int dims) const;
| |||
string TypeName(const int code) const;
| |||
string AttrType(const int kind) const;
| |||
};
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Construct an empty model |
| |||
//+------------------------------------------------------------------+
| |||
COmvModel::COmvModel(void)
| |||
{
| |||
//--- Clear every field so a query before loading reports empty
| |||
Clear();
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Drop everything the previous file filled |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::Clear(void)
| |||
{
| |||
//--- Zero the counts that bound every array
| |||
m_nodes = 0;
| |||
m_tensors = 0;
| |||
m_inputs = 0;
| |||
m_outputs = 0;
| |||
m_inners = 0;
| |||
//--- Release the stored structures
| |||
ArrayResize(m_node, 0);
| |||
ArrayResize(m_tensor, 0);
| |||
ArrayResize(m_input, 0);
| |||
ArrayResize(m_output, 0);
| |||
ArrayResize(m_inner, 0);
| |||
//--- Clear the header fields
| |||
m_irVersion = 0;
| |||
m_opset = 0;
| |||
m_producer = "";
| |||
m_producerVersion = "";
| |||
m_domain = "";
| |||
m_modelVersion = 0;
| |||
m_docString = "";
| |||
m_graphName = "";
| |||
m_graphDoc = "";
| |||
m_metas = 0;
| |||
m_gMetas = 0;
| |||
m_quants = 0;
| |||
m_funcs = 0;
| |||
m_hasTraining = false;
| |||
m_error = "";
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Walk raw weight bytes for their range and average |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
| |||
double &total, double &squares, double &lowest, double &highest)
| |||
{
| |||
//--- Seed the range from the first value that arrives
| |||
if(seen == 0)
| |||
{
| |||
lowest = v;
| |||
highest = v;
| |||
}
| |||
//--- Lower the floor when this value is smaller
| |||
if(v < lowest)
| |||
lowest = v;
| |||
//--- Raise the ceiling when this value is larger
| |||
if(v > highest)
| |||
highest = v;
| |||
//--- Add the value to the running totals
| |||
total += v;
| |||
squares += v * v;
| |||
//--- Count a value that is exactly zero
| |||
if(v == 0.0)
| |||
zeros++;
| |||
//--- Keep the value itself while room remains
| |||
if(tensor.kept < OMV_MAX_VALUES)
| |||
{
| |||
ArrayResize(tensor.val, tensor.kept + 1);
| |||
tensor.val[tensor.kept++] = v;
| |||
}
| |||
seen++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Publish what the accumulated values measured |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::Publish(OmvTensor &tensor, const int seen, const int zeros,
| |||
const double total, const double squares,
| |||
const double lowest, const double highest)
| |||
{
| |||
//--- Abort when the read stopped before any value arrived
| |||
if(seen <= 0)
| |||
return;
| |||
//--- Publish the range and the average
| |||
tensor.minValue = lowest;
| |||
tensor.maxValue = highest;
| |||
tensor.meanValue = total / seen;
| |||
//--- Take the spread as the root of the mean squared deviation
| |||
const double variance = squares / seen - tensor.meanValue * tensor.meanValue;
| |||
tensor.stdValue = (variance > 0.0) ? MathSqrt(variance) : 0.0;
| |||
//--- Take the share of the values that are exactly zero
| |||
tensor.sparsity = (double)zeros / seen;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Walk a raw byte payload at the width its type implies |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::RawStats(const uchar &raw[], const int from, const int to, OmvTensor &tensor)
| |||
{
| |||
//--- Take the width one value occupies for this element type
| |||
int width = 4;
| |||
if(tensor.dataType == 11 || tensor.dataType == 7 || tensor.dataType == 13)
| |||
width = 8;
| |||
else
| |||
if(tensor.dataType == 2 || tensor.dataType == 3 || tensor.dataType == 9)
| |||
width = 1;
| |||
else
| |||
if(tensor.dataType == 4 || tensor.dataType == 5 || tensor.dataType == 10)
| |||
width = 2;
| |||
//--- Count the whole values the payload holds
| |||
const int count = (to - from) / width;
| |||
if(count <= 0)
| |||
return;
| |||
//--- Point a cursor at the payload
| |||
CPbReader bytes;
| |||
if(!bytes.Attach(raw, from, to))
| |||
return;
| |||
//--- Start the running measures
| |||
double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
| |||
int zeros = 0, seen = 0;
| |||
//--- Visit every stored value
| |||
for(int i = 0; i < count; i++)
| |||
{
| |||
//--- Read the value the way its type is written
| |||
double v = 0.0;
| |||
if(tensor.dataType == 1)
| |||
v = (double)bytes.ReadFloat();
| |||
else
| |||
if(tensor.dataType == 11)
| |||
v = bytes.ReadDouble();
| |||
else
| |||
if(width == 8)
| |||
v = (double)(long)bytes.ReadFixed64();
| |||
else
| |||
if(width == 4)
| |||
v = (double)(int)bytes.ReadFixed32();
| |||
else
| |||
if(width == 2)
| |||
v = (double)(int)(bytes.ReadVarint() & 0xFFFF);
| |||
else
| |||
v = (double)bytes.ReadVarint();
| |||
//--- Stop on a broken read
| |||
if(bytes.Failed())
| |||
break;
| |||
//--- Fold the value into the running measures
| |||
Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
| |||
}
| |||
//--- Publish what the walk measured
| |||
Publish(tensor, seen, zeros, total, squares, lowest, highest);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Walk a typed payload the file used instead of raw bytes |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::PackedStats(const uchar &raw[], const int from, const int to,
| |||
const int field, OmvTensor &tensor)
| |||
{
| |||
//--- Point a cursor at the payload
| |||
CPbReader bytes;
| |||
if(!bytes.Attach(raw, from, to))
| |||
return;
| |||
//--- Start the running measures
| |||
double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
| |||
int zeros = 0, seen = 0;
| |||
//--- Read the payload the way the field it arrived in is written
| |||
while(!bytes.Eof())
| |||
{
| |||
//--- Take the next value in the packed run
| |||
double v = 0.0;
| |||
if(field == 4)
| |||
v = (double)bytes.ReadFloat();
| |||
else
| |||
if(field == 10)
| |||
v = bytes.ReadDouble();
| |||
else
| |||
if(field == 6)
| |||
break;
| |||
else
| |||
v = (double)(long)bytes.ReadVarint();
| |||
//--- Stop on a broken read
| |||
if(bytes.Failed())
| |||
break;
| |||
//--- Fold the value into the running measures
| |||
Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
| |||
}
| |||
//--- Publish what the walk measured
| |||
Publish(tensor, seen, zeros, total, squares, lowest, highest);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Join a list an attribute carried into one line |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::ReadList(const uchar &raw[], const int from, const int to, const int field)
| |||
{
| |||
//--- Point a cursor at the list
| |||
CPbReader items;
| |||
if(!items.Attach(raw, from, to))
| |||
return "";
| |||
//--- Join up to a handful of entries, then say how many remain
| |||
string out = "";
| |||
int shown = 0, total = 0;
| |||
while(!items.Eof())
| |||
{
| |||
//--- Read the next entry the way its field is written
| |||
string one = "";
| |||
if(field == 7)
| |||
one = DoubleToString((double)items.ReadFloat(), 4);
| |||
else
| |||
if(field == 8)
| |||
one = IntegerToString((long)items.ReadVarint());
| |||
else
| |||
items.ReadString(one);
| |||
//--- Stop on a broken read
| |||
if(items.Failed())
| |||
break;
| |||
total++;
| |||
//--- Keep the first few entries only, a row cannot hold more
| |||
if(shown >= 8)
| |||
continue;
| |||
out += (shown > 0 ? ", " : "") + one;
| |||
shown++;
| |||
}
| |||
//--- Say how many entries the line left out
| |||
if(total > shown)
| |||
out += ", ... " + IntegerToString(total - shown) + " more";
| |||
return out;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Describe a nested graph by what it actually holds |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::ReadSubgraph(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Point a cursor at the nested graph
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return "<graph>";
| |||
uchar inner[];
| |||
reader.Bytes(inner);
| |||
//--- Count what the nested graph carries and name its operators
| |||
string title = "";
| |||
string ops = "";
| |||
int nodes = 0, tensors = 0, shown = 0;
| |||
//--- Read every field of the nested graph
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Step over anything that is not a nested message
| |||
if(wire != PB_LENGTH)
| |||
{
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Locate the nested payload
| |||
int f2, t2;
| |||
if(!reader.ReadBlock(f2, t2))
| |||
break;
| |||
//--- Count a node and name the operator it applies
| |||
if(field == 1)
| |||
{
| |||
nodes++;
| |||
//--- Name only the first few, a line cannot hold more
| |||
if(shown < 6)
| |||
{
| |||
//--- Read the operator out of the node
| |||
CPbReader entry;
| |||
entry.Attach(inner, f2, t2);
| |||
while(!entry.Eof())
| |||
{
| |||
//--- Take the next field of the entry
| |||
int f3, w3;
| |||
if(!entry.ReadTag(f3, w3))
| |||
break;
| |||
//--- Keep the operator and stop reading this node
| |||
if(f3 == 4 && w3 == PB_LENGTH)
| |||
{
| |||
string op = "";
| |||
entry.ReadString(op);
| |||
ops += (shown > 0 ? ", " : "") + op;
| |||
shown++;
| |||
break;
| |||
}
| |||
if(!entry.SkipField(w3))
| |||
break;
| |||
}
| |||
}
| |||
}
| |||
//--- Count an initializer the nested graph carries
| |||
else
| |||
if(field == 5 || field == 15)
| |||
tensors++;
| |||
//--- Take the name the nested graph was given
| |||
else
| |||
if(field == 2)
| |||
{
| |||
//--- Decode the name from its own payload
| |||
uchar tmp[];
| |||
ArrayResize(tmp, t2 - f2);
| |||
for(int i = f2; i < t2; i++)
| |||
tmp[i - f2] = inner[i];
| |||
title = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
| |||
}
| |||
}
| |||
//--- Report the nested graph by its name, its size and what it runs
| |||
string out = "<graph";
| |||
if(StringLen(title) > 0)
| |||
out += " " + title;
| |||
out += ": " + IntegerToString(nodes) + " nodes, " +
| |||
IntegerToString(tensors) + " tensors";
| |||
if(StringLen(ops) > 0)
| |||
out += " (" + ops + (nodes > shown ? ", ..." : "") + ")";
| |||
return out + ">";
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Name the type an attribute declared itself to be |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::AttrType(const int kind) const
| |||
{
| |||
//--- Match the declared kind to the type the app reports for it
| |||
switch(kind)
| |||
{
| |||
case 1:
| |||
return "float32";
| |||
case 2:
| |||
return "int64";
| |||
case 3:
| |||
return "string";
| |||
case 4:
| |||
return "tensor";
| |||
case 5:
| |||
return "graph";
| |||
case 6:
| |||
return "float32[]";
| |||
case 7:
| |||
return "int64[]";
| |||
case 8:
| |||
return "string[]";
| |||
}
| |||
//--- Report nothing recognizable as undefined
| |||
return "undefined";
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read one attribute an operator was given |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer)
| |||
{
| |||
//--- Point a cursor at this attribute
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return false;
| |||
uchar inner[];
| |||
reader.Bytes(inner);
| |||
//--- Refuse an attribute beyond what the layer may hold
| |||
if(layer.attrCount >= OMV_MAX_ATTRS)
| |||
return false;
| |||
//--- Start with nothing read
| |||
const int slot = layer.attrCount;
| |||
layer.attrName[slot] = "";
| |||
layer.attrValue[slot] = "";
| |||
//--- Remember which kind of value the attribute declares
| |||
int kind = 0;
| |||
double asFloat = 0.0;
| |||
long asInt = 0;
| |||
string asText = "";
| |||
bool haveInt = false, haveFloat = false;
| |||
//--- Count the entries a repeated field delivers one at a time
| |||
int repeats = 0;
| |||
//--- Read every field of the attribute
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Store the attribute name
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
reader.ReadString(layer.attrName[slot]);
| |||
//--- Store a floating point value
| |||
else
| |||
if(field == 2 && wire == PB_FIXED32)
| |||
{
| |||
asFloat = (double)reader.ReadFloat();
| |||
haveFloat = true;
| |||
}
| |||
//--- Store a whole number value
| |||
else
| |||
if(field == 3 && wire == PB_VARINT)
| |||
{
| |||
asInt = (long)reader.ReadVarint();
| |||
haveInt = true;
| |||
}
| |||
//--- Store a text value
| |||
else
| |||
if(field == 4 && wire == PB_LENGTH)
| |||
reader.ReadString(asText);
| |||
//--- Name a nested tensor or graph rather than unfolding it
| |||
else
| |||
if((field == 5 || field == 6) && wire == PB_LENGTH)
| |||
{
| |||
//--- Step past the payload and record what it was
| |||
int n1, n2;
| |||
if(!reader.ReadBlock(n1, n2))
| |||
break;
| |||
asText = (field == 5) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
| |||
}
| |||
//--- Append one string, of which this field delivers many
| |||
else
| |||
if(field == 9 && wire == PB_LENGTH)
| |||
{
| |||
//--- Read this entry and join it to the ones before
| |||
string one = "";
| |||
if(!reader.ReadString(one))
| |||
break;
| |||
repeats++;
| |||
if(repeats <= 8)
| |||
asText += (repeats > 1 ? ", " : "") + one;
| |||
}
| |||
//--- Join a list of floats, whole numbers or strings
| |||
else
| |||
if((field == 7 || field == 8) && wire == PB_LENGTH)
| |||
{
| |||
//--- Locate the list and read it in the form it arrived
| |||
int l1, l2;
| |||
if(!reader.ReadBlock(l1, l2))
| |||
break;
| |||
asText = ReadList(inner, l1, l2, field);
| |||
}
| |||
//--- Name a list of tensors or graphs rather than unfolding it
| |||
else
| |||
if((field == 10 || field == 11) && wire == PB_LENGTH)
| |||
{
| |||
//--- Step past this entry and count it among its siblings
| |||
int n1, n2;
| |||
if(!reader.ReadBlock(n1, n2))
| |||
break;
| |||
repeats++;
| |||
//--- Describe the first entry, then say how many followed
| |||
if(repeats == 1)
| |||
asText = (field == 10) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
| |||
}
| |||
//--- Store which kind the attribute declares itself to be
| |||
else
| |||
if(field == 20 && wire == PB_VARINT)
| |||
kind = (int)reader.ReadVarint();
| |||
//--- Step over anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Say how many entries a repeated field delivered beyond those shown
| |||
if(repeats > 8)
| |||
asText += ", ... " + IntegerToString(repeats - 8) + " more";
| |||
else
| |||
if(repeats > 1 && (kind == 5 || kind == 6 || kind == 9 || kind == 10))
| |||
asText += " and " + IntegerToString(repeats - 1) + " more";
| |||
//--- Reject an attribute that never gave a name
| |||
if(StringLen(layer.attrName[slot]) == 0)
| |||
return false;
| |||
//--- Print the value the attribute carried, by the kind it declared
| |||
if(kind == 3 || kind >= 4 || StringLen(asText) > 0)
| |||
layer.attrValue[slot] = asText;
| |||
else
| |||
if(kind == 2 || (kind == 0 && haveInt))
| |||
layer.attrValue[slot] = IntegerToString(asInt);
| |||
else
| |||
if(kind == 1 || (kind == 0 && haveFloat))
| |||
layer.attrValue[slot] = DoubleToString(asFloat, 6);
| |||
//--- Record the kind so the row can report the type it opens to
| |||
layer.attrKind[slot] = kind;
| |||
//--- Report the attribute as read
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read one node into a layer |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParseNode(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Refuse a file that exceeds the node limit
| |||
if(m_nodes >= OMV_MAX_NODES)
| |||
return false;
| |||
//--- Point a cursor at this node
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return false;
| |||
//--- Start an empty layer
| |||
OmvNode layer;
| |||
layer.op = "";
| |||
layer.name = "";
| |||
layer.domain = "";
| |||
layer.docString = "";
| |||
layer.overload = "";
| |||
layer.attrCount = 0;
| |||
layer.feedCount = 0;
| |||
layer.emitCount = 0;
| |||
//--- Read every field of the node
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Skip anything that is not text
| |||
if(wire != PB_LENGTH)
| |||
{
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Read an attribute out of its own nested message
| |||
if(field == 5)
| |||
{
| |||
//--- Locate the attribute and decode it while room remains
| |||
int a1, a2;
| |||
if(!reader.ReadBlock(a1, a2))
| |||
break;
| |||
if(layer.attrCount < OMV_MAX_ATTRS)
| |||
if(ParseAttribute(raw, from + a1, from + a2, layer))
| |||
layer.attrCount++;
| |||
continue;
| |||
}
| |||
//--- Read the field as text
| |||
string text;
| |||
if(!reader.ReadString(text))
| |||
break;
| |||
//--- Store an input name
| |||
if(field == 1 && layer.feedCount < OMV_MAX_PORTS)
| |||
layer.feeds[layer.feedCount++] = text;
| |||
//--- Store an output name
| |||
else
| |||
if(field == 2 && layer.emitCount < OMV_MAX_PORTS)
| |||
layer.emits[layer.emitCount++] = text;
| |||
//--- Store the layer name
| |||
else
| |||
if(field == 3)
| |||
layer.name = text;
| |||
//--- Store the operator
| |||
else
| |||
if(field == 4)
| |||
layer.op = text;
| |||
//--- Store the operator set the operator belongs to
| |||
else
| |||
if(field == 7)
| |||
layer.domain = text;
| |||
//--- Store whatever the file says about this layer
| |||
else
| |||
if(field == 6)
| |||
layer.docString = text;
| |||
//--- Store the overload the operator was resolved to
| |||
else
| |||
if(field == 8)
| |||
layer.overload = text;
| |||
}
| |||
//--- Reject a node carrying no operator
| |||
if(StringLen(layer.op) == 0)
| |||
return false;
| |||
//--- Append the finished layer
| |||
ArrayResize(m_node, m_nodes + 1);
| |||
m_node[m_nodes++] = layer;
| |||
//--- Report success
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read one initializer into a weight tensor |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParseTensor(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Refuse a file that exceeds the tensor limit
| |||
if(m_tensors >= OMV_MAX_TENSORS)
| |||
return false;
| |||
//--- Point a cursor at this tensor
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return false;
| |||
//--- Keep the slice so nested payloads can be located
| |||
uchar inner[];
| |||
reader.Bytes(inner);
| |||
//--- Start an empty tensor
| |||
OmvTensor weights;
| |||
weights.name = "";
| |||
weights.dims = 0;
| |||
weights.dataType = 0;
| |||
weights.values = 0;
| |||
weights.minValue = 0.0;
| |||
weights.maxValue = 0.0;
| |||
weights.meanValue = 0.0;
| |||
weights.stdValue = 0.0;
| |||
weights.sparsity = 0.0;
| |||
weights.kept = 0;
| |||
weights.texts = 0;
| |||
weights.segmented = false;
| |||
weights.location = "default";
| |||
weights.category = "Initializer";
| |||
//--- Mark every payload the tensor might carry as not yet found
| |||
int rawFrom = -1, rawTo = -1;
| |||
int packFrom = -1, packTo = -1, packField = 0;
| |||
//--- Read every field of the tensor
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Store one axis length
| |||
if(field == 1 && wire == PB_VARINT)
| |||
{
| |||
//--- Keep the axis while room remains
| |||
if(weights.dims < OMV_MAX_DIMS)
| |||
weights.dim[weights.dims++] = (int)reader.ReadVarint();
| |||
else
| |||
reader.ReadVarint();
| |||
}
| |||
//--- Store several axis lengths packed together
| |||
else
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
{
| |||
//--- Locate the packed block
| |||
int f2, t2;
| |||
if(!reader.ReadBlock(f2, t2))
| |||
break;
| |||
//--- Read every length the block holds
| |||
CPbReader packed;
| |||
packed.Attach(inner, f2, t2);
| |||
while(!packed.Eof() && weights.dims < OMV_MAX_DIMS)
| |||
weights.dim[weights.dims++] = (int)packed.ReadVarint();
| |||
}
| |||
//--- Store the element type
| |||
else
| |||
if(field == 2 && wire == PB_VARINT)
| |||
weights.dataType = (int)reader.ReadVarint();
| |||
//--- Handle a tensor the file stored in segments
| |||
else
| |||
if(field == 3 && wire == PB_LENGTH)
| |||
{
| |||
//--- Step past the segment, only its presence is reported
| |||
int s1, s2;
| |||
if(!reader.ReadBlock(s1, s2))
| |||
break;
| |||
weights.segmented = true;
| |||
}
| |||
//--- Store the tensor name
| |||
else
| |||
if(field == 8 && wire == PB_LENGTH)
| |||
reader.ReadString(weights.name);
| |||
//--- Locate the raw byte payload
| |||
else
| |||
if(field == 9 && wire == PB_LENGTH)
| |||
{
| |||
if(!reader.ReadBlock(rawFrom, rawTo))
| |||
break;
| |||
}
| |||
//--- Collect one string, of which the field may carry many
| |||
else
| |||
if(field == 6 && wire == PB_LENGTH)
| |||
{
| |||
//--- Append it while room remains
| |||
string one = "";
| |||
if(!reader.ReadString(one))
| |||
break;
| |||
if(weights.texts < OMV_MAX_VALUES)
| |||
{
| |||
ArrayResize(weights.text, weights.texts + 1);
| |||
weights.text[weights.texts++] = one;
| |||
}
| |||
}
| |||
//--- Locate a typed payload, which the file may use instead
| |||
else
| |||
if(wire == PB_LENGTH &&
| |||
(field == 4 || field == 5 ||
| |||
field == 7 || field == 10 || field == 11))
| |||
{
| |||
//--- Remember which typed field carried the values
| |||
if(!reader.ReadBlock(packFrom, packTo))
| |||
break;
| |||
packField = field;
| |||
}
| |||
//--- Report values the file keeps somewhere else entirely
| |||
else
| |||
if(field == 14 && wire == PB_VARINT)
| |||
{
| |||
//--- A location of one means the weights sit outside the file
| |||
if((int)reader.ReadVarint() == 1)
| |||
weights.location = "external";
| |||
}
| |||
//--- Name the file the weights were moved out to
| |||
else
| |||
if(field == 13 && wire == PB_LENGTH)
| |||
{
| |||
//--- Read the entry and keep whatever names a location
| |||
int e1, e2;
| |||
if(!reader.ReadBlock(e1, e2))
| |||
break;
| |||
const string named = ReadEntry(inner, e1, e2);
| |||
if(StringLen(named) > 0)
| |||
weights.location = named;
| |||
}
| |||
//--- Step over anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Multiply every axis to size the block
| |||
weights.values = (weights.dims > 0) ? 1 : 0;
| |||
for(int i = 0; i < weights.dims; i++)
| |||
weights.values *= weights.dim[i];
| |||
//--- Summarize whichever payload the file actually used
| |||
if(rawFrom >= 0)
| |||
RawStats(inner, rawFrom, rawTo, weights);
| |||
else
| |||
if(packFrom >= 0)
| |||
PackedStats(inner, packFrom, packTo, packField, weights);
| |||
//--- Reject a tensor carrying no name
| |||
if(StringLen(weights.name) == 0)
| |||
return false;
| |||
//--- Append the finished tensor
| |||
ArrayResize(m_tensor, m_tensors + 1);
| |||
m_tensor[m_tensors++] = weights;
| |||
//--- Report success
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read a key and value pair into a pair of lists |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::ReadPair(const uchar &raw[], const int from, const int to,
| |||
string &keys[], string &values[], int &count)
| |||
{
| |||
//--- Refuse a pair beyond what the lists may hold
| |||
if(count >= OMV_MAX_META)
| |||
return;
| |||
//--- Point a cursor at the pair
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return;
| |||
//--- Read the key and the value it carries
| |||
string key = "", value = "";
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Store the key
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
reader.ReadString(key);
| |||
//--- Store the value
| |||
else
| |||
if(field == 2 && wire == PB_LENGTH)
| |||
reader.ReadString(value);
| |||
//--- Step over anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Keep only a pair that actually named something
| |||
if(StringLen(key) == 0)
| |||
return;
| |||
keys[count] = key;
| |||
values[count] = value;
| |||
count++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read an operator the file defines for itself |
| |||
//+------------------------------------------------------------------+
| |||
void COmvModel::ReadFunction(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Refuse a definition beyond what the list may hold
| |||
if(m_funcs >= OMV_MAX_FUNCS)
| |||
return;
| |||
//--- Point a cursor at the definition
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return;
| |||
//--- Count what the definition carries and name it
| |||
string named = "", domain = "";
| |||
int nodes = 0, ins = 0, outs = 0;
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Step over anything that is not a nested payload
| |||
if(wire != PB_LENGTH)
| |||
{
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Take the name the definition was given
| |||
if(field == 1)
| |||
reader.ReadString(named);
| |||
//--- Take the domain it belongs to
| |||
else
| |||
if(field == 9)
| |||
reader.ReadString(domain);
| |||
//--- Count the parts the definition is built from
| |||
else
| |||
{
| |||
//--- Locate the payload and count it by which field it arrived in
| |||
int f2, t2;
| |||
if(!reader.ReadBlock(f2, t2))
| |||
break;
| |||
if(field == 2)
| |||
ins++;
| |||
else
| |||
if(field == 3)
| |||
outs++;
| |||
else
| |||
if(field == 7)
| |||
nodes++;
| |||
}
| |||
}
| |||
//--- Keep only a definition that actually named itself
| |||
if(StringLen(named) == 0)
| |||
return;
| |||
m_funcName[m_funcs] = (StringLen(domain) > 0) ? domain + "." + named : named;
| |||
m_funcDetail[m_funcs] = IntegerToString(nodes) + " nodes, " +
| |||
IntegerToString(ins) + " in, " +
| |||
IntegerToString(outs) + " out";
| |||
m_funcs++;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read a key and value pair the file uses for metadata |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::ReadEntry(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Point a cursor at the pair
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return "";
| |||
//--- Read the key and the value it carries
| |||
string key = "", value = "";
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Store the key
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
reader.ReadString(key);
| |||
//--- Store the value
| |||
else
| |||
if(field == 2 && wire == PB_LENGTH)
| |||
reader.ReadString(value);
| |||
//--- Step over anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Report only the entry that names where the data sits
| |||
return (key == "location") ? value : "";
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read a sparse initializer through its values tensor |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParseSparse(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Point a cursor at the sparse tensor
| |||
CPbReader reader;
| |||
if(!reader.Attach(raw, from, to))
| |||
return false;
| |||
uchar inner[];
| |||
reader.Bytes(inner);
| |||
//--- Remember where the shape and the values sit
| |||
int valFrom = -1, valTo = -1;
| |||
int dim[OMV_MAX_DIMS];
| |||
int dims = 0;
| |||
//--- Read every field of the sparse tensor
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Locate the tensor holding the values themselves
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
{
| |||
if(!reader.ReadBlock(valFrom, valTo))
| |||
break;
| |||
}
| |||
//--- Store one axis of the shape the sparse tensor stands for
| |||
else
| |||
if(field == 3 && wire == PB_VARINT)
| |||
{
| |||
//--- Keep the axis while room remains
| |||
if(dims < OMV_MAX_DIMS)
| |||
dim[dims++] = (int)reader.ReadVarint();
| |||
else
| |||
reader.ReadVarint();
| |||
}
| |||
//--- Read several axes packed together
| |||
else
| |||
if(field == 3 && wire == PB_LENGTH)
| |||
{
| |||
//--- Locate the packed block and read every length it holds
| |||
int p1, p2;
| |||
if(!reader.ReadBlock(p1, p2))
| |||
break;
| |||
CPbReader packed;
| |||
packed.Attach(inner, p1, p2);
| |||
while(!packed.Eof() && dims < OMV_MAX_DIMS)
| |||
dim[dims++] = (int)packed.ReadVarint();
| |||
}
| |||
//--- Step over the indices and anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Nothing to report without the values tensor
| |||
if(valFrom < 0)
| |||
return false;
| |||
//--- Read the values as an ordinary tensor
| |||
if(!ParseTensor(inner, valFrom, valTo))
| |||
return false;
| |||
//--- Correct the entry to describe the sparse tensor it came from
| |||
const int slot = m_tensors - 1;
| |||
m_tensor[slot].category = "Sparse Initializer";
| |||
if(dims > 0)
| |||
{
| |||
//--- Report the shape the sparse tensor stands for, not the value run
| |||
m_tensor[slot].dims = dims;
| |||
for(int i = 0; i < dims; i++)
| |||
m_tensor[slot].dim[i] = dim[i];
| |||
m_tensor[slot].values = 1;
| |||
for(int i = 0; i < dims; i++)
| |||
m_tensor[slot].values *= dim[i];
| |||
}
| |||
//--- Report the sparse tensor as read
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read one value info into an exposed port |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port)
| |||
{
| |||
//--- Start an empty port
| |||
port.name = "";
| |||
port.dims = 0;
| |||
port.elemType = 0;
| |||
//--- Point a cursor at this value info
| |||
CPbReader value;
| |||
if(!value.Attach(raw, from, to))
| |||
return false;
| |||
//--- Keep the slice so nested payloads can be located
| |||
uchar level1[];
| |||
value.Bytes(level1);
| |||
//--- Read every field of the value info
| |||
while(!value.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!value.ReadTag(field, wire))
| |||
break;
| |||
//--- Store the port name
| |||
if(field == 1 && wire == PB_LENGTH)
| |||
{
| |||
value.ReadString(port.name);
| |||
continue;
| |||
}
| |||
//--- Step over anything but the type description
| |||
if(field != 2 || wire != PB_LENGTH)
| |||
{
| |||
if(!value.SkipField(wire))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Descend into the type description
| |||
int t1, t2;
| |||
if(!value.ReadBlock(t1, t2))
| |||
break;
| |||
CPbReader type;
| |||
type.Attach(level1, t1, t2);
| |||
uchar level2[];
| |||
type.Bytes(level2);
| |||
//--- Read every field of the type
| |||
while(!type.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int f2, w2;
| |||
if(!type.ReadTag(f2, w2))
| |||
break;
| |||
//--- Step over anything but the tensor description
| |||
if(f2 != 1 || w2 != PB_LENGTH)
| |||
{
| |||
if(!type.SkipField(w2))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Descend into the tensor description
| |||
int s1, s2;
| |||
if(!type.ReadBlock(s1, s2))
| |||
break;
| |||
CPbReader tt;
| |||
tt.Attach(level2, s1, s2);
| |||
uchar level3[];
| |||
tt.Bytes(level3);
| |||
//--- Read every field of the tensor description
| |||
while(!tt.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int f3, w3;
| |||
if(!tt.ReadTag(f3, w3))
| |||
break;
| |||
//--- Store the element type
| |||
if(f3 == 1 && w3 == PB_VARINT)
| |||
{
| |||
port.elemType = (int)tt.ReadVarint();
| |||
continue;
| |||
}
| |||
//--- Step over anything but the shape
| |||
if(f3 != 2 || w3 != PB_LENGTH)
| |||
{
| |||
if(!tt.SkipField(w3))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Descend into the shape
| |||
int p1, p2;
| |||
if(!tt.ReadBlock(p1, p2))
| |||
break;
| |||
CPbReader shape;
| |||
shape.Attach(level3, p1, p2);
| |||
uchar level4[];
| |||
shape.Bytes(level4);
| |||
//--- Read every axis the shape holds
| |||
while(!shape.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int f4, w4;
| |||
if(!shape.ReadTag(f4, w4))
| |||
break;
| |||
//--- Step over anything but an axis
| |||
if(f4 != 1 || w4 != PB_LENGTH)
| |||
{
| |||
if(!shape.SkipField(w4))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Descend into the axis
| |||
int d1, d2;
| |||
if(!shape.ReadBlock(d1, d2))
| |||
break;
| |||
CPbReader dim;
| |||
dim.Attach(level4, d1, d2);
| |||
//--- Read every field of the axis
| |||
while(!dim.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int f5, w5;
| |||
if(!dim.ReadTag(f5, w5))
| |||
break;
| |||
//--- Store the axis length
| |||
if(f5 == 1 && w5 == PB_VARINT)
| |||
{
| |||
if(port.dims < OMV_MAX_DIMS)
| |||
port.dim[port.dims++] = (int)dim.ReadVarint();
| |||
else
| |||
dim.ReadVarint();
| |||
}
| |||
//--- Step over a named axis
| |||
else
| |||
if(!dim.SkipField(w5))
| |||
break;
| |||
}
| |||
}
| |||
}
| |||
}
| |||
}
| |||
//--- Report a port that carries a name
| |||
return StringLen(port.name) > 0;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Read the graph into nodes, weights and ports |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::ParseGraph(const uchar &raw[], const int from, const int to)
| |||
{
| |||
//--- Point a cursor at the graph
| |||
CPbReader graph;
| |||
if(!graph.Attach(raw, from, to))
| |||
{
| |||
m_error = "graph out of range";
| |||
return false;
| |||
}
| |||
//--- Keep the slice so nested payloads can be located
| |||
uchar inner[];
| |||
graph.Bytes(inner);
| |||
//--- Read every field of the graph
| |||
while(!graph.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!graph.ReadTag(field, wire))
| |||
break;
| |||
//--- Step over anything that is not a nested message
| |||
if(wire != PB_LENGTH && field != 0)
| |||
{
| |||
if(!graph.SkipField(wire))
| |||
break;
| |||
continue;
| |||
}
| |||
//--- Locate the nested payload
| |||
int f2, t2;
| |||
if(!graph.ReadBlock(f2, t2))
| |||
break;
| |||
//--- Read a node
| |||
if(field == 1)
| |||
ParseNode(inner, f2, t2);
| |||
//--- Read an initializer
| |||
else
| |||
if(field == 5)
| |||
ParseTensor(inner, f2, t2);
| |||
//--- Read a sparse initializer, whose values sit one level deeper
| |||
else
| |||
if(field == 15)
| |||
ParseSparse(inner, f2, t2);
| |||
//--- Decode the graph name from its payload
| |||
else
| |||
if(field == 2)
| |||
{
| |||
uchar tmp[];
| |||
ArrayResize(tmp, t2 - f2);
| |||
for(int i = f2; i < t2; i++)
| |||
tmp[i - f2] = inner[i];
| |||
m_graphName = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
| |||
}
| |||
//--- Read a graph input
| |||
else
| |||
if(field == 11)
| |||
{
| |||
OmvPort port;
| |||
if(ParsePort(inner, f2, t2, port))
| |||
{
| |||
ArrayResize(m_input, m_inputs + 1);
| |||
m_input[m_inputs++] = port;
| |||
}
| |||
}
| |||
//--- Read a key and value pair the file attached to the graph
| |||
else
| |||
if(field == 16)
| |||
ReadPair(inner, f2, t2, m_gMetaKey, m_gMetaValue, m_gMetas);
| |||
//--- Count a quantization mapping the graph carries
| |||
else
| |||
if(field == 14)
| |||
m_quants++;
| |||
//--- Read whatever the file says about the graph
| |||
else
| |||
if(field == 10)
| |||
{
| |||
//--- Decode the text from its own payload
| |||
uchar tmp[];
| |||
ArrayResize(tmp, t2 - f2);
| |||
for(int i = f2; i < t2; i++)
| |||
tmp[i - f2] = inner[i];
| |||
m_graphDoc = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
| |||
}
| |||
//--- Read what the file says about a value passing between layers
| |||
else
| |||
if(field == 13)
| |||
{
| |||
OmvPort port;
| |||
if(ParsePort(inner, f2, t2, port))
| |||
{
| |||
ArrayResize(m_inner, m_inners + 1);
| |||
m_inner[m_inners++] = port;
| |||
}
| |||
}
| |||
//--- Read a graph output
| |||
else
| |||
if(field == 12)
| |||
{
| |||
OmvPort port;
| |||
if(ParsePort(inner, f2, t2, port))
| |||
{
| |||
ArrayResize(m_output, m_outputs + 1);
| |||
m_output[m_outputs++] = port;
| |||
}
| |||
}
| |||
}
| |||
//--- Report success
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Open a file and read its whole structure |
| |||
//+------------------------------------------------------------------+
| |||
bool COmvModel::Load(const string path)
| |||
{
| |||
//--- Drop any model held from a previous call
| |||
Clear();
| |||
//--- Read the whole file into memory
| |||
CPbReader reader;
| |||
if(!reader.LoadFile(path))
| |||
{
| |||
m_error = "cannot open " + path;
| |||
return false;
| |||
}
| |||
//--- Keep the bytes so nested payloads can be located
| |||
uchar raw[];
| |||
reader.Bytes(raw);
| |||
//--- Read every field of the model
| |||
while(!reader.Eof())
| |||
{
| |||
//--- Take the next field tag
| |||
int field, wire;
| |||
if(!reader.ReadTag(field, wire))
| |||
break;
| |||
//--- Store the format revision
| |||
if(field == 1 && wire == PB_VARINT)
| |||
m_irVersion = (long)reader.ReadVarint();
| |||
//--- Store the producing tool
| |||
else
| |||
if(field == 2 && wire == PB_LENGTH)
| |||
reader.ReadString(m_producer);
| |||
//--- Store the version of that tool
| |||
else
| |||
if(field == 3 && wire == PB_LENGTH)
| |||
reader.ReadString(m_producerVersion);
| |||
//--- Store the domain the model belongs to
| |||
else
| |||
if(field == 4 && wire == PB_LENGTH)
| |||
reader.ReadString(m_domain);
| |||
//--- Store the revision the model gives itself
| |||
else
| |||
if(field == 5 && wire == PB_VARINT)
| |||
m_modelVersion = (long)reader.ReadVarint();
| |||
//--- Store whatever the file says about itself
| |||
else
| |||
if(field == 6 && wire == PB_LENGTH)
| |||
reader.ReadString(m_docString);
| |||
//--- Store a key and value pair the file attached to the model
| |||
else
| |||
if(field == 14 && wire == PB_LENGTH)
| |||
{
| |||
//--- Read the pair and keep it while room remains
| |||
int e1, e2;
| |||
if(!reader.ReadBlock(e1, e2))
| |||
break;
| |||
ReadPair(raw, e1, e2, m_metaKey, m_metaValue, m_metas);
| |||
}
| |||
//--- Handle training state the file stores with the model
| |||
else
| |||
if(field == 20 && wire == PB_LENGTH)
| |||
{
| |||
//--- Step past it, only its presence is reported
| |||
int g1, g2;
| |||
if(!reader.ReadBlock(g1, g2))
| |||
break;
| |||
m_hasTraining = true;
| |||
}
| |||
//--- Store an operator the file defines for itself
| |||
else
| |||
if(field == 25 && wire == PB_LENGTH)
| |||
{
| |||
//--- Read enough of it to name it and say what it holds
| |||
int u1, u2;
| |||
if(!reader.ReadBlock(u1, u2))
| |||
break;
| |||
ReadFunction(raw, u1, u2);
| |||
}
| |||
//--- Read the graph itself
| |||
else
| |||
if(field == 7 && wire == PB_LENGTH)
| |||
{
| |||
int from, to;
| |||
if(!reader.ReadBlock(from, to))
| |||
break;
| |||
if(!ParseGraph(raw, from, to))
| |||
return false;
| |||
}
| |||
//--- Read the operator set version
| |||
else
| |||
if(field == 8 && wire == PB_LENGTH)
| |||
{
| |||
//--- Locate the operator set message
| |||
int from, to;
| |||
if(!reader.ReadBlock(from, to))
| |||
break;
| |||
//--- Read its version field
| |||
CPbReader ops;
| |||
ops.Attach(raw, from, to);
| |||
while(!ops.Eof())
| |||
{
| |||
int f2, w2;
| |||
if(!ops.ReadTag(f2, w2))
| |||
break;
| |||
if(f2 == 2 && w2 == PB_VARINT)
| |||
m_opset = (int)ops.ReadVarint();
| |||
else
| |||
if(!ops.SkipField(w2))
| |||
break;
| |||
}
| |||
}
| |||
//--- Step over anything else
| |||
else
| |||
if(!reader.SkipField(wire))
| |||
break;
| |||
}
| |||
//--- Report a file that could not be decoded
| |||
if(reader.Failed() && m_nodes == 0)
| |||
{
| |||
m_error = "malformed protobuf in " + path;
| |||
return false;
| |||
}
| |||
//--- Report a file that holds no graph
| |||
if(m_nodes == 0)
| |||
{
| |||
m_error = "no nodes found in " + path;
| |||
return false;
| |||
}
| |||
//--- Report success
| |||
return true;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Total every weight the file carries |
| |||
//+------------------------------------------------------------------+
| |||
int COmvModel::Parameters(void) const
| |||
{
| |||
//--- Add the value count of every tensor
| |||
int total = 0;
| |||
for(int i = 0; i < m_tensors; i++)
| |||
total += m_tensor[i].values;
| |||
//--- Return the running total
| |||
return total;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Render a shape as a bracketed list |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::DimsText(const int &dim[], const int dims) const
| |||
{
| |||
//--- Print empty brackets for an unshaped tensor
| |||
if(dims <= 0)
| |||
return "[]";
| |||
//--- Join every axis with a separator
| |||
string out = "[";
| |||
for(int i = 0; i < dims; i++)
| |||
{
| |||
//--- Separate all but the first axis
| |||
if(i > 0)
| |||
out += "x";
| |||
//--- Append this axis length
| |||
out += IntegerToString(dim[i]);
| |||
}
| |||
//--- Close the bracket
| |||
return out + "]";
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Name an ONNX element type code |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::TypeName(const int code) const
| |||
{
| |||
//--- Name each code the way the reader maps the ONNX enum
| |||
switch(code)
| |||
{
| |||
case 0:
| |||
return "undefined";
| |||
case 1:
| |||
return "float32";
| |||
case 2:
| |||
return "uint8";
| |||
case 3:
| |||
return "int8";
| |||
case 4:
| |||
return "uint16";
| |||
case 5:
| |||
return "int16";
| |||
case 6:
| |||
return "int32";
| |||
case 7:
| |||
return "int64";
| |||
case 8:
| |||
return "string";
| |||
case 9:
| |||
return "boolean";
| |||
case 10:
| |||
return "float16";
| |||
case 11:
| |||
return "float64";
| |||
case 12:
| |||
return "uint32";
| |||
case 13:
| |||
return "uint64";
| |||
case 14:
| |||
return "complex<float32>";
| |||
case 15:
| |||
return "complex<float64>";
| |||
case 16:
| |||
return "bfloat16";
| |||
}
| |||
//--- Print the raw code for anything unrecognized
| |||
return "type" + IntegerToString(code);
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Join the axes the way a shape row reports them |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::DimsBare(const int &dim[], const int dims) const
| |||
{
| |||
//--- Report nothing for a tensor the file never shaped
| |||
if(dims <= 0)
| |||
return "";
| |||
//--- Join every axis with a comma and no brackets
| |||
string out = "";
| |||
for(int i = 0; i < dims; i++)
| |||
{
| |||
//--- Separate all but the first axis
| |||
if(i > 0)
| |||
out += ",";
| |||
//--- Append this axis length
| |||
out += IntegerToString(dim[i]);
| |||
}
| |||
//--- Report the joined axes
| |||
return out;
| |||
}
| |||
| |||
//+------------------------------------------------------------------+
| |||
//| Render a shape the way a tensor type is written |
| |||
//+------------------------------------------------------------------+
| |||
string COmvModel::DimsList(const int &dim[], const int dims) const
| |||
{
| |||
//--- An unshaped tensor prints as empty brackets rather than nothing
| |||
if(dims <= 0)
| |||
return "[]";
| |||
//--- Separate the axes with commas the way a tensor type reads
| |||
string out = "[";
| |||
for(int i = 0; i < dims; i++)
| |||
{
| |||
//--- Separate all but the first axis
| |||
if(i > 0)
| |||
out += ",";
| |||
//--- Append this axis length
| |||
out += IntegerToString(dim[i]);
| |||
}
| |||
//--- Close the bracket
| |||
return out + "]";
| |||
}
| |||
| |||
#endif // OMV_MODEL_MQH
| |||
//+------------------------------------------------------------------+
|