1710 lines
No EOL
64 KiB
MQL5
1710 lines
No EOL
64 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| OMV Model.mqh |
|
|
//| Copyright 2026, Allan Munene Mutiiria. |
|
|
//| https://t.me/Forex_Algo_Trader |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2026, Allan Munene Mutiiria."
|
|
#property link "https://t.me/Forex_Algo_Trader"
|
|
|
|
#ifndef OMV_MODEL_MQH
|
|
#define OMV_MODEL_MQH
|
|
|
|
#include "OMV Protobuf.mqh"
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Capacity limits |
|
|
//+------------------------------------------------------------------+
|
|
#define OMV_MAX_NODES 4096 // Layers the graph may hold
|
|
#define OMV_MAX_TENSORS 4096 // Weight tensors the graph may hold
|
|
#define OMV_MAX_PORTS 64 // Names one node may carry per side
|
|
#define OMV_MAX_ATTRS 32 // Attributes one node may carry
|
|
#define OMV_MAX_META 32 // Key and value pairs the file may carry
|
|
#define OMV_MAX_FUNCS 64 // Custom operators the file may define
|
|
#define OMV_MAX_DIMS 16 // Axes one tensor may have
|
|
#define OMV_MAX_VALUES 4096 // Weights one tensor keeps for display
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Graph layer |
|
|
//+------------------------------------------------------------------+
|
|
struct OmvNode
|
|
{
|
|
// Name the operator this layer applies
|
|
string op;
|
|
// Name this layer was given
|
|
string name;
|
|
// Name the operator set this layer's operator comes from
|
|
string domain;
|
|
// Name each attribute the operator was given
|
|
string attrName[OMV_MAX_ATTRS];
|
|
// Print the value each attribute holds
|
|
string attrValue[OMV_MAX_ATTRS];
|
|
// Record which kind each attribute declared itself to be
|
|
int attrKind[OMV_MAX_ATTRS];
|
|
// Count the attributes present
|
|
int attrCount;
|
|
// Carry whatever the file says about this layer
|
|
string docString;
|
|
// Name the overload the operator was resolved to
|
|
string overload;
|
|
// List the names feeding this layer
|
|
string feeds[OMV_MAX_PORTS];
|
|
// Count the names present
|
|
int feedCount;
|
|
// List the names this layer produces
|
|
string emits[OMV_MAX_PORTS];
|
|
// Count the names present
|
|
int emitCount;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Weight tensor |
|
|
//+------------------------------------------------------------------+
|
|
struct OmvTensor
|
|
{
|
|
// Name the nodes refer to it by
|
|
string name;
|
|
// Measure each axis
|
|
int dim[OMV_MAX_DIMS];
|
|
// Count the axes present
|
|
int dims;
|
|
// Record the ONNX type code
|
|
int dataType;
|
|
// Multiply every axis to size the block
|
|
int values;
|
|
// Track the smallest weight held
|
|
double minValue;
|
|
// Track the largest weight held
|
|
double maxValue;
|
|
// Track the average weight
|
|
double meanValue;
|
|
// Track how far the weights spread from their average
|
|
double stdValue;
|
|
// Track what share of the weights are zero
|
|
double sparsity;
|
|
// Keep the weights themselves so the inspector can show them
|
|
double val[];
|
|
// Count the weights kept, which stops at the display limit
|
|
int kept;
|
|
// Report where the file keeps this tensor's values
|
|
string location;
|
|
// Name the category the file gives this tensor
|
|
string category;
|
|
// Keep the strings when the tensor holds text rather than numbers
|
|
string text[];
|
|
// Count the strings kept
|
|
int texts;
|
|
// Report whether the file stored this tensor in a segment
|
|
bool segmented;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Exposed graph port |
|
|
//+------------------------------------------------------------------+
|
|
struct OmvPort
|
|
{
|
|
// Name the graph exposes
|
|
string name;
|
|
// Measure each axis
|
|
int dim[OMV_MAX_DIMS];
|
|
// Count the axes present
|
|
int dims;
|
|
// Record the ONNX type code
|
|
int elemType;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| ONNX file reader |
|
|
//+------------------------------------------------------------------+
|
|
class COmvModel
|
|
{
|
|
private:
|
|
// Store the layers in file order
|
|
OmvNode m_node[];
|
|
// Store the weight tensors
|
|
OmvTensor m_tensor[];
|
|
// Store the graph inputs
|
|
OmvPort m_input[];
|
|
// Store the graph outputs
|
|
OmvPort m_output[];
|
|
// Store what the file says about the values passing between layers
|
|
OmvPort m_inner[];
|
|
// Count the layers held
|
|
int m_nodes;
|
|
// Count the tensors held
|
|
int m_tensors;
|
|
// Count the graph inputs held
|
|
int m_inputs;
|
|
// Count the graph outputs held
|
|
int m_outputs;
|
|
// Count the inner values held
|
|
int m_inners;
|
|
// Record the format revision
|
|
long m_irVersion;
|
|
// Record the operator set version
|
|
int m_opset;
|
|
// Name the tool that wrote the file
|
|
string m_producer;
|
|
// Record the version of the tool that wrote it
|
|
string m_producerVersion;
|
|
// Name the domain the model belongs to
|
|
string m_domain;
|
|
// Record the revision the model gives itself
|
|
long m_modelVersion;
|
|
// Carry whatever the file says about itself
|
|
string m_docString;
|
|
// Name the graph inside the file
|
|
string m_graphName;
|
|
// Carry whatever the file says about the graph
|
|
string m_graphDoc;
|
|
// Name each key the file attached to the model
|
|
string m_metaKey[OMV_MAX_META];
|
|
// Print the value each key carries
|
|
string m_metaValue[OMV_MAX_META];
|
|
// Count the pairs held
|
|
int m_metas;
|
|
// Name each operator the file defines for itself
|
|
string m_funcName[OMV_MAX_FUNCS];
|
|
// Describe what each defined operator holds
|
|
string m_funcDetail[OMV_MAX_FUNCS];
|
|
// Count the defined operators held
|
|
int m_funcs;
|
|
// Report whether the file carries training state
|
|
bool m_hasTraining;
|
|
// Name each key the file attached to the graph
|
|
string m_gMetaKey[OMV_MAX_META];
|
|
// Print the value each graph key carries
|
|
string m_gMetaValue[OMV_MAX_META];
|
|
// Count the graph pairs held
|
|
int m_gMetas;
|
|
// Count the quantization mappings the graph carries
|
|
int m_quants;
|
|
// Explain why the last load failed
|
|
string m_error;
|
|
|
|
bool ParseGraph(const uchar &raw[], const int from, const int to);
|
|
bool ParseNode(const uchar &raw[], const int from, const int to);
|
|
bool ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer);
|
|
string ReadList(const uchar &raw[], const int from, const int to, const int field);
|
|
string ReadSubgraph(const uchar &raw[], const int from, const int to);
|
|
string ReadEntry(const uchar &raw[], const int from, const int to);
|
|
void ReadPair(const uchar &raw[], const int from, const int to,
|
|
string &keys[], string &values[], int &count);
|
|
void ReadFunction(const uchar &raw[], const int from, const int to);
|
|
bool ParseTensor(const uchar &raw[], const int from, const int to);
|
|
bool ParseSparse(const uchar &raw[], const int from, const int to);
|
|
bool ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port);
|
|
void Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
|
|
double &total, double &squares,
|
|
double &lowest, double &highest);
|
|
void Publish(OmvTensor &tensor, const int seen, const int zeros,
|
|
const double total, const double squares,
|
|
const double lowest, const double highest);
|
|
void RawStats(const uchar &raw[], const int from, const int to,
|
|
OmvTensor &tensor);
|
|
void PackedStats(const uchar &raw[], const int from, const int to,
|
|
const int field, OmvTensor &tensor);
|
|
|
|
public:
|
|
COmvModel(void);
|
|
bool Load(const string path);
|
|
void Clear(void);
|
|
|
|
int Nodes(void) const { return m_nodes; }
|
|
int Tensors(void) const { return m_tensors; }
|
|
int Inputs(void) const { return m_inputs; }
|
|
int Outputs(void) const { return m_outputs; }
|
|
int Inners(void) const { return m_inners; }
|
|
long IrVersion(void) const { return m_irVersion; }
|
|
int Opset(void) const { return m_opset; }
|
|
string Producer(void) const { return m_producer; }
|
|
string ProducerVersion(void) const { return m_producerVersion; }
|
|
string Domain(void) const { return m_domain; }
|
|
long ModelVersion(void) const { return m_modelVersion; }
|
|
string DocString(void) const { return m_docString; }
|
|
string GraphName(void) const { return m_graphName; }
|
|
string GraphDoc(void) const { return m_graphDoc; }
|
|
int Metas(void) const { return m_metas; }
|
|
int GraphMetas(void) const { return m_gMetas; }
|
|
string GraphMetaKey(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaKey[i] : ""; }
|
|
string GraphMetaValue(const int i) const { return (i >= 0 && i < m_gMetas) ? m_gMetaValue[i] : ""; }
|
|
int Quants(void) const { return m_quants; }
|
|
string MetaKey(const int i) const { return (i >= 0 && i < m_metas) ? m_metaKey[i] : ""; }
|
|
string MetaValue(const int i) const { return (i >= 0 && i < m_metas) ? m_metaValue[i] : ""; }
|
|
int Funcs(void) const { return m_funcs; }
|
|
string FuncName(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcName[i] : ""; }
|
|
string FuncDetail(const int i) const { return (i >= 0 && i < m_funcs) ? m_funcDetail[i] : ""; }
|
|
bool HasTraining(void) const { return m_hasTraining; }
|
|
string Error(void) const { return m_error; }
|
|
int Parameters(void) const;
|
|
|
|
void NodeAt(const int i, OmvNode &out) const { if(i >= 0 && i < m_nodes) out = m_node[i]; }
|
|
void TensorAt(const int i, OmvTensor &out) const { if(i >= 0 && i < m_tensors) out = m_tensor[i]; }
|
|
void InputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inputs) out = m_input[i]; }
|
|
void OutputAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_outputs) out = m_output[i]; }
|
|
void InnerAt(const int i, OmvPort &out) const { if(i >= 0 && i < m_inners) out = m_inner[i]; }
|
|
string DimsText(const int &dim[], const int dims) const;
|
|
string DimsList(const int &dim[], const int dims) const;
|
|
string DimsBare(const int &dim[], const int dims) const;
|
|
string TypeName(const int code) const;
|
|
string AttrType(const int kind) const;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Construct an empty model |
|
|
//+------------------------------------------------------------------+
|
|
COmvModel::COmvModel(void)
|
|
{
|
|
//--- Clear every field so a query before loading reports empty
|
|
Clear();
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Drop everything the previous file filled |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::Clear(void)
|
|
{
|
|
//--- Zero the counts that bound every array
|
|
m_nodes = 0;
|
|
m_tensors = 0;
|
|
m_inputs = 0;
|
|
m_outputs = 0;
|
|
m_inners = 0;
|
|
//--- Release the stored structures
|
|
ArrayResize(m_node, 0);
|
|
ArrayResize(m_tensor, 0);
|
|
ArrayResize(m_input, 0);
|
|
ArrayResize(m_output, 0);
|
|
ArrayResize(m_inner, 0);
|
|
//--- Clear the header fields
|
|
m_irVersion = 0;
|
|
m_opset = 0;
|
|
m_producer = "";
|
|
m_producerVersion = "";
|
|
m_domain = "";
|
|
m_modelVersion = 0;
|
|
m_docString = "";
|
|
m_graphName = "";
|
|
m_graphDoc = "";
|
|
m_metas = 0;
|
|
m_gMetas = 0;
|
|
m_quants = 0;
|
|
m_funcs = 0;
|
|
m_hasTraining = false;
|
|
m_error = "";
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Walk raw weight bytes for their range and average |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::Accumulate(const double v, OmvTensor &tensor, int &seen, int &zeros,
|
|
double &total, double &squares, double &lowest, double &highest)
|
|
{
|
|
//--- Seed the range from the first value that arrives
|
|
if(seen == 0)
|
|
{
|
|
lowest = v;
|
|
highest = v;
|
|
}
|
|
//--- Lower the floor when this value is smaller
|
|
if(v < lowest)
|
|
lowest = v;
|
|
//--- Raise the ceiling when this value is larger
|
|
if(v > highest)
|
|
highest = v;
|
|
//--- Add the value to the running totals
|
|
total += v;
|
|
squares += v * v;
|
|
//--- Count a value that is exactly zero
|
|
if(v == 0.0)
|
|
zeros++;
|
|
//--- Keep the value itself while room remains
|
|
if(tensor.kept < OMV_MAX_VALUES)
|
|
{
|
|
ArrayResize(tensor.val, tensor.kept + 1);
|
|
tensor.val[tensor.kept++] = v;
|
|
}
|
|
seen++;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Publish what the accumulated values measured |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::Publish(OmvTensor &tensor, const int seen, const int zeros,
|
|
const double total, const double squares,
|
|
const double lowest, const double highest)
|
|
{
|
|
//--- Abort when the read stopped before any value arrived
|
|
if(seen <= 0)
|
|
return;
|
|
//--- Publish the range and the average
|
|
tensor.minValue = lowest;
|
|
tensor.maxValue = highest;
|
|
tensor.meanValue = total / seen;
|
|
//--- Take the spread as the root of the mean squared deviation
|
|
const double variance = squares / seen - tensor.meanValue * tensor.meanValue;
|
|
tensor.stdValue = (variance > 0.0) ? MathSqrt(variance) : 0.0;
|
|
//--- Take the share of the values that are exactly zero
|
|
tensor.sparsity = (double)zeros / seen;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Walk a raw byte payload at the width its type implies |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::RawStats(const uchar &raw[], const int from, const int to, OmvTensor &tensor)
|
|
{
|
|
//--- Take the width one value occupies for this element type
|
|
int width = 4;
|
|
if(tensor.dataType == 11 || tensor.dataType == 7 || tensor.dataType == 13)
|
|
width = 8;
|
|
else
|
|
if(tensor.dataType == 2 || tensor.dataType == 3 || tensor.dataType == 9)
|
|
width = 1;
|
|
else
|
|
if(tensor.dataType == 4 || tensor.dataType == 5 || tensor.dataType == 10)
|
|
width = 2;
|
|
//--- Count the whole values the payload holds
|
|
const int count = (to - from) / width;
|
|
if(count <= 0)
|
|
return;
|
|
//--- Point a cursor at the payload
|
|
CPbReader bytes;
|
|
if(!bytes.Attach(raw, from, to))
|
|
return;
|
|
//--- Start the running measures
|
|
double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
|
|
int zeros = 0, seen = 0;
|
|
//--- Visit every stored value
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
//--- Read the value the way its type is written
|
|
double v = 0.0;
|
|
if(tensor.dataType == 1)
|
|
v = (double)bytes.ReadFloat();
|
|
else
|
|
if(tensor.dataType == 11)
|
|
v = bytes.ReadDouble();
|
|
else
|
|
if(width == 8)
|
|
v = (double)(long)bytes.ReadFixed64();
|
|
else
|
|
if(width == 4)
|
|
v = (double)(int)bytes.ReadFixed32();
|
|
else
|
|
if(width == 2)
|
|
v = (double)(int)(bytes.ReadVarint() & 0xFFFF);
|
|
else
|
|
v = (double)bytes.ReadVarint();
|
|
//--- Stop on a broken read
|
|
if(bytes.Failed())
|
|
break;
|
|
//--- Fold the value into the running measures
|
|
Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
|
|
}
|
|
//--- Publish what the walk measured
|
|
Publish(tensor, seen, zeros, total, squares, lowest, highest);
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Walk a typed payload the file used instead of raw bytes |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::PackedStats(const uchar &raw[], const int from, const int to,
|
|
const int field, OmvTensor &tensor)
|
|
{
|
|
//--- Point a cursor at the payload
|
|
CPbReader bytes;
|
|
if(!bytes.Attach(raw, from, to))
|
|
return;
|
|
//--- Start the running measures
|
|
double lowest = 0.0, highest = 0.0, total = 0.0, squares = 0.0;
|
|
int zeros = 0, seen = 0;
|
|
//--- Read the payload the way the field it arrived in is written
|
|
while(!bytes.Eof())
|
|
{
|
|
//--- Take the next value in the packed run
|
|
double v = 0.0;
|
|
if(field == 4)
|
|
v = (double)bytes.ReadFloat();
|
|
else
|
|
if(field == 10)
|
|
v = bytes.ReadDouble();
|
|
else
|
|
if(field == 6)
|
|
break;
|
|
else
|
|
v = (double)(long)bytes.ReadVarint();
|
|
//--- Stop on a broken read
|
|
if(bytes.Failed())
|
|
break;
|
|
//--- Fold the value into the running measures
|
|
Accumulate(v, tensor, seen, zeros, total, squares, lowest, highest);
|
|
}
|
|
//--- Publish what the walk measured
|
|
Publish(tensor, seen, zeros, total, squares, lowest, highest);
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Join a list an attribute carried into one line |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::ReadList(const uchar &raw[], const int from, const int to, const int field)
|
|
{
|
|
//--- Point a cursor at the list
|
|
CPbReader items;
|
|
if(!items.Attach(raw, from, to))
|
|
return "";
|
|
//--- Join up to a handful of entries, then say how many remain
|
|
string out = "";
|
|
int shown = 0, total = 0;
|
|
while(!items.Eof())
|
|
{
|
|
//--- Read the next entry the way its field is written
|
|
string one = "";
|
|
if(field == 7)
|
|
one = DoubleToString((double)items.ReadFloat(), 4);
|
|
else
|
|
if(field == 8)
|
|
one = IntegerToString((long)items.ReadVarint());
|
|
else
|
|
items.ReadString(one);
|
|
//--- Stop on a broken read
|
|
if(items.Failed())
|
|
break;
|
|
total++;
|
|
//--- Keep the first few entries only, a row cannot hold more
|
|
if(shown >= 8)
|
|
continue;
|
|
out += (shown > 0 ? ", " : "") + one;
|
|
shown++;
|
|
}
|
|
//--- Say how many entries the line left out
|
|
if(total > shown)
|
|
out += ", ... " + IntegerToString(total - shown) + " more";
|
|
return out;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Describe a nested graph by what it actually holds |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::ReadSubgraph(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Point a cursor at the nested graph
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return "<graph>";
|
|
uchar inner[];
|
|
reader.Bytes(inner);
|
|
//--- Count what the nested graph carries and name its operators
|
|
string title = "";
|
|
string ops = "";
|
|
int nodes = 0, tensors = 0, shown = 0;
|
|
//--- Read every field of the nested graph
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Step over anything that is not a nested message
|
|
if(wire != PB_LENGTH)
|
|
{
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Locate the nested payload
|
|
int f2, t2;
|
|
if(!reader.ReadBlock(f2, t2))
|
|
break;
|
|
//--- Count a node and name the operator it applies
|
|
if(field == 1)
|
|
{
|
|
nodes++;
|
|
//--- Name only the first few, a line cannot hold more
|
|
if(shown < 6)
|
|
{
|
|
//--- Read the operator out of the node
|
|
CPbReader entry;
|
|
entry.Attach(inner, f2, t2);
|
|
while(!entry.Eof())
|
|
{
|
|
//--- Take the next field of the entry
|
|
int f3, w3;
|
|
if(!entry.ReadTag(f3, w3))
|
|
break;
|
|
//--- Keep the operator and stop reading this node
|
|
if(f3 == 4 && w3 == PB_LENGTH)
|
|
{
|
|
string op = "";
|
|
entry.ReadString(op);
|
|
ops += (shown > 0 ? ", " : "") + op;
|
|
shown++;
|
|
break;
|
|
}
|
|
if(!entry.SkipField(w3))
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
//--- Count an initializer the nested graph carries
|
|
else
|
|
if(field == 5 || field == 15)
|
|
tensors++;
|
|
//--- Take the name the nested graph was given
|
|
else
|
|
if(field == 2)
|
|
{
|
|
//--- Decode the name from its own payload
|
|
uchar tmp[];
|
|
ArrayResize(tmp, t2 - f2);
|
|
for(int i = f2; i < t2; i++)
|
|
tmp[i - f2] = inner[i];
|
|
title = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
|
}
|
|
}
|
|
//--- Report the nested graph by its name, its size and what it runs
|
|
string out = "<graph";
|
|
if(StringLen(title) > 0)
|
|
out += " " + title;
|
|
out += ": " + IntegerToString(nodes) + " nodes, " +
|
|
IntegerToString(tensors) + " tensors";
|
|
if(StringLen(ops) > 0)
|
|
out += " (" + ops + (nodes > shown ? ", ..." : "") + ")";
|
|
return out + ">";
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Name the type an attribute declared itself to be |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::AttrType(const int kind) const
|
|
{
|
|
//--- Match the declared kind to the type the app reports for it
|
|
switch(kind)
|
|
{
|
|
case 1:
|
|
return "float32";
|
|
case 2:
|
|
return "int64";
|
|
case 3:
|
|
return "string";
|
|
case 4:
|
|
return "tensor";
|
|
case 5:
|
|
return "graph";
|
|
case 6:
|
|
return "float32[]";
|
|
case 7:
|
|
return "int64[]";
|
|
case 8:
|
|
return "string[]";
|
|
}
|
|
//--- Report nothing recognizable as undefined
|
|
return "undefined";
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read one attribute an operator was given |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParseAttribute(const uchar &raw[], const int from, const int to, OmvNode &layer)
|
|
{
|
|
//--- Point a cursor at this attribute
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return false;
|
|
uchar inner[];
|
|
reader.Bytes(inner);
|
|
//--- Refuse an attribute beyond what the layer may hold
|
|
if(layer.attrCount >= OMV_MAX_ATTRS)
|
|
return false;
|
|
//--- Start with nothing read
|
|
const int slot = layer.attrCount;
|
|
layer.attrName[slot] = "";
|
|
layer.attrValue[slot] = "";
|
|
//--- Remember which kind of value the attribute declares
|
|
int kind = 0;
|
|
double asFloat = 0.0;
|
|
long asInt = 0;
|
|
string asText = "";
|
|
bool haveInt = false, haveFloat = false;
|
|
//--- Count the entries a repeated field delivers one at a time
|
|
int repeats = 0;
|
|
//--- Read every field of the attribute
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Store the attribute name
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
reader.ReadString(layer.attrName[slot]);
|
|
//--- Store a floating point value
|
|
else
|
|
if(field == 2 && wire == PB_FIXED32)
|
|
{
|
|
asFloat = (double)reader.ReadFloat();
|
|
haveFloat = true;
|
|
}
|
|
//--- Store a whole number value
|
|
else
|
|
if(field == 3 && wire == PB_VARINT)
|
|
{
|
|
asInt = (long)reader.ReadVarint();
|
|
haveInt = true;
|
|
}
|
|
//--- Store a text value
|
|
else
|
|
if(field == 4 && wire == PB_LENGTH)
|
|
reader.ReadString(asText);
|
|
//--- Name a nested tensor or graph rather than unfolding it
|
|
else
|
|
if((field == 5 || field == 6) && wire == PB_LENGTH)
|
|
{
|
|
//--- Step past the payload and record what it was
|
|
int n1, n2;
|
|
if(!reader.ReadBlock(n1, n2))
|
|
break;
|
|
asText = (field == 5) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
|
|
}
|
|
//--- Append one string, of which this field delivers many
|
|
else
|
|
if(field == 9 && wire == PB_LENGTH)
|
|
{
|
|
//--- Read this entry and join it to the ones before
|
|
string one = "";
|
|
if(!reader.ReadString(one))
|
|
break;
|
|
repeats++;
|
|
if(repeats <= 8)
|
|
asText += (repeats > 1 ? ", " : "") + one;
|
|
}
|
|
//--- Join a list of floats, whole numbers or strings
|
|
else
|
|
if((field == 7 || field == 8) && wire == PB_LENGTH)
|
|
{
|
|
//--- Locate the list and read it in the form it arrived
|
|
int l1, l2;
|
|
if(!reader.ReadBlock(l1, l2))
|
|
break;
|
|
asText = ReadList(inner, l1, l2, field);
|
|
}
|
|
//--- Name a list of tensors or graphs rather than unfolding it
|
|
else
|
|
if((field == 10 || field == 11) && wire == PB_LENGTH)
|
|
{
|
|
//--- Step past this entry and count it among its siblings
|
|
int n1, n2;
|
|
if(!reader.ReadBlock(n1, n2))
|
|
break;
|
|
repeats++;
|
|
//--- Describe the first entry, then say how many followed
|
|
if(repeats == 1)
|
|
asText = (field == 10) ? "<tensor>" : ReadSubgraph(inner, n1, n2);
|
|
}
|
|
//--- Store which kind the attribute declares itself to be
|
|
else
|
|
if(field == 20 && wire == PB_VARINT)
|
|
kind = (int)reader.ReadVarint();
|
|
//--- Step over anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Say how many entries a repeated field delivered beyond those shown
|
|
if(repeats > 8)
|
|
asText += ", ... " + IntegerToString(repeats - 8) + " more";
|
|
else
|
|
if(repeats > 1 && (kind == 5 || kind == 6 || kind == 9 || kind == 10))
|
|
asText += " and " + IntegerToString(repeats - 1) + " more";
|
|
//--- Reject an attribute that never gave a name
|
|
if(StringLen(layer.attrName[slot]) == 0)
|
|
return false;
|
|
//--- Print the value the attribute carried, by the kind it declared
|
|
if(kind == 3 || kind >= 4 || StringLen(asText) > 0)
|
|
layer.attrValue[slot] = asText;
|
|
else
|
|
if(kind == 2 || (kind == 0 && haveInt))
|
|
layer.attrValue[slot] = IntegerToString(asInt);
|
|
else
|
|
if(kind == 1 || (kind == 0 && haveFloat))
|
|
layer.attrValue[slot] = DoubleToString(asFloat, 6);
|
|
//--- Record the kind so the row can report the type it opens to
|
|
layer.attrKind[slot] = kind;
|
|
//--- Report the attribute as read
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read one node into a layer |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParseNode(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Refuse a file that exceeds the node limit
|
|
if(m_nodes >= OMV_MAX_NODES)
|
|
return false;
|
|
//--- Point a cursor at this node
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return false;
|
|
//--- Start an empty layer
|
|
OmvNode layer;
|
|
layer.op = "";
|
|
layer.name = "";
|
|
layer.domain = "";
|
|
layer.docString = "";
|
|
layer.overload = "";
|
|
layer.attrCount = 0;
|
|
layer.feedCount = 0;
|
|
layer.emitCount = 0;
|
|
//--- Read every field of the node
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Skip anything that is not text
|
|
if(wire != PB_LENGTH)
|
|
{
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Read an attribute out of its own nested message
|
|
if(field == 5)
|
|
{
|
|
//--- Locate the attribute and decode it while room remains
|
|
int a1, a2;
|
|
if(!reader.ReadBlock(a1, a2))
|
|
break;
|
|
if(layer.attrCount < OMV_MAX_ATTRS)
|
|
if(ParseAttribute(raw, from + a1, from + a2, layer))
|
|
layer.attrCount++;
|
|
continue;
|
|
}
|
|
//--- Read the field as text
|
|
string text;
|
|
if(!reader.ReadString(text))
|
|
break;
|
|
//--- Store an input name
|
|
if(field == 1 && layer.feedCount < OMV_MAX_PORTS)
|
|
layer.feeds[layer.feedCount++] = text;
|
|
//--- Store an output name
|
|
else
|
|
if(field == 2 && layer.emitCount < OMV_MAX_PORTS)
|
|
layer.emits[layer.emitCount++] = text;
|
|
//--- Store the layer name
|
|
else
|
|
if(field == 3)
|
|
layer.name = text;
|
|
//--- Store the operator
|
|
else
|
|
if(field == 4)
|
|
layer.op = text;
|
|
//--- Store the operator set the operator belongs to
|
|
else
|
|
if(field == 7)
|
|
layer.domain = text;
|
|
//--- Store whatever the file says about this layer
|
|
else
|
|
if(field == 6)
|
|
layer.docString = text;
|
|
//--- Store the overload the operator was resolved to
|
|
else
|
|
if(field == 8)
|
|
layer.overload = text;
|
|
}
|
|
//--- Reject a node carrying no operator
|
|
if(StringLen(layer.op) == 0)
|
|
return false;
|
|
//--- Append the finished layer
|
|
ArrayResize(m_node, m_nodes + 1);
|
|
m_node[m_nodes++] = layer;
|
|
//--- Report success
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read one initializer into a weight tensor |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParseTensor(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Refuse a file that exceeds the tensor limit
|
|
if(m_tensors >= OMV_MAX_TENSORS)
|
|
return false;
|
|
//--- Point a cursor at this tensor
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return false;
|
|
//--- Keep the slice so nested payloads can be located
|
|
uchar inner[];
|
|
reader.Bytes(inner);
|
|
//--- Start an empty tensor
|
|
OmvTensor weights;
|
|
weights.name = "";
|
|
weights.dims = 0;
|
|
weights.dataType = 0;
|
|
weights.values = 0;
|
|
weights.minValue = 0.0;
|
|
weights.maxValue = 0.0;
|
|
weights.meanValue = 0.0;
|
|
weights.stdValue = 0.0;
|
|
weights.sparsity = 0.0;
|
|
weights.kept = 0;
|
|
weights.texts = 0;
|
|
weights.segmented = false;
|
|
weights.location = "default";
|
|
weights.category = "Initializer";
|
|
//--- Mark every payload the tensor might carry as not yet found
|
|
int rawFrom = -1, rawTo = -1;
|
|
int packFrom = -1, packTo = -1, packField = 0;
|
|
//--- Read every field of the tensor
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Store one axis length
|
|
if(field == 1 && wire == PB_VARINT)
|
|
{
|
|
//--- Keep the axis while room remains
|
|
if(weights.dims < OMV_MAX_DIMS)
|
|
weights.dim[weights.dims++] = (int)reader.ReadVarint();
|
|
else
|
|
reader.ReadVarint();
|
|
}
|
|
//--- Store several axis lengths packed together
|
|
else
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
{
|
|
//--- Locate the packed block
|
|
int f2, t2;
|
|
if(!reader.ReadBlock(f2, t2))
|
|
break;
|
|
//--- Read every length the block holds
|
|
CPbReader packed;
|
|
packed.Attach(inner, f2, t2);
|
|
while(!packed.Eof() && weights.dims < OMV_MAX_DIMS)
|
|
weights.dim[weights.dims++] = (int)packed.ReadVarint();
|
|
}
|
|
//--- Store the element type
|
|
else
|
|
if(field == 2 && wire == PB_VARINT)
|
|
weights.dataType = (int)reader.ReadVarint();
|
|
//--- Handle a tensor the file stored in segments
|
|
else
|
|
if(field == 3 && wire == PB_LENGTH)
|
|
{
|
|
//--- Step past the segment, only its presence is reported
|
|
int s1, s2;
|
|
if(!reader.ReadBlock(s1, s2))
|
|
break;
|
|
weights.segmented = true;
|
|
}
|
|
//--- Store the tensor name
|
|
else
|
|
if(field == 8 && wire == PB_LENGTH)
|
|
reader.ReadString(weights.name);
|
|
//--- Locate the raw byte payload
|
|
else
|
|
if(field == 9 && wire == PB_LENGTH)
|
|
{
|
|
if(!reader.ReadBlock(rawFrom, rawTo))
|
|
break;
|
|
}
|
|
//--- Collect one string, of which the field may carry many
|
|
else
|
|
if(field == 6 && wire == PB_LENGTH)
|
|
{
|
|
//--- Append it while room remains
|
|
string one = "";
|
|
if(!reader.ReadString(one))
|
|
break;
|
|
if(weights.texts < OMV_MAX_VALUES)
|
|
{
|
|
ArrayResize(weights.text, weights.texts + 1);
|
|
weights.text[weights.texts++] = one;
|
|
}
|
|
}
|
|
//--- Locate a typed payload, which the file may use instead
|
|
else
|
|
if(wire == PB_LENGTH &&
|
|
(field == 4 || field == 5 ||
|
|
field == 7 || field == 10 || field == 11))
|
|
{
|
|
//--- Remember which typed field carried the values
|
|
if(!reader.ReadBlock(packFrom, packTo))
|
|
break;
|
|
packField = field;
|
|
}
|
|
//--- Report values the file keeps somewhere else entirely
|
|
else
|
|
if(field == 14 && wire == PB_VARINT)
|
|
{
|
|
//--- A location of one means the weights sit outside the file
|
|
if((int)reader.ReadVarint() == 1)
|
|
weights.location = "external";
|
|
}
|
|
//--- Name the file the weights were moved out to
|
|
else
|
|
if(field == 13 && wire == PB_LENGTH)
|
|
{
|
|
//--- Read the entry and keep whatever names a location
|
|
int e1, e2;
|
|
if(!reader.ReadBlock(e1, e2))
|
|
break;
|
|
const string named = ReadEntry(inner, e1, e2);
|
|
if(StringLen(named) > 0)
|
|
weights.location = named;
|
|
}
|
|
//--- Step over anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Multiply every axis to size the block
|
|
weights.values = (weights.dims > 0) ? 1 : 0;
|
|
for(int i = 0; i < weights.dims; i++)
|
|
weights.values *= weights.dim[i];
|
|
//--- Summarize whichever payload the file actually used
|
|
if(rawFrom >= 0)
|
|
RawStats(inner, rawFrom, rawTo, weights);
|
|
else
|
|
if(packFrom >= 0)
|
|
PackedStats(inner, packFrom, packTo, packField, weights);
|
|
//--- Reject a tensor carrying no name
|
|
if(StringLen(weights.name) == 0)
|
|
return false;
|
|
//--- Append the finished tensor
|
|
ArrayResize(m_tensor, m_tensors + 1);
|
|
m_tensor[m_tensors++] = weights;
|
|
//--- Report success
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read a key and value pair into a pair of lists |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::ReadPair(const uchar &raw[], const int from, const int to,
|
|
string &keys[], string &values[], int &count)
|
|
{
|
|
//--- Refuse a pair beyond what the lists may hold
|
|
if(count >= OMV_MAX_META)
|
|
return;
|
|
//--- Point a cursor at the pair
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return;
|
|
//--- Read the key and the value it carries
|
|
string key = "", value = "";
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Store the key
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
reader.ReadString(key);
|
|
//--- Store the value
|
|
else
|
|
if(field == 2 && wire == PB_LENGTH)
|
|
reader.ReadString(value);
|
|
//--- Step over anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Keep only a pair that actually named something
|
|
if(StringLen(key) == 0)
|
|
return;
|
|
keys[count] = key;
|
|
values[count] = value;
|
|
count++;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read an operator the file defines for itself |
|
|
//+------------------------------------------------------------------+
|
|
void COmvModel::ReadFunction(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Refuse a definition beyond what the list may hold
|
|
if(m_funcs >= OMV_MAX_FUNCS)
|
|
return;
|
|
//--- Point a cursor at the definition
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return;
|
|
//--- Count what the definition carries and name it
|
|
string named = "", domain = "";
|
|
int nodes = 0, ins = 0, outs = 0;
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Step over anything that is not a nested payload
|
|
if(wire != PB_LENGTH)
|
|
{
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Take the name the definition was given
|
|
if(field == 1)
|
|
reader.ReadString(named);
|
|
//--- Take the domain it belongs to
|
|
else
|
|
if(field == 9)
|
|
reader.ReadString(domain);
|
|
//--- Count the parts the definition is built from
|
|
else
|
|
{
|
|
//--- Locate the payload and count it by which field it arrived in
|
|
int f2, t2;
|
|
if(!reader.ReadBlock(f2, t2))
|
|
break;
|
|
if(field == 2)
|
|
ins++;
|
|
else
|
|
if(field == 3)
|
|
outs++;
|
|
else
|
|
if(field == 7)
|
|
nodes++;
|
|
}
|
|
}
|
|
//--- Keep only a definition that actually named itself
|
|
if(StringLen(named) == 0)
|
|
return;
|
|
m_funcName[m_funcs] = (StringLen(domain) > 0) ? domain + "." + named : named;
|
|
m_funcDetail[m_funcs] = IntegerToString(nodes) + " nodes, " +
|
|
IntegerToString(ins) + " in, " +
|
|
IntegerToString(outs) + " out";
|
|
m_funcs++;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read a key and value pair the file uses for metadata |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::ReadEntry(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Point a cursor at the pair
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return "";
|
|
//--- Read the key and the value it carries
|
|
string key = "", value = "";
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Store the key
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
reader.ReadString(key);
|
|
//--- Store the value
|
|
else
|
|
if(field == 2 && wire == PB_LENGTH)
|
|
reader.ReadString(value);
|
|
//--- Step over anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Report only the entry that names where the data sits
|
|
return (key == "location") ? value : "";
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read a sparse initializer through its values tensor |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParseSparse(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Point a cursor at the sparse tensor
|
|
CPbReader reader;
|
|
if(!reader.Attach(raw, from, to))
|
|
return false;
|
|
uchar inner[];
|
|
reader.Bytes(inner);
|
|
//--- Remember where the shape and the values sit
|
|
int valFrom = -1, valTo = -1;
|
|
int dim[OMV_MAX_DIMS];
|
|
int dims = 0;
|
|
//--- Read every field of the sparse tensor
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Locate the tensor holding the values themselves
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
{
|
|
if(!reader.ReadBlock(valFrom, valTo))
|
|
break;
|
|
}
|
|
//--- Store one axis of the shape the sparse tensor stands for
|
|
else
|
|
if(field == 3 && wire == PB_VARINT)
|
|
{
|
|
//--- Keep the axis while room remains
|
|
if(dims < OMV_MAX_DIMS)
|
|
dim[dims++] = (int)reader.ReadVarint();
|
|
else
|
|
reader.ReadVarint();
|
|
}
|
|
//--- Read several axes packed together
|
|
else
|
|
if(field == 3 && wire == PB_LENGTH)
|
|
{
|
|
//--- Locate the packed block and read every length it holds
|
|
int p1, p2;
|
|
if(!reader.ReadBlock(p1, p2))
|
|
break;
|
|
CPbReader packed;
|
|
packed.Attach(inner, p1, p2);
|
|
while(!packed.Eof() && dims < OMV_MAX_DIMS)
|
|
dim[dims++] = (int)packed.ReadVarint();
|
|
}
|
|
//--- Step over the indices and anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Nothing to report without the values tensor
|
|
if(valFrom < 0)
|
|
return false;
|
|
//--- Read the values as an ordinary tensor
|
|
if(!ParseTensor(inner, valFrom, valTo))
|
|
return false;
|
|
//--- Correct the entry to describe the sparse tensor it came from
|
|
const int slot = m_tensors - 1;
|
|
m_tensor[slot].category = "Sparse Initializer";
|
|
if(dims > 0)
|
|
{
|
|
//--- Report the shape the sparse tensor stands for, not the value run
|
|
m_tensor[slot].dims = dims;
|
|
for(int i = 0; i < dims; i++)
|
|
m_tensor[slot].dim[i] = dim[i];
|
|
m_tensor[slot].values = 1;
|
|
for(int i = 0; i < dims; i++)
|
|
m_tensor[slot].values *= dim[i];
|
|
}
|
|
//--- Report the sparse tensor as read
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read one value info into an exposed port |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParsePort(const uchar &raw[], const int from, const int to, OmvPort &port)
|
|
{
|
|
//--- Start an empty port
|
|
port.name = "";
|
|
port.dims = 0;
|
|
port.elemType = 0;
|
|
//--- Point a cursor at this value info
|
|
CPbReader value;
|
|
if(!value.Attach(raw, from, to))
|
|
return false;
|
|
//--- Keep the slice so nested payloads can be located
|
|
uchar level1[];
|
|
value.Bytes(level1);
|
|
//--- Read every field of the value info
|
|
while(!value.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!value.ReadTag(field, wire))
|
|
break;
|
|
//--- Store the port name
|
|
if(field == 1 && wire == PB_LENGTH)
|
|
{
|
|
value.ReadString(port.name);
|
|
continue;
|
|
}
|
|
//--- Step over anything but the type description
|
|
if(field != 2 || wire != PB_LENGTH)
|
|
{
|
|
if(!value.SkipField(wire))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Descend into the type description
|
|
int t1, t2;
|
|
if(!value.ReadBlock(t1, t2))
|
|
break;
|
|
CPbReader type;
|
|
type.Attach(level1, t1, t2);
|
|
uchar level2[];
|
|
type.Bytes(level2);
|
|
//--- Read every field of the type
|
|
while(!type.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int f2, w2;
|
|
if(!type.ReadTag(f2, w2))
|
|
break;
|
|
//--- Step over anything but the tensor description
|
|
if(f2 != 1 || w2 != PB_LENGTH)
|
|
{
|
|
if(!type.SkipField(w2))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Descend into the tensor description
|
|
int s1, s2;
|
|
if(!type.ReadBlock(s1, s2))
|
|
break;
|
|
CPbReader tt;
|
|
tt.Attach(level2, s1, s2);
|
|
uchar level3[];
|
|
tt.Bytes(level3);
|
|
//--- Read every field of the tensor description
|
|
while(!tt.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int f3, w3;
|
|
if(!tt.ReadTag(f3, w3))
|
|
break;
|
|
//--- Store the element type
|
|
if(f3 == 1 && w3 == PB_VARINT)
|
|
{
|
|
port.elemType = (int)tt.ReadVarint();
|
|
continue;
|
|
}
|
|
//--- Step over anything but the shape
|
|
if(f3 != 2 || w3 != PB_LENGTH)
|
|
{
|
|
if(!tt.SkipField(w3))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Descend into the shape
|
|
int p1, p2;
|
|
if(!tt.ReadBlock(p1, p2))
|
|
break;
|
|
CPbReader shape;
|
|
shape.Attach(level3, p1, p2);
|
|
uchar level4[];
|
|
shape.Bytes(level4);
|
|
//--- Read every axis the shape holds
|
|
while(!shape.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int f4, w4;
|
|
if(!shape.ReadTag(f4, w4))
|
|
break;
|
|
//--- Step over anything but an axis
|
|
if(f4 != 1 || w4 != PB_LENGTH)
|
|
{
|
|
if(!shape.SkipField(w4))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Descend into the axis
|
|
int d1, d2;
|
|
if(!shape.ReadBlock(d1, d2))
|
|
break;
|
|
CPbReader dim;
|
|
dim.Attach(level4, d1, d2);
|
|
//--- Read every field of the axis
|
|
while(!dim.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int f5, w5;
|
|
if(!dim.ReadTag(f5, w5))
|
|
break;
|
|
//--- Store the axis length
|
|
if(f5 == 1 && w5 == PB_VARINT)
|
|
{
|
|
if(port.dims < OMV_MAX_DIMS)
|
|
port.dim[port.dims++] = (int)dim.ReadVarint();
|
|
else
|
|
dim.ReadVarint();
|
|
}
|
|
//--- Step over a named axis
|
|
else
|
|
if(!dim.SkipField(w5))
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
//--- Report a port that carries a name
|
|
return StringLen(port.name) > 0;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Read the graph into nodes, weights and ports |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::ParseGraph(const uchar &raw[], const int from, const int to)
|
|
{
|
|
//--- Point a cursor at the graph
|
|
CPbReader graph;
|
|
if(!graph.Attach(raw, from, to))
|
|
{
|
|
m_error = "graph out of range";
|
|
return false;
|
|
}
|
|
//--- Keep the slice so nested payloads can be located
|
|
uchar inner[];
|
|
graph.Bytes(inner);
|
|
//--- Read every field of the graph
|
|
while(!graph.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!graph.ReadTag(field, wire))
|
|
break;
|
|
//--- Step over anything that is not a nested message
|
|
if(wire != PB_LENGTH && field != 0)
|
|
{
|
|
if(!graph.SkipField(wire))
|
|
break;
|
|
continue;
|
|
}
|
|
//--- Locate the nested payload
|
|
int f2, t2;
|
|
if(!graph.ReadBlock(f2, t2))
|
|
break;
|
|
//--- Read a node
|
|
if(field == 1)
|
|
ParseNode(inner, f2, t2);
|
|
//--- Read an initializer
|
|
else
|
|
if(field == 5)
|
|
ParseTensor(inner, f2, t2);
|
|
//--- Read a sparse initializer, whose values sit one level deeper
|
|
else
|
|
if(field == 15)
|
|
ParseSparse(inner, f2, t2);
|
|
//--- Decode the graph name from its payload
|
|
else
|
|
if(field == 2)
|
|
{
|
|
uchar tmp[];
|
|
ArrayResize(tmp, t2 - f2);
|
|
for(int i = f2; i < t2; i++)
|
|
tmp[i - f2] = inner[i];
|
|
m_graphName = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
|
}
|
|
//--- Read a graph input
|
|
else
|
|
if(field == 11)
|
|
{
|
|
OmvPort port;
|
|
if(ParsePort(inner, f2, t2, port))
|
|
{
|
|
ArrayResize(m_input, m_inputs + 1);
|
|
m_input[m_inputs++] = port;
|
|
}
|
|
}
|
|
//--- Read a key and value pair the file attached to the graph
|
|
else
|
|
if(field == 16)
|
|
ReadPair(inner, f2, t2, m_gMetaKey, m_gMetaValue, m_gMetas);
|
|
//--- Count a quantization mapping the graph carries
|
|
else
|
|
if(field == 14)
|
|
m_quants++;
|
|
//--- Read whatever the file says about the graph
|
|
else
|
|
if(field == 10)
|
|
{
|
|
//--- Decode the text from its own payload
|
|
uchar tmp[];
|
|
ArrayResize(tmp, t2 - f2);
|
|
for(int i = f2; i < t2; i++)
|
|
tmp[i - f2] = inner[i];
|
|
m_graphDoc = CharArrayToString(tmp, 0, t2 - f2, CP_UTF8);
|
|
}
|
|
//--- Read what the file says about a value passing between layers
|
|
else
|
|
if(field == 13)
|
|
{
|
|
OmvPort port;
|
|
if(ParsePort(inner, f2, t2, port))
|
|
{
|
|
ArrayResize(m_inner, m_inners + 1);
|
|
m_inner[m_inners++] = port;
|
|
}
|
|
}
|
|
//--- Read a graph output
|
|
else
|
|
if(field == 12)
|
|
{
|
|
OmvPort port;
|
|
if(ParsePort(inner, f2, t2, port))
|
|
{
|
|
ArrayResize(m_output, m_outputs + 1);
|
|
m_output[m_outputs++] = port;
|
|
}
|
|
}
|
|
}
|
|
//--- Report success
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Open a file and read its whole structure |
|
|
//+------------------------------------------------------------------+
|
|
bool COmvModel::Load(const string path)
|
|
{
|
|
//--- Drop any model held from a previous call
|
|
Clear();
|
|
//--- Read the whole file into memory
|
|
CPbReader reader;
|
|
if(!reader.LoadFile(path))
|
|
{
|
|
m_error = "cannot open " + path;
|
|
return false;
|
|
}
|
|
//--- Keep the bytes so nested payloads can be located
|
|
uchar raw[];
|
|
reader.Bytes(raw);
|
|
//--- Read every field of the model
|
|
while(!reader.Eof())
|
|
{
|
|
//--- Take the next field tag
|
|
int field, wire;
|
|
if(!reader.ReadTag(field, wire))
|
|
break;
|
|
//--- Store the format revision
|
|
if(field == 1 && wire == PB_VARINT)
|
|
m_irVersion = (long)reader.ReadVarint();
|
|
//--- Store the producing tool
|
|
else
|
|
if(field == 2 && wire == PB_LENGTH)
|
|
reader.ReadString(m_producer);
|
|
//--- Store the version of that tool
|
|
else
|
|
if(field == 3 && wire == PB_LENGTH)
|
|
reader.ReadString(m_producerVersion);
|
|
//--- Store the domain the model belongs to
|
|
else
|
|
if(field == 4 && wire == PB_LENGTH)
|
|
reader.ReadString(m_domain);
|
|
//--- Store the revision the model gives itself
|
|
else
|
|
if(field == 5 && wire == PB_VARINT)
|
|
m_modelVersion = (long)reader.ReadVarint();
|
|
//--- Store whatever the file says about itself
|
|
else
|
|
if(field == 6 && wire == PB_LENGTH)
|
|
reader.ReadString(m_docString);
|
|
//--- Store a key and value pair the file attached to the model
|
|
else
|
|
if(field == 14 && wire == PB_LENGTH)
|
|
{
|
|
//--- Read the pair and keep it while room remains
|
|
int e1, e2;
|
|
if(!reader.ReadBlock(e1, e2))
|
|
break;
|
|
ReadPair(raw, e1, e2, m_metaKey, m_metaValue, m_metas);
|
|
}
|
|
//--- Handle training state the file stores with the model
|
|
else
|
|
if(field == 20 && wire == PB_LENGTH)
|
|
{
|
|
//--- Step past it, only its presence is reported
|
|
int g1, g2;
|
|
if(!reader.ReadBlock(g1, g2))
|
|
break;
|
|
m_hasTraining = true;
|
|
}
|
|
//--- Store an operator the file defines for itself
|
|
else
|
|
if(field == 25 && wire == PB_LENGTH)
|
|
{
|
|
//--- Read enough of it to name it and say what it holds
|
|
int u1, u2;
|
|
if(!reader.ReadBlock(u1, u2))
|
|
break;
|
|
ReadFunction(raw, u1, u2);
|
|
}
|
|
//--- Read the graph itself
|
|
else
|
|
if(field == 7 && wire == PB_LENGTH)
|
|
{
|
|
int from, to;
|
|
if(!reader.ReadBlock(from, to))
|
|
break;
|
|
if(!ParseGraph(raw, from, to))
|
|
return false;
|
|
}
|
|
//--- Read the operator set version
|
|
else
|
|
if(field == 8 && wire == PB_LENGTH)
|
|
{
|
|
//--- Locate the operator set message
|
|
int from, to;
|
|
if(!reader.ReadBlock(from, to))
|
|
break;
|
|
//--- Read its version field
|
|
CPbReader ops;
|
|
ops.Attach(raw, from, to);
|
|
while(!ops.Eof())
|
|
{
|
|
int f2, w2;
|
|
if(!ops.ReadTag(f2, w2))
|
|
break;
|
|
if(f2 == 2 && w2 == PB_VARINT)
|
|
m_opset = (int)ops.ReadVarint();
|
|
else
|
|
if(!ops.SkipField(w2))
|
|
break;
|
|
}
|
|
}
|
|
//--- Step over anything else
|
|
else
|
|
if(!reader.SkipField(wire))
|
|
break;
|
|
}
|
|
//--- Report a file that could not be decoded
|
|
if(reader.Failed() && m_nodes == 0)
|
|
{
|
|
m_error = "malformed protobuf in " + path;
|
|
return false;
|
|
}
|
|
//--- Report a file that holds no graph
|
|
if(m_nodes == 0)
|
|
{
|
|
m_error = "no nodes found in " + path;
|
|
return false;
|
|
}
|
|
//--- Report success
|
|
return true;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Total every weight the file carries |
|
|
//+------------------------------------------------------------------+
|
|
int COmvModel::Parameters(void) const
|
|
{
|
|
//--- Add the value count of every tensor
|
|
int total = 0;
|
|
for(int i = 0; i < m_tensors; i++)
|
|
total += m_tensor[i].values;
|
|
//--- Return the running total
|
|
return total;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Render a shape as a bracketed list |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::DimsText(const int &dim[], const int dims) const
|
|
{
|
|
//--- Print empty brackets for an unshaped tensor
|
|
if(dims <= 0)
|
|
return "[]";
|
|
//--- Join every axis with a separator
|
|
string out = "[";
|
|
for(int i = 0; i < dims; i++)
|
|
{
|
|
//--- Separate all but the first axis
|
|
if(i > 0)
|
|
out += "x";
|
|
//--- Append this axis length
|
|
out += IntegerToString(dim[i]);
|
|
}
|
|
//--- Close the bracket
|
|
return out + "]";
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Name an ONNX element type code |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::TypeName(const int code) const
|
|
{
|
|
//--- Name each code the way the reader maps the ONNX enum
|
|
switch(code)
|
|
{
|
|
case 0:
|
|
return "undefined";
|
|
case 1:
|
|
return "float32";
|
|
case 2:
|
|
return "uint8";
|
|
case 3:
|
|
return "int8";
|
|
case 4:
|
|
return "uint16";
|
|
case 5:
|
|
return "int16";
|
|
case 6:
|
|
return "int32";
|
|
case 7:
|
|
return "int64";
|
|
case 8:
|
|
return "string";
|
|
case 9:
|
|
return "boolean";
|
|
case 10:
|
|
return "float16";
|
|
case 11:
|
|
return "float64";
|
|
case 12:
|
|
return "uint32";
|
|
case 13:
|
|
return "uint64";
|
|
case 14:
|
|
return "complex<float32>";
|
|
case 15:
|
|
return "complex<float64>";
|
|
case 16:
|
|
return "bfloat16";
|
|
}
|
|
//--- Print the raw code for anything unrecognized
|
|
return "type" + IntegerToString(code);
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Join the axes the way a shape row reports them |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::DimsBare(const int &dim[], const int dims) const
|
|
{
|
|
//--- Report nothing for a tensor the file never shaped
|
|
if(dims <= 0)
|
|
return "";
|
|
//--- Join every axis with a comma and no brackets
|
|
string out = "";
|
|
for(int i = 0; i < dims; i++)
|
|
{
|
|
//--- Separate all but the first axis
|
|
if(i > 0)
|
|
out += ",";
|
|
//--- Append this axis length
|
|
out += IntegerToString(dim[i]);
|
|
}
|
|
//--- Report the joined axes
|
|
return out;
|
|
}
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Render a shape the way a tensor type is written |
|
|
//+------------------------------------------------------------------+
|
|
string COmvModel::DimsList(const int &dim[], const int dims) const
|
|
{
|
|
//--- An unshaped tensor prints as empty brackets rather than nothing
|
|
if(dims <= 0)
|
|
return "[]";
|
|
//--- Separate the axes with commas the way a tensor type reads
|
|
string out = "[";
|
|
for(int i = 0; i < dims; i++)
|
|
{
|
|
//--- Separate all but the first axis
|
|
if(i > 0)
|
|
out += ",";
|
|
//--- Append this axis length
|
|
out += IntegerToString(dim[i]);
|
|
}
|
|
//--- Close the bracket
|
|
return out + "]";
|
|
}
|
|
|
|
#endif // OMV_MODEL_MQH
|
|
//+------------------------------------------------------------------+ |