CBitBuffer/CBitBuffer.mqh

//+------------------------------------------------------------------+
//|                                                   CBitBuffer.mqh |
//|                                        Copyright © 2025, Amr Ali |
//|                             https://www.mql5.com/en/users/amrali |
//+------------------------------------------------------------------+
#property copyright "Amr Ali"
#property link      "https://www.mql5.com/en/users/amrali"
#property version   "1.02"

// Updates:
// 2025.07.21 - v.1.01 : The CBitBuffer class actively prevents illegal attempts to write after a read operation has begun,
//                     : which helps in maintaining data integrity. Users receive a specific error (BIT_BUFFER_MIXED_OPERATION_ERROR)
//                     : when they attempt to mix operations.
// 2025.07.22 - v.1.02 : Changed design of the class to allow the mixed read/write operations, by correct handling of partial
//                     : flushes/refills, which helps in maintaining data integrity when switching between read and write modes.
//                     : Removed the variable (m_operationMode) and the error code (BIT_BUFFER_MIXED_OPERATION_ERROR).
//                     : Cleaned the code comments for better clarity of intent.
//                     : Updated examples in "CBitBuffer_Test.mq5" to cover more test cases.

//--- Bit size constants for MQL5 data types ---
enum ENUM_BIT_SIZES
  {
   BIT_SIZE_BOOL      = 1,
   BIT_SIZE_CHAR      = 8,
   BIT_SIZE_UCHAR     = 8,
   BIT_SIZE_SHORT     = 16,
   BIT_SIZE_USHORT    = 16,
   BIT_SIZE_INT       = 32,
   BIT_SIZE_UINT      = 32,
   BIT_SIZE_LONG      = 64,
   BIT_SIZE_ULONG     = 64,
   BIT_SIZE_DATETIME  = 64,
   BIT_SIZE_FLOAT     = 32,
   BIT_SIZE_DOUBLE    = 64
  };

//--- Error codes for CBitBuffer operations ---
enum ENUM_BIT_BUFFER_ERROR
  {
   BIT_BUFFER_NO_ERROR                 = 0,  // No error
   BIT_BUFFER_INVALID_BIT_COUNT        = 1,  // Invalid number of bits specified for read/write
   BIT_BUFFER_MEMORY_ALLOCATION_FAILED = 2,  // Memory allocation failure
   BIT_BUFFER_READ_OUT_OF_BOUNDS       = 3,  // Attempt to read beyond available data
   BIT_BUFFER_WRITE_FLUSH_FAILED       = 4,  // Write operation failed during internal buffer flush
   BIT_BUFFER_INVALID_READ_POSITION    = 5,  // Attempt to set read position out of valid bounds
   BIT_BUFFER_DATA_TOO_LARGE           = 6,  // Data length exceeds maximum allowed capacity
   BIT_BUFFER_READ_DATA_INCOMPLETE     = 7,  // Incomplete data found during read
   BIT_BUFFER_RAW_OPERATION_FAILED     = 8,  // Failed to copy raw buffer data
   BIT_BUFFER_FILE_SAVE_FAILED         = 9,  // File save failed
   BIT_BUFFER_FILE_LOAD_FAILED         = 10  // File load failed
  };

//+------------------------------------------------------------------+
//| CBitBuffer Class                                                 |
//| Reads/writes individual bits or bit sequences to ulong[] buffer. |
//| Supports efficient bit manipulation and mixed read/write ops.    |
//+------------------------------------------------------------------+
class CBitBuffer
  {
private:
   ulong             m_mainBuffer[];          // Main storage for bits (ulong = 64 bits)
   long              m_totalWrittenBits;      // Total bits written into buffer
   long              m_currentReadBitPosition;// Current bit position for reading
   ulong             m_writeBufferInternal;   // Internal buffer for accumulating bits before writing (up to 64 bits)
   int               m_writeBufferBitCounter; // Bits in m_writeBufferInternal
   ulong             m_readBufferInternal;    // Internal buffer for holding bits read from mainBuffer (up to 64 bits)
   int               m_readBufferBitCounter;  // Bits in m_readBufferInternal
   ENUM_BIT_BUFFER_ERROR m_lastError;         // Stores the last error code

private: // Constants for variable-length encoding (VLQ)
   enum ENUM_VLQ_CONSTANTS
     {
      VLQ_CONTINUATION_BIT    = 0x80, // MSB indicates more bytes follow
      VLQ_DATA_BITS           = 0x7F  // Lower 7 bits for data
     };

private: // Helper methods
   // Dynamically allocates memory for m_mainBuffer with exponential growth.
   bool              EnsureBufferCapacity(long requiredSize);
   // Flushes any pending writes from the write cache to the main buffer.
   bool              FlushWriteCache();
   // Refills the internal read buffer efficiently from the main buffer.
   bool              RefillReadBuffer();
   // Creates a 64-bit bitmask for the rightmost 'n' bits.
   ulong             Bitmask64(int n)  const { return (n == 64) ? 0xFFFFFFFFFFFFFFFF : ((1ULL << n) - 1); }

public: // Constructor and Destructor
                     CBitBuffer();
                     CBitBuffer(int initialCapacityBytes);
                    ~CBitBuffer();

public: // Core bit operations
   bool              WriteBit(bool bit);                   // Writes a single bit
   bool              WriteBits(ulong value, int numberOfBits); // Writes N bits from a ulong
   bool              ReadBit();                            // Reads a single bit
   ulong             ReadBits(int numberOfBits);           // Reads N bits as a ulong
   ulong             PeekBits(int numberOfBits);           // Reads N bits without advancing position

public: // Position and size management
   bool              SetReadPosition(long bitPosition);    // Sets read position in bits
   long              GetReadPosition()              const { return m_currentReadBitPosition; }
   bool              ResetReadPosition()                  { return SetReadPosition(0); } // Resets read position to 0
   bool              SkipBits(long bitsToSkip);            // Skips N bits from current read position
   long              GetTotalWrittenBits()          const { return m_totalWrittenBits + m_writeBufferBitCounter; } // Total bits written
   long              GetTotalBytesWritten()         const { return (GetTotalWrittenBits() + 7) / 8; }              // Total bytes written
   long              GetTotalBytesAllocated()       const { return ArraySize(m_mainBuffer) * sizeof(ulong); }      // Total allocated bytes
   long              GetRemainingReadBits()         const { return GetTotalWrittenBits() - GetReadPosition(); }    // Remaining bits to read

public: // Data type specific read/write operations
   bool              WriteBool(bool value)                { return WriteBit(value);                           }
   bool              WriteChar(char value)                { return WriteBits(value, BIT_SIZE_CHAR);           }
   bool              WriteUChar(uchar value)              { return WriteBits(value, BIT_SIZE_UCHAR);          }
   bool              WriteShort(short value)              { return WriteBits(value, BIT_SIZE_SHORT);          }
   bool              WriteUShort(ushort value)            { return WriteBits(value, BIT_SIZE_USHORT);         }
   bool              WriteInt(int value)                  { return WriteBits(value, BIT_SIZE_INT);            }
   bool              WriteUInt(uint value)                { return WriteBits(value, BIT_SIZE_UINT);           }
   bool              WriteLong(long value)                { return WriteBits(value, BIT_SIZE_LONG);           }
   bool              WriteULong(ulong value)              { return WriteBits(value, BIT_SIZE_ULONG);          }
   bool              WriteDatetime(datetime value)        { return WriteBits(value, BIT_SIZE_DATETIME);       }
   bool              WriteFloat(float value);              // Writes a 32-bit float
   bool              WriteDouble(double value);            // Writes a 64-bit double
   bool              WriteString(string value);            // Writes string with length prefix
   template<typename T>
   bool              WriteStruct(T &struct_object);        // Writes struct with length prefix

   bool              ReadBool()                           { return ReadBit();                                 }
   char              ReadChar()                           { return (char)ReadBits(BIT_SIZE_CHAR);             }
   uchar             ReadUChar()                          { return (uchar)ReadBits(BIT_SIZE_UCHAR);           }
   short             ReadShort()                          { return (short)ReadBits(BIT_SIZE_SHORT);           }
   ushort            ReadUShort()                         { return (ushort)ReadBits(BIT_SIZE_USHORT);         }
   int               ReadInt()                            { return (int)ReadBits(BIT_SIZE_INT);               }
   uint              ReadUInt()                           { return (uint)ReadBits(BIT_SIZE_UINT);             }
   long              ReadLong()                           { return (long)ReadBits(BIT_SIZE_LONG);             }
   ulong             ReadULong()                          { return ReadBits(BIT_SIZE_ULONG);                  }
   datetime          ReadDatetime()                       { return (datetime)ReadBits(BIT_SIZE_DATETIME);     }
   float             ReadFloat();                          // Reads a 32-bit float
   double            ReadDouble();                         // Reads a 64-bit double
   string            ReadString();                         // Reads string with length prefix
   template<typename T>
   T                 ReadStruct();                         // Reads struct with length prefix

public: // Variable-length encoding for integers (VLQ)
   bool              WriteVarInt(int value)               { ClearLastError(); return WriteVarULong((uint)((value << 1) ^ (value >> 31))); } // ZigZag + VLQ
   bool              WriteVarUInt(uint value)             { ClearLastError(); return WriteVarULong(value); } // VLQ
   bool              WriteVarLong(long value)             { ClearLastError(); return WriteVarULong((ulong)((value << 1) ^ (value >> 63))); } // ZigZag + VLQ
   bool              WriteVarULong(ulong value);           // Writes unsigned long using VLQ
   int               ReadVarInt();                         // Reads signed int using ZigZag + VLQ
   uint              ReadVarUInt();                        // Reads unsigned int using VLQ
   long              ReadVarLong();                        // Reads signed long using ZigZag + VLQ
   ulong             ReadVarULong();                       // Reads unsigned long using VLQ

public: // Buffer management
   void              Clear();                              // Clears buffer content and resets state
   bool              GetFinalizedBuffer(ulong &destinationArray[]); // Copies buffer content to array
   bool              SetRawBuffer(const ulong &sourceBuffer[], long numBits = -1); // Sets buffer content from array
   bool              Save(string filename, bool common = false); // Saves buffer to file
   bool              Load(string filename, bool common = false); // Loads buffer from file

public: // Error handling
   ENUM_BIT_BUFFER_ERROR GetLastError()           const { return m_lastError; }       // Returns last error code
   string            GetLastErrorString()         const;                               // Returns error description string
   void              ClearLastError()                   { m_lastError = BIT_BUFFER_NO_ERROR; } // Clears last error

   void              PrintHex(int start = 0, int count = WHOLE_ARRAY) const; // Prints main buffer in hex (debugging)
  };

//+------------------------------------------------------------------+
//| Default Constructor                                              |
//| Initializes the CBitBuffer instance.                             |
//+------------------------------------------------------------------+
CBitBuffer::CBitBuffer()
  {
   ArrayResize(m_mainBuffer, 1, 1024); // Start with a minimum of one ulong.
   m_totalWrittenBits         = 0;
   m_currentReadBitPosition   = 0;
   m_writeBufferInternal      = 0;
   m_writeBufferBitCounter    = 0;
   m_readBufferInternal       = 0;
   m_readBufferBitCounter     = 0;
   m_lastError                = BIT_BUFFER_NO_ERROR;
  }

//+------------------------------------------------------------------+
//| Constructor with Initial Capacity                                |
//| Initializes with specified byte capacity.                        |
//+------------------------------------------------------------------+
CBitBuffer::CBitBuffer(int initialCapacityBytes)
  {
   // Calculate required ulongs based on initial capacity in bytes.
   long initialUlongs = (initialCapacityBytes * 8 + 63) / 64;
   if (ArrayResize(m_mainBuffer, (int)initialUlongs) == -1)
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to resize main buffer to %d bytes.", m_lastError, initialCapacityBytes);
     }
   m_totalWrittenBits         = 0;
   m_currentReadBitPosition   = 0;
   m_writeBufferInternal      = 0;
   m_writeBufferBitCounter    = 0;
   m_readBufferInternal       = 0;
   m_readBufferBitCounter     = 0;
   m_lastError                = BIT_BUFFER_NO_ERROR;
  }

//+------------------------------------------------------------------+
//| Destructor: Releases the memory allocated for the main buffer.   |
//+------------------------------------------------------------------+
CBitBuffer::~CBitBuffer()
  {
   ArrayFree(m_mainBuffer);
  }

//---
//| PRIVATE HELPER METHODS
//---

//+------------------------------------------------------------------+
//| Ensures buffer has enough capacity (2x growth, min increment).   |
//+------------------------------------------------------------------+
bool CBitBuffer::EnsureBufferCapacity(long requiredSize)
  {
   if (requiredSize <= ArraySize(m_mainBuffer)) return true;

   long currentCapacity = ArraySize(m_mainBuffer);
   long newCapacity = MathMax(requiredSize, currentCapacity * 2);
   newCapacity = MathMax(newCapacity, currentCapacity + 256); // Grow by at least 256 ulongs

   if (ArrayResize(m_mainBuffer, (int)newCapacity, 1024) == -1)
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to resize main buffer to %d ulongs. Current: %d, Required: %d",
                  m_lastError, newCapacity, currentCapacity, requiredSize);
      return false;
     }
   return true;
  }

//+------------------------------------------------------------------+
//| Flushes accumulated bits from write buffer to main buffer.       |
//| Handles merging into partially filled ulongs.                    |
//+------------------------------------------------------------------+
bool CBitBuffer::FlushWriteCache()
  {
   if (m_writeBufferBitCounter == 0) return true;

   long targetArrIndex = m_totalWrittenBits / 64;
   int  bitOffsetInUlong = (int)(m_totalWrittenBits % 64);

   // Ensure capacity for the target ulong.
   if (!EnsureBufferCapacity(targetArrIndex + 1))
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to resize main buffer during flush.", m_lastError);
      return false;
     }

   // If the target ulong is partially filled, merge new bits. Otherwise, assign directly.
   if (bitOffsetInUlong != 0)
     {
      // Shift new bits into position and OR with existing bits.
      m_mainBuffer[targetArrIndex] |= (m_writeBufferInternal << bitOffsetInUlong);
     }
   else
     {
      // Target ulong is empty/aligned, just assign.
      m_mainBuffer[targetArrIndex] = m_writeBufferInternal;
     }

   m_totalWrittenBits += m_writeBufferBitCounter;
   m_writeBufferInternal = 0;
   m_writeBufferBitCounter = 0;

   return true;
  }

//+------------------------------------------------------------------+
//| Refills internal read buffer from main buffer.                   |
//| Handles unaligned read positions by shifting.                    |
//+------------------------------------------------------------------+
bool CBitBuffer::RefillReadBuffer()
  {
   long targetArrIndex = m_currentReadBitPosition / 64;
   int  bitOffsetInUlong = (int)(m_currentReadBitPosition % 64);

   // Check for read out of bounds.
   if (targetArrIndex >= ArraySize(m_mainBuffer))
     {
      m_lastError = BIT_BUFFER_READ_OUT_OF_BOUNDS;
      PrintFormat("Error %d: Attempt to read beyond main buffer size during RefillReadBuffer.", m_lastError);
      return false;
     }

   // Load the full ulong from the main buffer.
   m_readBufferInternal = m_mainBuffer[targetArrIndex];

   // If not 64-bit aligned, shift out already-read bits.
   if (bitOffsetInUlong != 0)
     {
      m_readBufferInternal >>= bitOffsetInUlong;
     }

   m_readBufferBitCounter = 64 - bitOffsetInUlong; // Update available bits in internal buffer.

   return true;
  }

//---
//| PUBLIC CORE WRITE METHODS
//---

//+------------------------------------------------------------------+
//| Writes a single boolean bit.                                     |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteBit(bool bitValue)
  {
   return WriteBits(bitValue ? 1 : 0, BIT_SIZE_BOOL);
  }

//+------------------------------------------------------------------+
//| Writes a specified number of bits from a ulong value.            |
//| Handles splitting writes across internal and main buffer.        |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteBits(ulong value, int numberOfBits)
  {
   ClearLastError();
   if (numberOfBits <= 0 || numberOfBits > 64)
     {
      m_lastError = BIT_BUFFER_INVALID_BIT_COUNT;
      PrintFormat("Error %d: WriteBits called with invalid numberOfBits (%d), must be 1-64.", m_lastError, numberOfBits);
      return false;
     }

   // Invalidate internal read buffer as we are writing.
   if (m_readBufferBitCounter > 0)
     {
      m_readBufferInternal = 0;
      m_readBufferBitCounter = 0;
     }

   // Mask value to only keep relevant bits for N < 64.
   if (numberOfBits < 64)
     {
      value &= Bitmask64(numberOfBits);
     }

   // Calculate the effective capacity remaining in the current ulong block
   // in the main buffer, considering m_totalWrittenBits alignment.
   int partialUlongBitsFlushed = (int)(m_totalWrittenBits % 64);
   int effectiveWriteBufferCapacity = 64 - partialUlongBitsFlushed;

   // Calculate how many bits can fit into the current internal write buffer
   // (which is implicitly filling the current ulong block).
   int spaceLeft = effectiveWriteBufferCapacity - m_writeBufferBitCounter;

   // Fast path: The new bits fit entirely into the current internal write buffer.
   if (numberOfBits <= spaceLeft)
     {
      // Shift the new bits into the internal buffer.
      // Bits are accumulated from the LSB (rightmost) side.
      m_writeBufferInternal |= value << m_writeBufferBitCounter;
      m_writeBufferBitCounter += numberOfBits;

      // If the internal buffer is now full (meaning the current ulong block is complete),
      // flush it to the main buffer.
      if (m_writeBufferBitCounter == effectiveWriteBufferCapacity)
        {
         if (!FlushWriteCache())
           {
            m_lastError = BIT_BUFFER_WRITE_FLUSH_FAILED;
            PrintFormat("Error %d: Flush failed after full buffer.", m_lastError);
            return false;
           }
        }
     }
   // Slow path: The new bits do not fit entirely, split across internal buffer and main buffer.
   else
     {
      // Fill the current internal buffer completely with the first part of 'value'.
      ulong maskForFirstPart = Bitmask64(spaceLeft);
      m_writeBufferInternal |= (value & maskForFirstPart) << m_writeBufferBitCounter;
      m_writeBufferBitCounter = effectiveWriteBufferCapacity; // Current ulong block is now full.

      // Flush the full (capacity) internal buffer to the main buffer.
      if (!FlushWriteCache())
        {
         m_lastError = BIT_BUFFER_WRITE_FLUSH_FAILED;
         PrintFormat("Error %d: Flush failed during split write (first part).", m_lastError);
         return false;
        }

      // Prepare remaining bits from the original 'value' for the next write.
      value >>= spaceLeft; // Shift out the bits that were just written.
      numberOfBits -= spaceLeft; // Reduce the count of bits remaining to write.

      // Shift remaining bits into the now-empty internal buffer (guaranteed to fit now).
      m_writeBufferInternal = value;
      m_writeBufferBitCounter = numberOfBits;
     }
   return true;
  }

//---
//| PUBLIC CORE READ METHODS
//---

//+------------------------------------------------------------------+
//| Reads a single bit.                                              |
//+------------------------------------------------------------------+
bool CBitBuffer::ReadBit()
  {
   return ReadBits(BIT_SIZE_BOOL) != 0;
  }

//+------------------------------------------------------------------+
//| Reads a specified number of bits as a ulong.                     |
//| Flushes write cache and refills read cache as needed.            |
//+------------------------------------------------------------------+
ulong CBitBuffer::ReadBits(int numberOfBits)
  {
   ClearLastError();
   if (numberOfBits <= 0 || numberOfBits > 64)
     {
      m_lastError = BIT_BUFFER_INVALID_BIT_COUNT;
      PrintFormat("Error %d: ReadBits called with invalid numberOfBits (%d), must be 1-64.", m_lastError, numberOfBits);
      return 0;
     }

   // Ensure pending writes are flushed to see latest data.
   if (m_writeBufferBitCounter > 0)
     {
      if (!FlushWriteCache())
        {
         m_lastError = BIT_BUFFER_WRITE_FLUSH_FAILED;
         PrintFormat("Error %d: Failed to flush write buffer before reading.", m_lastError);
         return 0;
        }
     }

   // Check for reading beyond written data.
   if (m_currentReadBitPosition + numberOfBits > m_totalWrittenBits)
     {
      m_lastError = BIT_BUFFER_READ_OUT_OF_BOUNDS;
      PrintFormat("Error %d: Attempt to read beyond written data. ReadPos:%d, Requested bits:%d, Remaining bits:%d",
                  m_lastError, m_currentReadBitPosition, numberOfBits, m_totalWrittenBits - m_currentReadBitPosition);
      return 0;
     }

   ulong readValue = 0;
   int bitsReadIntoResult = 0;

   while (bitsReadIntoResult < numberOfBits)
     {
      // Refill internal read buffer if empty.
      if (m_readBufferBitCounter == 0)
        {
         if (!RefillReadBuffer())
           {
            // RefillReadBuffer sets its own specific errors.
            return 0;
           }
        }

      // Copy available bits from internal read buffer.
      int bitsToTake = MathMin(numberOfBits - bitsReadIntoResult, m_readBufferBitCounter);
      ulong mask = Bitmask64(bitsToTake);
      ulong bits = m_readBufferInternal & mask;
      readValue |= (bits << bitsReadIntoResult);

      // Advance internal buffer and main read position.
      m_readBufferInternal >>= bitsToTake;
      m_readBufferBitCounter -= bitsToTake;
      m_currentReadBitPosition += bitsToTake;

      bitsReadIntoResult += bitsToTake;
     }

   return readValue;
  }

//+------------------------------------------------------------------+
//| Reads N bits without advancing read position.                    |
//+------------------------------------------------------------------+
ulong CBitBuffer::PeekBits(int numberOfBits)
  {
   // Save current read state.
   ulong tempReadBuffer   = m_readBufferInternal;
   int   tempReadCounter  = m_readBufferBitCounter;
   long  tempReadPosition = m_currentReadBitPosition;
   ENUM_BIT_BUFFER_ERROR tempLastError = m_lastError;

   // Perform read using ReadBits (handles flushes/refills).
   ulong peekedResult = ReadBits(numberOfBits);

   // Restore original read state.
   m_readBufferInternal     = tempReadBuffer;
   m_readBufferBitCounter   = tempReadCounter;
   m_currentReadBitPosition = tempReadPosition;
   m_lastError              = tempLastError;

   return peekedResult;
  }

//---
//| PUBLIC TYPE-SPECIFIC READ/WRITE METHODS
//---

//+------------------------------------------------------------------+
//| Writes a 32-bit float by converting to its uint bit pattern.     |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteFloat(float value)
  {
   union _f {float value; uint bits;} flt;
   flt.value = value;

   // Write the 32-bit representation of the float.
   return WriteBits(flt.bits, BIT_SIZE_FLOAT);
  }

//+------------------------------------------------------------------+
//| Reads a 32-bit float by converting uint bit pattern to float.    |
//+------------------------------------------------------------------+
float CBitBuffer::ReadFloat()
  {
   ClearLastError();
   // Read the 32-bit representation of float.
   uint bits = (uint)ReadBits(BIT_SIZE_FLOAT);
   if (m_lastError != BIT_BUFFER_NO_ERROR) return 0.0;

   // Union to interpret unsigned integer bits as a float.
   union _f {float value; uint bits;} flt;
   flt.bits = bits;
   return flt.value;
  }

//+------------------------------------------------------------------+
//| Writes a 64-bit double by converting to its ulong bit pattern.   |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteDouble(double value)
  {
   union _d {double value; ulong bits;} dbl;
   dbl.value = value;

   // Write the 64-bit representation of the double.
   return WriteBits(dbl.bits, BIT_SIZE_DOUBLE);
  }

//+------------------------------------------------------------------+
//| Reads a 64-bit double by converting ulong bit pattern to double. |
//+------------------------------------------------------------------+
double CBitBuffer::ReadDouble()
  {
   ClearLastError();
   // Read the 64-bit representation of double.
   ulong bits = ReadBits(BIT_SIZE_DOUBLE);
   if (m_lastError != BIT_BUFFER_NO_ERROR) return 0.0;

   // Union to interpret unsigned long bits as a double.
   union _d {double value; ulong bits;} dbl;
   dbl.bits = bits;
   return dbl.value;
  }

//+------------------------------------------------------------------+
//| Writes a string, prefixed by its UTF-8 byte length (ushort).     |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteString(string value)
  {
   ClearLastError();
   uchar utfBytes[];
   StringToCharArray(value, utfBytes, 0, -1, CP_UTF8);

   int lengthBytes = ArraySize(utfBytes) - 1; // Exclude null terminator
   if (lengthBytes > USHORT_MAX)
     {
      m_lastError = BIT_BUFFER_DATA_TOO_LARGE;
      PrintFormat("Error %d: String length %d exceeds max ushort (%d).", m_lastError, lengthBytes, USHORT_MAX);
      return false;
     }

   if (!WriteUShort((ushort)lengthBytes)) return false; // Write length

   for (int i = 0; i < lengthBytes; i++) // Write each byte
     {
      if (!WriteUChar(utfBytes[i])) return false;
     }
   return true;
  }

//+------------------------------------------------------------------+
//| Reads a string prefixed by its length. Decodes UTF-8 bytes.      |
//+------------------------------------------------------------------+
string CBitBuffer::ReadString()
  {
   ClearLastError();
   ushort lengthChars = ReadUShort();
   if (GetLastError() != BIT_BUFFER_NO_ERROR) return "";

   if (lengthChars == 0xFFFF) return NULL; // Null string
   if (lengthChars == 0) return "";       // Empty string

   long requiredBitsForString = (long)lengthChars * BIT_SIZE_UCHAR;
   if (requiredBitsForString > GetRemainingReadBits())
     {
      m_lastError = BIT_BUFFER_READ_DATA_INCOMPLETE;
      PrintFormat("Error %d: Not enough data for string (%d bits). ReadPos:%d, Remaining:%d",
                  m_lastError, requiredBitsForString, GetReadPosition(), GetRemainingReadBits());
      return "";
     }

   uchar utfBytes[];
   ArrayResize(utfBytes, lengthChars);

   for (int i = 0; i < lengthChars; i++)
     {
      utfBytes[i] = ReadUChar();
      if (GetLastError() != BIT_BUFFER_NO_ERROR) return "";
     }
   return CharArrayToString(utfBytes, 0, -1, CP_UTF8);
  }

//+------------------------------------------------------------------+
//| Writes a struct, prefixed by its byte length (ushort).           |
//+------------------------------------------------------------------+
template<typename T>
bool CBitBuffer::WriteStruct(T &struct_object)
  {
   ClearLastError();
   uchar bytes_arr[];
   ::ResetLastError();
   if(!StructToCharArray(struct_object, bytes_arr))
     {
      m_lastError = BIT_BUFFER_RAW_OPERATION_FAILED;
      PrintFormat("Error %d: StructToCharArray() failed. MQL5 Error: %d", m_lastError, ::GetLastError());
      return false;
     }

   int lengthBytes = ArraySize(bytes_arr);
   if (lengthBytes > USHORT_MAX)
     {
      m_lastError = BIT_BUFFER_DATA_TOO_LARGE;
      PrintFormat("Error %d: Struct length %d exceeds max ushort (%d).", m_lastError, lengthBytes, USHORT_MAX);
      return false;
     }

   if (!WriteUShort((ushort)lengthBytes)) return false; // Write length

   for (int i = 0; i < lengthBytes; i++) // Write each byte
     {
      if (!WriteUChar(bytes_arr[i])) return false;
     }
   return true;
  }

//+------------------------------------------------------------------+
//| Reads a struct prefixed by its byte length.                      |
//+------------------------------------------------------------------+
template<typename T>
T CBitBuffer::ReadStruct()
  {
   ClearLastError();
   T struct_result = {};
   ushort lengthBytes = ReadUShort();
   if (GetLastError() != BIT_BUFFER_NO_ERROR) return struct_result;

   long requiredBitsForStruct = (long)lengthBytes * BIT_SIZE_UCHAR;
   if (requiredBitsForStruct > GetRemainingReadBits())
     {
      m_lastError = BIT_BUFFER_READ_DATA_INCOMPLETE;
      PrintFormat("Error %d: Not enough data for struct (%d bits). ReadPos:%d, Remaining:%d",
                  m_lastError, requiredBitsForStruct, GetReadPosition(), GetRemainingReadBits());
      return struct_result;
     }

   uchar bytes_arr[];
   ArrayResize(bytes_arr, lengthBytes);

   for (int i = 0; i < lengthBytes; i++)
     {
      bytes_arr[i] = ReadUChar();
      if (GetLastError() != BIT_BUFFER_NO_ERROR) return struct_result;
     }

   ::ResetLastError();
   if(!CharArrayToStruct(struct_result, bytes_arr))
     {
      m_lastError = BIT_BUFFER_RAW_OPERATION_FAILED;
      PrintFormat("Error %d: CharArrayToStruct() failed. MQL5 Error: %d", m_lastError, ::GetLastError());
      return struct_result;
     }
   return struct_result;
  }

//---
//| PUBLIC VLQ ENCODING METHODS
//---

//+------------------------------------------------------------------+
//| Writes an unsigned long using Variable-Length Quantity (VLQ)     |
//| encoding. Smaller values use fewer bits.                         |
//+------------------------------------------------------------------+
bool CBitBuffer::WriteVarULong(ulong value)
  {
   ClearLastError();
   do
     {
      uchar byte = (uchar)(value & VLQ_DATA_BITS);
      value >>= 7;
      if (value > 0) byte |= VLQ_CONTINUATION_BIT; // Set MSB if more bytes follow
      if (!WriteUChar(byte)) return false;
     }
   while(value > 0);
   return true;
  }

//+------------------------------------------------------------------+
//| Reads an unsigned long using VLQ decoding.                       |
//+------------------------------------------------------------------+
ulong CBitBuffer::ReadVarULong()
  {
   ClearLastError();
   ulong value = 0;
   long shift = 0;
   uchar byte;

   do
     {
      byte = ReadUChar();
      if (GetLastError() != BIT_BUFFER_NO_ERROR) return 0;

      value |= (ulong)(byte & VLQ_DATA_BITS) << shift;
      if (shift >= 64) // Prevent overflow
        {
         m_lastError = BIT_BUFFER_DATA_TOO_LARGE;
         PrintFormat("Error %d: VLQ sequence malformed or too long.", m_lastError);
         return 0;
        }
      shift += 7;
     }
   while ((byte & VLQ_CONTINUATION_BIT) != 0); // Continue if MSB is set
   return value;
  }

//+------------------------------------------------------------------+
//| Reads an unsigned int using VLQ decoding.                        |
//+------------------------------------------------------------------+
uint CBitBuffer::ReadVarUInt()
  {
   ClearLastError();
   uint value = 0;
   int shift = 0;
   uchar byte;

   do
     {
      byte = ReadUChar();
      if (GetLastError() != BIT_BUFFER_NO_ERROR) return 0;

      value |= (uint)(byte & VLQ_DATA_BITS) << shift;
      if (shift >= 32) // Prevent overflow
        {
         m_lastError = BIT_BUFFER_DATA_TOO_LARGE;
         PrintFormat("Error %d: VLQ sequence malformed or too long (uint).", m_lastError);
         return 0;
        }
      shift += 7;
     }
   while ((byte & VLQ_CONTINUATION_BIT) != 0);
   return value;
  }

//+------------------------------------------------------------------+
//| Reads a signed long using ZigZag decoding + VLQ.                 |
//| Maps signed values to unsigned for efficient VLQ packing.        |
//+------------------------------------------------------------------+
long CBitBuffer::ReadVarLong()
  {
   // Read the VLQ encoded unsigned long.
   ulong unsignedValue = ReadVarULong();
   if (GetLastError() != BIT_BUFFER_NO_ERROR) return 0;

   // Perform ZigZag decoding:
   // For even unsigned: (unsigned >> 1)
   // For odd unsigned: ~(unsigned >> 1)
   return (long)((unsignedValue >> 1) ^ (-(long)(unsignedValue & 1)));
  }

//+------------------------------------------------------------------+
//| Reads a signed int using ZigZag decoding + VLQ.                  |
//+------------------------------------------------------------------+
int CBitBuffer::ReadVarInt()
  {
   // Read the VLQ encoded unsigned int.
   uint unsignedValue = ReadVarUInt();
   if (GetLastError() != BIT_BUFFER_NO_ERROR) return 0;

   // Perform ZigZag decoding for int.
   return (int)((unsignedValue >> 1) ^ (-(int)(unsignedValue & 1)));
  }

//---
//| PUBLIC POSITION MANAGEMENT METHODS
//---

//+------------------------------------------------------------------+
//| Sets the absolute read position within the buffer.               |
//+------------------------------------------------------------------+
bool CBitBuffer::SetReadPosition(long bitPosition)
  {
   ClearLastError();
   if (bitPosition < 0 || bitPosition > m_totalWrittenBits)
     {
      m_lastError = BIT_BUFFER_INVALID_READ_POSITION;
      PrintFormat("Error %d: Attempt to set read position out of bounds. Position: %d, TotalWrittenBits: %d",
                  m_lastError, bitPosition, m_totalWrittenBits);
      return false;
     }

   m_currentReadBitPosition = bitPosition;
   // Invalidate internal read buffer as its contents might not match the new position.
   m_readBufferInternal = 0;
   m_readBufferBitCounter = 0;
   return true;
  }

//+------------------------------------------------------------------+
//| Advances the read position by a specified number of bits.        |
//+------------------------------------------------------------------+
bool CBitBuffer::SkipBits(long bitsToSkip)
  {
   ClearLastError();
   if (bitsToSkip < 0)
     {
      m_lastError = BIT_BUFFER_INVALID_BIT_COUNT;
      PrintFormat("Error %d: SkipBits called with negative bitsToSkip (%d).", m_lastError, bitsToSkip);
      return false;
     }
   if (bitsToSkip == 0) return true;

   long newReadPos = m_currentReadBitPosition + bitsToSkip;
   if (newReadPos > m_totalWrittenBits)
     {
      // If the skip would go beyond the total written bits, cap it at the end.
      newReadPos = m_totalWrittenBits;
      m_lastError = BIT_BUFFER_READ_OUT_OF_BOUNDS;
      PrintFormat("Warning %d: Tried to skip beyond written data. Adjusted skip amount to prevent out-of-bounds.", m_lastError);
      return SetReadPosition(newReadPos);
     }

   return SetReadPosition(newReadPos); // Set the adjusted position
  }

//---
//| PUBLIC BUFFER MANAGEMENT METHODS
//---

//+------------------------------------------------------------------+
//| Clears buffer, keeping allocated memory for potential reuse.     |
//+------------------------------------------------------------------+
void CBitBuffer::Clear()
  {
   ZeroMemory(m_mainBuffer);
   m_totalWrittenBits         = 0;
   m_currentReadBitPosition   = 0;
   m_writeBufferInternal      = 0;
   m_writeBufferBitCounter    = 0;
   m_readBufferInternal       = 0;
   m_readBufferBitCounter     = 0;
   m_lastError                = BIT_BUFFER_NO_ERROR;
  }

//+------------------------------------------------------------------+
//| Copies finalized buffer content to a destination ulong array.    |
//| Flushes any pending writes first.                                |
//+------------------------------------------------------------------+
bool CBitBuffer::GetFinalizedBuffer(ulong &destinationArray[])
  {
   ClearLastError();
   if (m_writeBufferBitCounter > 0)
     {
      if (!FlushWriteCache())
        {
         m_lastError = BIT_BUFFER_WRITE_FLUSH_FAILED;
         PrintFormat("Error %d: Failed to flush write buffer before getting finalized buffer.", m_lastError);
         return false;
        }
     }

   long totalUlongsWritten = (m_totalWrittenBits + 63) / 64;
   if (totalUlongsWritten == 0)
     {
      ArrayFree(destinationArray);
      return true;
     }

   // Resize the destination array to hold the written data.
   if (ArrayResize(destinationArray, (int)totalUlongsWritten) == -1)
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to resize destination array to %d ulongs.", m_lastError, totalUlongsWritten);
      return false;
     }

   // Copy data from the internal main buffer to the destination array.
   ArrayCopy(destinationArray, m_mainBuffer, 0, 0, (int)totalUlongsWritten);
   return true;
  }

//+------------------------------------------------------------------+
//| Sets internal buffer content from a source ulong array.          |
//| Resets state and copies data.                                    |
//+------------------------------------------------------------------+
bool CBitBuffer::SetRawBuffer(const ulong &sourceBuffer[], long numBits = -1)
  {
   Clear();

   long sourceUlongCount = ArraySize(sourceBuffer);
   long effectiveNumBits = (numBits == -1) ? sourceUlongCount * 64 : numBits;

   if (effectiveNumBits < 0)
     {
      m_lastError = BIT_BUFFER_INVALID_BIT_COUNT;
      PrintFormat("Error %d: SetRawBuffer called with negative numBits (%d).", m_lastError, numBits);
      return false;
     }

   long requiredUlongs = (effectiveNumBits + 63) / 64;
   if (requiredUlongs > sourceUlongCount)
     {
      m_lastError = BIT_BUFFER_READ_DATA_INCOMPLETE;
      PrintFormat("Error %d: Source buffer too small for specified numBits. Source ulongs: %d, Required ulongs: %d.",
                  m_lastError, sourceUlongCount, requiredUlongs);
      return false;
     }

   // Ensure the main buffer has enough capacity.
   if (!EnsureBufferCapacity(requiredUlongs))
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to allocate memory for SetRawBuffer.", m_lastError);
      return false;
     }

   // Copy the raw data from the source buffer.
   ArrayCopy(m_mainBuffer, sourceBuffer, 0, 0, (int)requiredUlongs);
   m_totalWrittenBits = effectiveNumBits; // Set total written bits to the effective number of bits.
   return true;
  }

//+------------------------------------------------------------------+
//| Saves buffer content to a file as raw binary data.               |
//| The file is prefixed by the total number of written bits.        |
//+------------------------------------------------------------------+
bool CBitBuffer::Save(string filename, bool common = false)
  {
   ClearLastError();
   // Ensure all pending writes are flushed before saving.
   if (m_writeBufferBitCounter > 0)
     {
      if (!FlushWriteCache())
        {
         m_lastError = BIT_BUFFER_WRITE_FLUSH_FAILED;
         PrintFormat("Error %d: Failed to flush write buffer before saving file.", m_lastError);
         return false;
        }
     }

   int fileHandle = FileOpen(filename, FILE_WRITE | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0));
   if (fileHandle == INVALID_HANDLE)
     {
      m_lastError = BIT_BUFFER_FILE_SAVE_FAILED;
      PrintFormat("Error %d: Failed to open file '%s' for writing. MQL5 Error: %d", m_lastError, filename, ::GetLastError());
      return false;
     }

   long ulongsToWrite = (m_totalWrittenBits + 63) / 64;  // Round up to nearest ulong.
   if (ulongsToWrite == 0)
     {
      // If no bits were written, just close the file (it will be empty or 0-byte size).
      FileClose(fileHandle);
      return true;
     }

   // Write total bits first for correct loading.
   if (FileWriteLong(fileHandle, m_totalWrittenBits) != sizeof(long))
     {
      m_lastError = BIT_BUFFER_FILE_SAVE_FAILED;
      PrintFormat("Error %d: Failed to write totalWrittenBits to file '%s'.", m_lastError, filename);
      FileClose(fileHandle);
      return false;
     }

   // Write the main buffer content.
   if (FileWriteArray(fileHandle, m_mainBuffer, 0, (int)ulongsToWrite) != ulongsToWrite)
     {
      m_lastError = BIT_BUFFER_FILE_SAVE_FAILED;
      PrintFormat("Error %d: Failed to write buffer data to file '%s'. MQL5 Error: %d", m_lastError, filename, ::GetLastError());
      FileClose(fileHandle);
      return false;
     }

   FileClose(fileHandle);
   return true;
  }

//+------------------------------------------------------------------+
//| Loads buffer content from a file, expecting the total number     |
//| of bits as a prefix.                                             |
//+------------------------------------------------------------------+
bool CBitBuffer::Load(string filename, bool common = false)
  {
   Clear();

   int fileHandle = FileOpen(filename, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0));
   if (fileHandle == INVALID_HANDLE)
     {
      m_lastError = BIT_BUFFER_FILE_LOAD_FAILED;
      PrintFormat("Error %d: Failed to open file '%s' for reading. MQL5 Error: %d", m_lastError, filename, ::GetLastError());
      return false;
     }

   // Read the total number of bits first.
   long loadedTotalBits = FileReadLong(fileHandle);
   if (loadedTotalBits == 0 && FileIsEnding(fileHandle)) // Empty file case.
     {
      FileClose(fileHandle);
      return true;
     }

   if (loadedTotalBits < 0 || FileIsEnding(fileHandle)) // Error or unexpected end of file.
     {
      m_lastError = BIT_BUFFER_FILE_LOAD_FAILED;
      PrintFormat("Error %d: Failed to read totalWrittenBits from file '%s' or file is empty/corrupt.", m_lastError, filename);
      FileClose(fileHandle);
      return false;
     }

   long requiredUlongs = (loadedTotalBits + 63) / 64;

   // Ensure buffer capacity for the incoming data.
   if (!EnsureBufferCapacity(requiredUlongs))
     {
      m_lastError = BIT_BUFFER_MEMORY_ALLOCATION_FAILED;
      PrintFormat("Error %d: Failed to allocate memory for loading file '%s'.", m_lastError, filename);
      FileClose(fileHandle);
      return false;
     }

   // Read the main buffer content.
   if (FileReadArray(fileHandle, m_mainBuffer, 0, (int)requiredUlongs) != requiredUlongs)
     {
      m_lastError = BIT_BUFFER_FILE_LOAD_FAILED;
      PrintFormat("Error %d: Failed to read buffer data from file '%s'. MQL5 Error: %d", m_lastError, filename, ::GetLastError());
      FileClose(fileHandle);
      return false;
     }

   FileClose(fileHandle);
   m_totalWrittenBits = loadedTotalBits; // Set total written bits after successful load.
   return true;
  }

//+------------------------------------------------------------------+
//| Returns a string description for the given error code.           |
//+------------------------------------------------------------------+
string CBitBuffer::GetLastErrorString() const
  {
   switch (m_lastError)
     {
      case BIT_BUFFER_NO_ERROR:                 return "No error.";
      case BIT_BUFFER_INVALID_BIT_COUNT:        return "Invalid bit count specified (must be 1-64).";
      case BIT_BUFFER_MEMORY_ALLOCATION_FAILED: return "Memory allocation failed for the internal buffer.";
      case BIT_BUFFER_READ_OUT_OF_BOUNDS:       return "Attempt to read beyond available data.";
      case BIT_BUFFER_WRITE_FLUSH_FAILED:       return "Write operation failed during internal buffer flush.";
      case BIT_BUFFER_INVALID_READ_POSITION:    return "Attempt to set read position out of valid bounds (negative or beyond total written bits).";
      case BIT_BUFFER_DATA_TOO_LARGE:           return "Data length exceeds maximum allowed capacity (e.g., string/struct too large for ushort length prefix).";
      case BIT_BUFFER_READ_DATA_INCOMPLETE:     return "Incomplete data found during read, or declared length exceeds available data.";
      case BIT_BUFFER_RAW_OPERATION_FAILED:     return "Failed to copy raw buffer data to/from the internal buffer.";
      case BIT_BUFFER_FILE_SAVE_FAILED:         return "FileSave failed for the main buffer during save operation.";
      case BIT_BUFFER_FILE_LOAD_FAILED:         return "FileLoad failed for the main buffer during load operation.";
      default:                                  return "Unknown error code.";
     }
  }

//+------------------------------------------------------------------+
//| Prints internal main buffer in hexadecimal format (debugging).   |
//+------------------------------------------------------------------+
void CBitBuffer::PrintHex(int start = 0, int count = WHOLE_ARRAY) const
  {
   PrintFormat("Write Cache (Hex): 0x%.16llX (Bits: %d)", m_writeBufferInternal, m_writeBufferBitCounter);

   uchar b[];
   uint pos = 0;
   // Serialize the main buffer (m_mainBuffer) to bytes for printing.
   long ulongsToPrint = (m_totalWrittenBits + 63) / 64; // Round up to nearest ulong.
   long bytes = ulongsToPrint * 8;
   ArrayResize(b, (int)bytes);
   for (long i = 0; i < ulongsToPrint; i++)
     {
      ulong x = m_mainBuffer[i];
      // Extract bytes from ulong (LSB first for typical byte representation)
      b[pos++] = (uchar)((x >> 0) & 0xff);
      b[pos++] = (uchar)((x >> 8) & 0xff);
      b[pos++] = (uchar)((x >> 16) & 0xff);
      b[pos++] = (uchar)((x >> 24) & 0xff);
      b[pos++] = (uchar)((x >> 32) & 0xff);
      b[pos++] = (uchar)((x >> 40) & 0xff);
      b[pos++] = (uchar)((x >> 48) & 0xff);
      b[pos++] = (uchar)((x >> 56) & 0xff);
     }
   PrintFormat("Raw Buffer (Hex): (Bits: %d)", m_totalWrittenBits);
   // Helper function to print byte array in hex.
   ArrayPrintHex(b, start, count, 32); // Assuming ArrayPrintHex is defined globally or in a common include.
  }

//+------------------------------------------------------------------+
//| Global Helper Function: Converts an array to a hex string for printing.|
//| This function is assumed to be defined externally or in a common |
//| include file, as it's not part of the CBitBuffer class.          |
//+------------------------------------------------------------------+
template <typename T>
void ArrayPrintHex(const T &array[], int start = 0, int count = WHOLE_ARRAY, int wrap = 16)
  {
   string result = "";
   int size = ArraySize(array);
   if(start < 0)
      start = 0;
   if(count == WHOLE_ARRAY)
      count = size;
   size = MathMin((start + count), size);
   for(int i = 0; i < size-start; i++)
     {
      // insert a newline after each 'wrap' items
      if (i > 0 && (i % wrap) == 0)
         result += "\r\n";

      if(sizeof(T) < 8)
         result += StringFormat("%." + (string)(sizeof(T) * 2) + "X ",  array[start+i]);
      else
         result += StringFormat("%.16llX ",  array[start+i]);
     }
   //return(result);
   Print(result);
  }
//+------------------------------------------------------------------+