DiskBuffer.h

Go to the documentation of this file.

/***********************************************************************************\
* (c) Copyright 2026 CERN for the benefit of the LHCb and ATLAS collaborations      *
*                                                                                   *
* This software is distributed under the terms of the Apache version 2 licence,     *
* copied verbatim in the file "LICENSE".                                            *
*                                                                                   *
* In applying this licence, CERN does not waive the privileges and immunities       *
* granted to it by virtue of its status as an Intergovernmental Organization        *
* or submit itself to any jurisdiction.                                             *
\***********************************************************************************/
#pragma once
 
#include <cstdint>
#include <cstdio>
#include <filesystem>
#include <memory>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>

namespace Gaudi::NTuple::DiskBuffer {

  bool zstdAvailable();

  std::size_t maxBlockSize();
 
  namespace details {
    struct Close {
      void operator()( std::FILE* f ) const { std::fclose( f ); }
    };
  } // namespace details

  using FileHandle = std::unique_ptr<std::FILE, details::Close>;

  class Error : public std::runtime_error {
  public:
    Error( const std::filesystem::path& path, std::string_view what, int err = 0 );
    const std::filesystem::path& path() const { return m_path; }
    int                          code() const { return m_code; }
 
  private:
    std::filesystem::path m_path;
    int                   m_code;
  };
 
  class Writer {
  public:
    Writer( std::filesystem::path path, std::string_view header, int zstdLevel = 0, std::size_t blockSize = 64 * 1024 );
    ~Writer();
    Writer( const Writer& )            = delete;
    Writer& operator=( const Writer& ) = delete;
 
    void beginRecord();
    void append( const void* data, std::size_t bytes );
    void endRecord();

    void close();

    std::uint64_t entries() const { return m_entries; }
    std::uint64_t bytes() const { return m_bytes; }
    std::uint64_t                storedBytes() const { return m_stored; }
    int                          zstdLevel() const { return m_level; }
    std::size_t                  blockSize() const { return m_blockSize; }
    const std::filesystem::path& path() const { return m_path; }
 
  private:
    void flush();
    void write( const void* data, std::size_t bytes );
 
    std::filesystem::path m_path;
    FileHandle            m_file;
    int                   m_level;
    std::size_t           m_blockSize;
    std::vector<char>     m_buf;
    std::size_t           m_recordStart{ 0 }; // offset of the current record's length field
    bool                  m_inRecord{ false };
    std::uint64_t         m_entries{ 0 };
    std::uint64_t         m_bytes{ 0 };
    std::uint64_t         m_stored{ 0 };
    std::uint64_t         m_pending{ 0 };      // records in m_buf, counted once their block is written
    std::uint64_t         m_pendingBytes{ 0 }; // and their payload
  };
 
  class Reader {
  public:
    explicit Reader( std::filesystem::path path );
    ~Reader();
    Reader( const Reader& )            = delete;
    Reader& operator=( const Reader& ) = delete;
 
    const std::string& header() const { return m_header; }
    bool               compressed() const { return m_compressed; }

    bool next( std::span<const char>& record );
 
    std::uint64_t                entries() const { return m_entries; }
    const std::filesystem::path& path() const { return m_path; }
 
  private:
    std::size_t read( void* dest, std::size_t bytes );
    bool        readBlock();
 
    std::filesystem::path m_path;
    FileHandle            m_file;
    std::uintmax_t        m_size{ 0 }; // file size, 0 if unknown: bounds a block length
    std::string           m_header;
    bool                  m_compressed{ false };
    std::vector<char>     m_stored; // one block as read from the file
    std::vector<char>     m_block;  // the same block unpacked
    std::size_t           m_pos{ 0 };
    std::uint64_t         m_entries{ 0 };
  };
 
} // namespace Gaudi::NTuple::DiskBuffer