RootHistCnv::RCWNTupleCnv Class Reference

Converter of Column-wise NTuple into ROOT format. More...

#include </builds/gaudi/Gaudi/RootHistCnv/src/RCWNTupleCnv.h>

Inheritance diagram for RootHistCnv::RCWNTupleCnv:

Collaboration diagram for RootHistCnv::RCWNTupleCnv:

Classes
struct	Layout
	How the items of a tuple map onto the staging buffer and the TTree branches. More...
struct	Buffered

Public Member Functions
	RCWNTupleCnv (ISvcLocator *svc)
	Standard constructor.
StatusCode	initialize () override
StatusCode	finalize () override
StatusCode	updateRep (IOpaqueAddress *pAddr, DataObject *pObj) override
	Write the TTree; for a disk-buffered tuple, first build it from the buffer file.
Public Member Functions inherited from RootHistCnv::RNTupleCnv
StatusCode	initialize () override
	Initialize the converter.
StatusCode	finalize () override
	Finalize the converter.
StatusCode	createObj (IOpaqueAddress *pAddr, DataObject *&refpObj) override
	Create the transient representation of an object.
StatusCode	updateObj (IOpaqueAddress *pAddr, DataObject *refpObj) override
	Update the transient object from the other representation.
StatusCode	createRep (DataObject *pObj, IOpaqueAddress *&refpAddr) override
	Convert the transient object to the requested representation.
StatusCode	updateRep (IOpaqueAddress *pAddr, DataObject *pObj) override
	Update the converted representation of a transient object.
Public Member Functions inherited from RootHistCnv::RConverter
StatusCode	createRep (DataObject *pObj, IOpaqueAddress *&refpAddr) override
	Convert the transient object to the requested representation.
long	repSvcType () const override
StatusCode	error (const std::string &msg)
StatusCode	regTFile (const std::string &, const TFile *)
StatusCode	findTFile (const std::string &, TFile *&)
std::string	diskDirectory (const std::string &loc)
std::string	directory (const std::string &loc)
void	setDirectory (const std::string &loc)
void	setDiskDirectory (const std::string &loc)
StatusCode	createDirectory (const std::string &loc)
StatusCode	createDiskDirectory (const std::string &loc)
std::string	getDirectory ()
std::string	convertId (const std::string &)
Public Member Functions inherited from Converter
StatusCode	initialize () override
	Initialize the converter.
StatusCode	finalize () override
	Initialize the converter.
StatusCode	setDataProvider (IDataProviderSvc *svc) override
	Set Data provider service.
SmartIF< IDataProviderSvc > &	dataProvider () const override
	Get Data provider service.
StatusCode	setConversionSvc (IConversionSvc *svc) override
	Set conversion service the converter is connected to.
SmartIF< IConversionSvc >	conversionSvc () override
	Get conversion service the converter is connected to.
StatusCode	setAddressCreator (IAddressCreator *creator) override
	Set address creator facility.
SmartIF< IAddressCreator > &	addressCreator () const override
	Retrieve address creator facility.
const CLID &	objType () const override
	Retrieve the class type of objects the converter produces.
virtual long	i_repSvcType () const
	Retrieve the class type of the data store the converter uses.
StatusCode	createObj (IOpaqueAddress *pAddress, DataObject *&refpObject) override
	Create the transient representation of an object.
StatusCode	fillObjRefs (IOpaqueAddress *pAddress, DataObject *pObject) override
	Resolve the references of the created transient object.
StatusCode	updateObj (IOpaqueAddress *pAddress, DataObject *refpObject) override
	Update the transient object from the other representation.
StatusCode	updateObjRefs (IOpaqueAddress *pAddress, DataObject *pObject) override
	Update the references of an updated transient object.
StatusCode	createRep (DataObject *pObject, IOpaqueAddress *&refpAddress) override
	Convert the transient object to the requested representation.
StatusCode	fillRepRefs (IOpaqueAddress *pAddress, DataObject *pObject) override
	Resolve the references of the converted object.
StatusCode	updateRep (IOpaqueAddress *pAddress, DataObject *pObject) override
	Update the converted representation of a transient object.
StatusCode	updateRepRefs (IOpaqueAddress *pAddress, DataObject *pObject) override
	Update the references of an already converted object.
	Converter (long storage_type, const CLID &class_type, ISvcLocator *svc=0)
	Standard Constructor.
SmartIF< IService >	service (const std::string &name, const bool createIf=true) const
	Return a pointer to the service identified by name (or "type/name").
Public Member Functions inherited from implements< IConverter >
void const *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast.
StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface.
std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames.
	implements ()=default
	Default constructor.
implements &	operator= (const implements &)
	Assignment operator (do not touch the reference count).
unsigned long	addRef () const override
	Reference Interface instance.
unsigned long	release () const override
	Release Interface instance.
unsigned long	refCount () const override
	Current reference count.

Static Public Member Functions
static const CLID &	classID ()
	Inquire class type.
Static Public Member Functions inherited from RootHistCnv::RConverter
static long	storageType ()
	Inquire storage type.

Protected Member Functions
StatusCode	load (TTree *tree, INTuple *&refpObject) override
	Create the transient representation of an object.
StatusCode	book (const std::string &desc, INTuple *pObject, TTree *&tree) override
	Book the N tuple.
StatusCode	writeData (TTree *rtree, INTuple *pObject) override
	Write N tuple data.
StatusCode	readData (TTree *rtree, INTuple *pObject, long ievt) override
	Read N tuple data.
Protected Member Functions inherited from RootHistCnv::RNTupleCnv
	RNTupleCnv (ISvcLocator *svc, const CLID &clid)
	Standard constructor.
virtual std::string	rootVarType (int)
	Return ROOT type info:
Protected Member Functions inherited from RootHistCnv::RConverter
	RConverter (const CLID &clid, ISvcLocator *svc)
	Standard constructor.
virtual StatusCode	readObject (IOpaqueAddress *pAddr, DataObject *&refpObj)
	Create the transient representation of an object.
virtual TObject *	createPersistent (DataObject *pObj)
	Create the persistent representation of an object.
StatusCode	createAddress (DataObject *pObject, TDirectory *pDir, TObject *pTObject, IOpaqueAddress *&refpAddr)
	Create address of the transient object according to the requested representation.
StatusCode	createAddress (const std::string &rzdir, const CLID &clid, long id, TObject *pTobj, IOpaqueAddress *&refpAddress)
StatusCode	createAddress (const std::string &rzdir, const CLID &clid, const std::string &title, TObject *pTobj, IOpaqueAddress *&refpAddress)
TDirectory *	changeDirectory (DataObject *pObject)
	Switch to object directory (=Parent directory).
Protected Member Functions inherited from Converter
SmartIF< ISvcLocator > &	serviceLocator () const
	Retrieve pointer to service locator.
SmartIF< IMessageSvc > &	msgSvc () const
	Retrieve pointer to message service.
SmartIF< IDataManagerSvc > &	dataManager () const
	Get Data Manager service.
Protected Member Functions inherited from implements< IConverter >
unsigned long	decRef () const override

Private Member Functions
Layout	analyse (const INTuple &nt)
StatusCode	createTree (const std::string &desc, INTuple *nt, const Layout &layout, TTree *&rtree)
StatusCode	bookBuffered (const std::string &desc, INTuple *nt, Layout layout)
StatusCode	writeBuffered (Buffered &b, INTuple *nt)
StatusCode	replay (IOpaqueAddress *pAddr, INTuple *nt, Buffered &b)
StatusCode	bufferDirectory (std::filesystem::path &dir)
void	releaseBufferDir ()
Buffered *	buffered (INTuple *nt)
bool	replayed (INTuple *nt)

Private Attributes
std::mutex	m_mutex
	guards m_buffered, m_replayed and m_bufferDir
std::map< INTuple *, Buffered >	m_buffered
std::set< INTuple * >	m_replayed
	tuples whose tree has been built; cannot be written again
std::filesystem::path	m_bufferDir
unsigned	m_nextBufferId { 0 }
bool	m_bookDirect { false }
	force the columnar path in book()
bool	m_diskBuffer { false }
	NTupleSvc.DiskBuffer* of the service owning this stream, read once in initialize().
std::string	m_diskBufferDirectory
int	m_diskBufferLevel { 0 }
std::size_t	m_diskBufferBlockSize { 0 }

Additional Inherited Members
Public Types inherited from Converter
using	Factory = Gaudi::PluginService::Factory<IConverter*( ISvcLocator* )>
Public Types inherited from implements< IConverter >
using	base_class
	Typedef to this class.
using	extend_interfaces_base
	Typedef to the base of this class.
using	iids
Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids
	take union of the ext_iids of all Interfaces...
Protected Attributes inherited from RootHistCnv::RNTupleCnv
SmartIF< INTupleSvc >	m_ntupleSvc
	Reference to N tuple service.
Protected Attributes inherited from implements< IConverter >
std::atomic_ulong	m_refCount
	Reference counter.

Detailed Description

Converter of Column-wise NTuple into ROOT format.

Author

Charles Leggett

Definition at line 37 of file RCWNTupleCnv.h.

Constructor & Destructor Documentation

RCWNTupleCnv()

RootHistCnv::RCWNTupleCnv::RCWNTupleCnv ( ISvcLocator * svc )

inline

Standard constructor.

Definition at line 42 of file RCWNTupleCnv.h.

: RNTupleCnv( svc, classID() ) {}

Member Function Documentation

analyse()

RootHistCnv::RCWNTupleCnv::Layout RootHistCnv::RCWNTupleCnv::analyse ( const INTuple & nt )

private

Definition at line 225 of file RCWNTupleCnv.cpp.

{
  MsgStream   log( msgSvc(), "RCWNTupleCnv" );
  Layout      layout;
  const auto& items = nt.items();
  layout.items.reserve( items.size() );
  long size = 0;
 
  for ( const auto& i : items ) {
    Layout::Item li;
    const long   oldsize = size;
 
    visit( *i, [&]( const auto& data ) {
      analyzeItem( this->rootVarType( data.type() ), &data, li.leaflist, li.block, li.name, li.rangeLower,
                   li.rangeUpper, size );
    } );
 
    li.bufPos = oldsize;
    li.bufLen = size - oldsize;
    if ( i->hasIndex() ) {
      auto idx    = std::find( items.begin(), items.end(), i->indexItem() );
      li.indexPos = idx != items.end() ? idx - items.begin() : -1;
    }
    const long elemBytes = i->length() > 0 ? i->size() / i->length() : 0;
    li.unitBytes         = elemBytes * ( i->ndim() == 2 ? i->dim( 0 ) : 1 ); // a matrix is counted in columns
    li.maxUnits          = li.unitBytes > 0 ? li.bufLen / li.unitBytes : 0;
 
    log << MSG::VERBOSE << "item: " << li.leaflist << " type " << i->type() << " blk: " << li.block
        << " var: " << li.name << " rng: " << li.rangeLower << " " << li.rangeUpper << " sz: " << size << " "
        << li.bufLen << " buf_pos: " << li.bufPos << endmsg;
 
    layout.items.push_back( std::move( li ) );
  }
  layout.rowBytes = size;
  return layout;
}

book()

StatusCode RootHistCnv::RCWNTupleCnv::book ( const std::string & desc, INTuple * pObject, TTree *& tree )

overrideprotectedvirtual

Book the N tuple.

Implements RootHistCnv::RNTupleCnv.

Definition at line 361 of file RCWNTupleCnv.cpp.

{
  rtree         = nullptr;
  Layout layout = analyse( *nt );
 
  if ( m_diskBuffer && !m_bookDirect ) return bookBuffered( desc, nt, std::move( layout ) );
 
  StatusCode sc = createTree( desc, nt, layout, rtree );
  if ( sc ) {
    MsgStream log( msgSvc(), "RCWNTupleCnv" );
    log << MSG::INFO << "Booked TTree with ID: " << desc << " \"" << nt->title() << "\" in directory " << getDirectory()
        << endmsg;
  }
  return sc;
}

bookBuffered()

StatusCode RootHistCnv::RCWNTupleCnv::bookBuffered ( const std::string & desc, INTuple * nt, Layout layout )

private

Definition at line 428 of file RCWNTupleCnv.cpp.

{
  MsgStream   log( msgSvc(), "RCWNTupleCnv" );
  const auto& items = nt->items();
  for ( size_t k = 0; k < items.size(); ++k ) {
    const auto& li = layout.items[k];
    if ( items[k]->hasIndex() && ( li.indexPos < 0 || li.indexPos >= long( k ) ) ) {
      log << MSG::ERROR << "DiskBuffer: index " << items[k]->index() << " must be added before " << items[k]->name()
          << " in " << desc << endmsg;
      return StatusCode::FAILURE;
    }
  }
  std::string id = desc;
  if ( auto obj = dynamic_cast<DataObject*>( nt ); obj && obj->registry() ) id = obj->registry()->identifier();
 
  std::lock_guard lock( m_mutex );
  fs::path        dir;
  if ( auto sc = bufferDirectory( dir ); !sc ) return sc;
  const fs::path path = dir / fmt::format( "{:04}.ntbuf", m_nextBufferId++ );
 
  Buffered b{ desc, std::move( layout ), nullptr };
  try {
    b.writer = std::make_unique<DiskBuffer::Writer>( path, describe( id, *nt, b.layout ), m_diskBufferLevel,
                                                     m_diskBufferBlockSize );
  } catch ( const DiskBuffer::Error& e ) {
    log << MSG::ERROR << e.what() << endmsg;
    return StatusCode::FAILURE;
  }
  log << MSG::INFO << "Buffering TTree with ID: " << desc << " \"" << nt->title() << "\" in directory "
      << getDirectory() << " to " << path.string() << endmsg;
  m_buffered.emplace( nt, std::move( b ) );
  return StatusCode::SUCCESS;
}

bufferDirectory()

StatusCode RootHistCnv::RCWNTupleCnv::bufferDirectory ( std::filesystem::path & dir )

private

Definition at line 379 of file RCWNTupleCnv.cpp.

{
  if ( !m_bufferDir.empty() ) {
    dir = m_bufferDir;
    return StatusCode::SUCCESS;
  }
  MsgStream log( msgSvc(), "RCWNTupleCnv" );
 
  TFile* file = gDirectory ? gDirectory->GetFile() : nullptr;
  if ( !file ) {
    log << MSG::ERROR << "DiskBuffer: " << getDirectory() << " is not inside a file" << endmsg;
    return StatusCode::FAILURE;
  }
  TUrl     url( file->GetName(), kTRUE );
  fs::path base;
  if ( m_diskBufferDirectory.empty() && std::string_view( url.GetProtocol() ) == "file" ) {
    base = url.GetFile();
  } else {
    base =
        fs::path( m_diskBufferDirectory.empty() ? "." : m_diskBufferDirectory ) / fs::path( url.GetFile() ).filename();
  }
  base += ".ntbuf";
 
  {
    std::lock_guard lock( s_dirsMutex );
    if ( !s_claimedDirs.insert( base ).second ) {
      log << MSG::ERROR << "DiskBuffer directory " << base << " is already used by another output stream" << endmsg;
      return StatusCode::FAILURE;
    }
  }
  std::error_code ec;
  if ( fs::exists( base, ec ) ) {
    log << MSG::WARNING << "Removing stale DiskBuffer directory " << base << endmsg;
    fs::remove_all( base, ec );
  }
  if ( !ec ) fs::create_directories( base, ec );
  if ( ec ) {
    log << MSG::ERROR << "Cannot create DiskBuffer directory " << base << ": " << ec.message() << endmsg;
    std::lock_guard lock( s_dirsMutex );
    s_claimedDirs.erase( base );
    return StatusCode::FAILURE;
  }
  m_bufferDir = base;
  dir         = base;
  return StatusCode::SUCCESS;
}

buffered()

RootHistCnv::RCWNTupleCnv::Buffered * RootHistCnv::RCWNTupleCnv::buffered ( INTuple * nt )

private

Definition at line 464 of file RCWNTupleCnv.cpp.

{
  std::lock_guard lock( m_mutex );
  auto            it = m_buffered.find( nt );
  return it != m_buffered.end() ? &it->second : nullptr;
}

classID()

const CLID & RootHistCnv::RCWNTupleCnv::classID ( )

inlinestatic

Inquire class type.

Definition at line 40 of file RCWNTupleCnv.h.

{ return CLID_ColumnWiseTuple; }

createTree()

StatusCode RootHistCnv::RCWNTupleCnv::createTree ( const std::string & desc, INTuple * nt, const Layout & layout, TTree *& rtree )

private

Definition at line 264 of file RCWNTupleCnv.cpp.

{
  MsgStream log( msgSvc(), "RCWNTupleCnv" );
  rtree = new TTree( desc.c_str(), nt->title().c_str() );
  log << MSG::VERBOSE << "created tree id: " << rtree->GetName() << "  title: " << nt->title() << " desc: " << desc
      << endmsg;
 
  // Make a new buffer, and tell the ntuple where it is
  const long size = layout.rowBytes;
  char*      buff = nt->setBuffer( new char[size] );
 
  log << MSG::VERBOSE << "Created buffer size: " << size << " at " << (void*)buff << endmsg;
 
  // Zero out the buffer to make ROOT happy
  std::fill_n( buff, size, 0 );
 
  Gaudi::Property<int> basket_size( "BasketSize", 32000 );
  m_ntupleSvc.as<IProperty>()->getProperty( &basket_size ).ignore();
 
  // Loop over items, creating a new branch for each one;
  for ( const auto& li : layout.items ) {
 
    char* buf_pos = buff + li.bufPos;
 
    auto br = new TBranch( rtree, li.name.c_str(), buf_pos, li.leaflist.c_str(), basket_size );
    if ( li.block != "AUTO_BLK" ) {
      std::string title = li.block + "::" + br->GetTitle();
      br->SetTitle( title.c_str() );
    }
 
    log << MSG::DEBUG << "adding TBranch  " << br->GetTitle() << "  at " << (void*)buf_pos << endmsg;
 
    // for index items with a limited range. Must be a TLeafI!
    if ( li.rangeLower < li.rangeUpper ) {
      TLeafI*  index = nullptr;
      TObject* tobj  = br->GetListOfLeaves()->FindObject( li.name.c_str() );
      if ( tobj->IsA()->InheritsFrom( "TLeafI" ) ) {
        index = dynamic_cast<TLeafI*>( tobj );
 
        if ( index ) {
          index->SetMaximum( li.rangeUpper );
          // FIXME -- add for next version of ROOT
          // index->SetMinimum( li.rangeLower );
        } else {
          log << MSG::ERROR << "Could dynamic cast to TLeafI: " << li.name << endmsg;
        }
      }
    }
 
    rtree->GetListOfBranches()->Add( br );
  }
 
  return StatusCode::SUCCESS;
}

finalize()

StatusCode RootHistCnv::RCWNTupleCnv::finalize ( )

override

Definition at line 635 of file RCWNTupleCnv.cpp.

{
  {
    std::lock_guard lock( m_mutex );
    if ( !m_buffered.empty() ) {
      MsgStream log( msgSvc(), "RCWNTupleCnv" );
      for ( auto& [nt, b] : m_buffered ) {
        // flush first, so the file left behind holds every row that was written
        try {
          b.writer->close();
        } catch ( const DiskBuffer::Error& e ) { log << MSG::ERROR << e.what() << endmsg; }
        log << MSG::ERROR << "TTree " << b.desc << " was never built from its disk buffer " << b.writer->path().string()
            << endmsg;
      }
      m_buffered.clear();
    }
    m_replayed.clear();
    releaseBufferDir();
  }
  return RNTupleCnv::finalize();
}

initialize()

StatusCode RootHistCnv::RCWNTupleCnv::initialize ( )

override

Definition at line 322 of file RCWNTupleCnv.cpp.

{
  StatusCode sc = RNTupleCnv::initialize();
  if ( !sc ) return sc;
  MsgStream log( msgSvc(), "RCWNTupleCnv" );
 
  // the DiskBuffer properties belong to the tuple service owning this stream, which is our data provider
  SmartIF<IProperty> owner = dataProvider().as<IProperty>();
  if ( !owner ) owner = m_ntupleSvc.as<IProperty>(); // not owned by a tuple service: keep the lookup by name
  Gaudi::Property<bool>        diskBuffer( "DiskBuffer", false );
  Gaudi::Property<std::string> directory( "DiskBufferDirectory", "" );
  Gaudi::Property<int>         level( "DiskBufferCompression", 3 );
  Gaudi::Property<int>         blockSize( "DiskBufferBlockSize", 64 * 1024 );
  if ( owner ) {
    for ( auto* p :
          std::initializer_list<Gaudi::Details::PropertyBase*>{ &diskBuffer, &directory, &level, &blockSize } )
      owner->getProperty( p ).ignore();
  }
 
  m_diskBuffer          = diskBuffer;
  m_diskBufferDirectory = directory;
  m_diskBufferLevel     = level;
  m_diskBufferBlockSize = static_cast<std::size_t>( blockSize.value() );
  if ( !m_diskBuffer ) return sc;
 
  if ( m_diskBufferLevel != 0 && !DiskBuffer::zstdAvailable() ) {
    log << MSG::WARNING << "DiskBuffer: zstd not available in this build, not compressing" << endmsg;
    m_diskBufferLevel = 0;
  }
  if ( blockSize <= 0 || m_diskBufferBlockSize > DiskBuffer::maxBlockSize() ) {
    log << MSG::ERROR << "DiskBuffer: DiskBufferBlockSize must be between 1 and " << DiskBuffer::maxBlockSize()
        << endmsg;
    return StatusCode::FAILURE;
  }
  return sc;
}

load()

StatusCode RootHistCnv::RCWNTupleCnv::load ( TTree * tree, INTuple *& refpObject )

overrideprotectedvirtual

Create the transient representation of an object.

Implements RootHistCnv::RNTupleCnv.

Definition at line 688 of file RCWNTupleCnv.cpp.

{
  MsgStream log( msgSvc(), "RCWNTupleCnv::load" );
 
  StatusCode status;
 
  NTuple::Tuple* pObj = nullptr;
 
  std::string title = tree->GetTitle();
  log << MSG::VERBOSE << "loading CWNT " << title << " at: " << tree << endmsg;
 
  status        = m_ntupleSvc->create( CLID_ColumnWiseTuple, title, pObj );
  INTuple* ntup = dynamic_cast<INTuple*>( pObj );
  if ( !ntup ) { log << MSG::ERROR << "cannot dynamic cast to INTuple" << endmsg; }
 
  INTupleItem* item = nullptr;
 
  std::string itemName, indexName, item_type, itemTitle, blockName;
  // long numEnt, numVar;
  long                                size, totsize = 0;
  std::vector<std::pair<TLeaf*, int>> itemList;
 
  // numEnt = (int)tree->GetEntries();
  // numVar = tree->GetNbranches();
 
  // loop over all branches (==leaves)
  TObjArray* lbr = tree->GetListOfBranches();
  TIter      bitr( lbr );
  while ( TObject* tobjb = bitr() ) {
 
    TBranch* br = dynamic_cast<TBranch*>( tobjb );
    itemTitle   = br->GetTitle();
 
    int ipos = itemTitle.find( "::" );
    if ( ipos >= 0 ) {
      blockName = itemTitle.substr( 0, ipos );
    } else {
      blockName = "";
    }
 
    TObjArray* lf = br->GetListOfLeaves();
 
    TIter litr( lf );
    while ( TObject* tobj = litr() ) {
 
      bool hasRange   = false;
      int  indexRange = 0;
      int  itemSize;
      item = nullptr;
 
      TLeaf* tl = dynamic_cast<TLeaf*>( tobj );
      if ( !tl ) {
        log << MSG::ERROR << "cannot dynamic cast to TLeaf" << endmsg;
        return StatusCode::FAILURE;
      }
      itemName = tl->GetName();
 
      if ( blockName != "" ) {
        log << MSG::DEBUG << "loading NTuple item " << blockName << "/" << itemName;
      } else {
        log << MSG::DEBUG << "loading NTuple item " << itemName;
      }
 
      int    arraySize{ 0 };
      TLeaf* indexLeaf = tl->GetLeafCounter( arraySize );
 
      if ( arraySize == 0 ) { log << MSG::ERROR << "TLeaf counter size = 0. This should not happen!" << endmsg; }
 
      if ( indexLeaf ) {
        // index Arrays and Matrices
 
        indexName  = indexLeaf->GetName();
        indexRange = indexLeaf->GetMaximum();
        itemSize   = indexRange * tl->GetLenType() * arraySize;
 
        log << "[" << indexName;
 
        // Just for Matrices
        if ( arraySize != 1 ) { log << "][" << arraySize; }
        log << "]";
 
      } else {
        itemSize = tl->GetLenType() * arraySize;
 
        indexName = "";
 
        if ( arraySize == 1 ) {
          // Simple items
        } else {
          // Arrays of constant size
          log << "[" << arraySize << "]";
        }
      }
 
      log << endmsg;
 
      size = itemSize;
      totsize += size;
 
      hasRange = tl->IsRange();
 
      itemList.emplace_back( tl, itemSize );
 
      // Integer
      if ( tobj->IsA()->InheritsFrom( "TLeafI" ) ) {
 
        TLeafI* tli = dynamic_cast<TLeafI*>( tobj );
        if ( tli ) {
          if ( tli->IsUnsigned() ) {
            unsigned long min = 0, max = 0;
            if ( hasRange ) {
              min = tli->GetMinimum();
              max = tli->GetMaximum();
            }
 
            item = createNTupleItem( itemName, blockName, indexName, indexRange, arraySize, min, max, ntup, hasRange );
          } else {
            long min = 0, max = 0;
            if ( hasRange ) {
              min = tli->GetMinimum();
              max = tli->GetMaximum();
            }
 
            item = createNTupleItem( itemName, blockName, indexName, indexRange, arraySize, min, max, ntup, hasRange );
          }
        } else {
          log << MSG::ERROR << "cannot dynamic cast to TLeafI" << endmsg;
        }
 
        // Float
      } else if ( tobj->IsA()->InheritsFrom( "TLeafF" ) ) {
        float min = 0., max = 0.;
 
        TLeafF* tlf = dynamic_cast<TLeafF*>( tobj );
        if ( tlf ) {
          if ( hasRange ) {
            min = float( tlf->GetMinimum() );
            max = float( tlf->GetMaximum() );
          }
        } else {
          log << MSG::ERROR << "cannot dynamic cast to TLeafF" << endmsg;
        }
 
        item = createNTupleItem( itemName, blockName, indexName, indexRange, arraySize, min, max, ntup, hasRange );
 
        // Double
      } else if ( tobj->IsA()->InheritsFrom( "TLeafD" ) ) {
        double min = 0., max = 0.;
 
        TLeafD* tld = dynamic_cast<TLeafD*>( tobj );
        if ( tld ) {
          if ( hasRange ) {
            min = tld->GetMinimum();
            max = tld->GetMaximum();
          }
        } else {
          log << MSG::ERROR << "cannot dynamic cast to TLeafD" << endmsg;
        }
 
        item = createNTupleItem( itemName, blockName, indexName, indexRange, arraySize, min, max, ntup, hasRange );
 
      } else {
        log << MSG::ERROR << "Uknown data type" << endmsg;
      }
 
      if ( item ) {
        ntup->add( item ).ignore();
      } else {
        log << MSG::ERROR << "Unable to create ntuple item \"" << itemName << "\"" << endmsg;
      }
 
    } // end litr
  }   // end bitr
 
  log << MSG::DEBUG << "Total buffer size of NTuple: " << totsize << " Bytes." << endmsg;
 
  char* buf    = ntup->setBuffer( new char[totsize] );
  char* bufpos = buf;
 
  int ts = 0;
  for ( const auto& iitr : itemList ) {
    TLeaf* leaf  = iitr.first;
    int    isize = iitr.second;
 
    log << MSG::VERBOSE << "setting TBranch " << leaf->GetBranch()->GetName() << " buffer at " << (void*)bufpos
        << endmsg;
 
    leaf->GetBranch()->SetAddress( (void*)bufpos );
 
    //        //testing
    //        if (leaf->IsA()->InheritsFrom("TLeafI")) {
    //      for (int ievt=0; ievt<5; ievt++) {
    //        leaf->GetBranch()->GetEvent(ievt);
    //        int *idat = (int*)bufpos;
    //        log << MSG::WARNING << leaf->GetName() << ": " << ievt << "   "
    //            << *idat << endmsg;
 
    //      }
    //        }
 
    ts += isize;
 
    bufpos += isize;
  }
 
  if ( totsize != ts ) { log << MSG::ERROR << "buffer size mismatch: " << ts << "  " << totsize << endmsg; }
 
  refpObject = ntup;
 
  return StatusCode::SUCCESS;
}

readData()

StatusCode RootHistCnv::RCWNTupleCnv::readData ( TTree * rtree, INTuple * pObject, long ievt )

overrideprotectedvirtual

Read N tuple data.

Implements RootHistCnv::RNTupleCnv.

Definition at line 659 of file RCWNTupleCnv.cpp.

{
  if ( !rtree ) {
    MsgStream log( msgSvc(), "RCWNTupleCnv::readData" );
    log << MSG::ERROR << "cannot read a disk-buffered N-tuple: its TTree is only built at finalize" << endmsg;
    return StatusCode::FAILURE;
  }
  if ( ievt >= rtree->GetEntries() ) {
    MsgStream log( msgSvc(), "RCWNTupleCnv::readData" );
    log << MSG::ERROR << "no more entries in tree to read. max: " << rtree->GetEntries() << "  current: " << ievt
        << endmsg;
    return StatusCode::FAILURE;
  }
 
  rtree->GetEvent( ievt );
  ievt++;
 
  // copy data from ntup->buffer() to ntup->items()->buffer()
  auto& items = ntup->items();
  std::accumulate( begin( items ), end( items ), const_cast<const char*>( ntup->buffer() ),
                   []( const char* src, INTupleItem* i ) {
                     return src + visit( *i, [src]( auto& item ) { return loadItem( src, item ); } );
                   } );
 
  return StatusCode::SUCCESS;
}

releaseBufferDir()

void RootHistCnv::RCWNTupleCnv::releaseBufferDir ( )

private

Definition at line 625 of file RCWNTupleCnv.cpp.

{
  if ( m_bufferDir.empty() ) return;
  std::lock_guard lock( s_dirsMutex );
  s_claimedDirs.erase( m_bufferDir );
  m_bufferDir.clear();
}

replay()

StatusCode RootHistCnv::RCWNTupleCnv::replay ( IOpaqueAddress * pAddr, INTuple * nt, Buffered & b )

private

Definition at line 553 of file RCWNTupleCnv.cpp.

{
  MsgStream              log( msgSvc(), "RCWNTupleCnv" );
  GlobalDirectoryRestore restore;
  const auto             t0      = std::chrono::steady_clock::now();
  const std::string      desc    = b.desc;
  const fs::path         path    = b.writer->path();
  TDirectory*            pDir    = (TDirectory*)pAddr->ipar()[0];
  TTree*                 tree    = nullptr;
  StatusCode             sc      = StatusCode::SUCCESS;
  std::uint64_t          entries = 0, bytes = 0, stored = 0;
 
  try {
    b.writer->close();
    entries = b.writer->entries();
    bytes   = b.writer->bytes();
    stored  = b.writer->storedBytes();
    b.writer.reset(); // free its write cache before the tree is built
    if ( nt->items().size() != b.layout.items.size() )
      throw std::runtime_error( "N-tuple changed shape after its first write" );
    if ( !pDir ) throw std::runtime_error( "no output directory" );
    pDir->cd();
    sc = createTree( desc, nt, b.layout, tree );
    if ( sc ) {
      const auto& items = nt->items();
      char*       buf   = nt->buffer();
      // the part of a slot not stored in a record is whatever reset() leaves there
      nt->reset();
      std::vector<char> defaults( b.layout.rowBytes );
      for ( size_t k = 0; k < items.size(); ++k )
        std::memcpy( defaults.data() + b.layout.items[k].bufPos, items[k]->buffer(), b.layout.items[k].bufLen );
 
      DiskBuffer::Reader    reader( path );
      std::span<const char> rec;
      while ( reader.next( rec ) ) {
        unpackRow( rec, b.layout, items, buf, defaults.data() );
        tree->Fill();
      }
      if ( reader.entries() != entries )
        throw std::runtime_error( fmt::format( "expected {} entries, found {}", entries, reader.entries() ) );
      if ( tree->Write( "", TObject::kOverwrite ) == 0 ) throw std::runtime_error( "TTree::Write failed" );
    }
  } catch ( const std::exception& e ) {
    log << MSG::ERROR << "Failed to build TTree " << desc << " from disk buffer " << path.string() << ": " << e.what()
        << endmsg;
    sc = StatusCode::FAILURE;
  }
  delete tree;
  nt->setBuffer( nullptr );
 
  std::error_code ec;
  if ( sc ) fs::remove( path, ec );
  {
    std::lock_guard lock( m_mutex );
    m_buffered.erase( nt );
    m_replayed.insert( nt );
    if ( m_buffered.empty() ) {
      fs::remove( m_bufferDir, ec ); // only when empty, i.e. nothing was left behind for inspection
      releaseBufferDir();
    }
  }
  if ( sc ) {
    const double secs = std::chrono::duration<double>( std::chrono::steady_clock::now() - t0 ).count();
    log << MSG::INFO << "Built TTree with ID: " << desc << " \"" << nt->title() << "\" in directory " << pDir->GetPath()
        << " from disk buffer: " << entries << " entries, " << bytes << " bytes (" << stored << " on disk), "
        << fmt::format( "{:.2f}", secs ) << " s" << endmsg;
  }
  return sc;
}

replayed()

bool RootHistCnv::RCWNTupleCnv::replayed ( INTuple * nt )

private

Definition at line 473 of file RCWNTupleCnv.cpp.

{
  std::lock_guard lock( m_mutex );
  return m_replayed.count( nt ) != 0;
}

updateRep()

StatusCode RootHistCnv::RCWNTupleCnv::updateRep ( IOpaqueAddress * pAddr, DataObject * pObj )

override

Write the TTree; for a disk-buffered tuple, first build it from the buffer file.

Definition at line 534 of file RCWNTupleCnv.cpp.

{
  if ( !pAddr ) {
    // never written: the base class books it and writes one default row; keep that on the direct path
    m_bookDirect  = true;
    StatusCode sc = RNTupleCnv::updateRep( pAddr, pObj );
    m_bookDirect  = false;
    return sc;
  }
  INTuple*  nt = dynamic_cast<INTuple*>( pObj );
  Buffered* b  = nt ? buffered( nt ) : nullptr;
  if ( b ) return replay( pAddr, nt, *b );
  // an earlier save() already built and wrote the tree, there is nothing left to do
  if ( nt && replayed( nt ) ) return StatusCode::SUCCESS;
  return RNTupleCnv::updateRep( pAddr, pObj );
}

writeBuffered()

StatusCode RootHistCnv::RCWNTupleCnv::writeBuffered ( Buffered & b, INTuple * nt )

private

Definition at line 513 of file RCWNTupleCnv.cpp.

{
  const auto& items = nt->items();
  if ( items.size() != b.layout.items.size() ) {
    MsgStream log( msgSvc(), "RCWNTupleCnv" );
    log << MSG::ERROR << "N-tuple " << b.desc << " changed shape after its first write" << endmsg;
    return StatusCode::FAILURE;
  }
  try {
    packRow( *b.writer, b.layout, items );
  } catch ( const DiskBuffer::Error& e ) {
    MsgStream log( msgSvc(), "RCWNTupleCnv" );
    log << MSG::ERROR << "Disk buffer write failed: " << e.what() << endmsg;
    return StatusCode::FAILURE;
  }
  nt->reset();
  return StatusCode::SUCCESS;
}

writeData()

StatusCode RootHistCnv::RCWNTupleCnv::writeData ( TTree * rtree, INTuple * pObject )

overrideprotectedvirtual

Write N tuple data.

Implements RootHistCnv::RNTupleCnv.

Definition at line 483 of file RCWNTupleCnv.cpp.

{
  if ( !rtree ) {
    if ( Buffered* b = buffered( nt ) ) return writeBuffered( *b, nt );
    MsgStream log( msgSvc(), "RCWNTupleCnv" );
    if ( replayed( nt ) ) {
      // save() built the tree and closed the buffer; there is nowhere to put this row
      log << MSG::ERROR << "N-tuple \"" << nt->title()
          << "\" was already written from its disk buffer: with NTupleSvc.DiskBuffer a tuple cannot be filled after "
             "it has been saved. Row lost."
          << endmsg;
    } else {
      log << MSG::ERROR << "No TTree and no disk buffer for N-tuple \"" << nt->title() << "\"" << endmsg;
    }
    return StatusCode::FAILURE;
  }
 
  // Fill the tree;
  const auto& items = nt->items();
  std::accumulate( begin( items ), end( items ), nt->buffer(), []( char* dest, const INTupleItem* i ) {
    return dest + visit( *i, [dest]( const auto& item ) { return saveItem( dest, item ); } );
  } );
 
  rtree->Fill();
  nt->reset();
  return StatusCode::SUCCESS;
}

Member Data Documentation

m_bookDirect

bool RootHistCnv::RCWNTupleCnv::m_bookDirect { false }

private

force the columnar path in book()

Definition at line 99 of file RCWNTupleCnv.h.

{ false }; 

m_bufferDir

std::filesystem::path RootHistCnv::RCWNTupleCnv::m_bufferDir

private

Definition at line 97 of file RCWNTupleCnv.h.

m_buffered

std::map<INTuple*, Buffered> RootHistCnv::RCWNTupleCnv::m_buffered

private

Definition at line 95 of file RCWNTupleCnv.h.

m_diskBuffer

bool RootHistCnv::RCWNTupleCnv::m_diskBuffer { false }

private

NTupleSvc.DiskBuffer* of the service owning this stream, read once in initialize().

Definition at line 102 of file RCWNTupleCnv.h.

{ false };

m_diskBufferBlockSize

std::size_t RootHistCnv::RCWNTupleCnv::m_diskBufferBlockSize { 0 }

private

Definition at line 105 of file RCWNTupleCnv.h.

{ 0 };

m_diskBufferDirectory

std::string RootHistCnv::RCWNTupleCnv::m_diskBufferDirectory

private

Definition at line 103 of file RCWNTupleCnv.h.

m_diskBufferLevel

int RootHistCnv::RCWNTupleCnv::m_diskBufferLevel { 0 }

private

Definition at line 104 of file RCWNTupleCnv.h.

{ 0 };

m_mutex

std::mutex RootHistCnv::RCWNTupleCnv::m_mutex

private

guards m_buffered, m_replayed and m_bufferDir

Definition at line 94 of file RCWNTupleCnv.h.

m_nextBufferId

unsigned RootHistCnv::RCWNTupleCnv::m_nextBufferId { 0 }

private

Definition at line 98 of file RCWNTupleCnv.h.

{ 0 };

m_replayed

std::set<INTuple*> RootHistCnv::RCWNTupleCnv::m_replayed

private

tuples whose tree has been built; cannot be written again

Definition at line 96 of file RCWNTupleCnv.h.

---

The documentation for this class was generated from the following files:

• RootHistCnv/src/RCWNTupleCnv.h
• RootHistCnv/src/RCWNTupleCnv.cpp