| The Gaudi Framework v32r1 (f65d50dc)

merge.C
Go to the documentation of this file.
1 #ifndef GAUDIROOTCNV_ROOTDATABASEMERGER_H
2 # define GAUDIROOTCNV_ROOTDATABASEMERGER_H
3
4 // C++ includes
5 # include <map>
6 # include <memory>
7 # include <string>
8 # include <vector>
9
10 // Forward declarations
11 class TTree;
12 class TFile;
13 class TBranch;
14
15 /*
16 * Gaudi namespace declaration
17 */
18 namespace Gaudi {
19
20 typedef int MergeStatus;
21 enum MergeStatusEnum { MERGE_ERROR = 0, MERGE_SUCCESS = 1 };
22
23 struct ContainerSection {
24 int start;
25 int length;
26 };
27
33 class RootDatabaseMerger {
34
35 typedef std::vector<ContainerSection> ContainerSections;
36 typedef std::map<std::string, ContainerSections> DatabaseSections;
37
38 DatabaseSections m_sections;
39 std::unique_ptr<TFile> m_output;
40 bool m_treeSections = false;
41
42 union uuid_data {
43 unsigned char buf[16];
44 unsigned short sbuf[8];
45 unsigned int ibuf[4];
46 };
47
48 public:
50 RootDatabaseMerger();
52 virtual ~RootDatabaseMerger();
54 bool exists( const std::string& fid ) const;
56 MergeStatus create( const std::string& fid );
58 MergeStatus attach( const std::string& fid );
60 MergeStatus close();
62 MergeStatus merge( const std::string& fid );
64 void dumpSections();
66 MergeStatus saveSections();
68 MergeStatus createFID();
69
71 MergeStatus copyAllTrees( TFile* source );
73 MergeStatus copyTree( TFile* source, const std::string& name );
75 MergeStatus copyBranch( TTree* src_tree, TTree* out_tree, const std::string& name );
77 MergeStatus copyRefs( TFile* source, const std::string& name );
79 MergeStatus addSections( TTree* in, TTree* out );
80 };
81 } // End namespace Gaudi
82
83 #endif // GAUDIROOTCNV_ROOTDATABASEMERGER_H
84
85 #ifndef GAUDIROOTCNV_ROOTDATABASEMERGER_H
86 # include "RootDatabaseMerger.h"
87 #endif // GAUDIROOTCNV_ROOTDATABASEMERGER_H
88 #include "TBranch.h"
89 #include "TFile.h"
90 #include "TInterpreter.h"
91 #include "TKey.h"
92 #include "TLeaf.h"
93 #include "TROOT.h"
94 #include "TSystem.h"
95 #include "TTree.h"
96 #include "TTreeCloner.h"
97 #include "TUUID.h"
98 #ifdef _WIN32
99 #else
100 # include <libgen.h>
101 #endif
102
103 using namespace Gaudi;
104 using namespace std;
105
106 namespace {
107 static bool s_dbg = true;
108 }
109
111 RootDatabaseMerger::RootDatabaseMerger() = default;
112
114 RootDatabaseMerger::~RootDatabaseMerger() { close(); }
115
117 bool RootDatabaseMerger::exists( const std::string& fid ) const {
118 Bool_t result = gSystem->AccessPathName( fid.c_str(), kFileExists );
119 // if ( s_dbg ) ::printf("File %s %s!\n",fid.c_str(),result == kFALSE ? "EXISTS" : "DOES NOT EXIST");
120 return result == kFALSE;
121 }
122
124 MergeStatus RootDatabaseMerger::attach( const string& file_id ) {
125 const char* fid = file_id.c_str();
126 if ( m_output ) {
127 ::printf( "+++ Another output file %s is already open. Request denied.\n", m_output->GetName() );
128 return MERGE_ERROR;
129 } else if ( !exists( file_id ) ) {
130 ::printf( "+++ Cannot attach output file %s --- file does not exist.\n", fid );
131 return MERGE_ERROR;
132 }
133 m_output.reset( TFile::Open( fid, "UPDATE" ) );
134 if ( m_output && !m_output->IsZombie() ) {
135 // if ( s_dbg ) ::printf("+++ Update output file %s.\n",fid);
136 // if ( s_dbg ) ::printf("+++ Update output file.\n");
137 return MERGE_SUCCESS;
138 }
139 ::printf( "+++ Failed to open new output file %s.\n", fid );
140 return MERGE_ERROR;
141 }
142
144 MergeStatus RootDatabaseMerger::create( const string& fid ) {
145 if ( m_output ) {
146 ::printf( "+++ Another output file %s is already open. Request denied.\n", m_output->GetName() );
147 return MERGE_ERROR;
148 } else if ( exists( fid ) ) {
149 ::printf( "+++ Cannot create output file %s --- file already exists.\n", fid.c_str() );
150 return MERGE_ERROR;
151 }
152 m_output.reset( TFile::Open( fid.c_str(), "RECREATE" ) );
153 if ( m_output && !m_output->IsZombie() ) {
154 TTree* t1 = new TTree( "Sections", "Root Section data" );
155 TTree* t2 = new TTree( "Refs", "Root Section data" );
156 if ( t1 && t2 ) {
157 t1->Branch( "Sections", 0, "Sections/C" );
158 t2->Branch( "Links", 0, "Links/C" );
159 t2->Branch( "Params", 0, "Params/C" );
160 t2->Branch( "Databases", 0, "Databases/C" );
161 t2->Branch( "Containers", 0, "Containers/C" );
162 if ( s_dbg ) ::printf( "+++ Opened new output file %s.\n", fid.c_str() );
163 return MERGE_SUCCESS;
164 }
165 }
166 ::printf( "+++ Failed to open new output file %s.\n", fid.c_str() );
167 return MERGE_ERROR;
168 }
169
171 MergeStatus RootDatabaseMerger::createFID() {
172 static const char* fmt = "FID=%08lX-%04hX-%04hX-%02hhX%02hhX-%02hhX%02hhX%02hhX%02hhX%02hhX%02hhX";
173 if ( m_output ) {
174 TTree* t = (TTree*)m_output->Get( "Refs" );
175 if ( t ) {
176 uuid_data d;
177 char text[256];
178 TUUID uuid;
179 TBranch* b = t->GetBranch( "Params" );
180 if ( b ) {
181 uuid.GetUUID( d.buf );
182 sprintf( text, fmt, d.ibuf[0], d.sbuf[2], d.sbuf[3], d.buf[8], d.buf[9], d.buf[10], d.buf[11], d.buf[12],
183 d.buf[13], d.buf[14], d.buf[15] );
184 b->SetAddress( text );
185 b->Fill();
186 t->Write();
187 if ( s_dbg ) ::printf( "+++ Added new GUID %s to merge file.\n", text );
188 return MERGE_SUCCESS;
189 }
190 }
191 }
192 ::printf( "+++ Failed to add new GUID to merge file.\n" );
193 return MERGE_ERROR;
194 }
195
197 MergeStatus RootDatabaseMerger::close() {
198 if ( m_output ) {
199 // if ( s_dbg ) ::printf("+++ Closing merge file: %s.\n",m_output->GetName());
200 if ( s_dbg ) ::printf( "+++ Closing merge file.\n" );
201 m_output->Write();
202 m_output->Purge();
203 if ( s_dbg ) m_output->ls();
204 m_output->Close();
205 m_output.reset();
206 }
207 return MERGE_SUCCESS;
208 }
209
211 MergeStatus RootDatabaseMerger::saveSections() {
212 if ( m_output ) {
213 int nb, total = 0, nbytes = 0;
214 char text[1024];
215 TTree* t = (TTree*)m_output->Get( "Sections" );
216 if ( t ) {
217 TBranch* b = t->GetBranch( "Sections" );
218 if ( b ) {
219 b->SetAddress( text );
220 for ( const auto& i : m_sections ) {
221 string cont = i.first;
222 for ( const auto& j : i.second ) {
223 ::sprintf( text, "[CNT=%s][START=%d][LEN=%d]", cont.c_str(), j.start, j.length );
224 nb = t->Fill();
225 ++total;
226 if ( nb > 0 )
227 nbytes += nb;
228 else
229 ::printf( "+++ Failed to update Sections tree with new entries. [WRITE_ERROR]\n" );
230 }
231 }
232 ::sprintf( text, "[END-OF-SECTION]" );
233 nb = t->Fill();
234 ++total;
235 if ( nb > 0 )
236 nbytes += nb;
237 else
238 ::printf( "+++ Failed to update Sections branch with new entries. [WRITE_ERROR]\n" );
239 t->Write();
240 if ( s_dbg ) ::printf( "+++ Added %d Sections entries with %d bytes in total.\n", total, nbytes );
241 return MERGE_SUCCESS;
242 }
243 ::printf( "+++ Failed to update Sections tree with new entries. [NO_OUTPUT_BRANCH]\n" );
244 return MERGE_ERROR;
245 }
246 ::printf( "+++ Failed to update Sections tree with new entries. [NO_OUTPUT_TREE]\n" );
247 return MERGE_ERROR;
248 }
249 ::printf( "+++ Failed to update Sections tree with new entries. [NO_OUTPUT_FILE]\n" );
250 return MERGE_ERROR;
251 }
252
254 void RootDatabaseMerger::dumpSections() {
255 for ( const auto& i : m_sections ) {
256 int cnt = 0;
257 string prefix = i.first;
258 for ( const auto& j : i.second ) {
259 char text[1024];
260 ::sprintf( text, "['%s'][%d]", prefix.c_str(), cnt++ );
261 if ( s_dbg ) {
262 ::printf( "+++ section %-55s Start:%8d ... %8d [%d entries]\n", text, j.start, j.start + j.length, j.length );
263 }
264 }
265 }
266 }
267
269 MergeStatus RootDatabaseMerger::merge( const string& fid ) {
270 if ( m_output ) {
271 std::unique_ptr<TFile> source{TFile::Open( fid.c_str() )};
272 if ( source && !source->IsZombie() ) {
273 size_t idx = fid.rfind( '/' );
274 ::printf( "+++ Start merging input file:%s\n",
275 idx != string::npos ? fid.substr( idx + 1 ).c_str() : fid.c_str() );
276 if ( copyAllTrees( source.get() ) == MERGE_SUCCESS ) {
277 if ( copyRefs( source.get(), "Refs" ) == MERGE_SUCCESS ) {
278 source->Close();
279 return MERGE_SUCCESS;
280 }
281 }
282 }
283 ::printf( "+++ Cannot open input file:%s\n", source->GetName() );
284 m_output->cd();
285 return MERGE_ERROR;
286 }
287 ::printf( "+++ No valid output file present. Merge request refused for fid:%s.\n", fid.c_str() );
288 return MERGE_ERROR;
289 }
290
292 MergeStatus RootDatabaseMerger::addSections( TTree* in, TTree* out ) {
293 if ( m_treeSections ) {
294 ContainerSection s;
295 s.start = (int)( out ? out->GetEntries() : 0 );
296 s.length = (int)in->GetEntries();
297 m_sections[in->GetName()].push_back( s );
298 return MERGE_SUCCESS;
299 }
300
301 TObjArray* a_in = in->GetListOfBranches();
302 for ( int i = 0, n = a_in->GetLast(); i < n; ++i ) {
303 TBranch* b_in = (TBranch*)a_in->At( i );
304 TBranch* b_out = out ? out->GetBranch( b_in->GetName() ) : 0;
305 if ( !out || b_out ) {
306 ContainerSection s;
307 s.start = (int)( b_out ? b_out->GetEntries() : 0 );
308 s.length = (int)b_in->GetEntries();
309 m_sections[b_in->GetName()].push_back( s );
310 continue;
311 }
312 ::printf( "+++ Cannot merge incompatible branches:%s.\n", b_in->GetName() );
313 return MERGE_ERROR;
314 }
315 return MERGE_SUCCESS;
316 }
317
319 MergeStatus RootDatabaseMerger::copyBranch( TTree* src_tree, TTree* out_tree, const string& name ) {
320 char text[4096];
321 TBranch* s = src_tree->GetBranch( name.c_str() );
322 TBranch* o = out_tree->GetBranch( name.c_str() );
323 if ( s && o ) {
324 s->SetAddress( text );
325 o->SetAddress( text );
326 for ( Long64_t i = 0, n = s->GetEntries(); i < n; ++i ) {
327 s->GetEntry( i );
328 o->Fill();
329 }
330 return MERGE_SUCCESS;
331 }
332 return MERGE_ERROR;
333 }
334
336 MergeStatus RootDatabaseMerger::copyAllTrees( TFile* source ) {
337 TIter nextkey( source->GetListOfKeys() );
338 // m_treeSections = true;
339 for ( TKey* key = (TKey*)nextkey(); key; key = (TKey*)nextkey() ) {
340 const char* classname = key->GetClassName();
341 TClass* cl = gROOT->GetClass( classname );
342 if ( !cl ) continue;
343 if ( cl->InheritsFrom( "TTree" ) ) {
344 string name = key->GetName();
345 if ( name == "Refs" ) continue;
346 m_treeSections = 0 == ::strncmp( key->GetName(), "<local>_", 7 );
347 printf( "+++ Copy Tree:%s %d\n", name.c_str(), int( m_treeSections ) );
348 if ( copyTree( source, name ) != MERGE_SUCCESS ) {
349 m_treeSections = false;
350 return MERGE_ERROR;
351 }
352 }
353 }
354 m_treeSections = false;
355 return MERGE_SUCCESS;
356 }
357
359 MergeStatus RootDatabaseMerger::copyTree( TFile* source, const string& name ) {
360 TTree* src_tree = (TTree*)source->Get( name.c_str() );
361 if ( src_tree ) {
362 Long64_t src_entries = src_tree->GetEntries();
363 TTree* out_tree = (TTree*)m_output->Get( name.c_str() );
364 m_output->cd();
365 if ( 0 == src_tree->GetEntries() ) { src_tree->SetEntries( 1 ); }
366 addSections( src_tree, out_tree );
367 if ( !out_tree ) {
368 out_tree = src_tree->CloneTree( -1, "fast" );
369 if ( s_dbg ) ::printf( "+++ Created new Tree %s.\n", out_tree->GetName() );
370 out_tree->Write();
371 delete out_tree;
372 return MERGE_SUCCESS;
373 }
374 m_output->GetObject( name.c_str(), out_tree );
375 TTreeCloner cloner( src_tree, out_tree, "fast" );
376 if ( cloner.IsValid() ) {
377 Long64_t out_entries = out_tree->GetEntries();
378 out_tree->SetEntries( out_entries + src_entries );
379 Bool_t res = cloner.Exec();
380 if ( s_dbg ) ::printf( "+++ Merged tree: %s res=%d\n", out_tree->GetName(), res );
381 out_tree->Write();
382 delete out_tree;
383 return MERGE_SUCCESS;
384 } else {
385 // Fast cloning is not possible for this input TTree.
386 // ... see TTree::CloneTree for example of recovery code ...
387 ::printf( "+++ Got a tree where fast cloning is not possible -- operation failed.\n" );
388 return MERGE_ERROR;
389 }
390 #if 0
391 Long64_t nb = out_tree->CopyEntries(src_tree,-1,"fast");
392 Long64_t out_entries = out_tree->GetEntries();
393 out_tree->SetEntries(out_entries+src_entries);
394 out_tree->Write();
395 if ( s_dbg ) ::printf("+++ Merged tree: %s res=%lld\n",out_tree->GetName(),nb);
396 return MERGE_SUCCESS;
397 #endif
398 }
399 return MERGE_ERROR;
400 }
401
403 MergeStatus RootDatabaseMerger::copyRefs( TFile* source, const string& name ) {
404 TTree* src_tree = (TTree*)source->Get( name.c_str() );
405 if ( src_tree ) {
406 TTree* out_tree = (TTree*)m_output->Get( name.c_str() );
407 if ( out_tree ) {
408 addSections( src_tree, out_tree );
409 copyBranch( src_tree, out_tree, "Links" );
410 copyBranch( src_tree, out_tree, "Params" );
411 copyBranch( src_tree, out_tree, "Containers" );
412 copyBranch( src_tree, out_tree, "Databases" );
413 out_tree->Write();
414 return MERGE_SUCCESS;
415 }
416 }
417 return MERGE_ERROR;
418 }
419
420 int merge( const char* target, const char* source, bool fixup = false, bool dbg = true ) {
421 s_dbg = dbg;
422 // s_dbg = true;
423 #if 0
424 static bool first = true;
425 if ( first ) {
426 first = false;
427 gSystem->Load("libCintex");
428 gInterpreter->ProcessLine("Cintex::Enable()");
429 //gSystem->Load("libGaudiKernelDict");
430 //gSystem->Load("libGaudiExamplesDict");
431 }
432 #endif
433 // printf("+++ Target:%s\n+++ Source file:%s Fixup:%s Dbg:%s\n",
434 // target,source, fixup ? "YES" : "NO",s_dbg ? "YES" : "NO");
435 RootDatabaseMerger m;
436 MergeStatus ret = m.exists( target ) ? m.attach( target ) : m.create( target );
437 if ( ret == MERGE_SUCCESS ) {
438 ret = m.merge( source );
439 if ( ret == MERGE_SUCCESS ) {
440 m.dumpSections();
441 if ( fixup ) m.createFID();
442 m.saveSections();
443 return m.close();
444 }
445 }
446 ::printf( "+++ Cannot open output file:%s\n", target );
447 return 0;
448 }
Gaudi::RootDatabaseMerger::RootDatabaseMerger
RootDatabaseMerger()
Standard constructor.
Gaudi::RootDatabaseMerger::copyAllTrees
MergeStatus copyAllTrees(TFile *source)
Copy all data trees from the input file to the output file.
Definition: merge.C:336
Gaudi::RootDatabaseMerger::uuid_data::ibuf
unsigned int ibuf[4]
Definition: merge.C:45
gaudiComponentHelp.name
name
Definition: gaudiComponentHelp.py:41
Gaudi::RootDatabaseMerger::addSections
MergeStatus addSections(TTree *in, TTree *out)
Add section information for the next merge step.
Definition: merge.C:292
Gaudi::RootDatabaseMerger::create
MergeStatus create(const std::string &fid)
Create new output file.
Definition: merge.C:144
std::string::rfind
T rfind(T... args)
Gaudi::RootDatabaseMerger::saveSections
MergeStatus saveSections()
Save new sections to the output file.
Definition: merge.C:211
merge
int merge(const char *target, const char *source, bool fixup=false, bool dbg=true)
Definition: merge.C:420
std
STL namespace.
Gaudi::RootDatabaseMerger
Definition: merge.C:33
Gaudi::RootDatabaseMerger::uuid_data
Definition: merge.C:42
Gaudi::RootDatabaseMerger::exists
bool exists(const std::string &fid) const
Check if a database exists.
Definition: merge.C:117
std::map< std::string, ContainerSections >
std::string
STL class.
GaudiPluginService.cpluginsvc.n
n
Definition: cpluginsvc.py:210
Gaudi::MergeStatusEnum
MergeStatusEnum
Definition: merge.C:21
Gaudi::MERGE_SUCCESS
Definition: merge.C:21
Gaudi::RootDatabaseMerger::m_output
std::unique_ptr< TFile > m_output
Definition: merge.C:39
Gaudi::RootDatabaseMerger::copyTree
MergeStatus copyTree(TFile *source, const std::string &name)
Copy one single tree from the input file to the output file.
Definition: merge.C:359
Gaudi::RootDatabaseMerger::merge
MergeStatus merge(const std::string &fid)
Merge new input to existing output.
Definition: merge.C:269
Gaudi::RootDatabaseMerger::createFID
MergeStatus createFID()
Create and add new FID to the newly merged file.
Definition: merge.C:171
Gaudi::ContainerSection::start
int start
Definition: merge.C:24
Gaudi::RootDatabaseMerger::~RootDatabaseMerger
virtual ~RootDatabaseMerger()
Default destructor.
Definition: merge.C:114
Gaudi::RootDatabaseMerger::close
MergeStatus close()
Close output file.
Definition: merge.C:197
Gaudi::Units::m
constexpr double m
Definition: SystemOfUnits.h:92
Gaudi::RootDatabaseMerger::dumpSections
void dumpSections()
Dump collected database sections.
Definition: merge.C:254
Gaudi::RootDatabaseMerger::uuid_data::sbuf
unsigned short sbuf[8]
Definition: merge.C:44
Gaudi::RootDatabaseMerger::DatabaseSections
std::map< std::string, ContainerSections > DatabaseSections
Definition: merge.C:36
gaudirun.prefix
string prefix
Definition: gaudirun.py:331
Gaudi::RootDatabaseMerger::ContainerSections
std::vector< ContainerSection > ContainerSections
Definition: merge.C:35
Gaudi::MERGE_ERROR
Definition: merge.C:21
std::string::length
T length(T... args)
std::vector
STL class.
Gaudi::RootDatabaseMerger::attach
MergeStatus attach(const std::string &fid)
Attach to existing output file for further merging.
Definition: merge.C:124
std::unique_ptr< TFile >
Gaudi::RootDatabaseMerger::copyBranch
MergeStatus copyBranch(TTree *src_tree, TTree *out_tree, const std::string &name)
Copy single reference branch.
Definition: merge.C:319
Gaudi::RootDatabaseMerger::uuid_data::buf
unsigned char buf[16]
Definition: merge.C:43
std::string::c_str
T c_str(T... args)
gaudirun.s
string s
Definition: gaudirun.py:316
std::string::substr
T substr(T... args)
Gaudi::ContainerSection::length
int length
Definition: merge.C:25
Gaudi::RootDatabaseMerger::copyRefs
MergeStatus copyRefs(TFile *source, const std::string &name)
Copy one single tree from the input file to the output file.
Definition: merge.C:403
Gaudi::RootDatabaseMerger::m_treeSections
bool m_treeSections
Definition: merge.C:40
Gaudi::MergeStatus
int MergeStatus
Definition: merge.C:20
Gaudi::RootDatabaseMerger::m_sections
DatabaseSections m_sections
Definition: merge.C:38
Gaudi::ContainerSection
Definition: merge.C:23
std::sprintf
T sprintf(T... args)
Gaudi
Header file for std:chrono::duration-based Counters.
Definition: __init__.py:1