20 #include "RConfigure.h"
50 ClassImp(TProofBenchRunCPU);
55 TProofBenchRunCPU::TProofBenchRunCPU(TPBHistType *histtype, Int_t nhists,
56 TDirectory* dirproofbench, TProof* proof,
57 TProofNodes* nodes, Long64_t nevents, Int_t ntries,
58 Int_t start, Int_t stop, Int_t step, Int_t draw,
60 : TProofBenchRun(proof, kPROOF_BenchSelCPUDef),
61 fHistType(histtype), fNHists(nhists),
62 fNEvents(nevents), fNTries(ntries), fStart(start), fStop(stop),
63 fStep(step), fDraw(draw), fDebug(debug), fDirProofBench(dirproofbench),
64 fNodes(nodes), fListPerfPlots(0),
65 fCanvas(0), fProfile_perfstat_event(0), fHist_perfstat_event(0),
66 fProfile_perfstat_evtmax(0), fNorm_perfstat_evtmax(0),
67 fProfile_queryresult_event(0), fNorm_queryresult_event(0), fProfile_cpu_eff(0),
68 fProfLegend(0), fNormLegend(0), fName(0)
70 if (TestBit(kInvalidObject)) {
71 Error(
"TProofBenchRunCPU",
"problems validating PROOF session or enabling selector PAR");
77 if (!fNodes) fNodes =
new TProofNodes(fProof);
79 if (stop == -1) fStop = fNodes->GetNWorkersCluster();
81 fListPerfPlots =
new TList;
83 gEnv->SetValue(
"Proof.StatsTrace",1);
84 gStyle->SetOptStat(0);
90 TProofBenchRunCPU::~TProofBenchRunCPU()
94 SafeDelete(fListPerfPlots);
97 SafeDelete(fProfLegend);
98 SafeDelete(fNormLegend);
104 void TProofBenchRunCPU::BuildHistos(Int_t start, Int_t stop, Int_t step, Bool_t nx)
107 Int_t quotient = (stop - start) / step;
108 Int_t ndiv = quotient + 1;
109 Double_t ns_min = start - step/2.;
110 Double_t ns_max = quotient*step + start + step/2.;
112 fProfLegend =
new TLegend(0.1, 0.8, 0.3, 0.9);
113 fNormLegend =
new TLegend(0.7, 0.8, 0.9, 0.9);
115 TString axtitle(
"Active Workers"), namelab(GetName()), sellab(GetSelName());
117 axtitle =
"Active Workers/Node";
118 namelab.Form(
"x_%s", GetName());
120 if (fSelName == kPROOF_BenchSelCPUDef)
121 sellab.Form(
"%s_%s", GetSelName(), GetNameStem().Data());
126 name.Form(
"Prof_%s_PS_MaxEvts_%s", namelab.Data(), sellab.Data());
127 title.Form(
"Profile %s PerfStat Event - %s", namelab.Data(), sellab.Data());
128 fProfile_perfstat_evtmax =
new TProfile(name, title, ndiv, ns_min, ns_max);
129 fProfile_perfstat_evtmax->SetDirectory(fDirProofBench);
130 fProfile_perfstat_evtmax->GetYaxis()->SetTitle(
"Events/sec");
131 fProfile_perfstat_evtmax->GetXaxis()->SetTitle(axtitle);
132 fProfile_perfstat_evtmax->SetMarkerStyle(23);
133 fProfile_perfstat_evtmax->SetMarkerColor(2);
134 if ((o = fListPerfPlots->FindObject(name))) {
135 fListPerfPlots->Remove(o);
138 fListPerfPlots->Add(fProfile_perfstat_evtmax);
139 fProfLegend->AddEntry(fProfile_perfstat_evtmax,
"Maximum");
142 name.Form(
"Prof_%s_PS_Evts_%s", namelab.Data(), sellab.Data());
143 title.Form(
"Profile %s PerfStat Event - %s", namelab.Data(), sellab.Data());
144 fProfile_perfstat_event =
new TProfile(name, title, ndiv, ns_min, ns_max);
145 fProfile_perfstat_event->SetDirectory(fDirProofBench);
146 fProfile_perfstat_event->GetYaxis()->SetTitle(
"Events/sec");
147 fProfile_perfstat_event->GetXaxis()->SetTitle(axtitle);
148 fProfile_perfstat_event->SetMarkerStyle(21);
149 if ((o = fListPerfPlots->FindObject(name))) {
150 fListPerfPlots->Remove(o);
153 fListPerfPlots->Add(fProfile_perfstat_event);
154 fProfLegend->AddEntry(fProfile_perfstat_event,
"Average");
157 name.Form(
"Hist_%s_PS_Evts_%s", namelab.Data(), sellab.Data());
158 title.Form(
"Histogram %s PerfStat Event - %s", namelab.Data(), sellab.Data());
159 fHist_perfstat_event =
new TH2D(name, title, ndiv, ns_min, ns_max, 100, 0, 0);
160 fHist_perfstat_event->SetDirectory(fDirProofBench);
161 fHist_perfstat_event->GetYaxis()->SetTitle(
"Events/sec");
162 fHist_perfstat_event->GetXaxis()->SetTitle(axtitle);
163 fHist_perfstat_event->SetMarkerStyle(7);
164 if ((o = fListPerfPlots->FindObject(name))) {
165 fListPerfPlots->Remove(o);
168 fListPerfPlots->Add(fHist_perfstat_event);
171 name.Form(
"Norm_%s_PS_MaxEvts_%s", namelab.Data(), sellab.Data());
172 title.Form(
"Profile %s Normalized PerfStat Event - %s", namelab.Data(), sellab.Data());
173 fNorm_perfstat_evtmax =
new TProfile(name, title, ndiv, ns_min, ns_max);
174 fNorm_perfstat_evtmax->SetDirectory(fDirProofBench);
175 fNorm_perfstat_evtmax->GetYaxis()->SetTitle(
"Events/sec");
176 fNorm_perfstat_evtmax->GetXaxis()->SetTitle(axtitle);
177 fNorm_perfstat_evtmax->SetMarkerStyle(23);
178 fNorm_perfstat_evtmax->SetMarkerColor(2);
179 if ((o = fListPerfPlots->FindObject(name))) {
180 fListPerfPlots->Remove(o);
183 fListPerfPlots->Add(fNorm_perfstat_evtmax);
184 fNormLegend->AddEntry(fNorm_perfstat_evtmax,
"Maximum");
187 name.Form(
"Prof_%s_QR_Evts_%s", namelab.Data(), sellab.Data());
188 title.Form(
"Profile %s QueryResult Event - %s", namelab.Data(), sellab.Data());
189 fProfile_queryresult_event =
new TProfile(name, title, ndiv, ns_min, ns_max);
190 fProfile_queryresult_event->SetDirectory(fDirProofBench);
191 fProfile_queryresult_event->GetYaxis()->SetTitle(
"Events/sec");
192 fProfile_queryresult_event->GetXaxis()->SetTitle(axtitle);
193 fProfile_queryresult_event->SetMarkerStyle(22);
194 if ((o = fListPerfPlots->FindObject(name))) {
195 fListPerfPlots->Remove(o);
198 fListPerfPlots->Add(fProfile_queryresult_event);
201 name.Form(
"Norm_%s_QR_Evts_%s", namelab.Data(), sellab.Data());
202 title.Form(
"Profile %s Normalized QueryResult Event - %s", namelab.Data(), sellab.Data());
203 fNorm_queryresult_event =
new TProfile(name, title, ndiv, ns_min, ns_max);
204 fNorm_queryresult_event->SetDirectory(fDirProofBench);
205 fNorm_queryresult_event->GetYaxis()->SetTitle(
"Events/sec");
206 fNorm_queryresult_event->GetXaxis()->SetTitle(axtitle);
207 fNorm_queryresult_event->SetMarkerStyle(22);
208 if ((o = fListPerfPlots->FindObject(name))) {
209 fListPerfPlots->Remove(o);
212 fListPerfPlots->Add(fNorm_queryresult_event);
213 fNormLegend->AddEntry(fNorm_queryresult_event,
"Average");
216 name.Form(
"Prof_%s_CPU_eff_%s", namelab.Data(), sellab.Data());
217 title.Form(
"Profile %s CPU efficiency - %s", namelab.Data(), sellab.Data());
218 fProfile_cpu_eff =
new TProfile(name, title, ndiv, ns_min, ns_max);
219 fProfile_cpu_eff->SetDirectory(fDirProofBench);
220 fProfile_cpu_eff->GetYaxis()->SetTitle(
"Efficiency");
221 fProfile_cpu_eff->GetXaxis()->SetTitle(axtitle);
222 fProfile_cpu_eff->SetMarkerStyle(22);
223 if ((o = fListPerfPlots->FindObject(name))) {
224 fListPerfPlots->Remove(o);
227 fListPerfPlots->Add(fProfile_cpu_eff);
244 void TProofBenchRunCPU::Run(Long64_t nevents, Int_t start, Int_t stop,
245 Int_t step, Int_t ntries, Int_t debug, Int_t draw)
248 Error(
"Run",
"Proof not set");
252 nevents = (nevents == -1) ? fNEvents : nevents;
253 start = (start == -1) ? fStart : start;
254 stop = (stop == -1) ? fStop : stop;
255 step = (step == -1) ? fStep : step;
256 ntries = (ntries == -1) ? fNTries : ntries;
257 debug = (debug == -1) ? fDebug : debug;
258 draw = (draw == -1) ? fDraw : draw;
268 Int_t minnworkersanode = fNodes->GetMinWrksPerNode();
269 if (stop > minnworkersanode) stop = minnworkersanode;
273 if (!TClass::GetClass(fSelName)) {
275 if (fSelName == kPROOF_BenchSelCPUDef) {
277 TString par = TString::Format(
"%s/%s%s.par", TROOT::GetEtcDir().Data(), kPROOF_BenchParDir, kPROOF_BenchCPUSelPar);
278 Info(
"Run",
"Uploading '%s' ...", par.Data());
279 if (fProof->UploadPackage(par) != 0) {
280 Error(
"Run",
"problems uploading '%s' - cannot continue", par.Data());
283 Info(
"Run",
"Enabling '%s' ...", kPROOF_BenchCPUSelPar);
284 if (fProof->EnablePackage(kPROOF_BenchCPUSelPar) != 0) {
285 Error(
"Run",
"problems enabling '%s' - cannot continue", kPROOF_BenchCPUSelPar);
289 if (fParList.IsNull()) {
290 Error(
"Run",
"you should load the class '%s' before running the benchmark", fSelName.Data());
295 while (fParList.Tokenize(par, from,
",")) {
296 Info(
"Run",
"Uploading '%s' ...", par.Data());
297 if (fProof->UploadPackage(par) != 0) {
298 Error(
"Run",
"problems uploading '%s' - cannot continue", par.Data());
301 Info(
"Run",
"Enabling '%s' ...", par.Data());
302 if (fProof->EnablePackage(par) != 0) {
303 Error(
"Run",
"problems enabling '%s' - cannot continue", par.Data());
310 if (!TClass::GetClass(fSelName)) {
311 Error(
"Run",
"failed to load '%s'", fSelName.Data());
317 BuildHistos(start, stop, step, nx);
320 if (!fCanvas) fCanvas =
new TCanvas(
"Canvas");
325 fCanvas->Divide(2,1);
327 TString perfstats_name =
"PROOF_PerfStats";
332 Info(
"Run",
"Running CPU-bound tests; %d ~ %d active worker(s)/node,"
333 " every %d worker(s)/node.", start, stop, step);
335 Info(
"Run",
"Running CPU-bound tests; %d ~ %d active worker(s),"
336 " every %d worker(s).", start, stop, step);
341 Int_t nnodes = fNodes->GetNNodes();
342 Int_t ncores = fNodes->GetNCores();
344 Double_t ymi = -1., ymx = -1., emx = -1.;
345 for (Int_t nactive = start; nactive <= stop; nactive += step) {
348 Int_t ncoren = (nactive < ncores) ? nactive : ncores;
354 workers.Form(
"%dx", nactive);
355 nw = fNodes->ActivateWorkers(workers);
357 nw = fNodes->ActivateWorkers(nactive);
360 Error(
"Run",
"could not activate the requested number of"
361 " workers/node on the cluster; skipping the test point"
362 " (%d workers/node)", nactive);
366 for (Int_t j = 0; j < ntries; j++) {
369 Info(
"Run",
"Running CPU-bound tests with %d active worker(s)/node;"
370 " trial %d/%d", nactive, j + 1, ntries);
372 Info(
"Run",
"Running CPU-bound tests with %d active worker(s);"
373 " trial %d/%d", nactive, j + 1, ntries);
378 nevents_all=nevents*nactive*nnodes;
380 nevents_all=nevents*nactive;
384 fProof->Process(fSelName, nevents_all, fSelOption);
386 TList *l = fProof->GetOutputList();
390 if (l) t =
dynamic_cast<TTree*
>(l->FindObject(perfstats_name.Data()));
394 FillPerfStatPerfPlots(t, nactive);
396 TProofPerfAnalysis pfa(t);
397 Double_t pf_eventrate = pfa.GetEvtRateAvgMax();
399 fProfile_perfstat_evtmax->Fill(nactive, pf_eventrate);
401 fProfile_perfstat_evtmax->SetMaximum(emx*1.6);
402 fProfile_perfstat_evtmax->SetMinimum(0.);
403 fProfile_perfstat_evtmax->Draw(
"L");
408 Double_t nert = nx ? pf_eventrate/nactive/nnodes : pf_eventrate/nactive;
409 fNorm_perfstat_evtmax->Fill(nactive, nert);
410 Double_t y1 = fNorm_perfstat_evtmax->GetBinContent(1);
411 Double_t e1 = fNorm_perfstat_evtmax->GetBinError(1);
412 Double_t dy = 5 * e1;
413 if (dy / y1 < 0.2) dy = y1 * 0.2;
414 if (dy > y1) dy = y1*.999999;
415 if (ymi < 0.) ymi = y1 - dy;
416 if (fNorm_perfstat_evtmax->GetBinContent(nactive) < ymi)
417 ymi = fNorm_perfstat_evtmax->GetBinContent(nactive) / 2.;
418 if (ymx < 0.) ymx = y1 + dy;
419 if (fNorm_perfstat_evtmax->GetBinContent(nactive) > ymx)
420 ymx = fNorm_perfstat_evtmax->GetBinContent(nactive) * 1.5;
421 fNorm_perfstat_evtmax->SetMaximum(ymx);
422 fNorm_perfstat_evtmax->SetMinimum(ymi);
423 fCanvas->cd(npad + 1);
424 fNorm_perfstat_evtmax->Draw(
"L");
429 TString newname = TString::Format(
"%s_%s_%dwrks%dthtry", t->GetName(), GetName(), nactive, j);
432 if (debug && fDirProofBench->IsWritable()){
433 TDirectory *curdir = gDirectory;
434 TString dirn = nx ?
"RunCPUx" :
"RunCPU";
435 if (!fDirProofBench->GetDirectory(dirn))
436 fDirProofBench->mkdir(dirn,
"RunCPU results");
437 if (fDirProofBench->cd(dirn)) {
438 t->SetDirectory(fDirProofBench);
442 Warning(
"Run",
"cannot cd to subdirectory '%s' to store the results!", dirn.Data());
449 Warning(
"Run",
"%s: tree not found", perfstats_name.Data());
451 Error(
"Run",
"PROOF output list is empty!");
456 const char *drawopt = t ?
"LSAME" :
"L";
457 TQueryResult *queryresult = fProof->GetQueryResult();
459 queryresult->Print(
"F");
460 TDatime qr_start = queryresult->GetStartTime();
461 TDatime qr_end = queryresult->GetEndTime();
462 Float_t qr_proc = queryresult->GetProcTime();
464 Long64_t qr_entries = queryresult->GetEntries();
467 Double_t qr_eventrate = qr_entries / Double_t(qr_proc);
468 if (qr_eventrate > emx) emx = qr_eventrate;
471 Float_t qr_cpu_eff = -1.;
473 qr_cpu_eff = queryresult->GetUsedCPU() / ncoren / qr_proc ;
474 fProfile_cpu_eff->Fill(nactive, qr_cpu_eff);
475 Printf(
"cpu_eff: %f", qr_cpu_eff);
479 fProfile_queryresult_event->Fill(nactive, qr_eventrate);
481 fProfile_queryresult_event->Draw(drawopt);
485 Double_t nert = nx ? qr_eventrate/nactive/nnodes : qr_eventrate/nactive;
486 fNorm_queryresult_event->Fill(nactive, nert);
488 Double_t y1 = fNorm_queryresult_event->GetBinContent(1);
489 Double_t e1 = fNorm_queryresult_event->GetBinError(1);
490 Double_t dy = 5 * e1;
491 if (dy / y1 < 0.2) dy = y1 * 0.2;
492 if (dy > y1) dy = y1*.999999;
493 if (ymi < 0.) ymi = y1 - dy;
494 if (fNorm_queryresult_event->GetBinContent(nactive) < ymi)
495 ymi = fNorm_queryresult_event->GetBinContent(nactive) / 2.;
496 if (ymx < 0.) ymx = y1 + dy;
497 if (fNorm_queryresult_event->GetBinContent(nactive) > ymx)
498 ymx = fNorm_queryresult_event->GetBinContent(nactive) * 1.5;
499 fNorm_queryresult_event->SetMaximum(ymx);
501 fNorm_queryresult_event->SetMinimum(0.);
503 fNorm_queryresult_event->Draw(drawopt);
506 Warning(
"Run",
"TQueryResult not found!");
515 fProfile_queryresult_event->SetMaximum(1.6*emx);
516 fProfile_queryresult_event->DrawCopy(
"L");
517 fProfile_perfstat_evtmax->DrawCopy(
"LSAME");
519 fCanvas->cd(npad + 1);
520 fNorm_queryresult_event->DrawCopy(
"L");
521 fNorm_perfstat_evtmax->DrawCopy(
"LSAME");
526 if (fDirProofBench && fDirProofBench->IsWritable()){
527 TDirectory *curdir = gDirectory;
528 TString dirn = nx ?
"RunCPUx" :
"RunCPU";
529 if (!fDirProofBench->GetDirectory(dirn))
530 fDirProofBench->mkdir(dirn,
"RunCPU results");
531 if (fDirProofBench->cd(dirn)) {
532 fListPerfPlots->Write(0, kOverwrite);
533 fListPerfPlots->SetOwner(kFALSE);
534 fListPerfPlots->Clear();
536 Warning(
"Run",
"cannot cd to subdirectory '%s' to store the results!", dirn.Data());
544 void TProofBenchRunCPU::FillPerfStatPerfPlots(TTree* t, Int_t nactive)
555 if (!fProfile_perfstat_event){
556 Error(
"FillPerfStatPerfPlots",
"no perfstat profile found");
561 if (!fHist_perfstat_event){
562 Error(
"FillPerfStatPerfPlots",
"no perfstat histogram found");
568 TPerfEvent* pep = &pe;
569 t->SetBranchAddress(
"PerfEvents",&pep);
570 Long64_t entries = t->GetEntries();
572 Double_t event_rate_packet = 0;
574 for (Long64_t k=0; k<entries; k++) {
579 if (pe.fEvtNode.Contains(
"."))
continue;
581 if (pe.fType == TVirtualPerfStats::kPacket){
582 if (pe.fProcTime != 0.0) {
583 event_rate_packet = pe.fEventsProcessed / pe.fProcTime;
584 fHist_perfstat_event->Fill(Double_t(nactive), event_rate_packet);
595 void TProofBenchRunCPU::Print(Option_t* option)
const
597 Printf(
"+++ TProofBenchRunCPU +++++++++++++++++++++++++++++++++++++++++");
598 Printf(
"Name = %s", fName.Data());
599 if (fProof) fProof->Print(option);
600 Printf(
"fHistType = k%s", GetNameStem().Data());
601 Printf(
"fNHists = %d", fNHists);
602 Printf(
"fNEvents = %lld", fNEvents);
603 Printf(
"fNTries = %d", fNTries);
604 Printf(
"fStart = %d", fStart);
605 Printf(
"fStop = %d", fStop);
606 Printf(
"fStep = %d", fStep);
607 Printf(
"fDraw = %d", fDraw);
608 Printf(
"fDebug = %d", fDebug);
610 Printf(
"fDirProofBench = %s", fDirProofBench->GetPath());
611 if (fNodes) fNodes->Print(option);
612 if (fListPerfPlots) fListPerfPlots->Print(option);
614 Printf(
"Performance Canvas: Name = %s Title = %s",
615 fCanvas->GetName(), fCanvas->GetTitle());
616 Printf(
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
622 void TProofBenchRunCPU::DrawPerfPlots()
625 if (!fCanvas) fCanvas =
new TCanvas(
"Canvas");
630 Int_t nprofiles = fListPerfPlots->GetSize();
632 fCanvas->Divide(1,nprofiles);
634 Int_t nside = (Int_t)TMath::Sqrt((Float_t)nprofiles);
635 nside = (nside*nside<nprofiles)?nside+1:nside;
636 fCanvas->Divide(nside,nside);
640 TIter nxt(fListPerfPlots);
642 while ((profile=(TProfile*)(nxt()))){
653 void TProofBenchRunCPU::SetHistType(TPBHistType *histtype)
655 fHistType = histtype;
656 fName.Form(
"%sCPU", GetNameStem().Data());
662 TString TProofBenchRunCPU::GetNameStem()
const
664 TString namestem(
"+++undef+++");
666 switch (fHistType->GetType()) {
667 case TPBHistType::kHist1D:
670 case TPBHistType::kHist2D:
673 case TPBHistType::kHist3D:
676 case TPBHistType::kHistAll:
677 namestem =
"HistAll";
689 Int_t TProofBenchRunCPU::SetParameters()
692 Error(
"SetParameters",
"proof not set; Doing nothing");
696 if (!fHistType) fHistType =
new TPBHistType(TPBHistType::kHist1D);
697 fProof->AddInput(fHistType);
698 fProof->SetParameter(
"PROOF_BenchmarkNHists", fNHists);
699 fProof->SetParameter(
"PROOF_BenchmarkDraw", Int_t(fDraw));
706 Int_t TProofBenchRunCPU::DeleteParameters()
709 Error(
"DeleteParameters",
"proof not set; Doing nothing");
712 if (fProof->GetInputList()) {
713 TObject *type = fProof->GetInputList()->FindObject(
"PROOF_Benchmark_HistType");
714 if (type) fProof->GetInputList()->Remove(type);
716 fProof->DeleteParameters(
"PROOF_BenchmarkNHists");
717 fProof->DeleteParameters(
"PROOF_BenchmarkDraw");