46 fBatchEnv =
new TEnv(
".KVBatchrc");
50 fWorkDirInit = fWorkDirEnd = 0;
53 fProofMode = EProofMode::None;
57 fUseBaseClassSubmitTask =
kFALSE;
65 KVDataAnalyser::~KVDataAnalyser()
86 fUserClassOptions =
"";
88 fBatchSystem =
nullptr;
91 fProofMode = EProofMode::None;
111 if (CheckTaskVariables()) {
114 fBatchSystem->SubmitTask(
this);
117 if (
BatchMode() && fBatchSystem && !fParent) fBatchSystem->Print(
"log");
118 if (!RunningInLaunchDirectory() && fParent) {
122 ScanWorkingDirectory(&fWorkDirInit);
124 if (!PreSubmitCheck())
return;
134 if (fUseBaseClassSubmitTask)
139 if (!RunningInLaunchDirectory() && fParent) {
143 ScanWorkingDirectory(&fWorkDirEnd);
146 CopyAnalysisResultsToLaunchDirectory();
148 if (
BatchMode() && fBatchSystem && !fParent) {
182 if (NeedToChooseWhatToDo())
184 else if (NeedToChooseWhatToAnalyse())
185 ChooseWhatToAnalyse();
208 if (
BatchMode()) ReadBatchEnvFile(
Form(
".%s", GetBatchName()));
212 if (fTask->WithUserClass() && fUserClass !=
ClassName()) {
214 if (fUserClass ==
"") {
218 if (!CheckIfUserClassIsValid(fUserClassAlternativeBaseClass)) {
219 cout <<
"============> Warning <=============" << endl;
220 cout << GetUserClass() <<
" is not a valid " << fTask->GetUserBaseClass() << endl;
221 cout <<
"Analysis aborted." << endl;
222 cout <<
"====================================" << endl;
228 if (nbEventToRead < 0) {
229 ChooseNbEventToRead();
241 if (NeedToChooseWhatToDo()) {
245 if (NeedToChooseWhatToAnalyse()) {
298 fIncludes +=
id.
Data();
300 if (!curIncDir.
Contains(
id.Data())) {
301 cout <<
"Include path \"" <<
id.Data() <<
"\" added." << endl;
336 while (!slib.
End() && !loaded) {
339 Info(
"SetUserLibraries",
"%s already load",
id.Data());
346 Info(
"SetUserLibraries",
"Library \"%s\"added.",
id.Data());
349 fLibraries +=
id.Data();
365 SetNbEventToRead(-1);
366 while (nbEventToRead < 0) {
367 cout <<
"Give the number of events to read [<RET>=all]:" << endl;
380 Data() <<
"\" is not a number. Please retry." << endl;
417 while (!fUserClass.Length()) {
418 cout <<
"Give the name of the analysis class derived from " << fTask->GetUserBaseClass() <<
":" << endl;
419 fUserClass.ReadLine(cin);
477 TObject* o = GetInstanceOfUserClass(alternative_base_class);
525 fBatchSystem =
nullptr;
564 if (!fTask->CheckUserBaseClassIsLoaded())
return 0x0;
567 TPluginHandler* ph =
gROOT->GetPluginManager()->FindHandler(fTask->GetUserBaseClass(), fUserClass.Data());
575 if (DoUserClassFilesExist()) {
578 cmd.
Form(
".L %s%s", fUserClassImp.Data(), GetACliCMode());
581 cl =
gROOT->GetClass(fUserClass.Data());
585 Info(
"GetInstanceOfUserClass",
"Class %s is unknown and no source files available",
591 Info(
"GetInstanceOfUserClass",
"Compilation of class %s failed. Correct the mistakes and try again",
595 if (!
cl->GetBaseClass(fTask->GetUserBaseClass())) {
597 if (alternative_base_class ==
"") {
598 Info(
"GetInstanceOfUserClass",
"Class %s does not inherit from correct base class (%s), or compilation of class %s failed. Correct the mistakes and try again",
599 fUserClass.Data(), fTask->GetUserBaseClass(), fUserClass.Data());
604 bool got_good_base =
false;
606 alternative_base_class.
Begin(
",");
607 while (!alternative_base_class.
End()) {
608 good_base = alternative_base_class.
Next(
kTRUE);
609 if (
cl->GetBaseClass(good_base)) {
610 got_good_base =
true;
615 Info(
"GetInstanceOfUserClass",
"Class %s inherits from alternative base class %s: OK!",
616 fUserClass.Data(), good_base.
Data());
619 Info(
"GetInstanceOfUserClass",
"Class %s does not inherit from task-defined base class (%s) or any provided alternative base classes (%s), or compilation of class %s failed. Correct the mistakes and try again",
620 fUserClass.Data(), fTask->GetUserBaseClass(), alternative_base_class.
Data(), fUserClass.Data());
629 Info(
"GetInstanceOfUserClass",
"Found plugin handler for class %s",
634 Info(
"GetInstanceOfUserClass",
"KVBase::LoadPlugin failed for %s", fUserClass.Data());
640 if (obj->
InheritsFrom(fTask->GetUserBaseClass()))
return obj;
641 Info(
"GetInstanceOfUserClass",
"%s does not inherit from %s", fUserClass.Data(), fTask->GetUserBaseClass());
645 Info(
"GetInstanceOfUserClass",
"constructor not OK for %s", fUserClass.Data());
668 fUserClassIsOK = CheckIfUserClassIsValid();
671 fUserClassIsOK =
kTRUE;
692 fBatchEnv =
new TEnv(
Form(
".%s", jobname));
694 fBatchEnv->SetValue(
"BatchSystem", fBatchSystem->GetName());
695 fBatchSystem->WriteBatchEnvFile(fBatchEnv);
697 fBatchEnv->SetValue(
"AnalysisTask", fTask->GetType());
699 if (fTask->WithUserClass()) {
700 fBatchEnv->SetValue(
"UserClass", GetUserClass());
701 if (fUserClassImp ==
"" || fUserClassDec ==
"") {
702 if (!DoUserClassFilesExist()) {
703 Warning(
"WriteBatchEnvFile",
"Source files for user class %s do not exist. Job will not work.",
707 fBatchEnv->SetValue(
"UserClassOptions", fUserClassOptions);
708 fBatchEnv->SetValue(
"UserClassImp", fUserClassImp);
709 fBatchEnv->SetValue(
"UserClassDec", fUserClassDec);
713 if (fUserClassOptions !=
"") fBatchEnv->SetValue(
"UserClassOptions", fUserClassOptions);
715 fBatchEnv->SetValue(
"NbToRead", (
Double_t)nbEventToRead);
717 if (fIncludes.Length()) {
718 fBatchEnv->SetValue(
"UserIncludes", fIncludes.Data());
720 if (fLibraries.Length()) {
721 fBatchEnv->SetValue(
"UserLibraries", fLibraries.Data());
724 if (save) fBatchEnv->SaveLevel(
kEnvUser);
744 fBatchEnv =
new TEnv(filename);
745 KVString val = fBatchEnv->GetValue(
"AnalysisTask",
"");
755 Error(
"ReadBatchEnvFile",
"Name of analysis task not given");
759 Error(
"ReadBatchEnvFile",
"Analysis task \"%s\"not found for dataset %s",
764 nbEventToRead = (
Long64_t)fBatchEnv->GetValue(
"NbToRead", -1);
765 SetUserIncludes(fBatchEnv->GetValue(
"UserIncludes",
""));
766 SetUserLibraries(fBatchEnv->GetValue(
"UserLibraries",
""));
769 if (strcmp(fBatchEnv->GetValue(
"BatchSystem",
""),
"")) {
773 fBatchSystem->SetAnalyser(
this);
777 if (fTask->WithUserClass()) {
778 fUserClass = fBatchEnv->GetValue(
"UserClass",
"");
779 if (fUserClass ==
"") {
780 Error(
"ReadBatchEnvFile",
"Name of user class not given");
783 fUserClassOptions = fBatchEnv->GetValue(
"UserClassOptions",
"");
784 fUserClassImp = fBatchEnv->GetValue(
"UserClassImp",
"");
785 if (fUserClassImp ==
"") {
786 Error(
"ReadBatchEnvFile",
"Name of user class implementation file not given");
789 fUserClassDec = fBatchEnv->GetValue(
"UserClassDec",
"");
790 if (fUserClass ==
"") {
791 Error(
"ReadBatchEnvFile",
"Name of user class header file not given");
794 fUserClassAlternativeBaseClass = fBatchEnv->GetValue(
"UserClassAlternativeBaseClass",
"");
798 if (!RunningInLaunchDirectory()) {
813 fUserClassOptions = fBatchEnv->GetValue(
"UserClassOptions",
"");
866 KVString task_data_analyser = fTask->GetDataAnalyser();
867 Info(
"SubmitTask",
"fTask->GetDataAnalyser()=%s", task_data_analyser.
Data());
868 unique_ptr<KVDataAnalyser> the_analyser;
869 if (task_data_analyser ==
"UserClass") {
874 the_analyser.reset(GetAnalyser(fTask->GetDataAnalyser()));
876 if (!the_analyser.get())
877 Fatal(
"SubmitTask",
"the_analyser is 0x0, go to crash");
879 set_up_analyser_for_task(the_analyser.get());
881 if (fTask->WithUserClass()) {
882 the_analyser->SetUserClass(GetUserClass(),
kFALSE);
883 the_analyser->SetUserClassOptions(fUserClassOptions);
885 else if (strcmp(fTask->GetUserBaseClass(),
"")) the_analyser->SetUserClass(fTask->GetUserBaseClass(),
kFALSE);
890 the_analyser->CheckTaskVariables();
891 if (!fChoseRunMode) ChooseRunningMode();
894 the_analyser->SetBatchName(GetBatchName());
895 the_analyser->SetBatchSystem(fBatchSystem);
896 the_analyser->SetProofMode(GetProofMode());
926 if (fUserClass.Length()) tmp.
ReplaceAll(
"$UserClass", fUserClass.Data());
927 else if (fTask) tmp.
ReplaceAll(
"$UserClass", fTask->GetDataAnalyser());
938 static TString keywords =
"Will be expanded: $Date, $User, $UserClass";
991 if (!
BatchMode() || !fBatchEnv)
return "";
997 TString launchDir = GetLaunchDirectory();
1013 TEnv stats(GetBatchStatusFileName());
1014 stats.
SetValue(
"TotalEvents", totev);
1015 stats.
SetValue(
"EventsRead", evread);
1033 TString stats = GetBatchStatusFileName();
1049 return (!(nevents % fTask->GetStatusUpdateInterval()) && nevents);
1061 cout <<
" +++ " << nevents <<
" events processed +++ " << endl;
1064 cout <<
" ------------- Process infos -------------" << endl;
1065 printf(
" CpuSys = %f s. CpuUser = %f s. ResMem = %f MB VirtMem = %f MB\n",
1085 fChoseRunMode =
kTRUE;
1088 cout << endl <<
"Run in Interactive or Batch mode (I or B) ? : ";
1091 while (tmp !=
"i" && tmp !=
"I" && tmp !=
"b" && tmp !=
"B");
1098 cout << endl <<
"Choose the batch system to use : " << endl;
1101 cout <<
"(enter a number) : " << endl;
1106 fBatchSystem->
Clear();
1120 if (*ls)
delete (*ls);
1139 if (!fWorkDirInit || !fWorkDirEnd)
return;
1141 TIter next_new_file(fWorkDirEnd);
1143 while ((
file = next_new_file())) {
1144 if (!fWorkDirInit->FindObject(
file->GetName())) {
1154 Info(
"CopyAnalysisResultsToLaunchDirectory",
"Copying analysis results file :\n%s ---> %s",
1158 Info(
"CopyAnalysisResultsToLaunchDirectory",
"File copied correctly");
1161 Info(
"CopyAnalysisResultsToLaunchDirectory",
" **** ERROR copying file !!! ");
1182 tt->GetUserInfo()->Add(
new TEnv());
1183 TEnv* kvenv = (
TEnv*)
tt->GetUserInfo()->FindObject(
"TEnv");
1199 #ifdef WITH_BZR_INFOS
1200 kvenv->
SetValue(
"KVBase::bzrRevisionId()", KVBase::bzrRevisionId(),
kEnvUser);
1201 kvenv->
SetValue(
"KVBase::bzrRevisionDate()", KVBase::bzrRevisionDate(),
kEnvUser);
1202 kvenv->
SetValue(
"KVBase::bzrBranchNick()", KVBase::bzrBranchNick(),
kEnvUser);
1203 kvenv->
SetValue(
"KVBase::bzrRevisionNumber()", KVBase::bzrRevisionNumber());
1204 kvenv->
SetValue(
"KVBase::bzrIsBranchClean()", KVBase::bzrIsBranchClean());
1206 #ifdef WITH_GIT_INFOS
1255 ::Warning(
"KVDataAnalyser::RunAnalyser",
"No plugin %s found for KVDataAnalyser",
1300 jdl->
SetValue(
"AnalysisTask", fTask->GetType());
1301 jdl->
SetValue(
"PROOFMode", GetProofMode());
void AssignAndDelete(TString &target, char *tobedeleted)
KVBatchSystemManager * gBatchSystemManager
KVBatchSystem * gBatchSystem
KVDataAnalyser * gDataAnalyser
KVDataSetManager * gDataSetManager
Defines macros for standard error messages.
ClassImp(KVPartitionList) void KVPartitionList
Initialisation.
KVString GetValue(KVString &l, char c)
char * Form(const char *fmt,...)
R__EXTERN TSystem * gSystem
static Bool_t FindClassSourceFiles(const Char_t *class_name, KVString &imp_file, KVString &dec_file, const Char_t *dir_name=".")
static const Char_t * GetKVBuildDate()
Returns KaliVeda build date.
static const Char_t * gitCommit()
Returns last git commit of sources.
static const Char_t * GetKVBuildUser()
Returns username of person who performed build.
static const Char_t * gitBranch()
Returns git branch of sources.
static const Char_t * GetKVSourceDir()
Returns top-level directory of source tree used for build.
static void PrintSplashScreen()
Prints welcome message and infos on version etc.
static TPluginHandler * LoadPlugin(const Char_t *base, const Char_t *uri="0")
static const Char_t * GetKVVersion()
Returns KaliVeda version string.
Handles list of all available batch systems for processing non-interactive data analysis tasks.
void Print(Option_t *opt="") const
KVBatchSystem * GetBatchSystem(const Char_t *name)
Get batch system by name.
virtual const Char_t * GetJobName() const
virtual void ReadBatchEnvFile(TEnv *)
virtual void Clear(Option_t *opt="")
Manager class which sets up and runs data analysis tasks.
const Char_t * GetACliCMode()
void DoStatusUpdate(Long64_t nevents) const
Print infos on events treated, disk usage, memory usage.
virtual TString ExpandAutoBatchName(const Char_t *format) const
void SetParent(KVDataAnalyser *da)
virtual void PostRunReset()
virtual void Run()
Check all task variables, then run analyser.
virtual void AddJobDescriptionList(TList *)
void ChooseNbEventToRead()
Ask user to set number of events to read.
Bool_t RunningInLaunchDirectory()
void SetUserLibraries(const Char_t *libs=0)
void SetUserClass(const Char_t *kvs, Bool_t check=kTRUE)
virtual Bool_t CheckIfUserClassIsValid(const KVString &alternative_base_class="")
TString GetPathToFileInLaunchDirectory(const TString &) const
void WriteBatchInfo(TTree *)
virtual void RunMenus()
Run data analyser in menu-driven mode.
virtual Bool_t CheckTaskVariables()
Bool_t DoUserClassFilesExist()
const Char_t * GetLaunchDirectory() const
void SetAnalysisTask(KVDataAnalysisTask *at)
virtual const Char_t * GetRecognisedAutoBatchNameKeywords() const
virtual void set_up_analyser_for_task(KVDataAnalyser *the_analyser)
virtual void WriteBatchEnvFile(const Char_t *, Bool_t sav=kTRUE)
virtual Bool_t CheckStatusUpdateInterval(Long64_t nevents) const
virtual Bool_t CheckWhatToAnalyseAndHow()
virtual void SubmitTask()
const Char_t * GetBatchStatusFileName() const
Returns full path to file used to store status of running batch jobs.
virtual void ChooseWhatToAnalyse()
TO IMPLEMENT ?
static Bool_t fCleanAbort
flag to force abort of processing
static KVDataAnalyser * GetAnalyser(const Char_t *plugin)
static Bool_t IsRunningBatchAnalysis()
static void RunAnalyser(const Char_t *plugin="")
void ScanWorkingDirectory(TList **)
void ChooseUserClass()
Choose the user's analysis class.
void CopyAnalysisResultsToLaunchDirectory()
void DeleteBatchStatusFile() const
Delete batch status file (and backup - '.bak') for batch job.
virtual Bool_t ReadBatchEnvFile(const Char_t *)
void SetUserIncludes(const Char_t *incDirs=0)
virtual void ChooseWhatToDo()
TO IMPLEMENT ?
void UpdateBatchStatusFile(Int_t totev, Int_t evread, TString disk) const
Update infos in batch status file.
KVBatchSystem * fBatchSystem
batch system to use for submission of job
TObject * GetInstanceOfUserClass(const KVString &alternative_base_class="")
void SetNbEventToRead(Long64_t nb=0)
Define and manage data analysis tasks.
virtual const Char_t * GetPrereq() const
Manage all datasets contained in a given data repository.
virtual Bool_t Init(KVDataRepository *=0)
KVDataAnalysisTask * GetAnalysisTaskAny(const Char_t *keywords) const
Handles lists of named parameters with different types, a list of KVNamedParameter objects.
void SetValue(const Char_t *name, value_type value)
Extension of ROOT TString class which allows backwards compatibility with ROOT v3....
void Begin(TString delim) const
KVString Next(Bool_t strip_whitespace=kFALSE) const
virtual Int_t GetEntries() const
virtual void SetOwner(Bool_t enable=kTRUE)
const char * AsSQLString() const
virtual const char * GetValue(const char *name, const char *dflt) const
virtual void SetValue(const char *name, const char *value, EEnvLevel level=kEnvChange, const char *type=nullptr)
THashList * GetTable() const
virtual void SaveLevel(EEnvLevel level)
virtual TObject * At(Int_t idx) const
virtual const char * GetName() const
const TString & GetString() const
virtual const char * GetName() const
virtual Bool_t InheritsFrom(const char *classname) const
Longptr_t ExecPlugin(int nargs, const T &... params)
std::istream & ReadToDelim(std::istream &str, char delim='\n')
Bool_t EndsWith(const char *pat, ECaseCompare cmp=kExact) const
TObjArray * Tokenize(const TString &delim) const
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
const char * Data() const
void Form(const char *fmt,...)
TString & Remove(EStripType s, char c)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
TString & ReplaceAll(const char *s1, const char *s2)
std::istream & ReadLine(std::istream &str, Bool_t skipWhite=kTRUE)
virtual TList * GetListOfFiles() const
virtual const char * GetBuildNode() const
virtual const char * GetBuildCompilerVersion() const
virtual const char * GetBuildDir() const
virtual void AddIncludePath(const char *includePath)
virtual int CopyFile(const char *from, const char *to, Bool_t overwrite=kFALSE)
virtual const char * Getenv(const char *env)
virtual const char * GetIncludePath()
virtual char * ConcatFileName(const char *dir, const char *name)
virtual int Load(const char *module, const char *entry="", Bool_t system=kFALSE)
virtual UserGroup_t * GetUserInfo(const char *user=nullptr)
virtual int GetProcInfo(ProcInfo_t *info) const
virtual const char * HostName()
virtual const char * WorkingDirectory()
virtual const char * GetBuildCompiler() const
virtual const char * GetLibraries(const char *regexp="", const char *option="", Bool_t isRegexp=kTRUE)
virtual const char * GetBuildArch() const
virtual int Unlink(const char *name)
RooCmdArg ClassName(const char *name)
RooCmdArg BatchMode(bool flag=true)