Logo ROOT   6.30.04
Reference Guide
 All Namespaces Files Pages
TDataSetManagerFile.cxx
Go to the documentation of this file.
1 // @(#)root/base:$Id$
2 // Author: Jan Fiete Grosse-Oetringhaus, 04.06.07
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 //////////////////////////////////////////////////////////////////////////
13 // //
14 // TDataSetManagerFile //
15 // //
16 // Implementation of TDataSetManager handling datasets from root //
17 // files under a specific directory path //
18 // //
19 //////////////////////////////////////////////////////////////////////////
20 #include <sys/stat.h>
21 
22 #include "TDataSetManagerFile.h"
23 
24 #include "Riostream.h"
25 #include "TDatime.h"
26 #include "TEnv.h"
27 #include "TFileCollection.h"
28 #include "TFileInfo.h"
29 #include "TFile.h"
30 #include "TFileStager.h"
31 #include "TLockFile.h"
32 #include "TMap.h"
33 #include "TRegexp.h"
34 #include "TMD5.h"
35 #include "TMacro.h"
36 #include "TMessage.h"
37 #include "TSystem.h"
38 #include "TError.h"
39 #include "TPRegexp.h"
40 #include "TVirtualMonitoring.h"
41 #include "TObjArray.h"
42 #include "THashList.h"
43 #include "TKey.h"
44 #include "TTree.h"
45 #include "TParameter.h"
46 
47 struct LsTreeEntry_t {
48  TObjString *fGrp; // Group
49  TObjString *fUsr; // User
50  TObjString *fMd5; // Checksum
51  TObjString *fLss; // 'ls' string
52  Long64_t fMtime; // modification time
53  LsTreeEntry_t(const char *g, const char *u, const char *cs, const char *ls, Long64_t m) :
54  fMtime(m) { fGrp = new TObjString(g); fUsr = new TObjString(u);
55  fMd5 = new TObjString(cs); fLss = new TObjString(ls); }
56  ~LsTreeEntry_t() { SafeDelete(fGrp); SafeDelete(fUsr); SafeDelete(fMd5); SafeDelete(fLss);}
57 };
58 
59 ClassImp(TDataSetManagerFile);
60 
61 ////////////////////////////////////////////////////////////////////////////////
62 ///
63 /// Main constructor
64 
65 TDataSetManagerFile::TDataSetManagerFile(const char *group,
66  const char *user, const char *ins)
67  : TDataSetManager(group, user, ins)
68 {
69  // Parse options
70  ParseInitOpts(ins);
71 
72  // Init the instance
73  Init();
74 }
75 
76 ////////////////////////////////////////////////////////////////////////////////
77 ///
78 /// Main constructor
79 
80 TDataSetManagerFile::TDataSetManagerFile(const char *ins)
81  : TDataSetManager("", "", ins)
82 {
83  // Parse options
84  ParseInitOpts(ins);
85 
86  // Init the instance
87  Init();
88 }
89 
90 ////////////////////////////////////////////////////////////////////////////////
91 /// Do the real inititialization
92 
93 void TDataSetManagerFile::Init()
94 {
95  fIsRemote = kFALSE;
96  if (!fUser.IsNull() && !fGroup.IsNull() && !fDataSetDir.IsNull()) {
97 
98  // Make sure that the dataset dir exists
99  TString dir;
100  dir.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
101  if (gSystem->AccessPathName(dir)) {
102  if (gSystem->mkdir(dir, kTRUE) != 0) {
103  TString emsg = dir;
104  // Read only dataset info system: switch to COMMON
105  fUser = fCommonUser;
106  fGroup = fCommonGroup;
107  ResetBit(TDataSetManager::kCheckQuota);
108  ResetBit(TDataSetManager::kAllowRegister);
109  ResetBit(TDataSetManager::kAllowVerify);
110  ResetBit(TDataSetManager::kTrustInfo);
111  dir.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
112  if (gSystem->AccessPathName(dir)) {
113  Error("Init",
114  "could not attach to a valid the dataset dir; paths tried:");
115  Error("Init", " %s", emsg.Data());
116  Error("Init", " %s", dir.Data());
117  SetBit(TObject::kInvalidObject);
118  return;
119  }
120  }
121  else if (fOpenPerms) {
122 
123  // Directory creation was OK: let's open permissions if requested
124  TString t;
125  Int_t rr = 0;
126 
127  t.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
128  rr += gSystem->Chmod(t.Data(), 0777);
129 
130  t.Form("%s/%s", fDataSetDir.Data(), fGroup.Data());
131  rr += gSystem->Chmod(t.Data(), 0777);
132 
133  rr += gSystem->Chmod(fDataSetDir.Data(), 0777);
134 
135  if (rr < 0) {
136  t.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(),
137  fUser.Data());
138  Warning("Init",
139  "problems setting perms of dataset directory %s (#%d)",
140  t.Data(), TSystem::GetErrno());
141  }
142 
143  }
144  }
145 
146  // If not in sandbox, construct the base URI using session defaults
147  // (group, user) (syntax: /group/user/dsname[#[subdir/]objname])
148  if (!TestBit(TDataSetManager::kIsSandbox))
149  fBase.SetUri(TString(Form("/%s/%s/", fGroup.Data(), fUser.Data())));
150 
151  // Local or remote?
152  TString locPath;
153  TFile::EFileType pathType = TFile::GetType(fDataSetDir, "READ", &locPath);
154  if (pathType == TFile::kLocal) {
155  // Remote Url pointing to this machine
156  fDataSetDir = locPath;
157  if (gDebug > 0)
158  Info("Init", "repository '%s' is local", fDataSetDir.Data());
159  } else if (pathType != TFile::kDefault && pathType != TFile::kFile) {
160  fIsRemote = kTRUE;
161  if (gDebug > 0)
162  Info("Init", "repository '%s' is remote", fDataSetDir.Data());
163  }
164 
165  // Read locking path from kDataSet_LockLocation
166  TString lockloc = TString::Format("%s/%s", fDataSetDir.Data(), kDataSet_LockLocation);
167  if (!gSystem->AccessPathName(lockloc, kReadPermission)) {
168  // Open the file in RAW mode
169  lockloc += "?filetype=raw";
170  TFile *f = TFile::Open(lockloc);
171  if (f && !(f->IsZombie())) {
172  const Int_t blen = 8192;
173  char buf[blen];
174  Long64_t rest = f->GetSize();
175  while (rest > 0) {
176  Long64_t len = (rest > blen - 1) ? blen - 1 : rest;
177  if (f->ReadBuffer(buf, len)) {
178  fDataSetLockFile = "";
179  break;
180  }
181  buf[len] = '\0';
182  fDataSetLockFile += buf;
183  rest -= len;
184  }
185  f->Close();
186  SafeDelete(f);
187  fDataSetLockFile.ReplaceAll("\n","");
188  } else {
189  lockloc.ReplaceAll("?filetype=raw", "");
190  Warning("Init", "could not open remore file '%s' with the lock location", lockloc.Data());
191  }
192  }
193  if (fDataSetLockFile.IsNull()) {
194  fDataSetLockFile.Form("%s-dataset-lock", fDataSetDir.Data());
195  fDataSetLockFile.ReplaceAll("/","%");
196  fDataSetLockFile.ReplaceAll(":","%");
197  fDataSetLockFile.Insert(0, TString::Format("%s/", gSystem->TempDirectory()));
198  }
199  if (!fDataSetLockFile.IsNull() && fIsRemote) {
200  TUrl lu(fDataSetLockFile, kTRUE);
201  if (!strcmp(lu.GetProtocol(), "file")) {
202  // Add host and port
203  TUrl u(fDataSetDir);
204  TString srv(fDataSetDir);
205  srv.Remove(srv.Index(u.GetFile()));
206  fDataSetLockFile.Insert(0, srv);
207  }
208  }
209  }
210 
211  // Limit in seconds after a lock automatically expires
212  fLockFileTimeLimit = 120;
213 
214  // Default validity of the cache
215  fCacheUpdatePeriod = gEnv->GetValue("ProofDataSet.CacheUpdatePeriod", 0);
216 
217  // If the MSS url was not given, check if one is defined via env
218  if (fMSSUrl.IsNull())
219  fMSSUrl = gEnv->GetValue("ProofDataSet.MSSUrl", "");
220  // Default highest priority for xrd-backends
221  fStageOpts = gEnv->GetValue("DataSet.StageOpts", "p=3");
222 
223  // File to check updates and its locking path
224  fListFile.Form("%s/%s", fDataSetDir.Data(), kDataSet_DataSetList);
225 
226  // Init the local cache directory if the repository is remote
227  fUseCache = kFALSE;
228  fLocalCacheDir = "";
229  InitLocalCache();
230 }
231 ////////////////////////////////////////////////////////////////////////////////
232 /// Init the local cache if required
233 
234 void TDataSetManagerFile::InitLocalCache()
235 {
236  fUseCache = (fIsRemote) ? kTRUE : kFALSE;
237 
238  // Check if the caller has given specific instructions
239  TString useCache;
240  if (TestBit(TDataSetManager::kUseCache)) useCache = "yes";
241  if (TestBit(TDataSetManager::kDoNotUseCache)) useCache = "no";
242  if (useCache.IsNull()) useCache = gEnv->GetValue("DataSet.UseCache", "");
243  if (useCache.IsNull() && gSystem->Getenv("DATASETCACHE"))
244  useCache = gSystem->Getenv("DATASETCACHE");
245  useCache.ToLower();
246  if (!useCache.IsNull())
247  fUseCache = (useCache == "no" || useCache == "0") ? kFALSE : kTRUE;
248 
249  if (fUseCache) {
250  fLocalCacheDir = gSystem->Getenv("DATASETLOCALCACHEDIR");
251  if (fLocalCacheDir.IsNull())
252  fLocalCacheDir = gEnv->GetValue("DataSet.LocalCacheDir", "");
253  if (!fLocalCacheDir.IsNull()) {
254  // Make sure that the non-default local cache directory exists and is writable
255  if (gSystem->AccessPathName(fLocalCacheDir)) {
256  if (gSystem->mkdir(fLocalCacheDir, kTRUE) != 0) {
257  // Switch to default
258  Warning("InitLocalCache",
259  "non-default local cache directory '%s' could not be created"
260  " - switching to default", fLocalCacheDir.Data());
261  fLocalCacheDir = "";
262  }
263  }
264  if (!fLocalCacheDir.IsNull() &&
265  gSystem->AccessPathName(fLocalCacheDir, kWritePermission)) {
266  Warning("InitLocalCache",
267  "non-default local cache directory '%s' is not writable"
268  " - switching to default",
269  fDataSetDir.Data());
270  fLocalCacheDir = "";
271  }
272  }
273  // If not defined yet try the (unique) default
274  if (fLocalCacheDir.IsNull()) {
275  // Add something related to fDataSetDir
276  TString uds(fDataSetDir.Data());
277  uds.ReplaceAll("/","%");
278  uds.ReplaceAll(":","%");
279  if (TString(gSystem->TempDirectory()).EndsWith(fUser.Data())) {
280  fLocalCacheDir.Form("%s/%s/%s", gSystem->TempDirectory(),
281  kDataSet_LocalCache, uds.Data());
282  } else {
283  fLocalCacheDir.Form("%s/%s/%s/%s", gSystem->TempDirectory(),
284  fUser.Data(), kDataSet_LocalCache, uds.Data());
285  }
286  // Make sure that the local cache dir exists and is writable
287  if (gSystem->AccessPathName(fLocalCacheDir) && gSystem->mkdir(fLocalCacheDir, kTRUE) != 0) {
288  // Disable
289  Warning("InitLocalCache",
290  "local cache directory '%s' could not be created"
291  " - disabling cache", fLocalCacheDir.Data());
292  fUseCache = kFALSE;
293  }
294  if (!fLocalCacheDir.IsNull() &&
295  gSystem->AccessPathName(fLocalCacheDir, kWritePermission)) {
296  Warning("InitLocalCache",
297  "local cache directory '%s' is not writable - disabling cache",
298  fDataSetDir.Data());
299  fUseCache = kFALSE;
300  }
301  if (!fUseCache) fLocalCacheDir = "";
302  }
303  }
304  // Done
305  return;
306 }
307 
308 ////////////////////////////////////////////////////////////////////////////////
309 /// Parse the input string and set the init bits accordingly
310 /// Format is
311 /// dir:<datasetdir> [mss:<mss-url>] [opt:<base-options>]
312 /// The <datasetdir> is mandatory.
313 /// See TDataSetManager::ParseInitOpts for the available
314 /// base options.
315 /// The base options are already initialized by the base constructor
316 
317 void TDataSetManagerFile::ParseInitOpts(const char *ins)
318 {
319  SetBit(TObject::kInvalidObject);
320  fOpenPerms = kFALSE;
321 
322  // Needs something in
323  if (!ins || strlen(ins) <= 0) return;
324 
325  // Extract elements
326  Int_t from = 0;
327  TString s(ins), tok;
328  while (s.Tokenize(tok, from, " ")) {
329  if (tok.BeginsWith("dir:"))
330  fDataSetDir = tok(4, tok.Length());
331  if (tok.BeginsWith("mss:"))
332  fMSSUrl = tok(4, tok.Length());
333  if (tok == "perms:open")
334  fOpenPerms = kTRUE;
335  }
336 
337  // The directory is mandatory
338  if (fDataSetDir.IsNull()) return;
339 
340  // Object is valid
341  ResetBit(TObject::kInvalidObject);
342 }
343 
344 ////////////////////////////////////////////////////////////////////////////////
345 /// Returns path of the indicated dataset. The extension is '.root' for all files
346 /// except for 'dsName==ls' which have extension '.txt'.
347 /// If 'local' is kTRUE the local cache path is returned instead in the form
348 /// <cachedir>/<group>.<user>.<dsName>.<ext>.
349 /// NB: contains a static TString for result, so copy result before using twice.
350 
351 const char *TDataSetManagerFile::GetDataSetPath(const char *group,
352  const char *user,
353  const char *dsName,
354  TString &md5path, Bool_t local)
355 {
356  if (fgCommonDataSetTag == group)
357  group = fCommonGroup;
358 
359  if (fgCommonDataSetTag == user)
360  user = fCommonUser;
361 
362  const char *ext = (!strcmp(dsName, "ls")) ? ".txt" : ".root";
363  static TString result;
364  if (local) {
365  result.Form("%s/%s.%s.%s%s", fLocalCacheDir.Data(), group, user, dsName, ext);
366  md5path.Form("%s/%s.%s.%s.md5sum", fLocalCacheDir.Data(), group, user, dsName);
367  } else {
368  result.Form("%s/%s/%s/%s%s", fDataSetDir.Data(), group, user, dsName, ext);
369  md5path.Form("%s/%s/%s/%s.md5sum", fDataSetDir.Data(), group, user, dsName);
370  }
371  if (gDebug > 0)
372  Info("GetDataSetPath","paths: %s, %s ", result.Data(), md5path.Data());
373  return result;
374 }
375 
376 ////////////////////////////////////////////////////////////////////////////////
377 /// Save into the <datasetdir>/kDataSet_DataSetList file the name of the updated
378 /// or created or modified dataset. For still existing datasets, fill the
379 /// modification date in seconds anf the checksum.
380 /// Returns 0 on success, -1 on error
381 
382 Int_t TDataSetManagerFile::NotifyUpdate(const char *group, const char *user,
383  const char *dsName, Long_t mtime, const char *checksum)
384 {
385  // Update / create list for the owner
386  Long_t lsmtime = 0;
387  TString lschecksum;
388  Int_t lsrc = -1;
389  if ((lsrc = CreateLsFile(group, user, lsmtime, lschecksum)) < 0) {
390  Warning("NotifyUpdate", "problems (re-)creating the dataset lists for '/%s/%s'",
391  group, user);
392  }
393 
394  { TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
395  TString dspath = TString::Format("/%s/%s/%s", group, user, dsName);
396  // Check if the global file exists
397  Bool_t hasListFile = gSystem->AccessPathName(fListFile) ? kFALSE : kTRUE;
398  // Load the info in form of TMacro
399  TMD5 *oldMd5 = 0, *newMd5 = 0;
400  if (hasListFile && !(oldMd5 = TMD5::FileChecksum(fListFile.Data()))) {
401  Error("NotifyUpdate", "problems calculating old checksum of %s", fListFile.Data());
402  return -1;
403  }
404  // Create the TMacro object, filling it with the existing file, if any
405  TMacro mac;
406  if (hasListFile) mac.ReadFile(fListFile.Data());
407  // Locate the line to change or delete
408  TObjString *os = mac.GetLineWith(dspath);
409  if (os) {
410  // Delete the line if required so
411  if (!strcmp(checksum, "removed")) {
412  mac.GetListOfLines()->Remove(os);
413  SafeDelete(os);
414  } else {
415  // Update the information
416  os->SetString(TString::Format("%ld %s %s", mtime, dspath.Data(), checksum));
417  }
418  } else {
419  if (!strcmp(checksum, "removed")) {
420  Warning("NotifyUpdate", "entry for removed dataset '%s' not found!", dspath.Data());
421  } else {
422  // Add new line
423  mac.AddLine(TString::Format("%ld %s %s", mtime, dspath.Data(), checksum));
424  }
425  }
426  // Locate the ls line now, is needed
427  TString lspath = TString::Format("/%s/%s/ls", group, user);
428  os = mac.GetLineWith(lspath);
429  if (os) {
430  // Delete the line if required so
431  if (lsrc == 1) {
432  mac.GetListOfLines()->Remove(os);
433  SafeDelete(os);
434  } else {
435  // Update the information
436  os->SetString(TString::Format("%ld %s %s", lsmtime, lspath.Data(), lschecksum.Data()));
437  }
438  } else {
439  if (lsrc == 0) {
440  // Add new line
441  mac.AddLine(TString::Format("%ld %s %s", lsmtime, lspath.Data(), lschecksum.Data()));
442  }
443  }
444  // Write off the new content
445  mac.SaveSource(fListFile.Data());
446  if (fOpenPerms) {
447  if (gSystem->Chmod(fListFile.Data(), 0666) < 0) {
448  Warning("NotifyUpdate",
449  "can't set permissions of dataset list file %s (#%d)",
450  fListFile.Data(), TSystem::GetErrno());
451  }
452  }
453  if (!(newMd5 = TMD5::FileChecksum(fListFile.Data()))) {
454  Error("NotifyUpdate", "problems calculating new checksum of %s", fListFile.Data());
455  SafeDelete(oldMd5);
456  return -1;
457  }
458  if (oldMd5 && (*newMd5 == *oldMd5))
459  Warning("NotifyUpdate", "checksum for %s did not change!", fListFile.Data());
460  // Cleanup
461  SafeDelete(oldMd5);
462  SafeDelete(newMd5);
463  }
464  // Done
465  return 0;
466 }
467 
468 ////////////////////////////////////////////////////////////////////////////////
469 /// Create or recreate the dataset lists for 'uri'.
470 /// The list are saved in text form in 'uri'/ls.txt for fast browsing and in
471 /// 'uri'/ls.root in form of TMacro for optimized and portable transfer.
472 /// Return 0 on success, 1 if the file was empty, -1 on error
473 
474 Int_t TDataSetManagerFile::CreateLsFile(const char *group, const char *user,
475  Long_t &mtime, TString &checksum)
476 {
477  mtime = 0;
478  checksum = "";
479  // Create temporary file; we cannot lock now because we would (dead-)lock
480  // during ShowDataSets
481  TString tmpfile;
482  tmpfile.Form("%s/%s/%s/ls.tmp.txt", fDataSetDir.Data(), group, user);
483 
484  // Redirect output to 'tmpfile'
485  RedirectHandle_t rh;
486  if (gSystem->RedirectOutput(tmpfile.Data(), "w", &rh) != 0) {
487  Error("CreateLsFile", "problems redirecting output to %s (errno: %d)",
488  tmpfile.Data(), TSystem::GetErrno());
489  return -1;
490  }
491  // Create the list
492  TString uri;
493  uri.Form("/%s/%s", group, user);
494  ShowDataSets(uri, "forcescan:noheader:");
495  // Restore output to standard streams
496  if (gSystem->RedirectOutput(0, 0, &rh) != 0) {
497  Error("CreateLsFile", "problems restoring output to standard streams (errno: %d)",
498  TSystem::GetErrno());
499  return -1;
500  }
501  // We can lock now
502  TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
503  // Rename the temp file
504  TString lsfile;
505  lsfile.Form("%s/%s/%s/ls.txt", fDataSetDir.Data(), group, user);
506  // Remove the file, if existing
507  if (!gSystem->AccessPathName(lsfile) && gSystem->Unlink(lsfile) != 0) {
508  Error("CreateLsFile", "problems unlinking old file '%s' (errno: %d)",
509  lsfile.Data(), TSystem::GetErrno());
510  return -1;
511  }
512  // Save the new file only if non empty
513  FileStat_t st;
514  if (gSystem->GetPathInfo(tmpfile, st) == 0 && st.fSize > 0) {
515  if (gSystem->Rename(tmpfile, lsfile) != 0) {
516  Error("CreateLsFile", "problems renaming '%s' to '%s' (errno: %d)",
517  tmpfile.Data(), lsfile.Data(), TSystem::GetErrno());
518  return -1;
519  }
520 #ifndef WIN32
521  // Make sure that the ownership and permissions are those expected
522  FileStat_t udirst;
523  if (!fIsRemote && gSystem->GetPathInfo(gSystem->DirName(tmpfile), udirst) == 0) {
524  if (chown(lsfile.Data(), udirst.fUid, udirst.fGid) != 0) {
525  Warning("CreateLsFile", "problems setting ownership on file '%s' (errno: %d)",
526  lsfile.Data(), TSystem::GetErrno());
527  }
528  if (fOpenPerms) {
529  if (gSystem->Chmod(lsfile.Data(), 0666) < 0) {
530  Warning("NotifyUpdate",
531  "can't set permissions of list file %s (#%d)",
532  lsfile.Data(), TSystem::GetErrno());
533  }
534  }
535  else if (chmod(lsfile.Data(), 0644) != 0) {
536  Warning("CreateLsFile", "problems setting permissions on file '%s' (errno: %d)",
537  lsfile.Data(), TSystem::GetErrno());
538  }
539  }
540 #endif
541  mtime = st.fMtime;
542  TMD5 *md5 = TMD5::FileChecksum(lsfile);
543  if (!md5) {
544  Error("CreateLsFile", "problems calculating checksum for '%s'", lsfile.Data());
545  } else {
546  checksum = md5->AsString();
547  SafeDelete(md5);
548  }
549  } else {
550  if (!gSystem->AccessPathName(tmpfile) && gSystem->Unlink(tmpfile) != 0) {
551  Error("CreateLsFile", "problems unlinking temp file '%s' (errno: %d)",
552  tmpfile.Data(), TSystem::GetErrno());
553  return -1;
554  }
555  // No datasets anymore
556  return 1;
557  }
558  // Done
559  return 0;
560 }
561 
562 ////////////////////////////////////////////////////////////////////////////////
563 /// Adds the dataset in the folder of group, user to the list in target.
564 /// If dsName is defined, only the information about the specified dataset
565 /// is processed.
566 ///
567 /// The unsigned int 'option' is forwarded to GetDataSet and BrowseDataSet.
568 /// Available options (to be .or.ed):
569 /// kPrint print the dataset content
570 /// kQuotaUpdate update quotas
571 /// kExport use export naming
572 /// kList get a list of dataset names
573 ///
574 /// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
575 /// NB2: for options "kPrint" and "kQuotaUpdate" return is null.
576 
577 Bool_t TDataSetManagerFile::BrowseDataSets(const char *group, const char *user,
578  const char *dsName,
579  UInt_t option, TObject *target)
580 {
581  TString userDirPath;
582  userDirPath.Form("%s/%s/%s", fDataSetDir.Data(), group, user);
583  void *userDir = gSystem->OpenDirectory(userDirPath);
584  if (!userDir)
585  return kFALSE;
586 
587  // Options
588  Bool_t printing = (option & kPrint) ? kTRUE : kFALSE;
589  Bool_t exporting = (option & kExport) ? kTRUE : kFALSE;
590  Bool_t updating = (option & kQuotaUpdate) ? kTRUE : kFALSE;
591  Bool_t printout = (printing && (option & kDebug)) ? kTRUE : kFALSE;
592  Bool_t listing = (option & kList) ? kTRUE : kFALSE;
593 
594  // If printing is required add kReadShort to the options
595  if (printing || updating)
596  option |= kReadShort;
597 
598  // The last three options are mutually exclusive
599  if (((Int_t)printing + (Int_t)exporting + (Int_t)updating + (Int_t)listing) > 1) {
600  Error("BrowseDataSets",
601  "only one of kPrint, kQuotaUpdate, kExport or kList can be specified at once");
602  return kFALSE;
603  }
604  Bool_t fillmap = (!exporting && !printing && !updating) ? kTRUE : kFALSE;
605 
606  // Output object
607  TMap *outmap = (fillmap || exporting || listing) ? (TMap *)target : (TMap *)0;
608  TList *outlist = (printing) ? (TList *)target : (TList *)0;
609 
610  TRegexp rg("^[^./][^/]*.root$"); //check that it is a root file, not starting with "."
611 
612  TRegexp *reds = 0;
613  if (dsName && strlen(dsName) > 0) reds = new TRegexp(dsName, kTRUE);
614 
615  TMap *userMap = 0, *datasetMap = 0;
616  // loop over datasets
617  const char *dsEnt = 0;
618  while ((dsEnt = gSystem->GetDirEntry(userDir))) {
619  TString datasetFile(dsEnt);
620  if (datasetFile.Index(rg) != kNPOS) {
621  TString datasetName(datasetFile(0, datasetFile.Length()-5));
622 
623  // Check dataset name, if required
624  if (reds && datasetName.Index(*reds) == kNPOS) continue;
625 
626  if (gDebug > 0)
627  Info("GetDataSets", "found dataset %s of user %s in group %s",
628  datasetName.Data(), user, group);
629 
630  TFileCollection *fileList = GetDataSet(group, user, datasetName, option);
631  if (!fileList) {
632  Error("GetDataSets", "dataset %s (user %s, group %s) could not be opened",
633  datasetName.Data(), user, group);
634  continue;
635  }
636  if (gDebug > 0)
637  fileList->Print();
638 
639  // We found a dataset, now add it to the map
640 
641  // COMMON dataset transition
642  const char *mapGroup = group;
643  if (fCommonGroup == mapGroup)
644  mapGroup = fgCommonDataSetTag.Data();
645  const char *mapUser = user;
646  if (fCommonUser == mapUser)
647  mapUser = fgCommonDataSetTag.Data();
648 
649  if (fillmap && !listing && outmap) {
650  if (!(userMap = dynamic_cast<TMap*> (outmap->GetValue(mapGroup)))) {
651  userMap = new TMap;
652  userMap->SetOwner();
653  outmap->Add(new TObjString(mapGroup), userMap);
654  }
655 
656  if (!(datasetMap = dynamic_cast<TMap*> (userMap->GetValue(mapUser)))) {
657  datasetMap = new TMap;
658  datasetMap->SetOwner();
659  userMap->Add(new TObjString(mapUser), datasetMap);
660  }
661  }
662 
663  // Action depends on option
664  if (exporting) {
665 
666  // Just format the dataset name with group and user
667  TString dsNameFormatted(Form("/%s/%s/%s", mapGroup,
668  mapUser, datasetName.Data()));
669  if (outmap)
670  outmap->Add(new TObjString(dsNameFormatted), fileList);
671 
672  } else if (updating) {
673 
674  // Update quotas
675  GetQuota(mapGroup, mapUser, datasetName.Data(), fileList);
676 
677  } else if (printing) {
678 
679  // Prepare the output list
680  if (outlist) {
681  TString dsNameFormatted(Form("/%s/%s/%s", mapGroup,
682  mapUser, datasetName.Data()));
683  // Magic number?
684  if (dsNameFormatted.Length() < 42)
685  dsNameFormatted.Resize(42);
686 
687  // Done
688  outlist->Add(fileList->ExportInfo(dsNameFormatted));
689  if (printout) {
690  // Recreating the LS file
691  TObjString *os = (TObjString *) outlist->Last();
692  if (os) Printf("%s", os->GetName());
693  }
694  }
695  } else if (listing) {
696 
697  // Just a list of available datasets
698  if (outmap) {
699  outmap->Add(new TObjString(TString::Format("/%s/%s/%s", mapGroup, mapUser, datasetName.Data())),
700  new TObjString(""));
701  }
702  } else {
703  if (fillmap && datasetMap)
704  datasetMap->Add(new TObjString(datasetName), fileList);
705  }
706  }
707  }
708  gSystem->FreeDirectory(userDir);
709  SafeDelete(reds);
710 
711  return kTRUE;
712 }
713 
714 ////////////////////////////////////////////////////////////////////////////////
715 /// General purpose call to go through the existing datasets.
716 /// If <user> is 0 or "*", act on all datasets for the given <group>.
717 /// If <group> is 0 or "*", act on all datasets.
718 /// If <dsName> is defined, only the information about the specified dataset
719 /// is processed.
720 /// Action depends on option; available options:
721 ///
722 /// kExport Return a TMap object containing all the information about
723 /// datasets in the form:
724 /// { <group>, <map of users> }
725 /// |
726 /// { <map of datasets>, <dataset>}
727 /// (<dataset> are TFileCollection objects)
728 /// kShowDefault as kExport with in addition a default selection including
729 /// the datasets from the current user, the ones from the group
730 /// and the common ones
731 ///
732 /// kPrint print the dataset content; no output is returned
733 /// kList get a list of available dataset names
734 /// kForceScan Re-open files while processing kPrint (do not use the
735 /// pre-processed information)
736 /// kNoHeaderPrint Labelling header is not printed
737 /// kQuotaUpdate update {group, user} quotas; no output is returned
738 ///
739 /// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
740 /// NB2: for options "kPrint" and "kQuoatUpdate" return is null.
741 
742 TMap *TDataSetManagerFile::GetDataSets(const char *group, const char *user,
743  const char *dsName, UInt_t option)
744 {
745  if (group && fgCommonDataSetTag == group)
746  group = fCommonGroup;
747 
748  if (user && fgCommonDataSetTag == user)
749  user = fCommonUser;
750 
751  // Special treatment for the COMMON user
752  Bool_t notCommonUser = kTRUE;
753  if ((user && fCommonUser == user) &&
754  (group && fCommonGroup == group)) notCommonUser = kFALSE;
755 
756  // convert * to "nothing"
757  if (group && (strcmp(group, "*") == 0 || !group[0]))
758  group = 0;
759  if (user && (strcmp(user, "*") == 0 || !user[0]))
760  user = 0;
761 
762  Bool_t printing = (option & kPrint) ? kTRUE : kFALSE;
763  Bool_t forcescan = (option & kForceScan) ? kTRUE : kFALSE;
764  Bool_t printheader = (option & kNoHeaderPrint) ? kFALSE : kTRUE;
765  Bool_t exporting = (option & kExport) ? kTRUE : kFALSE;
766  Bool_t updating = (option & kQuotaUpdate) ? kTRUE : kFALSE;
767  Bool_t refreshingls = (option & kRefreshLs) ? kTRUE : kFALSE;
768  Bool_t listing = (option & kList) ? kTRUE : kFALSE;
769 
770  // The last three options are mutually exclusive
771  if (((Int_t)printing + (Int_t)exporting + (Int_t)updating + (Int_t)listing) > 1) {
772  Error("GetDataSets", "only one of '?P', '?Q', '?E' or '?L' can be specified at once");
773  return 0;
774  }
775 
776  TObject *result = 0;
777  if (printing) {
778  // The output is a list of strings
779  TList *ol = new TList();
780  ol->SetOwner();
781  result = ol;
782  } else if (exporting || !updating || listing) {
783  TMap *om = new TMap;
784  om->SetOwner();
785  result = om;
786  }
787 
788  if (gDebug > 0)
789  Info("GetDataSets", "opening dir %s", fDataSetDir.Data());
790 
791  Long_t m;
792  TString s;
793  if (option & kShowDefault) {
794  // Add the common ones
795  if (refreshingls) {
796  if (CreateLsFile(fCommonGroup, fCommonUser, m, s) != 0)
797  Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
798  fCommonGroup.Data(), fCommonUser.Data());
799  } else if (!printing || forcescan || (printing &&
800  FillLsDataSet(fCommonGroup, fCommonUser, dsName, (TList *)result, option) != 0)) {
801  BrowseDataSets(fCommonGroup, fCommonUser, dsName, option, result);
802  }
803  user = 0;
804  } else {
805  // Fill the information at least once
806  if (!notCommonUser) notCommonUser = kTRUE;
807  }
808 
809  // Fill the information only once
810  if (notCommonUser) {
811  // group, user defined, no looping needed
812  if (user && group && strchr(user, '*') && strchr(group, '*')) {
813  if (refreshingls) {
814  if (CreateLsFile(group, user, m, s) != 0)
815  Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
816  group, user);
817  } else if (!printing || forcescan || (printing &&
818  FillLsDataSet(group, user, dsName, (TList *)result, option) != 0)) {
819  BrowseDataSets(group, user, dsName, option, result);
820  }
821  if (!printing) return (TMap *)result;
822  } else {
823  TRegexp *reg = (group && strlen(group) > 0) ? new TRegexp(group, kTRUE) : 0;
824  TRegexp *reu = (user && strlen(user) > 0) ? new TRegexp(user, kTRUE) : 0;
825  // Loop needed, either on the local cache or on the real thing
826  if (printing && !forcescan &&
827  fUseCache && CheckLocalCache(group, user, 0, option) == 0) {
828  // Loop on the local cache
829  Int_t from = 0;
830  TString locupdate, dsn, grp, usr;
831  locupdate.Form("%s/%s", fLocalCacheDir.Data(), kDataSet_DataSetList);
832  TMacro uptmac(locupdate);
833  TIter nxl(uptmac.GetListOfLines());
834  TObjString *os = 0;
835  while ((os = (TObjString *) nxl())) {
836  if (!(os->GetString().Contains("/ls"))) continue;
837  from = 0;
838  if (!(os->GetString().Tokenize(dsn, from, " "))) continue;
839  if (!(os->GetString().Tokenize(dsn, from, " "))) continue;
840  from = 0;
841  // Get group and apply filter on group
842  if (!(dsn.Tokenize(grp, from, "/")) || (reg && (grp.Index(*reg) == kNPOS))) continue;
843  // Get user and apply filter on user
844  if (!(dsn.Tokenize(usr, from, "/")) || (reu && (usr.Index(*reu) == kNPOS))) continue;
845  // Now get the info
846  if (FillLsDataSet(grp, usr, dsName, (TList *)result, option) != 0) {
847  // Use the standard way opening all the files
848  BrowseDataSets(grp, usr, dsName, option, result);
849  }
850  }
851  } else {
852  // Loop needed on the real thing
853  void *dataSetDir = 0;
854  if ((dataSetDir = gSystem->OpenDirectory(fDataSetDir))) {
855  // loop over groups
856  const char *eg = 0;
857  while ((eg = gSystem->GetDirEntry(dataSetDir))) {
858 
859  if (strcmp(eg, ".") == 0 || strcmp(eg, "..") == 0)
860  continue;
861 
862  if (reg && (TString(eg).Index(*reg) == kNPOS))
863  continue;
864 
865  TString groupDirPath;
866  groupDirPath.Form("%s/%s", fDataSetDir.Data(), eg);
867 
868  // Make sure it is a directory
869  FileStat_t dirSt;
870  if (gSystem->GetPathInfo(groupDirPath, dirSt) != 0 || !R_ISDIR(dirSt.fMode))
871  continue;
872 
873  void *groupDir = gSystem->OpenDirectory(groupDirPath);
874  if (!groupDir)
875  continue;
876 
877  // loop over users
878  const char *eu = 0;
879  while ((eu = gSystem->GetDirEntry(groupDir))) {
880 
881  if (strcmp(eu, ".") == 0 || strcmp(eu, "..") == 0)
882  continue;
883 
884  if (reu && (TString(eu).Index(*reu) == kNPOS))
885  continue;
886 
887  // If we have the ls.macro use that
888  if (refreshingls) {
889  if (CreateLsFile(eg, eu, m, s) != 0)
890  Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
891  eg, eu);
892  } else if (!printing || forcescan || (printing &&
893  FillLsDataSet(eg, eu, dsName, (TList *)result, option) != 0)) {
894  // Use the standard way opening all the files
895  BrowseDataSets(eg, eu, dsName, option, result);
896  }
897  }
898  gSystem->FreeDirectory(groupDir);
899  }
900  gSystem->FreeDirectory(dataSetDir);
901  }
902  }
903  SafeDelete(reg);
904  SafeDelete(reu);
905  }
906  }
907  // Print the result, if required
908  if (printing) {
909  TList *output = (TList *)result;
910  output->Sort();
911  if (printheader) {
912  Printf("Dataset repository: %s", fDataSetDir.Data());
913  Printf("Dataset URI | # Files | Default tree | # Events | Disk | Staged");
914  }
915  TIter iter4(output);
916  TObjString *os = 0;
917  while ((os = dynamic_cast<TObjString*> (iter4()))) {
918  if (os->GetString().BeginsWith("file:")) {
919  // Path of the file to be browsed
920  TString path(os->GetString()(5, os->GetString().Length()));
921  RedirectHandle_t rh(path.Data());
922  gSystem->ShowOutput(&rh);
923  fflush(stderr);
924  } else {
925  // Simple line
926  Printf("%s", os->String().Data());
927  }
928  }
929  // Cleanup
930  SafeDelete(output);
931  result = 0;
932  }
933 
934  return (TMap *)result;
935 }
936 
937 ////////////////////////////////////////////////////////////////////////////////
938 /// Check for the 'ls.txt' for 'group' and 'user' and fill the path for the
939 /// ls file in 'out'.
940 /// If 'dsname' is defined, open the file and extract the relevant line.
941 /// Return 0 on success, -1 on failure
942 
943 Int_t TDataSetManagerFile::FillLsDataSet(const char *group, const char *user,
944  const char *dsname, TList *out, UInt_t option)
945 {
946  // Check inputs
947  if (!group || strlen(group) <= 0 || !user || strlen(user) <= 0 || !out) {
948  Error("FillLsDataSet", "at least one of the inputs is invalid (%s,%s,%p)", group, user, out);
949  return -1;
950  }
951 
952  // File with the TMacro
953  Int_t crc = -1;
954  TString lsfile, lsmd5file;
955  if (!fUseCache || (fUseCache && (crc = CheckLocalCache(group, user, "ls", option)) <= 0)) {
956  Bool_t local = (crc == 0) ? kTRUE : kFALSE;
957  lsfile = GetDataSetPath(group, user, "ls", lsmd5file, local);
958  } else {
959  // The dataset does not exist anymore
960  return 0;
961  }
962 
963  if (gSystem->AccessPathName(lsfile, kFileExists)) {
964  if (gDebug > 0)
965  Info("FillLsDataSet", "file '%s' does not exists", lsfile.Data());
966  return -1;
967  }
968  if (gSystem->AccessPathName(lsfile, kReadPermission)) {
969  Warning("FillLsDataSet", "file '%s' exists cannot be read (permission denied)", lsfile.Data());
970  return -1;
971  }
972 
973  if (dsname && strlen(dsname) > 0) {
974  // Read the macro
975  TMacro *mac = new TMacro(lsfile.Data());
976  if (!mac) {
977  Error("FillLsDataSet", "could not initialize TMacro from '%s'", lsfile.Data());
978  return -1;
979  }
980  // Prepare the string to search for
981  TString fullname = TString::Format("/%s/%s/%s", group, user, dsname);
982  Bool_t wc = (fullname.Contains("*")) ? kTRUE : kFALSE;
983  if (wc) fullname.ReplaceAll("*", ".*");
984  TRegexp reds(fullname);
985  TIter nxl(mac->GetListOfLines());
986  TObjString *o;
987  Int_t nf = 0;
988  while ((o = (TObjString *) nxl())) {
989  if (o->GetString().Index(reds) != kNPOS) {
990  out->Add(o->Clone());
991  nf++;
992  if (!wc) break;
993  }
994  }
995  if (nf > 0 && gDebug > 0)
996  Info("FillLsDataSet", "no match for dataset uri '/%s/%s/%s'", group, user, dsname);
997  // Delete the macro
998  SafeDelete(mac);
999  } else {
1000  // Fill in the file information
1001  out->Add(new TObjString(TString::Format("file:%s", lsfile.Data())));
1002  }
1003  // Done
1004  return 0;
1005 }
1006 
1007 ////////////////////////////////////////////////////////////////////////////////
1008 ///
1009 /// Returns the dataset <dsName> of user <user> in group <group> .
1010 /// If checksum is non-zero, it will contain the pointer to a TMD5 sum object
1011 /// with the checksum of the file, has to be deleted by the user.
1012 /// If option has the bi kReadShort set, the shortobject is read, that does not
1013 /// contain the list of files. This is much faster.
1014 
1015 TFileCollection *TDataSetManagerFile::GetDataSet(const char *group,
1016  const char *user,
1017  const char *dsName,
1018  UInt_t option,
1019  TMD5 **checksum)
1020 {
1021  TFileCollection *fileList = 0;
1022  Bool_t readshort = (option & kReadShort) ? kTRUE : kFALSE;
1023  // Check is the file is in the cache
1024  Int_t crc = -1;
1025  TString path, md5path;
1026  if (readshort || !fUseCache ||
1027  (!readshort && fUseCache && (crc = CheckLocalCache(group, user, dsName, option)) <= 0)) {
1028  Bool_t local = (crc == 0) ? kTRUE : kFALSE;
1029  path = GetDataSetPath(group, user, dsName, md5path, local);
1030  } else {
1031  // The dataset does not exist (anymore?)
1032  if (gDebug > 0)
1033  Info("GetDataSet", "dataset %s does not exist", path.Data());
1034  return fileList;
1035  }
1036 
1037  // Now we lock because we are going to use the file, if there
1038  TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
1039 
1040  // Check if the file can be opened for reading
1041  if (gSystem->AccessPathName(path, kFileExists)) {
1042  if (gDebug > 0)
1043  Info("GetDataSet", "file '%s' does not exists", path.Data());
1044  return fileList;
1045  }
1046  if (gSystem->AccessPathName(path, kReadPermission)) {
1047  Warning("GetDataSet", "file '%s' exists cannot be read (permission denied)", path.Data());
1048  return fileList;
1049  }
1050 
1051  // Get checksum
1052  if (checksum) {
1053  // save md5 sum
1054  *checksum = TMD5::ReadChecksum(md5path);
1055  if (!(*checksum)) {
1056  Error("GetDataSet", "could not get checksum of %s from %s", path.Data(), md5path.Data());
1057  return fileList;
1058  }
1059  }
1060 
1061  TFile *f = TFile::Open(path.Data());
1062  if (!f) {
1063  Error("GetDataSet", "could not open file %s", path.Data());
1064  if (checksum) SafeDelete(*checksum);
1065  return fileList;
1066  }
1067 
1068  if (option & kReadShort)
1069  fileList = dynamic_cast<TFileCollection*> (f->Get("dataset_short"));
1070 
1071  if (!fileList)
1072  fileList = dynamic_cast<TFileCollection*> (f->Get("dataset"));
1073 
1074  f->Close();
1075  SafeDelete(f);
1076 
1077  return fileList;
1078 }
1079 
1080 ////////////////////////////////////////////////////////////////////////////////
1081 /// Check if the local cache information for group, user, dsName is up-to-date
1082 /// If not, make the relevant updates
1083 /// Return 0 if OK, 1 if the dataset does not exists anymore, -1 on failure
1084 
1085 Int_t TDataSetManagerFile::CheckLocalCache(const char *group, const char *user,
1086  const char *dsName, UInt_t option)
1087 {
1088  // Check first if the global update info is uptodate
1089  static TMacro *uptmac = 0;
1090  Bool_t need_last_update = (option & kNoCacheUpdate) ? kFALSE : kTRUE;
1091  TString locupdtim, locupdate, remupdate;
1092  locupdtim.Form("%s/%s.update", fLocalCacheDir.Data(), kDataSet_DataSetList);
1093  locupdate.Form("%s/%s", fLocalCacheDir.Data(), kDataSet_DataSetList);
1094  remupdate.Form("%s/%s", fDataSetDir.Data(), kDataSet_DataSetList);
1095  need_last_update = (gSystem->AccessPathName(locupdate)) ? kTRUE : need_last_update;
1096  TDatime now;
1097  UInt_t tnow = now.Convert();
1098  FileStat_t timst, locst, remst;
1099  if (need_last_update && !gSystem->AccessPathName(locupdtim)) {
1100  if (gSystem->GetPathInfo(locupdtim, timst) == 0) {
1101  need_last_update = kFALSE;
1102  if ((Int_t)tnow > timst.fMtime + fCacheUpdatePeriod) need_last_update = kTRUE;
1103  }
1104  }
1105  if (need_last_update) {
1106  if (gSystem->GetPathInfo(remupdate, remst) != 0) {
1107  Error("CheckLocalCache", "cannot get info for remote file '%s' - ignoring", remupdate.Data());
1108  return -1;
1109  }
1110  if (gSystem->GetPathInfo(locupdate, locst) == 0) {
1111  need_last_update = kFALSE;
1112  if (remst.fMtime > locst.fMtime) {
1113  need_last_update = kTRUE;
1114  } else {
1115  if (!gSystem->AccessPathName(locupdtim))
1116  if (gSystem->Utime(locupdtim, tnow, 0) != 0)
1117  Warning("CheckLocalCache",
1118  "cannot set modification time on file '%s' (errno: %d)",
1119  locupdtim.Data(), TSystem::GetErrno());
1120  }
1121  }
1122  }
1123  // Get the file, if needed
1124  if (need_last_update) {
1125  if (!TFile::Cp(remupdate, locupdate, kFALSE)) {
1126  Error("CheckLocalCache", "cannot get remote file '%s' - ignoring", remupdate.Data());
1127  return -1;
1128  }
1129  // Set the modification time
1130  if (gSystem->Utime(locupdate, remst.fMtime, 0) != 0) {
1131  Warning("CheckLocalCache", "cannot set modification time on file '%s' (errno: %d)",
1132  locupdate.Data(), TSystem::GetErrno());
1133  }
1134  // Touch or create the file to control updates
1135  if (gSystem->AccessPathName(locupdtim)) {
1136  FILE *ftim = fopen(locupdtim.Data(), "w");
1137  if (!ftim) {
1138  Warning("CheckLocalCache", "problems create file '%s' (errno: %d)",
1139  locupdtim.Data(), TSystem::GetErrno());
1140  } else {
1141  if (fclose(ftim) != 0)
1142  Warning("CheckLocalCache", "problems close file '%s' (errno: %d)",
1143  locupdtim.Data(), TSystem::GetErrno());
1144  if (gSystem->Utime(locupdtim, now.Convert(), 0) != 0)
1145  Warning("CheckLocalCache",
1146  "cannot set modification time on file '%s' (errno: %d)",
1147  locupdtim.Data(), TSystem::GetErrno());
1148  }
1149  }
1150  // Update macro info
1151  SafeDelete(uptmac);
1152  uptmac = new TMacro(locupdate);
1153  } else {
1154  // Touch or create the file to control updates
1155  if (gSystem->AccessPathName(locupdtim)) {
1156  FILE *ftim = fopen(locupdtim.Data(), "w");
1157  if (!ftim) {
1158  Warning("CheckLocalCache", "problems create file '%s' (errno: %d)",
1159  locupdtim.Data(), TSystem::GetErrno());
1160  } else {
1161  if (fclose(ftim) != 0)
1162  Warning("CheckLocalCache", "problems close file '%s' (errno: %d)",
1163  locupdtim.Data(), TSystem::GetErrno());
1164  if (gSystem->GetPathInfo(locupdate, locst) == 0) {
1165  if (gSystem->Utime(locupdtim, locst.fMtime, 0) != 0)
1166  Warning("CheckLocalCache",
1167  "cannot set modification time on file '%s' (errno: %d)",
1168  locupdtim.Data(), TSystem::GetErrno());
1169  } else {
1170  Warning("CheckLocalCache", "cannot get info for file '%s'"
1171  " - will not touch '%s'", locupdate.Data(), locupdtim.Data());
1172  }
1173  }
1174  }
1175  if (!uptmac) uptmac = new TMacro(locupdate);
1176  }
1177 
1178  // If we are just interested in the global dataset list we are done
1179  if (!dsName || strlen(dsName) <= 0)
1180  return 0;
1181 
1182  // Read the information
1183  TString ds, locpath, path, locmd5path, md5path, remmd5s;
1184  TMD5 *locmd5 = 0;
1185  // The paths ...
1186  path = GetDataSetPath(group, user, dsName, md5path);
1187  locpath = GetDataSetPath(group, user, dsName, locmd5path, kTRUE);
1188  ds.Form("/%s/%s/%s", group, user, dsName);
1189  TObjString *os = uptmac->GetLineWith(ds);
1190  if (!os) {
1191  // DataSet does not exist anymore
1192  if (strcmp(dsName, "ls"))
1193  Warning("CheckLocalCache", "dataset '%s' does not exists anymore", ds.Data());
1194  return 1;
1195  }
1196  // Extract the relevant information
1197  TString s;
1198  Int_t from = 0;
1199  while (os->GetString().Tokenize(s, from, " ")) {
1200  if (!s.IsDigit() && s != ds) {
1201  remmd5s = s;
1202  }
1203  }
1204  if (remmd5s == "---") {
1205  // DataSet does not exist anymore
1206  if (strcmp(dsName, "ls"))
1207  Warning("CheckLocalCache", "dataset '%s' does not exists anymore", ds.Data());
1208  return 1;
1209  }
1210  Bool_t need_update = (option & kNoCacheUpdate) ? kFALSE : kTRUE;
1211  if (!gSystem->AccessPathName(locpath)) {
1212  if (need_update) {
1213  need_update = kFALSE;
1214  locmd5 = TMD5::ReadChecksum(locmd5path);
1215  if (!locmd5 && !(locmd5 = TMD5::FileChecksum(locpath))) {
1216  Warning("CheckLocalCache", "cannot get checksum of '%s' - assuming match failed", ds.Data());
1217  need_update = kTRUE;
1218  } else {
1219  if (remmd5s != locmd5->AsString()) need_update = kTRUE;
1220  }
1221  }
1222  } else {
1223  need_update = kTRUE;
1224  }
1225  // Get the file, if needed
1226  if (need_update) {
1227  SafeDelete(locmd5);
1228  if (!TFile::Cp(path, locpath, kFALSE)) {
1229  Error("CheckLocalCache", "cannot get remote file '%s' - ignoring", path.Data());
1230  return -1;
1231  }
1232  // Calculate and save the new checksum
1233  locmd5 = TMD5::FileChecksum(locpath);
1234  if (locmd5) {
1235  if (remmd5s != locmd5->AsString())
1236  Warning("CheckLocalCache", "checksum for freshly downloaded file '%s' does not match the"
1237  " one posted in '%s'", locpath.Data(), kDataSet_DataSetList);
1238  if (TMD5::WriteChecksum(locmd5path, locmd5) != 0)
1239  Warning("CheckLocalCache", "problems saving checksum to '%s' (errno: %d)",
1240  locmd5path.Data(), TSystem::GetErrno());
1241  } else {
1242  Warning("CheckLocalCache", "problems calculating checksum for '%s'", locpath.Data());
1243  }
1244  }
1245  SafeDelete(locmd5);
1246  // Done
1247  return 0;
1248 }
1249 
1250 ////////////////////////////////////////////////////////////////////////////////
1251 /// Clear cached information matching uri
1252 
1253 Int_t TDataSetManagerFile::ClearCache(const char *uri)
1254 {
1255  // Open the top directory
1256  void *dirp = gSystem->OpenDirectory(fLocalCacheDir.Data());
1257  if (!dirp) {
1258  Error("ClearCache", "cannot open directory '%s' (errno: %d)",
1259  fLocalCacheDir.Data(), TSystem::GetErrno());
1260  return -1;
1261  }
1262  TRegexp *re = 0;
1263  if (uri && strlen(uri) > 0) {
1264  if (strcmp(uri, "*") && strcmp(uri, "/*") && strcmp(uri, "/*/") &&
1265  strcmp(uri, "/*/*") && strcmp(uri, "/*/*/") && strcmp(uri, "/*/*/*")) {
1266  TString u(uri);
1267  // Remove leading '/'
1268  if (u(0) == '/') u.Remove(0,1);
1269  // Change '/' to '%'
1270  u.ReplaceAll("/", ".");
1271  // Init the regular expression
1272  u.ReplaceAll("*", ".*");
1273  re = new TRegexp(u.Data());
1274  }
1275  }
1276 
1277  Printf(" Dataset repository: %s", fDataSetDir.Data());
1278  Printf(" Local cache directory: %s", fLocalCacheDir.Data());
1279 
1280  Long64_t totsz = 0, nf = 0;
1281  FileStat_t st;
1282  TString path;
1283  const char *e = 0;
1284  while ((e = gSystem->GetDirEntry(dirp))) {
1285  // Skip basic entries
1286  if (!strcmp(e,".") || !strcmp(e,"..")) continue;
1287  // Apply regular expression, if requested
1288  if (re && TString(e).Index(*re) == kNPOS) continue;
1289  // Group directory
1290  path.Form("%s/%s", fLocalCacheDir.Data(), e);
1291  // Get file information
1292  if (gSystem->GetPathInfo(path, st) != 0) {
1293  Warning("ShowCache", "problems 'stat'-ing '%s' (errno: %d)",
1294  path.Data(), TSystem::GetErrno());
1295  continue;
1296  }
1297  // Count
1298  totsz += st.fSize;
1299  nf++;
1300  // Remove the file
1301  if (gSystem->Unlink(path) != 0) {
1302  Warning("ClearCache", "problems unlinking '%s' (errno: %d)",
1303  path.Data(), TSystem::GetErrno());
1304  }
1305  }
1306  gSystem->FreeDirectory(dirp);
1307  SafeDelete(re);
1308 
1309  // Notify totals
1310  Printf(" %lld bytes (%lld files) have been freed", totsz, nf);
1311 
1312  // Done
1313  return 0;
1314 }
1315 
1316 ////////////////////////////////////////////////////////////////////////////////
1317 /// Show cached information matching uri
1318 
1319 Int_t TDataSetManagerFile::ShowCache(const char *uri)
1320 {
1321  // Open the top directory
1322  void *dirp = gSystem->OpenDirectory(fLocalCacheDir.Data());
1323  if (!dirp) {
1324  Error("ShowCache", "cannot open directory '%s' (errno: %d)",
1325  fLocalCacheDir.Data(), TSystem::GetErrno());
1326  return -1;
1327  }
1328  TRegexp *re = 0;
1329  if (uri && strlen(uri) > 0) {
1330  if (strcmp(uri, "*") && strcmp(uri, "/*") && strcmp(uri, "/*/") &&
1331  strcmp(uri, "/*/*") && strcmp(uri, "/*/*/") && strcmp(uri, "/*/*/*")) {
1332  TString u(uri);
1333  // Remove leading '/'
1334  if (u(0) == '/') u.Remove(0,1);
1335  // Change '/' to '%'
1336  u.ReplaceAll("/", ".");
1337  // Init the regular expression
1338  u.ReplaceAll("*", ".*");
1339  re = new TRegexp(u.Data());
1340  }
1341  }
1342 
1343  Printf(" Dataset repository: %s", fDataSetDir.Data());
1344  Printf(" Local cache directory: %s", fLocalCacheDir.Data());
1345  Printf(" Last modified Size(bytes) File");
1346 
1347  Long64_t totsz = 0, nf = 0;
1348  FileStat_t st;
1349  TString path, sz;
1350  const char *e = 0;
1351  while ((e = gSystem->GetDirEntry(dirp))) {
1352  // Skip basic entries
1353  if (!strcmp(e,".") || !strcmp(e,"..")) continue;
1354  // Apply regular expression, if requested
1355  if (re && TString(e).Index(*re) == kNPOS) continue;
1356  // Group directory
1357  path.Form("%s/%s", fLocalCacheDir.Data(), e);
1358  // Get file information
1359  if (gSystem->GetPathInfo(path, st) != 0) {
1360  Warning("ShowCache", "problems 'stat'-ing '%s' (errno: %d)",
1361  path.Data(), TSystem::GetErrno());
1362  continue;
1363  }
1364  // Count
1365  totsz += st.fSize;
1366  nf++;
1367  // Get modification time in human readable form
1368  TDatime tmod(st.fMtime);
1369  sz.Form("%lld", st.fSize);
1370  sz.Resize(12);
1371  Printf(" %s %s %s", tmod.AsSQLString(), sz.Data(), e);
1372  }
1373  gSystem->FreeDirectory(dirp);
1374  SafeDelete(re);
1375 
1376  // Notify totals
1377  Printf(" %lld files, %lld bytes", nf, totsz);
1378 
1379  // Done
1380  return 0;
1381 }
1382 
1383 ////////////////////////////////////////////////////////////////////////////////
1384 ///
1385 /// Writes indicated dataset.
1386 /// If option has the bit kFileMustExist set, the file must still exist,
1387 /// otherwise the new dataset is not written (returns 3 in this case).
1388 /// If checksum is non-zero the files current checksum is checked against it,
1389 /// if it does not match the file is not written (the function returns 2 in this
1390 /// case, if the file has disappeared it is also not written (i.e. checksum
1391 /// implies the bit kFileMustExist set in option).
1392 /// Returns != 0 for success, 0 for error
1393 
1394 Int_t TDataSetManagerFile::WriteDataSet(const char *group, const char *user,
1395  const char *dsName, TFileCollection *dataset,
1396  UInt_t option, TMD5 *checksum)
1397 {
1398  TString md5path, path, md5sum;
1399  Long_t mtime = 0;
1400  { TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
1401 
1402  Bool_t checkIfExists = ((option & kFileMustExist) || checksum) ? kTRUE : kFALSE;
1403 
1404  path = GetDataSetPath(group, user, dsName, md5path);
1405 
1406  if (checkIfExists) {
1407  // check if file still exists, otherwise it was deleted in the meanwhile and is not written here
1408  Long_t tmp;
1409  if (gSystem->GetPathInfo(path, 0, (Long_t*) 0, 0, &tmp) != 0) {
1410  if (gDebug > 0)
1411  Info("WriteDataSet", "Dataset disappeared. Discarding update.");
1412  return 3;
1413  }
1414  }
1415 
1416  if (checksum) {
1417  // verify md5 sum, otherwise the file was changed in the meanwhile and is not overwritten here
1418  TMD5 *checksum2 = TMD5::FileChecksum(path);
1419  if (!checksum2) {
1420  Error("WriteDataSet", "Could not get checksum of %s", path.Data());
1421  return 0;
1422  }
1423 
1424  Bool_t checksumAgrees = (*checksum == *checksum2);
1425  delete checksum2;
1426 
1427  if (!checksumAgrees) {
1428  if (gDebug > 0)
1429  Info("WriteDataSet", "Dataset changed. Discarding update.");
1430  return 2;
1431  }
1432  }
1433 
1434  // write first in ".<file>" then rename to recover from crash during writing
1435  TString tempFile(path);
1436  Int_t index = -1;
1437  while (tempFile.Index("/", index+1) >= 0)
1438  index = tempFile.Index("/", index+1);
1439 
1440  tempFile.Insert(index+1, ".");
1441 
1442  TFile *f = TFile::Open(tempFile, "RECREATE");
1443  if (!f) {
1444  Error("WriteDataSet", "Could not open dataset for writing %s", tempFile.Data());
1445  return 0;
1446  }
1447 
1448  // write full TFileCollection
1449  dataset->Write("dataset", TObject::kSingleKey | TObject::kOverwrite);
1450 
1451  // write only metadata
1452  THashList *list = dataset->GetList();
1453  dataset->SetList(0);
1454  dataset->Write("dataset_short", TObject::kSingleKey | TObject::kOverwrite);
1455 
1456  f->Close();
1457  delete f;
1458 
1459  // Restore full list
1460  dataset->SetList(list);
1461 
1462  // file is written, rename to real filename
1463  if (gSystem->Rename(tempFile, path) != 0) {
1464  Error("WriteDataSet", "renaming %s to %s failed; dataset might be corrupted",
1465  tempFile.Data(), path.Data());
1466  // Cleanup any MD5 sum information
1467  if (!gSystem->AccessPathName(md5path, kWritePermission) && gSystem->Unlink(md5path) != 0)
1468  Error("WriteDataSet", "unlink of %s failed", md5path.Data());
1469  return 0;
1470  }
1471  else if (fOpenPerms) {
1472  if (gSystem->Chmod(path.Data(), 0666) < 0) {
1473  Warning("NotifyUpdate",
1474  "can't set permissions of dataset file %s (#%d)",
1475  path.Data(), TSystem::GetErrno());
1476  }
1477  }
1478 
1479  // Save md5 sum, otherwise the file was changed in the meanwhile and is not overwritten here
1480  if (ChecksumDataSet(path, md5path, md5sum) != 0) {
1481  Error("WriteDataSet", "problems calculating checksum of %s", path.Data());
1482  return 0;
1483  }
1484  else if (fOpenPerms) {
1485  if (gSystem->Chmod(md5path.Data(), 0666) < 0) {
1486  Warning("NotifyUpdate",
1487  "can't set permissions of dataset MD5 checksum file %s (#%d)",
1488  md5path.Data(), TSystem::GetErrno());
1489  }
1490  }
1491 
1492  FileStat_t st;
1493  if (gSystem->GetPathInfo(path, st) != 0) {
1494  Error("WriteDataSet", "could not 'stat' the version of '%s'!", path.Data());
1495  return 0;
1496  }
1497  mtime= st.fMtime;
1498  }
1499 
1500  // The repository was updated
1501  if (NotifyUpdate(group, user, dsName, mtime, md5sum) != 0)
1502  Warning("WriteDataSet", "problems notifying update with 'NotifyUpdate'");
1503 
1504  return 1;
1505 }
1506 
1507 ////////////////////////////////////////////////////////////////////////////////
1508 /// Calculate the checksum of the indicated dataset at 'path' and save it to the
1509 /// appropriate file 'md5path'. The MD5 string is returned in 'md5sum'.
1510 /// Return 0 on success, -1 on error.
1511 
1512 Int_t TDataSetManagerFile::ChecksumDataSet(const char *path,
1513  const char *md5path, TString &checksum)
1514 {
1515  checksum = "";
1516  // Check inputs
1517  if (!path || strlen(path) <= 0 || !md5path || strlen(md5path) <= 0) {
1518  Error("ChecksumDataSet", "one or more inputs are invalid ('%s','%s')",
1519  path, md5path);
1520  return -1;
1521  }
1522  // Calculate md5 sum
1523  TMD5 *md5sum = TMD5::FileChecksum(path);
1524  if (!md5sum) {
1525  Error("ChecksumDataSet", "problems calculating checksum of '%s'", path);
1526  return -1;
1527  }
1528  // Save it to a file
1529  if (TMD5::WriteChecksum(md5path, md5sum) != 0) {
1530  Error("ChecksumDataSet", "problems saving checksum to '%s'", md5path);
1531  SafeDelete(md5sum);
1532  return -1;
1533  }
1534  // Fill output
1535  checksum = md5sum->AsString();
1536  // Done
1537  SafeDelete(md5sum);
1538  return 0;
1539 }
1540 
1541 ////////////////////////////////////////////////////////////////////////////////
1542 /// Removes the indicated dataset
1543 
1544 Bool_t TDataSetManagerFile::RemoveDataSet(const char *group, const char *user,
1545  const char *dsName)
1546 {
1547  TString md5path, path;
1548  { TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
1549 
1550  path = GetDataSetPath(group, user, dsName, md5path);
1551  Int_t rc = 0;
1552  // Remove the main file
1553  if ((rc = gSystem->Unlink(path)) != 0)
1554  Warning("RemoveDataSet", "problems removing main file '%s' (errno: %d)",
1555  path.Data(), TSystem::GetErrno());
1556  // Remove the checksum file
1557  if (gSystem->Unlink(md5path) != 0)
1558  Warning("RemoveDataSet", "problems removing chcksum file '%s' (errno: %d)",
1559  md5path.Data(), TSystem::GetErrno());
1560  }
1561 
1562  // The repository was updated
1563  if (gSystem->AccessPathName(path, kFileExists)) {
1564  if (NotifyUpdate(group, user, dsName, 0, "removed") != 0)
1565  Warning("RemoveDataSet", "problems notifying update with 'NotifyUpdate'");
1566  // Success
1567  return kTRUE;
1568  }
1569  // Failure
1570  return kFALSE;
1571 }
1572 
1573 ////////////////////////////////////////////////////////////////////////////////
1574 /// Checks if the indicated dataset exits
1575 
1576 Bool_t TDataSetManagerFile::ExistsDataSet(const char *group, const char *user,
1577  const char *dsName)
1578 {
1579  TLockFile lock(fDataSetLockFile, fLockFileTimeLimit);
1580 
1581  TString md5path, path(GetDataSetPath(group, user, dsName, md5path));
1582 
1583  return (gSystem->AccessPathName(path) == kFALSE);
1584 }
1585 
1586 ////////////////////////////////////////////////////////////////////////////////
1587 /// Register a dataset, perfoming quota checkings and verification, if required.
1588 /// If a dataset with the same name already exists the action fails unless 'opts'
1589 /// contains 'O', in which case the old dataset is overwritten, or contains 'U',
1590 /// in which case 'newDataSet' is added to the existing dataset (duplications are
1591 /// ignored, if any).
1592 /// If 'opts' contains 'V' the dataset files are also verified (if the dataset manager
1593 /// is configured to allow so). By default the dataset is not verified.
1594 /// If 'opts' contains 'T' the in the dataset object (status bits, meta,...)
1595 /// is trusted, i.e. not reset (if the dataset manager is configured to allow so).
1596 /// Returns 0 on success, -1 on failure
1597 
1598 Int_t TDataSetManagerFile::RegisterDataSet(const char *uri,
1599  TFileCollection *newDataSet,
1600  const char *opts)
1601 {
1602  if (!TestBit(TDataSetManager::kAllowRegister))
1603  return -1;
1604 
1605  // Get the dataset name
1606  TString dsName;
1607  if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE) == kFALSE) {
1608  Error("RegisterDataSet", "problem parsing uri: %s", uri);
1609  return -1;
1610  }
1611 
1612  // The dataset
1613  TFileCollection *dataSet = newDataSet;
1614  // Check option
1615  TString opt(opts);
1616  // If in update mode, retrieve the existing dataset, if any
1617  if (opt.Contains("U", TString::kIgnoreCase)) {
1618  // Fail if it exists already
1619  if (ExistsDataSet(fGroup, fUser, dsName)) {
1620  // Retrieve the dataset
1621  if (!(dataSet = GetDataSet(fGroup, fUser, dsName))) {
1622  // Dataset name does exist
1623  Warning("RegisterDataSet",
1624  "dataset '%s' claimed to exists but retrieval failed - ignoring", uri);
1625  dataSet = newDataSet;
1626  } else {
1627  // Add new dataset to existing one
1628  dataSet->Add(newDataSet);
1629  }
1630  }
1631  } else if (!opt.Contains("O", TString::kIgnoreCase)) {
1632  // Fail if it exists already
1633  if (ExistsDataSet(fGroup, fUser, dsName)) {
1634  //Dataset name does exist
1635  Error("RegisterDataSet", "dataset '%s' exists already", uri);
1636  return -1;
1637  }
1638  }
1639 
1640  // A temporary list to hold the unique members (i.e. the very set)
1641  TList *uniqueFileList = new TList();
1642  TIter nextFile(dataSet->GetList());
1643  TFileInfo *prevFile = (TFileInfo*)nextFile();
1644  uniqueFileList->Add(prevFile);
1645  while (TFileInfo *obj = (TFileInfo*)nextFile()) {
1646  // Add entities only once to the temporary list
1647  if (!uniqueFileList->FindObject(obj->GetFirstUrl()->GetUrl()))
1648  uniqueFileList->Add(obj);
1649  }
1650 
1651  // Clear dataSet and add contents of uniqueFileList needed otherwise
1652  // THashList deletes the objects even when nodelete is set
1653  dataSet->GetList()->SetOwner(0);
1654  dataSet->GetList()->Clear("nodelete");
1655  dataSet->GetList()->SetOwner(1);
1656  dataSet->GetList()->AddAll(uniqueFileList);
1657  uniqueFileList->SetOwner(kFALSE);
1658  delete uniqueFileList;
1659 
1660  // Enforce certain settings
1661  Bool_t reset = kTRUE;
1662  if (opt.Contains("T", TString::kIgnoreCase)) {
1663  if (!TestBit(TDataSetManager::kTrustInfo)) {
1664  Warning("RegisterDataSet", "configured to not trust the information"
1665  " provided by users: ignoring request");
1666  } else {
1667  reset = kFALSE;
1668  }
1669  }
1670  if (reset) {
1671  dataSet->SetName(dsName);
1672  dataSet->ResetBitAll(TFileInfo::kStaged);
1673  dataSet->ResetBitAll(TFileInfo::kCorrupted);
1674  dataSet->RemoveMetaData();
1675  }
1676 
1677  // Verify the dataset if required
1678  if (opt.Contains("V", TString::kIgnoreCase)) {
1679  if (TestBit(TDataSetManager::kAllowVerify)) {
1680  // Reopen files and notify
1681  if (TDataSetManager::ScanDataSet(dataSet, 1, 0, 0, kTRUE ) < 0) {
1682  Error("RegisterDataSet", "problems verifying the dataset");
1683  return -1;
1684  }
1685  } else {
1686  Warning("RegisterDataSet", "user-driven verification not allowed: ignoring request");
1687  }
1688  }
1689 
1690  // Update accumulated information
1691  dataSet->Update(fAvgFileSize);
1692 
1693  if (TestBit(TDataSetManager::kCheckQuota)) {
1694  if (dataSet->GetTotalSize() <= 0) {
1695  Error("RegisterDataSet", "datasets without size information are not accepted:");
1696  if (fAvgFileSize < 0) {
1697  Error("RegisterDataSet", "you may want to define an average"
1698  " file size to get an estimated dataset size");
1699  }
1700  return -1;
1701  }
1702  // now check the quota
1703  UpdateUsedSpace();
1704  Long64_t used = GetGroupUsed(fGroup) + dataSet->GetTotalSize();
1705 
1706  Info("RegisterDataSet", "your group %s uses %.1f GB + %.1f GB for the new dataset; "
1707  "the available quota is %.1f GB", fGroup.Data(),
1708  (Float_t) GetGroupUsed(fGroup) / 1073741824,
1709  (Float_t) dataSet->GetTotalSize() / 1073741824,
1710  (Float_t) GetGroupQuota(fGroup) / 1073741824);
1711  if (used > GetGroupQuota(fGroup)) {
1712  Error("RegisterDataSet", "quota exceeded");
1713  return -1;
1714  }
1715  }
1716 
1717  Bool_t success = WriteDataSet(fGroup, fUser, dsName, dataSet);
1718  if (!success)
1719  Error("RegisterDataSet", "could not write dataset: %s", dsName.Data());
1720 
1721  // Done
1722  return ((success) ? 0 : -1);
1723 }
1724 ////////////////////////////////////////////////////////////////////////////////
1725 /// Scans the dataset indicated by <uri> and returns the number of missing files.
1726 /// Returns -1 if any failure occurs, >= 0 on success.
1727 /// For more details, see documentation of
1728 /// ScanDataSet(TFileCollection *dataset, const char *option)
1729 
1730 Int_t TDataSetManagerFile::ScanDataSet(const char *uri, UInt_t opt)
1731 {
1732  TString dsName, dsTree;
1733  if ((opt & kSetDefaultTree)) {
1734  if (TestBit(TDataSetManager::kAllowRegister)) {
1735  if (ParseUri(uri, 0, 0, &dsName, &dsTree, kTRUE)) {
1736  TFileCollection *dataset = GetDataSet(fGroup, fUser, dsName);
1737  if (!dataset) return -1;
1738  dataset->SetDefaultTreeName(dsTree.Data());
1739  Int_t rc = WriteDataSet(fGroup, fUser, dsName, dataset);
1740  delete dataset;
1741  return (rc == 0) ? -1 : 0;
1742  }
1743  }
1744  } else {
1745  if (TestBit(TDataSetManager::kAllowVerify)) {
1746  if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE, kTRUE)) {
1747  if (!(dsName.Contains("*"))) {
1748  if (ScanDataSet(fGroup, fUser, dsName, opt) > 0)
1749  return GetNDisapparedFiles();
1750  } else {
1751  TString luri = TString::Format("/%s/%s/%s", fGroup.Data(), fUser.Data(), dsName.Data());
1752  TMap *fcs = GetDataSets(luri, kList);
1753  if (!fcs) return -1;
1754  fcs->Print();
1755  Int_t ndisappeared = 0;
1756  TIter nxd(fcs);
1757  TObjString *d = 0;
1758  while ((d = (TObjString *) nxd())) {
1759  if (!(d->GetString().IsNull())) {
1760  TString dsn(d->GetName());
1761  if (dsn.Contains("/")) dsn.Remove(0, dsn.Last('/') + 1);
1762  if (ScanDataSet(fGroup, fUser, dsn, opt) > 0) {
1763  ndisappeared += GetNDisapparedFiles();
1764  } else {
1765  Warning("ScanDataSet", "problems processing dataset: %s", d->GetName());
1766  }
1767  } else {
1768  Warning("ScanDataSet", "empty string found in map while processing: %s", uri);
1769  }
1770  }
1771  SafeDelete(fcs);
1772  return ndisappeared;
1773  }
1774  }
1775  }
1776  }
1777  return -1;
1778 }
1779 
1780 ////////////////////////////////////////////////////////////////////////////////
1781 /// See documentation of ScanDataSet(TFileCollection *dataset, UInt_t option)
1782 
1783 Int_t TDataSetManagerFile::ScanDataSet(const char *group, const char *user,
1784  const char *dsName, UInt_t option)
1785 {
1786  if (!TestBit(TDataSetManager::kAllowVerify))
1787  return -1;
1788 
1789  TFileCollection *dataset = GetDataSet(group, user, dsName);
1790  if (!dataset)
1791  return -1;
1792 
1793  // File selection
1794  Int_t fopt = ((option & kAllFiles)) ? -1 : 0;
1795  if (fopt >= 0) {
1796  if ((option & kStagedFiles)) {
1797  fopt = 10;
1798  } else {
1799  if ((option & kReopen)) fopt++;
1800  if ((option & kTouch)) fopt++;
1801  }
1802  if ((option & kNoStagedCheck)) fopt += 100;
1803  } else {
1804  if ((option & kStagedFiles) || (option & kReopen) || (option & kTouch)) {
1805  Warning("ScanDataSet", "kAllFiles mode: ignoring kStagedFiles or kReopen"
1806  " or kTouch requests");
1807  }
1808  if ((option & kNoStagedCheck)) fopt -= 100;
1809  }
1810 
1811  // Type of action
1812  Int_t sopt = ((option & kNoAction)) ? -1 : 0;
1813  if (sopt >= 0) {
1814  if ((option & kLocateOnly) && (option & kStageOnly)) {
1815  Error("ScanDataSet", "kLocateOnly and kStageOnly cannot be processed concurrently");
1816  return -1;
1817  }
1818  if ((option & kLocateOnly)) sopt = 1;
1819  if ((option & kStageOnly)) sopt = 2;
1820  } else if ((option & kLocateOnly) || (option & kStageOnly)) {
1821  Warning("ScanDataSet", "kNoAction mode: ignoring kLocateOnly or kStageOnly requests");
1822  }
1823 
1824  Bool_t dbg = ((option & kDebug)) ? kTRUE : kFALSE;
1825  // Do the scan
1826  Int_t result = TDataSetManager::ScanDataSet(dataset, fopt, sopt, 0, dbg,
1827  &fNTouchedFiles, &fNOpenedFiles, &fNDisappearedFiles,
1828  (TList *)0, fAvgFileSize, fMSSUrl.Data(), -1, fStageOpts.Data());
1829  if (result == 2) {
1830  if (WriteDataSet(group, user, dsName, dataset) == 0) {
1831  delete dataset;
1832  return -2;
1833  }
1834  }
1835  delete dataset;
1836 
1837  return result;
1838 }
1839 
1840 ////////////////////////////////////////////////////////////////////////////////
1841 ///
1842 /// Returns all datasets for the <group> and <user> specified by <uri>.
1843 /// If <user> is 0, it returns all datasets for the given <group>.
1844 /// If <group> is 0, it returns all datasets.
1845 /// The returned TMap contains:
1846 /// <group> --> <map of users> --> <map of datasets> --> <dataset> (TFileCollection)
1847 ///
1848 /// The unsigned int 'option' is forwarded to GetDataSet and BrowseDataSet.
1849 /// Available options (to be .or.ed):
1850 /// kShowDefault a default selection is shown that include the ones from
1851 /// the current user, the ones from the group and the common ones
1852 /// kPrint print the dataset content
1853 /// kQuotaUpdate update quotas
1854 /// kExport use export naming
1855 ///
1856 /// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
1857 /// NB2: for options "kPrint" and "kQuoatUpdate" return is null.
1858 
1859 TMap *TDataSetManagerFile::GetDataSets(const char *uri, UInt_t option)
1860 {
1861  TString dsUser, dsGroup, dsName;
1862 
1863  if (((option & kPrint) || (option & kExport)) && strlen(uri) <= 0)
1864  option |= kShowDefault;
1865 
1866  if (ParseUri(uri, &dsGroup, &dsUser, &dsName, 0, kFALSE, kTRUE))
1867  return GetDataSets(dsGroup, dsUser, dsName, option);
1868  return (TMap *)0;
1869 }
1870 
1871 ////////////////////////////////////////////////////////////////////////////////
1872 /// Utility function used in various methods for user dataset upload.
1873 
1874 TFileCollection *TDataSetManagerFile::GetDataSet(const char *uri, const char *opts)
1875 {
1876  TString dsUser, dsGroup, dsName, ss(opts);
1877 
1878  TFileCollection *fc = 0;
1879  if (!strchr(uri, '*')) {
1880  if (!ParseUri(uri, &dsGroup, &dsUser, &dsName)) return fc;
1881  UInt_t opt = (ss.Contains("S:") || ss.Contains("short:")) ? kReadShort : 0;
1882  ss.ReplaceAll("S:","");
1883  ss.ReplaceAll("short:","");
1884  fc = GetDataSet(dsGroup, dsUser, dsName, opt);
1885  } else {
1886  TMap *fcs = GetDataSets(uri);
1887  if (!fcs) return fc;
1888  TIter nxd(fcs);
1889  TObject *k = 0;
1890  TFileCollection *xfc = 0;
1891  while ((k = nxd()) && (xfc = (TFileCollection *) fcs->GetValue(k))) {
1892  if (!fc) {
1893  // The first one
1894  fc = xfc;
1895  fcs->Remove(k);
1896  } else {
1897  // Add
1898  fc->Add(xfc);
1899  }
1900  }
1901  }
1902 
1903  if (fc && !ss.IsNull()) {
1904  // Build up the subset
1905  TFileCollection *sfc = 0;
1906  TString s;
1907  Int_t from = 0;
1908  while (ss.Tokenize(s, from, ",")) {
1909  TFileCollection *xfc = fc->GetFilesOnServer(s.Data());
1910  if (xfc) {
1911  if (sfc) {
1912  sfc->Add(xfc);
1913  delete xfc;
1914  } else {
1915  sfc = xfc;
1916  }
1917  }
1918  }
1919  // Cleanup
1920  delete fc;
1921  fc = sfc;
1922  }
1923  // Done
1924  return fc;
1925 }
1926 
1927 ////////////////////////////////////////////////////////////////////////////////
1928 /// Removes the indicated dataset
1929 
1930 Bool_t TDataSetManagerFile::RemoveDataSet(const char *uri)
1931 {
1932  TString dsName;
1933 
1934  if (TestBit(TDataSetManager::kAllowRegister)) {
1935  if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE)) {
1936  Bool_t rc = RemoveDataSet(fGroup, fUser, dsName);
1937  if (rc) return kTRUE;
1938  Error("RemoveDataSet", "error removing dataset %s", dsName.Data());
1939  }
1940  }
1941  return kFALSE;
1942 }
1943 
1944 ////////////////////////////////////////////////////////////////////////////////
1945 /// Checks if the indicated dataset exits
1946 
1947 Bool_t TDataSetManagerFile::ExistsDataSet(const char *uri)
1948 {
1949  TString dsUser, dsGroup, dsName;
1950 
1951  if (ParseUri(uri, &dsGroup, &dsUser, &dsName))
1952  return ExistsDataSet(dsGroup, dsUser, dsName);
1953  return kFALSE;
1954 }
1955 
1956 ////////////////////////////////////////////////////////////////////////////////
1957 /// updates the used space maps
1958 
1959 void TDataSetManagerFile::UpdateUsedSpace()
1960 {
1961  // Clear used space entries
1962  fGroupUsed.DeleteAll();
1963  fUserUsed.DeleteAll();
1964 
1965  // Scan the existing datasets
1966  GetDataSets(0, 0, 0, (UInt_t)kQuotaUpdate);
1967 }
1968 
1969 ////////////////////////////////////////////////////////////////////////////////
1970 /// Gets last dataset modification time. Returns -1 on error, or number of
1971 /// seconds since epoch on success
1972 
1973 Long_t TDataSetManagerFile::GetModTime(const char *uri)
1974 {
1975  TString group, user, name, md5path;
1976  if (!ParseUri(uri, &group, &user, &name)) {
1977  return -1;
1978  }
1979 
1980  TString path( GetDataSetPath(group, user, name, md5path) );
1981 
1982  Long_t modTime;
1983  if (gSystem->GetPathInfo(path.Data(),
1984  (Long_t *)0, (Long_t *)0, (Long_t *)0, &modTime)) {
1985  return -1;
1986  }
1987 
1988  return modTime;
1989 }