Add/use createDirectoriesIfAbsent method, robust against concurrent calls

Common/Utils/include/CommonUtils/FileSystemUtils.h

@@ -16,6 +16,7 @@
 
 #include <vector>
 #include <string>
+#include <unistd.h>
 
 namespace o2::utils
 {
@@ -28,6 +29,9 @@ std::vector<std::string> listFiles(std::string const& dir, std::string const& se
 // same in the current dir
 std::vector<std::string> listFiles(std::string const& searchpattern);
 
+// create path if absent, account for eventual concurrent creation
+void createDirectoriesIfAbsent(std::string const& path);
+
 } // namespace o2::utils
 
 #endif //O2_FILEITERATOR_H

Common/Utils/src/FileFetcher.cxx

@@ -14,6 +14,7 @@
 
 #include "CommonUtils/FileFetcher.h"
 #include "CommonUtils/StringUtils.h"
+#include "CommonUtils/FileSystemUtils.h"
 #include "Framework/Logger.h"
 #include <filesystem>
 #include <fstream>
@@ -55,7 +56,7 @@ FileFetcher::FileFetcher(const std::string& input, const std::string& selRegex,
         throw std::runtime_error(fmt::format("remote files asked but copy cmd \"{}\" is not valid", mCopyCmd));
       }
       try {
-        fs::create_directories(mCopyDirName);
+        o2::utils::createDirectoriesIfAbsent(mCopyDirName);
       } catch (...) {
         throw std::runtime_error(fmt::format("failed to create scratch directory {}", mCopyDirName));
       }

Common/Utils/src/FileSystemUtils.cxx

@@ -18,6 +18,8 @@
 #include <vector>
 #include <regex>
 #include <iostream>
+#include <unistd.h>
+#include <fmt/format.h>
 
 namespace o2::utils
 {
@@ -53,4 +55,11 @@ std::vector<std::string> listFiles(std::string const& searchpattern)
   return listFiles("./", searchpattern);
 }
 
+void createDirectoriesIfAbsent(std::string const& path)
+{
+  if (!std::filesystem::create_directories(path) && !std::filesystem::is_directory(path)) {
+    throw std::runtime_error(fmt::format("Failed to create {} directory", path));
+  }
+}
+
 } // namespace o2::utils

Detectors/CTF/workflow/src/CTFWriterSpec.cxx

@@ -22,6 +22,7 @@
 #include "CTFWorkflow/CTFWriterSpec.h"
 #include "DetectorsCommonDataFormats/CTFHeader.h"
 #include "CommonUtils/NameConf.h"
+#include "CommonUtils/FileSystemUtils.h"
 #include "DetectorsCommonDataFormats/EncodedBlocks.h"
 #include "DetectorsCommonDataFormats/FileMetaData.h"
 #include "CommonUtils/StringUtils.h"
@@ -226,14 +227,7 @@ void CTFWriterSpec::init(InitContext& ic)
     }
   }
   mChkSize = std::max(size_t(mMinSize * 1.1), mMaxSize);
-  if (!std::filesystem::exists(LOCKFileDir)) {
-    if (!std::filesystem::create_directories(LOCKFileDir)) {
-      usleep(10); // protection in case the directory was being created by other process at the time of query
-      if (!std::filesystem::exists(LOCKFileDir)) {
-        throw std::runtime_error(fmt::format("Failed to create {} directory", LOCKFileDir));
-      }
-    }
-  }
+  o2::utils::createDirectoriesIfAbsent(LOCKFileDir);
 
   if (mCreateDict) { // make sure that there is no local dictonary
     for (int id = 0; id < DetID::nDetectors; id++) {
@@ -501,13 +495,8 @@ void CTFWriterSpec::prepareTFTreeAndFile(const o2::header::DataHeader* dh)
     }
     if (mCreateRunEnvDir && !mEnvironmentID.empty()) {
       ctfDir += fmt::format("{}_{}/", mEnvironmentID, mRun);
-      if (!std::filesystem::exists(ctfDir)) {
-        if (!std::filesystem::create_directories(ctfDir)) {
-          throw std::runtime_error(fmt::format("Failed to create {} directory", ctfDir));
-        } else {
-          LOG(info) << "Created {} directory for CTFs output" << ctfDir;
-        }
-      }
+      o2::utils::createDirectoriesIfAbsent(ctfDir);
+      LOG(info) << "Created {} directory for CTFs output" << ctfDir;
     }
     mCurrentCTFFileName = o2::base::NameConf::getCTFFileName(mRun, dh->firstTForbit, dh->tfCounter);
     mCurrentCTFFileNameFull = fmt::format("{}{}", ctfDir, mCurrentCTFFileName);

Detectors/ITSMFT/ITS/workflow/src/ThresholdCalibratorSpec.cxx

@@ -12,6 +12,7 @@
 /// @file   ThresholdCalibratorSpec.cxx
 
 #include "ITSWorkflow/ThresholdCalibratorSpec.h"
+#include "CommonUtils/FileSystemUtils.h"
 
 #ifdef WITH_OPENMP
 #include <omp.h>
@@ -140,13 +141,8 @@ void ITSThresholdCalibrator::initThresholdTree(bool recreate /*=true*/)
 {
   // Create output directory to store output
   std::string dir = this->mOutputDir + fmt::format("{}_{}/", this->mEnvironmentID, this->mRunNumber);
-  if (!std::filesystem::exists(dir)) {
-    if (!std::filesystem::create_directories(dir)) {
-      throw std::runtime_error("Failed to create " + dir + " directory");
-    } else {
-      LOG(info) << "Created " << dir << " directory for threshold output";
-    }
-  }
+  o2::utils::createDirectoriesIfAbsent(dir);
+  LOG(info) << "Created " << dir << " directory for threshold output";
 
   std::string filename = dir + std::to_string(this->mRunNumber) + '_' +
                          std::to_string(this->mFileNumber) + '_' + this->mHostname + "_modSel" + std::to_string(mChipModSel) + ".root.part";

EventVisualisation/Workflow/src/FileProducer.cxx

@@ -14,6 +14,7 @@
 /// \author julian.myrcha@cern.ch
 
 #include "EveWorkflow/FileProducer.h"
+#include "CommonUtils/FileSystemUtils.h"
 
 #include <deque>
 #include <iostream>
@@ -49,7 +50,7 @@ FileProducer::FileProducer(const std::string& path, int filesInFolder, const std
   this->mFilesInFolder = filesInFolder;
   this->mPath = path;
   this->mName = name;
-  std::filesystem::create_directories(this->mPath); // create folder if not exists (fails if no rights)
+  o2::utils::createDirectoriesIfAbsent(path); // create folder if not exists (fails if no rights)
 }
 
 std::string FileProducer::newFileName() const