From 6e43c1e051ef084db88820951f69725fb040c109 Mon Sep 17 00:00:00 2001 From: miller86 Date: Sat, 25 Jun 2022 08:14:31 -0700 Subject: [PATCH 01/14] Support tar files --- src/databases/ZipWrapper/ZipWrapper.xml | 9 +++++++++ src/databases/ZipWrapper/ZipWrapperPluginInfo.C | 10 ++++++++++ .../ZipWrapper/avtZipWrapperFileFormatInterface.C | 10 +++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/databases/ZipWrapper/ZipWrapper.xml b/src/databases/ZipWrapper/ZipWrapper.xml index 5ae416ff522..5e972f737cc 100644 --- a/src/databases/ZipWrapper/ZipWrapper.xml +++ b/src/databases/ZipWrapper/ZipWrapper.xml @@ -3,7 +3,16 @@ *.gz *.bz2 + *.xz *.zip + *.tar.gz + *.tgz + *.tar.bz2 + *.tbz2 + *.tar.bz + *.tbz + *.tar.xz + *.txz avtZipWrapperFileFormatInterface.C diff --git a/src/databases/ZipWrapper/ZipWrapperPluginInfo.C b/src/databases/ZipWrapper/ZipWrapperPluginInfo.C index 87772b31cfe..760e1dd7362 100644 --- a/src/databases/ZipWrapper/ZipWrapperPluginInfo.C +++ b/src/databases/ZipWrapper/ZipWrapperPluginInfo.C @@ -120,7 +120,17 @@ ZipWrapperGeneralPluginInfo::GetDefaultFilePatterns() const { std::vector defaultPatterns; defaultPatterns.push_back("*.gz"); + defaultPatterns.push_back("*.bz"); defaultPatterns.push_back("*.bz2"); + defaultPatterns.push_back("*.xz"); + defaultPatterns.push_back("*.tar.gz"); + defaultPatterns.push_back("*.tar.bz"); + defaultPatterns.push_back("*.tar.bz2"); + defaultPatterns.push_back("*.tar.xz"); + defaultPatterns.push_back("*.tgz"); + defaultPatterns.push_back("*.tbz"); + defaultPatterns.push_back("*.tbz2"); + defaultPatterns.push_back("*.txz"); defaultPatterns.push_back("*.zip"); return defaultPatterns; diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 276a06456aa..937921638aa 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -736,8 +736,16 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac } #else - if (ext == ".gz") + // have to handle tar archives first in this logic + if (ext == ".tar.gz" || ext == ".tgz" || + ext == ".tar.bz" || ext == ".tbz" || + ext == ".tar.bz2" || ext == ".tbz2") || + ext == ".tar.xz || ext == ".txz") + dcmd = "tar xvf"; + else if (ext == ".gz") dcmd = "gunzip -f"; + else if (ext == ".xz") + dcmd = "unxz -f"; else if (ext == ".bz") dcmd = "bunzip -f"; else if (ext == ".bz2") From 631876134b341824c6645b1c9c51c512a96280d3 Mon Sep 17 00:00:00 2001 From: miller86 Date: Sat, 25 Jun 2022 09:19:44 -0700 Subject: [PATCH 02/14] fix syntax errors --- src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 937921638aa..ab07c256bfb 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -739,8 +739,8 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac // have to handle tar archives first in this logic if (ext == ".tar.gz" || ext == ".tgz" || ext == ".tar.bz" || ext == ".tbz" || - ext == ".tar.bz2" || ext == ".tbz2") || - ext == ".tar.xz || ext == ".txz") + ext == ".tar.bz2" || ext == ".tbz2" || + ext == ".tar.xz" || ext == ".txz") dcmd = "tar xvf"; else if (ext == ".gz") dcmd = "gunzip -f"; From 6791b9f6d64b68b1d2c5c0164e36450264e43f57 Mon Sep 17 00:00:00 2001 From: miller86 Date: Fri, 16 Sep 2022 17:00:33 -0700 Subject: [PATCH 03/14] add dir logic for tarfiles --- .../avtZipWrapperFileFormatInterface.C | 82 ++++++++++++++++--- .../avtZipWrapperFileFormatInterface.h | 11 +-- 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index ab07c256bfb..921da76817a 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -23,6 +23,7 @@ #include #include #else +#include #include #include // for WIFEXITED and WEXITSTATUS #endif @@ -61,6 +62,11 @@ vector avtZipWrapperFileFormatInterface::objL int avtZipWrapperFileFormatInterface::maxDecompressedFiles = 50; static bool atExiting = false; +typedef struct DecompressedFileInfo { + string rmDirname; + avtFileFormatInterface *iface; +} DecompressedFileInfo; + // **************************************************************************** // Class: avtZWGenercDatabase // @@ -202,21 +208,69 @@ class avtZipWrapperFileFormat : public avtMTMDFileFormat // **************************************************************************** static void FreeUpCacheSlot(void *item) { - avtZWFileFormatInterface *iface = (avtZWFileFormatInterface*) item; + DecompressedFileInfo *finfo = (DecompressedFileInfo*) item; + avtZWFileFormatInterface *iface = (avtZWFileFormatInterface*) finfo->iface; string filename = iface->GetFilename(0); debug5 << "Removing decompressed file \"" << filename << "\"" << endl; delete iface; errno = 0; - if (unlink(filename.c_str()) != 0 && errno != ENOENT) + static int issuedWarnings = 0; + + if (finfo->rmDirname == "") + { + if (unlink(filename.c_str()) != 0 && errno != ENOENT) + { + if (issuedWarnings < 5) + { + debug5 << "Unable to unlink() decompressed file \"" << filename << "\"" << endl; + debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + cerr << "Unable to remove decompressed file \"" << filename << "\"" << endl; + cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + issuedWarnings++; + } + } + } + else { - static int issuedWarning = 0; - if (issuedWarning < 5) + DIR *dirp = opendir(finfo->rmDirname.c_str()); + if (!dirp) { - debug5 << "Unable to unlink() decompressed file \"" << filename << "\"" << endl; - debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - cerr << "Unable to remove decompressed file \"" << filename << "\"" << endl; - cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - issuedWarning++; + if (issuedWarnings < 5) + { + debug5 << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; + cerr << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; + issuedWarnings++; + } + + } + dirent *dp; + while ((dp = readdir(dirp))) + { + errno = 0; + if (unlink(dp->d_name) != 0 && errno != ENOENT) + { + if (issuedWarnings < 5) + { + debug5 << "Unable to unlink() \"" << dp->d_name << "\"" << endl; + debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + cerr << "Unable to unlink() \"" << dp->d_name << "\"" << endl; + cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + issuedWarnings++; + } + } + } + closedir(dirp); + errno = 0; + if (rmdir(finfo->rmDirname.c_str()) != 0) + { + if (issuedWarnings < 5) + { + debug5 << "Unable to rmdir() \"" << finfo->rmDirname << "\"" << endl; + debug5 << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + cerr << "Unable to rmdir() \"" << finfo->rmDirname << "\"" << endl; + cerr << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + issuedWarnings++; + } } } } @@ -701,7 +755,8 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac if (decompressedFilesCache.exists(compressedName)) { debug5 << "Found interface object for file \"" << compressedName << "\" in cache" << endl; - avtFileFormatInterface *retval = decompressedFilesCache[compressedName]; + DecompressedFileInfo *finfo = decompressedFilesCache[compressedName]; + avtFileFormatInterface *retval = finfo->iface; // Always update the interface object to whatever the dummy format thinks // is right before returning the object for use. UpdateRealFileFormatInterface(retval); @@ -883,9 +938,14 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac if (!dontCache) { + DecompressedFileInfo *finfo = new DecompressedFileInfo(); + finfo->iface = realInterface; + finfo->rmDirname = ""; + if (dcmd.substr(0,5) == "tar x") + finfo->rmDirname = bname; UpdateRealFileFormatInterface(realInterface); realInterface->SetDatabaseMetaData(&mdCopy, 0); - decompressedFilesCache[compressedName] = realInterface; + decompressedFilesCache[compressedName] = finfo; } return realInterface; diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.h b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.h index fe443390d3d..33092c86f5e 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.h +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.h @@ -20,6 +20,7 @@ class avtZipWrapperFileFormat; class DBOptionsAttributes; class CommonDatabasePluginInfo; class DatabasePluginManager; +struct DecompressedFileInfo; // **************************************************************************** // Class: avtZipWrapperFileFormatInterface @@ -130,8 +131,8 @@ class avtZipWrapperFileFormatInterface : public avtFileFormatInterface { return false; } protected: - virtual int GetNumberOfFileFormats(void) { return 1; }; - virtual avtFileFormat *GetFormat(int) const; + virtual int GetNumberOfFileFormats(void) { return 1; }; + virtual avtFileFormat *GetFormat(int) const; private: static void Initialize(int procNum, int procCount, @@ -140,8 +141,8 @@ class avtZipWrapperFileFormatInterface : public avtFileFormatInterface static void CleanUpAtExit(); static std::vector objList; - static std::string tmpDir; - static std::string decompCmd; + static std::string tmpDir; + static std::string decompCmd; static int maxDecompressedFiles; int procNum; @@ -163,7 +164,7 @@ class avtZipWrapperFileFormatInterface : public avtFileFormatInterface std::string pluginId; bool realPluginWasLoadedByMe; - MRUCache + MRUCache decompressedFilesCache; }; From fc80a3429c115387a890f6438734415bb6a62d47 Mon Sep 17 00:00:00 2001 From: miller86 Date: Fri, 16 Sep 2022 17:05:05 -0700 Subject: [PATCH 04/14] add missing delete --- src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 921da76817a..9a57741f113 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -213,11 +213,11 @@ static void FreeUpCacheSlot(void *item) string filename = iface->GetFilename(0); debug5 << "Removing decompressed file \"" << filename << "\"" << endl; delete iface; - errno = 0; static int issuedWarnings = 0; if (finfo->rmDirname == "") { + errno = 0; if (unlink(filename.c_str()) != 0 && errno != ENOENT) { if (issuedWarnings < 5) @@ -273,6 +273,7 @@ static void FreeUpCacheSlot(void *item) } } } + delete finfo; } // **************************************************************************** From 3e8c9c26226ef6530b28e199eab72c5029b1c1b8 Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 20 Sep 2022 15:36:49 -0700 Subject: [PATCH 05/14] add opt for dir-per-timestep --- src/tools/data/datagen/multidir.C | 53 +++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/tools/data/datagen/multidir.C b/src/tools/data/datagen/multidir.C index 072bf3e5f8d..91e999db5a5 100644 --- a/src/tools/data/datagen/multidir.C +++ b/src/tools/data/datagen/multidir.C @@ -53,6 +53,13 @@ float g_Center[2]; // // Mark C. Miller, Tue Jul 17 20:12:32 PDT 2012 // Add option controlling use of absolute file paths in multi-block objects. +// +// Mark C. Miller, Tue Sep 20 13:12:48 PDT 2022 +// Added dirPerTimestep option. Originally, this program created a directory +// for each domain and then wrote timesteps out in each domain. Thats fine +// but not typically what our application codes do. Instead, they write a +// directory per timestep with all domain files in that directory. If +// dirPerTimestep is true, this program will behave that way. // **************************************************************************** class ProgramOptions @@ -62,7 +69,8 @@ public: isteps(40), jsteps(40), driver(DB_PDB), - useAbsoluteFilePaths(false) + useAbsoluteFilePaths(false), + dirPerTimestep(false) { } @@ -70,6 +78,7 @@ public: int jsteps; int driver; bool useAbsoluteFilePaths; + bool dirPerTimestep; }; // **************************************************************************** @@ -135,7 +144,7 @@ GotoDirectory(const std::string &dirname) } // **************************************************************************** -// Function: DomainFileName +// Function: InstanceFileName // // Purpose: // Returns the domain filename at a given cycle. @@ -155,10 +164,13 @@ GotoDirectory(const std::string &dirname) // **************************************************************************** std::string -DomainFileName(int d) +InstanceFileName(int c, int d, ProgramOptions const &opt) { char filename[100]; - snprintf(filename, 100, "TIME%04d.silo", d); + if (opt.dirPerTimestep) + snprintf(filename, 100, "DOMAIN%03d.silo", d); + else + snprintf(filename, 100, "TIME%05d.silo", c); return filename; } @@ -183,10 +195,14 @@ DomainFileName(int d) // **************************************************************************** std::string -DirectoryName(int d) +DirectoryName(int c, int d, ProgramOptions const &opt) { char dirname[100]; - snprintf(dirname, 100, "p%05d", d); + + if (opt.dirPerTimestep) + snprintf(dirname, 100, "c%05d", c); + else + snprintf(dirname, 100, "p%03d", d); return dirname; } @@ -288,7 +304,7 @@ WriteDomainFile(int cycle, int dom, const ProgramOptions &opt) M.CreateNodalData("Array_comps/Array_002", RadialCos2, "cm"); M.CreateNodalData("Array_comps/Array_003", Wavy, "cm"); - std::string filename = DomainFileName(cycle); + std::string filename = InstanceFileName(cycle, dom, opt); DBfile *db; db = DBCreate(filename.c_str(), DB_CLOBBER, DB_LOCAL, "This is one domain of a dataset that tests multimeshes and " @@ -322,7 +338,7 @@ WriteMasterFile(int cycle, const ProgramOptions &opt) #define MAX_STRING 500 char root[MAX_STRING]; - snprintf(root, MAX_STRING, "multidir%04d.root", cycle); + snprintf(root, MAX_STRING, "multidir%05d.root", cycle); DBfile *db = DBCreate(root, DB_CLOBBER, DB_LOCAL, "Master file of a dataset that tests multimeshes and " "multivars with various directory heirarchy levels", opt.driver); @@ -338,10 +354,10 @@ WriteMasterFile(int cycle, const ProgramOptions &opt) std::string cwd = GetCurrentDirectory() + "/"; - std::string filename = DomainFileName(cycle); for(int dom = 0; dom < MAX_DOMAINS; ++dom) { - std::string dirname = DirectoryName(dom); + std::string dirname = DirectoryName(cycle, dom, opt); + std::string filename = InstanceFileName(cycle, dom, opt); meshNames[dom] = new char[MAX_STRING]; snprintf(meshNames[dom], MAX_STRING, "%s%s/%s:/Mesh", opt.useAbsoluteFilePaths?cwd.c_str():"", @@ -422,10 +438,10 @@ void WriteCycle(int cycle, const ProgramOptions &opt) { std::string topdir = GetCurrentDirectory(); - - for(int dom = 0; dom < MAX_DOMAINS; ++dom) + + for (int dom = 0; dom < MAX_DOMAINS; ++dom) { - std::string dirname = DirectoryName(dom); + std::string dirname = DirectoryName(cycle, dom, opt); GotoDirectory(dirname); // Create the file @@ -450,6 +466,9 @@ WriteCycle(int cycle, const ProgramOptions &opt) // // Mark C. Miller, Tue Jul 17 20:12:15 PDT 2012 // Add option to use absolute file paths for multi-block objects. +// +// Mark C. Miller, Tue Sep 20 13:15:06 PDT 2022 +// Add dirpts option // **************************************************************************** int @@ -460,12 +479,14 @@ main(int argc, char *argv[]) // Look through command line args. for(int j = 1; j < argc; ++j) { - if (strcmp(argv[j], "DB_HDF5") == 0) + if (!strncmp(argv[j], "DB_HDF5", 7)) opt.driver = DB_HDF5; - else if (strcmp(argv[j], "DB_PDB") == 0) + else if (!strncmp(argv[j], "DB_PDB", 6)) opt.driver = DB_PDB; - else if (strcmp(argv[j], "-abspaths") == 0) + else if (!strncmp(argv[j], "-abspaths", 9)) opt.useAbsoluteFilePaths = true; + else if (!strncmp(argv[j], "-dirpts", 7)) + opt.dirPerTimestep = true; else fprintf(stderr,"Unrecognized driver name \"%s\"\n", argv[j]); } From 8b259c2d1c5ceb4412789431848935dd9ebfc99f Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 20 Sep 2022 15:37:15 -0700 Subject: [PATCH 06/14] adjust typos --- .../ZipWrapper/avtZipWrapperFileFormatInterface.C | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 9a57741f113..42fd2adb41f 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -211,7 +211,7 @@ static void FreeUpCacheSlot(void *item) DecompressedFileInfo *finfo = (DecompressedFileInfo*) item; avtZWFileFormatInterface *iface = (avtZWFileFormatInterface*) finfo->iface; string filename = iface->GetFilename(0); - debug5 << "Removing decompressed file \"" << filename << "\"" << endl; + debug5 << "Removing decompressed entry \"" << filename << "\"" << endl; delete iface; static int issuedWarnings = 0; @@ -273,6 +273,7 @@ static void FreeUpCacheSlot(void *item) } } } + delete finfo; } @@ -635,11 +636,7 @@ avtZipWrapperFileFormatInterface::~avtZipWrapperFileFormatInterface() decompressedFilesCache.clear(); delete dummyFileFormat; - - // We use FreeUpCacheSlot here even though dummyInterface isn't cached - // because FreeUpCacheSlot is where all the logic for deleting the - // decompressed file associated with an interface resides. - FreeUpCacheSlot(dummyInterface); + delete dummyInterface; // if this is the last instance we have, finalize the class too if (objList.size() == 0) From 7c8bc6e8a8e750a247e4ee0422db1be93242e290 Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 20 Sep 2022 15:54:19 -0700 Subject: [PATCH 07/14] match root/dir stems --- src/tools/data/datagen/multidir.C | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tools/data/datagen/multidir.C b/src/tools/data/datagen/multidir.C index 91e999db5a5..f05724c3864 100644 --- a/src/tools/data/datagen/multidir.C +++ b/src/tools/data/datagen/multidir.C @@ -338,7 +338,10 @@ WriteMasterFile(int cycle, const ProgramOptions &opt) #define MAX_STRING 500 char root[MAX_STRING]; - snprintf(root, MAX_STRING, "multidir%05d.root", cycle); + if (opt.dirPerTimestep) + snprintf(root, MAX_STRING, "%s.root", DirectoryName(cycle, 0, opt).c_str()); + else + snprintf(root, MAX_STRING, "multidir%05d.root", cycle); DBfile *db = DBCreate(root, DB_CLOBBER, DB_LOCAL, "Master file of a dataset that tests multimeshes and " "multivars with various directory heirarchy levels", opt.driver); From ab38562d7e4cd33be63154d0b4ae4dbc6f4abf61 Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 20 Sep 2022 16:16:54 -0700 Subject: [PATCH 08/14] add xz to regexes --- .../ZipWrapper/avtZipWrapperFileFormatInterface.C | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 42fd2adb41f..c239f7284c8 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -547,9 +547,9 @@ avtZipWrapperFileFormatInterface::avtZipWrapperFileFormatInterface( // // Make sure the necessary real plugin is loaded. // - string ext = StringHelpers::ExtractRESubstr(inputFileList[0][0].c_str(), "<\\.(gz|bz|bz2|zip)$>"); + string ext = StringHelpers::ExtractRESubstr(inputFileList[0][0].c_str(), "<\\.(xz|gz|bz|bz2|zip)$>"); const char *bname = FileFunctions::Basename(inputFileList[0][0].c_str()); - string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(gz|bz|bz2|zip)$> \\1"); + string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(xz|gz|bz|bz2|zip)$> \\1"); // Save the pointer to the plugin manager. pluginManager = zwinfo->GetPluginManager(); @@ -762,9 +762,9 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac } debug5 << "Interface object for file \"" << compressedName << "\" not in cache" << endl; - string ext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(gz|bz|bz2|zip)$>"); + string ext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(xz|gz|bz|bz2|zip)$>"); const char *bname = FileFunctions::Basename(compressedName.c_str()); - string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(gz|bz|bz2|zip)$> \\1"); + string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(xz|gz|bz|bz2|zip)$> \\1"); string dcmd = decompCmd; if (dcmd == "") From 9e6d204335780e319a0d3eb26bbc93d099558e0e Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 20 Sep 2022 18:52:57 -0700 Subject: [PATCH 09/14] fix abs paths --- .../avtZipWrapperFileFormatInterface.C | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index c239f7284c8..637c4aafab7 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -240,22 +240,29 @@ static void FreeUpCacheSlot(void *item) debug5 << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; cerr << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; issuedWarnings++; + goto done; } - } dirent *dp; while ((dp = readdir(dirp))) { errno = 0; - if (unlink(dp->d_name) != 0 && errno != ENOENT) + if (string(dp->d_name) == ".") + continue; + if (string(dp->d_name) == "..") + continue; + string tmp = finfo->rmDirname + "/" + string(dp->d_name); +printf("unlinking \"%s\"\n", tmp.c_str()); + if (unlink(tmp.c_str()) != 0 && errno != ENOENT) { if (issuedWarnings < 5) { - debug5 << "Unable to unlink() \"" << dp->d_name << "\"" << endl; + debug5 << "Unable to unlink() \"" << tmp << "\"" << endl; debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - cerr << "Unable to unlink() \"" << dp->d_name << "\"" << endl; + cerr << "Unable to unlink() \"" << tmp << "\"" << endl; cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; issuedWarnings++; + goto done; } } } @@ -274,6 +281,8 @@ static void FreeUpCacheSlot(void *item) } } +done: + delete finfo; } @@ -547,9 +556,9 @@ avtZipWrapperFileFormatInterface::avtZipWrapperFileFormatInterface( // // Make sure the necessary real plugin is loaded. // - string ext = StringHelpers::ExtractRESubstr(inputFileList[0][0].c_str(), "<\\.(xz|gz|bz|bz2|zip)$>"); + string ext = StringHelpers::ExtractRESubstr(inputFileList[0][0].c_str(), "<\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$>"); const char *bname = FileFunctions::Basename(inputFileList[0][0].c_str()); - string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(xz|gz|bz|bz2|zip)$> \\1"); + string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\1"); // Save the pointer to the plugin manager. pluginManager = zwinfo->GetPluginManager(); @@ -564,7 +573,9 @@ avtZipWrapperFileFormatInterface::avtZipWrapperFileFormatInterface( pluginId = ""; dummyInterface = 0; const bool searchAllPlugins = true; - vector ids = pluginManager->GetMatchingPluginIds(dcname.c_str(), searchAllPlugins); + //vector ids = pluginManager->GetMatchingPluginIds(dcname.c_str(), searchAllPlugins); +#warning FIXME + vector ids = pluginManager->GetMatchingPluginIds("foo.silo", searchAllPlugins); for (size_t i = 0; i < ids.size() && dummyInterface == 0; i++) { realPluginWasLoadedByMe = pluginManager->LoadSinglePluginNow(ids[i]); @@ -762,9 +773,9 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac } debug5 << "Interface object for file \"" << compressedName << "\" not in cache" << endl; - string ext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(xz|gz|bz|bz2|zip)$>"); + string ext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$>"); const char *bname = FileFunctions::Basename(compressedName.c_str()); - string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(xz|gz|bz|bz2|zip)$> \\1"); + string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\1"); string dcmd = decompCmd; if (dcmd == "") @@ -918,6 +929,8 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac #endif string newfname = tmpDir + VISIT_SLASH_STRING + dcname; + if (dcmd.substr(0,5) == "tar x") + newfname = newfname + ".root"; // its in a sub-dir of same name const char *tmpstr = newfname.c_str(); vector dummyPlugins; @@ -940,7 +953,7 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac finfo->iface = realInterface; finfo->rmDirname = ""; if (dcmd.substr(0,5) == "tar x") - finfo->rmDirname = bname; + finfo->rmDirname = tmpDir + VISIT_SLASH_STRING + dcname; UpdateRealFileFormatInterface(realInterface); realInterface->SetDatabaseMetaData(&mdCopy, 0); decompressedFilesCache[compressedName] = finfo; From 322333757b3e141141f996b0a2fac375ad4c8081 Mon Sep 17 00:00:00 2001 From: miller86 Date: Mon, 3 Oct 2022 09:52:42 -0700 Subject: [PATCH 10/14] add circ. cache for return strings --- src/common/misc/FileFunctions.C | 102 ++++++++++++++++++-------------- src/common/misc/FileFunctions.h | 9 +++ 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/src/common/misc/FileFunctions.C b/src/common/misc/FileFunctions.C index e1aa1a7bbc6..3ac337bda16 100644 --- a/src/common/misc/FileFunctions.C +++ b/src/common/misc/FileFunctions.C @@ -11,6 +11,7 @@ #include #include +#include #include @@ -30,10 +31,39 @@ #include #endif - const int STATIC_BUF_SIZE = 4096; static char StaticStringBuf[STATIC_BUF_SIZE]; +// +// very simple circular cache for short-lived strings returned to callers +// +static size_t const max_retstrs = 32; +static char const * retstrbuf[max_retstrs]; +static char const * save_returned_string(char const * const retstr) +{ + static size_t n = 0; + int modn = (int) (n++ % max_retstrs); + if (retstr == 0) // to wholly free memory if ever needed. + { + for (n = 0; n < max_retstrs; n++) + { + if (retstrbuf[n]) + free((void*)retstrbuf[n]); + } + n = 0; + return 0; + } + if (retstrbuf[modn]) + free((void*)retstrbuf[modn]); + retstrbuf[modn] = strdup(retstr); + return retstrbuf[modn]; +} + +static std::string save_returned_string(std::string const &retstr) +{ + return std::string(save_returned_string(retstr.c_str())); +} + // **************************************************************************** // Method: FileFunctions::VisItStat // @@ -591,13 +621,11 @@ basename(char const *path, int& start, char const *suffix=0) if (path == 0) { - strcpy(StaticStringBuf, "."); - return StaticStringBuf; + return save_returned_string("."); } else if (*path == '\0') { - strcpy(StaticStringBuf, "."); - return StaticStringBuf; + return save_returned_string("."); } else { @@ -609,8 +637,7 @@ basename(char const *path, int& start, char const *suffix=0) // deal with string too large if (n == STATIC_BUF_SIZE) { - strcpy(StaticStringBuf, "."); - return StaticStringBuf; + return save_returned_string("."); } // backup, skipping over all trailing slash chars @@ -622,8 +649,7 @@ basename(char const *path, int& start, char const *suffix=0) if (j == -1) { start = -1; - strcpy(StaticStringBuf, VISIT_SLASH_STRING); - return StaticStringBuf; + return save_returned_string(VISIT_SLASH_STRING); } // backup to just after next slash char @@ -648,7 +674,7 @@ basename(char const *path, int& start, char const *suffix=0) StaticStringBuf[k-n] = '\0'; } - return StaticStringBuf; + return save_returned_string(StaticStringBuf); } } @@ -669,9 +695,7 @@ FileFunctions::Basename(char const *path, char const *suffix) { suffString = suffix; } - std::string fbn = Basename(std::string(path), suffString); - strcpy(StaticStringBuf, fbn.c_str()); - return StaticStringBuf; + return save_returned_string(Basename(path, suffString.c_str())); #else int dummy1; return basename(path, dummy1, suffix); @@ -735,13 +759,12 @@ FileFunctions::Dirname(const char *path) { // preserve the previous assumption of 'current directory' if 'path' // contains no directory information - strcpy(StaticStringBuf, "."); + return save_returned_string("."); } else { - strcpy(StaticStringBuf, fsp.parent_path().string().c_str()); + return save_returned_string(fsp.parent_path().string().c_str()); } - return StaticStringBuf; #else int start; @@ -750,13 +773,11 @@ FileFunctions::Dirname(const char *path) if (start == -1) { - strcpy(StaticStringBuf, VISIT_SLASH_STRING); - return StaticStringBuf; + return save_returned_string(VISIT_SLASH_STRING); } else if (start == 0) { - strcpy(StaticStringBuf, "."); - return StaticStringBuf; + return save_returned_string("."); } else { @@ -768,7 +789,7 @@ FileFunctions::Dirname(const char *path) StaticStringBuf[i-1] = '\0'; else StaticStringBuf[i] = '\0'; - return StaticStringBuf; + return save_returned_string(StaticStringBuf); } #endif } @@ -782,8 +803,7 @@ FileFunctions::Dirname(const std::string &path) { // preserve the previous assumption of 'current directory' if 'path' // contains no directory information - strcpy(StaticStringBuf, "."); - return StaticStringBuf; + return save_returned_string("."); } else { @@ -886,9 +906,7 @@ FileFunctions::Normalize(const char *path, const char *pathSep) if (retval == "" && !noCharsRemainingToBackup) retval = "."; - StaticStringBuf[0] = '\0'; - strcat(StaticStringBuf, retval.c_str()); - return StaticStringBuf; + return save_returned_string(retval.c_str()); } std::string @@ -926,36 +944,30 @@ FileFunctions::Absname(const char *cwd_context, const char *path, // cwd_context is null or empty string if (!cwd_context || cwd_context[0] == '\0') { - if (!path) return StaticStringBuf; - if (path[0] != pathSep[0]) return StaticStringBuf; + if (!path) return save_returned_string(""); + if (path[0] != pathSep[0]) return save_returned_string(""); - std::string npath(Normalize(path, pathSep)); - strcpy(StaticStringBuf, npath.c_str()); - return StaticStringBuf; + return save_returned_string(Normalize(path, pathSep)); } // path is null or empty string if (!path || path[0] == '\0') { - if (!cwd_context) return StaticStringBuf; - if (cwd_context[0] != pathSep[0]) return StaticStringBuf; + if (!cwd_context) return save_returned_string(""); + if (cwd_context[0] != pathSep[0]) return save_returned_string(""); - std::string ncwd(Normalize(cwd_context, pathSep)); - strcpy(StaticStringBuf, ncwd.c_str()); - return StaticStringBuf; + return save_returned_string(Normalize(cwd_context, pathSep)); } if (path[0] == pathSep[0]) { - std::string npath(Normalize(path, pathSep)); - strcpy(StaticStringBuf, npath.c_str()); - return StaticStringBuf; + return save_returned_string(Normalize(path, pathSep)); } #ifndef _WIN32 if (cwd_context[0] != pathSep[0]) { - return StaticStringBuf; + return save_returned_string(""); } #else if(cwd_context[0] == '.') @@ -963,19 +975,17 @@ FileFunctions::Absname(const char *cwd_context, const char *path, if (PathIsRelative(path)) { if(_fullpath(StaticStringBuf, ".\\", _MAX_PATH) != NULL) - return StaticStringBuf; + return save_returned_string(""); else - return path; + return save_returned_string(path); } - return path; + return save_returned_string(path); } #endif // Catenate path to cwd_context and then Normalize the result std::string path2 = std::string(cwd_context) + std::string(pathSep) + std::string(path); - std::string npath = Normalize(path2.c_str(), pathSep); - strcpy(StaticStringBuf, npath.c_str()); - return StaticStringBuf; + return save_returned_string(Normalize(path2.c_str(), pathSep)); } std::string diff --git a/src/common/misc/FileFunctions.h b/src/common/misc/FileFunctions.h index b72547fc19f..e8c916d5765 100644 --- a/src/common/misc/FileFunctions.h +++ b/src/common/misc/FileFunctions.h @@ -79,12 +79,21 @@ typedef enum FILE_TYPE_NOT_KNOWN } FileType; +typedef enum +{ + REMOVE_ENTRY, // file or empty dir + REMOVE_LEAFAN, // dir and children none of which are dirs + REMOVE_TREE // dir and all things below it +} RemoveMode; + VisItStat_t* const FILE_TYPE_DONT_STAT = ((VisItStat_t*)0x1); PermissionsResult MISC_API CheckPermissions(const std::string &filename); int MISC_API VisItStat(const std::string &filename, VisItStat_t *buf); int MISC_API VisItFstat(int fd, VisItStat_t *buf); +int MISC_API VisItRemove(char const *path, RemoveMode mode = REMOVE_ENTRY); +int MISC_API VisItRemove(std::string const &path, RemoveMode mode = REMOVE_ENTRY); bool MISC_API ReadAndProcessDirectory(const std::string &, ProcessDirectoryCallback *, void * = 0, From 9fc30fad5dc3e4074da4b13926c3b628fa09a8d4 Mon Sep 17 00:00:00 2001 From: miller86 Date: Mon, 3 Oct 2022 09:53:59 -0700 Subject: [PATCH 11/14] match root/dir name --- src/tools/data/datagen/multidir.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/data/datagen/multidir.C b/src/tools/data/datagen/multidir.C index f05724c3864..bbcde66184d 100644 --- a/src/tools/data/datagen/multidir.C +++ b/src/tools/data/datagen/multidir.C @@ -339,7 +339,7 @@ WriteMasterFile(int cycle, const ProgramOptions &opt) char root[MAX_STRING]; if (opt.dirPerTimestep) - snprintf(root, MAX_STRING, "%s.root", DirectoryName(cycle, 0, opt).c_str()); + snprintf(root, MAX_STRING, "%s.silo", DirectoryName(cycle, 0, opt).c_str()); else snprintf(root, MAX_STRING, "multidir%05d.root", cycle); DBfile *db = DBCreate(root, DB_CLOBBER, DB_LOCAL, From 116eb4b114728ad639e1a95c8ee22fe61ac6e96d Mon Sep 17 00:00:00 2001 From: miller86 Date: Mon, 3 Oct 2022 09:54:11 -0700 Subject: [PATCH 12/14] optimizing caching --- .../avtZipWrapperFileFormatInterface.C | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 637c4aafab7..9b000a186d3 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -252,7 +252,6 @@ static void FreeUpCacheSlot(void *item) if (string(dp->d_name) == "..") continue; string tmp = finfo->rmDirname + "/" + string(dp->d_name); -printf("unlinking \"%s\"\n", tmp.c_str()); if (unlink(tmp.c_str()) != 0 && errno != ENOENT) { if (issuedWarnings < 5) @@ -450,7 +449,7 @@ avtZipWrapperFileFormatInterface::Initialize(int procNum, int procCount, char procNumStr[32]; snprintf(procNumStr, sizeof(procNumStr), "_%04d", procNum); tmpDir = tmpDir + VISIT_SLASH_STRING + "visitzw_" + userName + "_" + - string(VisItInit::GetComponentName()) + +// string(VisItInit::GetComponentName()) + (procCount > 1 ? string(procNumStr) : ""); debug5 << "ZipWrapper is using \"" << tmpDir << "\" as the temporary directory" << endl; @@ -560,6 +559,7 @@ avtZipWrapperFileFormatInterface::avtZipWrapperFileFormatInterface( const char *bname = FileFunctions::Basename(inputFileList[0][0].c_str()); string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\1"); + // Save the pointer to the plugin manager. pluginManager = zwinfo->GetPluginManager(); @@ -573,9 +573,7 @@ avtZipWrapperFileFormatInterface::avtZipWrapperFileFormatInterface( pluginId = ""; dummyInterface = 0; const bool searchAllPlugins = true; - //vector ids = pluginManager->GetMatchingPluginIds(dcname.c_str(), searchAllPlugins); -#warning FIXME - vector ids = pluginManager->GetMatchingPluginIds("foo.silo", searchAllPlugins); + vector ids = pluginManager->GetMatchingPluginIds(dcname.c_str(), searchAllPlugins); for (size_t i = 0; i < ids.size() && dummyInterface == 0; i++) { realPluginWasLoadedByMe = pluginManager->LoadSinglePluginNow(ids[i]); @@ -773,9 +771,14 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac } debug5 << "Interface object for file \"" << compressedName << "\" not in cache" << endl; - string ext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$>"); - const char *bname = FileFunctions::Basename(compressedName.c_str()); - string dcname = StringHelpers::ExtractRESubstr(bname, "<(.*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\1"); + string zext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$>"); + string vext = StringHelpers::ExtractRESubstr(compressedName.c_str(), "<(.*)(\\..*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\2"); + string bname = FileFunctions::Basename(compressedName); + string dcname = StringHelpers::ExtractRESubstr(bname.c_str(), "<(.*)\\.(txz|tgz|tbz|tbz2|xz|gz|bz|bz2|zip)$> \\1"); + string bnameNoExt = FileFunctions::Basename(compressedName, vext+zext); +printf("bnameNoExt = \"%s\"\n",bnameNoExt.c_str()); +printf("zext = \"%s\"\n", zext.c_str()); +printf("vext = \"%s\"\n", vext.c_str()); string dcmd = decompCmd; if (dcmd == "") @@ -801,20 +804,20 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac #else // have to handle tar archives first in this logic - if (ext == ".tar.gz" || ext == ".tgz" || - ext == ".tar.bz" || ext == ".tbz" || - ext == ".tar.bz2" || ext == ".tbz2" || - ext == ".tar.xz" || ext == ".txz") + if (zext == ".tar.gz" || zext == ".tgz" || + zext == ".tar.bz" || zext == ".tbz" || + zext == ".tar.bz2" || zext == ".tbz2" || + zext == ".tar.xz" || zext == ".txz") dcmd = "tar xvf"; - else if (ext == ".gz") + else if (zext == ".gz") dcmd = "gunzip -f"; - else if (ext == ".xz") + else if (zext == ".xz") dcmd = "unxz -f"; - else if (ext == ".bz") + else if (zext == ".bz") dcmd = "bunzip -f"; - else if (ext == ".bz2") + else if (zext == ".bz2") dcmd = "bunzip2 -f"; - else if (ext == "zip") + else if (zext == "zip") dcmd = "unzip -o"; #endif } @@ -909,8 +912,9 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac #else char tmpcmd[1024]; snprintf(tmpcmd, sizeof(tmpcmd), "cd %s ; cp %s . ; touch %s.lck ; %s %s ; rm -f %s.lck", - tmpDir.c_str(), compressedName.c_str(), dcname.c_str(), dcmd.c_str(), bname, dcname.c_str()); + tmpDir.c_str(), compressedName.c_str(), dcname.c_str(), dcmd.c_str(), bname.c_str(), dcname.c_str()); debug5 << "Using decompression command: \"" << tmpcmd << "\"" << endl; +#ifdef MDSERVER int ret = system(tmpcmd); if (WIFEXITED(ret)) { @@ -926,11 +930,10 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac { EXCEPTION1(InvalidFilesException, "Decompression command exited abnormally"); } +#endif #endif string newfname = tmpDir + VISIT_SLASH_STRING + dcname; - if (dcmd.substr(0,5) == "tar x") - newfname = newfname + ".root"; // its in a sub-dir of same name const char *tmpstr = newfname.c_str(); vector dummyPlugins; @@ -953,7 +956,7 @@ avtZipWrapperFileFormatInterface::GetRealInterface(int ts, int dom, bool dontCac finfo->iface = realInterface; finfo->rmDirname = ""; if (dcmd.substr(0,5) == "tar x") - finfo->rmDirname = tmpDir + VISIT_SLASH_STRING + dcname; + finfo->rmDirname = tmpDir + VISIT_SLASH_STRING + bnameNoExt; UpdateRealFileFormatInterface(realInterface); realInterface->SetDatabaseMetaData(&mdCopy, 0); decompressedFilesCache[compressedName] = finfo; From 5d3c66e5c26b59cdf52c6bc104d40afe0cf35aae Mon Sep 17 00:00:00 2001 From: miller86 Date: Wed, 14 Dec 2022 22:47:46 -0800 Subject: [PATCH 13/14] rmdir recurse --- .../avtZipWrapperFileFormatInterface.C | 106 +++++++++--------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 9b000a186d3..82ecd7f2772 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -197,6 +197,58 @@ class avtZipWrapperFileFormat : public avtMTMDFileFormat }; +static void +RemoveDirTree(char const *dirname) +{ + static int issuedWarnings = 0; + + DIR *dirp = opendir(dirname); + if (!dirp) + { + if (issuedWarnings < 5) + { + debug5 << "Unable to descend into directory \"" << dirname << "\" to remove its contents." << endl; + cerr << "Unable to descend into directory \"" << dirname << "\" to remove its contents." << endl; + issuedWarnings++; + } + return; + } + + // Remove contents of this directory. + struct dirent *dp; + while ((dp = readdir(dirp))) + { + if (!strcmp(dp->d_name, ".")) + continue; + if (!strcmp(dp->d_name, "..")) + continue; + string tmp = string(dirname) + "/" + string(dp->d_name); + + // try to unlink this entry. + if (unlink(tmp.c_str()) == 0) + continue; + + // If the above unlink fails, its probably a dir we need to recruse on. + RemoveDirTree(tmp.c_str()); + } + closedir(dirp); + + // Now, remove the directory itself. + errno = 0; + if (rmdir(dirname) != 0) + { + if (issuedWarnings < 5) + { + int errnotmp = errno; + debug5 << "Unable to rmdir() \"" << dirname << "\"" << endl; + debug5 << "rmdir() reported errno=" << errnotmp << ", \"" << strerror(errnotmp) << "\"" << endl; + cerr << "Unable to rmdir() \"" << dirname << "\"" << endl; + cerr << "rmdir() reported errno=" << errnotmp << ", \"" << strerror(errnotmp) << "\"" << endl; + issuedWarnings++; + } + } +} + // **************************************************************************** // Static Function: FreeUpCacheSlot // @@ -222,66 +274,20 @@ static void FreeUpCacheSlot(void *item) { if (issuedWarnings < 5) { + int errnotmp = errno; debug5 << "Unable to unlink() decompressed file \"" << filename << "\"" << endl; - debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + debug5 << "unlink() reported errno=" << errnotmp << ", \"" << strerror(errnotmp) << "\"" << endl; cerr << "Unable to remove decompressed file \"" << filename << "\"" << endl; - cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + cerr << "unlink() reported errno=" << errnotmp << ", \"" << strerror(errnotmp) << "\"" << endl; issuedWarnings++; } } } else { - DIR *dirp = opendir(finfo->rmDirname.c_str()); - if (!dirp) - { - if (issuedWarnings < 5) - { - debug5 << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; - cerr << "Unable to descend into directory \"" << finfo->rmDirname << "\" to remove its contents." << endl; - issuedWarnings++; - goto done; - } - } - dirent *dp; - while ((dp = readdir(dirp))) - { - errno = 0; - if (string(dp->d_name) == ".") - continue; - if (string(dp->d_name) == "..") - continue; - string tmp = finfo->rmDirname + "/" + string(dp->d_name); - if (unlink(tmp.c_str()) != 0 && errno != ENOENT) - { - if (issuedWarnings < 5) - { - debug5 << "Unable to unlink() \"" << tmp << "\"" << endl; - debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - cerr << "Unable to unlink() \"" << tmp << "\"" << endl; - cerr << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - issuedWarnings++; - goto done; - } - } - } - closedir(dirp); - errno = 0; - if (rmdir(finfo->rmDirname.c_str()) != 0) - { - if (issuedWarnings < 5) - { - debug5 << "Unable to rmdir() \"" << finfo->rmDirname << "\"" << endl; - debug5 << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - cerr << "Unable to rmdir() \"" << finfo->rmDirname << "\"" << endl; - cerr << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; - issuedWarnings++; - } - } + RemoveDirTree(finfo->rmDirname.c_str()); } -done: - delete finfo; } From ba943a2d32632ff2e633519792cf229767b64cef Mon Sep 17 00:00:00 2001 From: miller86 Date: Fri, 16 Dec 2022 10:06:24 -0800 Subject: [PATCH 14/14] fix rm decompressed --- .../avtZipWrapperFileFormatInterface.C | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C index 82ecd7f2772..cc2f6d33eb4 100644 --- a/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C +++ b/src/databases/ZipWrapper/avtZipWrapperFileFormatInterface.C @@ -452,26 +452,32 @@ avtZipWrapperFileFormatInterface::Initialize(int procNum, int procCount, << " decompressed files " << (procCount > 1 ? "per-processor" : "") << " at any one time." << endl; - char procNumStr[32]; - snprintf(procNumStr, sizeof(procNumStr), "_%04d", procNum); - tmpDir = tmpDir + VISIT_SLASH_STRING + "visitzw_" + userName + "_" + -// string(VisItInit::GetComponentName()) + - (procCount > 1 ? string(procNumStr) : ""); + char pidStr[32]; + snprintf(pidStr, sizeof(pidStr), "_%lu", (unsigned long) +#ifdef WIN32 + _getpid()); +#else + getpid()); +#endif + if (tmpDir.back() == string(VISIT_SLASH_STRING).back()) + tmpDir.pop_back(); + tmpDir = tmpDir + VISIT_SLASH_STRING + "visitzw_" + userName + + "_" + string(VisItInit::GetComponentName()) + + string(pidStr); debug5 << "ZipWrapper is using \"" << tmpDir << "\" as the temporary directory" << endl; // Make the temporary directory // (will have different name on mdserver and engine) - errno = 0; #ifdef WIN32 - if(_mkdir(tmpDir.c_str()) != 0 && errno != EEXIST) + if(_mkdir(tmpDir.c_str()) != 0) #else - if (mkdir(tmpDir.c_str(), 0777) != 0 && errno != EEXIST) + if (mkdir(tmpDir.c_str(), 0777)) #endif - { + int errnotmp = errno; static char errMsg[1024]; snprintf(errMsg, sizeof(errMsg), "mkdir failed with errno=%d (\"%s\")", - errno, strerror(errno)); + errnotmp, strerror(errnotmp)); EXCEPTION1(InvalidFilesException, errMsg); } @@ -493,17 +499,20 @@ avtZipWrapperFileFormatInterface::Initialize(int procNum, int procCount, void avtZipWrapperFileFormatInterface::Finalize() { - errno = 0; + if (tmpDir == "") + return; + #ifdef WIN32 - if (_rmdir(tmpDir.c_str()) != 0 && errno != ENOENT) + if (_rmdir(tmpDir.c_str()) != 0) #else - if (rmdir(tmpDir.c_str()) != 0 && errno != ENOENT) + if (rmdir(tmpDir.c_str()) != 0) #endif { + int errnotmp = errno; debug5 << "Unable to remove temporary directory \"" << tmpDir << "\"" << endl; - debug5 << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + debug5 << "rmdir() reported errno=" << errnotmp << " (\"" << strerror(errnotmp) << "\")" << endl; cerr << "Unable to remove temporary directory \"" << tmpDir << "\"" << endl; - cerr << "rmdir() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + cerr << "rmdir() reported errno=" << errnotmp << " (\"" << strerror(errnotmp) << "\")" << endl; } } @@ -649,6 +658,11 @@ avtZipWrapperFileFormatInterface::~avtZipWrapperFileFormatInterface() // we must explicitly call clear cache here to ensure items in it are // deleted *before* we enter Finalize decompressedFilesCache.clear(); + if (tmpDir.size()) + { + RemoveDirTree(tmpDir.c_str()); + tmpDir = ""; + } delete dummyFileFormat; delete dummyInterface; @@ -910,19 +924,27 @@ printf("vext = \"%s\"\n", vext.c_str()); EXCEPTION1(InvalidFilesException, "Decompression command exited abnormally"); } // delete the .bat file - if (unlink(tmpcmd) != 0 && errno != ENOENT) + if (unlink(tmpcmd) != 0) { + int errnotmp = errno; debug5 << "Unable to unlink() bat file \"" << tmpcmd << "\"" << endl; - debug5 << "unlink() reported errno=" << errno << " (\"" << strerror(errno) << "\")" << endl; + debug5 << "unlink() reported errno=" << errnotmp << " (\"" << strerror(errnotmp) << "\")" << endl; } #else char tmpcmd[1024]; snprintf(tmpcmd, sizeof(tmpcmd), "cd %s ; cp %s . ; touch %s.lck ; %s %s ; rm -f %s.lck", tmpDir.c_str(), compressedName.c_str(), dcname.c_str(), dcmd.c_str(), bname.c_str(), dcname.c_str()); debug5 << "Using decompression command: \"" << tmpcmd << "\"" << endl; -#ifdef MDSERVER + + // check that we have a shell to run a system command + if (system(0) == 0) + { + EXCEPTION1(InvalidDBTypeException, "No shell for system() to decompress files."); + } + + // Ok, run the decompression commands int ret = system(tmpcmd); - if (WIFEXITED(ret)) + if (ret != -1 && ret != 127 && WIFEXITED(ret)) { if (WEXITSTATUS(ret) != 0) { @@ -936,7 +958,6 @@ printf("vext = \"%s\"\n", vext.c_str()); { EXCEPTION1(InvalidFilesException, "Decompression command exited abnormally"); } -#endif #endif string newfname = tmpDir + VISIT_SLASH_STRING + dcname;