commit dbbd2afaf6410ef19291da7d0d18b16407571023 Author: david.sorber Date: Tue Jan 30 15:52:33 2024 -0500 WIP; adding options to preserve file/directory permissions, owner, and group. diff --git a/src/CopyManager.cc b/src/CopyManager.cc index e59d95d..245a9a6 100644 --- a/src/CopyManager.cc +++ b/src/CopyManager.cc @@ -28,6 +28,7 @@ DBS::CopyManager::CopyManager(CopyManagerOptions& options) : r_options(options), m_terminate(false), m_running(false), + m_iterationError(false), p_mainThread(nullptr), p_counterThread(nullptr), m_totalEntries(0), @@ -148,17 +149,24 @@ void DBS::CopyManager::mainThreadBody() // Recursively iterate over the source directory and add each entry to the // queue const std::string& srcPath(r_options.getSrcPath()); - for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath)) + try { - // Flow control -- delay until the entry queue is below the specified - // threshold size before proceeding - while (m_entryQueue.size() >= r_options.getFlowControlThreshold()) - { - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - } + for (const auto &dirEntry: sfs::recursive_directory_iterator(srcPath)) { + // Flow control -- delay until the entry queue is below the specified + // threshold size before proceeding + while (m_entryQueue.size() >= r_options.getFlowControlThreshold()) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } -// std::cout << "PATH: " << dirEntry << std::endl; - m_entryQueue.push_back(dirEntry); + // std::cout << "PATH: " << dirEntry << std::endl; + m_entryQueue.push_back(dirEntry); + } + } + catch (const std::filesystem::filesystem_error& exp) + { + LOG(error) << "Exception while iterating... terminating"; + m_iterationError = true; + terminate(); } // Add copy thread terminators @@ -195,6 +203,9 @@ void DBS::CopyManager::copyThreadBody() std::error_code errorCode; int fromFd = 0; int toFd = 0; + std::string srcPath; + uid_t eUid = geteuid(); + gid_t eGid = getegid(); while (true) { @@ -204,12 +215,28 @@ void DBS::CopyManager::copyThreadBody() { break; } + if (m_terminate) + { + continue; + } + // TODO: restructure this if (entry.is_directory()) { - // Directory entry -// const char* destDirPath = entry.path().string().c_str(); -// rc = stat(destDirPath, &statBuf); + //--- Directory entry + + // Stat directory if preserving perms, owner, or group + if (r_options.getPreservePerms() || r_options.getPreserveOwner() || + r_options.getPreserveGroup()) + { + srcPath.assign(entry.path().string()); + rc = stat(srcPath.c_str(), &statBuf); + if (rc != 0) + { + LOG(error) << "Unable to stat(): " << srcPath << " -- " + << errno << " -- " << std::strerror(errno); + } + } // Create destination path auto destDirPath = sfs::path{r_options.getDestPath()} / @@ -226,12 +253,36 @@ void DBS::CopyManager::copyThreadBody() } } + // Preserve permissions + if (r_options.getPreservePerms() && rc == 0) + { + rc = chmod(destDirPath.string().c_str(), statBuf.st_mode); + if (rc != 0) + { + LOG(error) << "Unable to chmod(): " << destDirPath << " -- " + << errno << " -- " << std::strerror(errno); + } + } + + // Preserve owner / group + if (r_options.getPreserveOwner() || r_options.getPreserveGroup()) + { + uid_t newUid = r_options.getPreserveOwner() ? statBuf.st_uid : eUid; + gid_t newGid = r_options.getPreserveGroup() ? statBuf.st_gid : eGid; + rc = chown(srcPath.c_str(), newUid, newGid); + if (rc != 0) + { + LOG(error) << "Unable to chown(): " << srcPath << " -- " + << errno << " -- " << std::strerror(errno); + } + } + incProcessedEntries(); LOG(debug) << "Created directory: " << entry.path(); } else if (entry.is_regular_file()) { - // Regular file entry + //--- Regular file entry // Remove filename from file path auto fileDirPath = entry.path(); @@ -288,6 +339,10 @@ void DBS::CopyManager::copyThreadBody() while (rc > 0) { rc = sendfile(toFd, fromFd, 0, count); + if (m_terminate) + { + break; + } } if (rc < 0) { @@ -300,8 +355,11 @@ void DBS::CopyManager::copyThreadBody() close(toFd); incProcessedEntries(); - LOG(debug) << "Copy: " << entry.path().string() - << " to: " << destFilePath.string(); + if (! m_terminate) + { + LOG(debug) << "Copy: " << entry.path().string() + << " to: " << destFilePath.string(); + } } else { @@ -316,15 +374,29 @@ void DBS::CopyManager::copyThreadBody() void DBS::CopyManager::counterThreadBody() { const std::string& srcPath(r_options.getSrcPath()); - for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath)) + try { - if (dirEntry.is_directory() || dirEntry.is_regular_file()) - { - incTotalEntries(); - } - else + for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath)) { - incInvalidEntries(); + if (m_terminate) + { + break; + } + + if (dirEntry.is_directory() || dirEntry.is_regular_file()) + { + incTotalEntries(); + } + else + { + incInvalidEntries(); + } } } + catch (const std::filesystem::filesystem_error& exp) + { + LOG(error) << "Exception while iterating... terminating"; + m_iterationError = true; + terminate(); + } } \ No newline at end of file diff --git a/src/CopyManager.h b/src/CopyManager.h index 9f12ec4..e27af95 100644 --- a/src/CopyManager.h +++ b/src/CopyManager.h @@ -49,6 +49,12 @@ public: */ void run(); + inline void terminate() + { + // TODO: something else??? + m_terminate = true; + } + /** * Wait for the main thread to complete. Should be called after run(). * @return @@ -60,6 +66,11 @@ public: return m_running; } + inline bool iterationError() const + { + return m_iterationError; + } + inline uint64_t getTotalEntries() { return m_totalEntries; @@ -114,6 +125,7 @@ private: ProtectedQueue m_entryQueue; std::atomic_bool m_terminate; std::atomic_bool m_running; + std::atomic_bool m_iterationError; std::thread* p_mainThread; std::vector m_copyThreads; diff --git a/src/CopyManagerOptions.cc b/src/CopyManagerOptions.cc index cee39ff..e0a7f46 100644 --- a/src/CopyManagerOptions.cc +++ b/src/CopyManagerOptions.cc @@ -18,7 +18,10 @@ DBS::CopyManagerOptions::CopyManagerOptions( int logLevel, uint32_t flowCtrlThreshold, uint32_t numProcThreads, - bool copyTopLevel) + bool copyTopLevel, + bool preservePerms, + bool preserveOwner, + bool preserveGroup) : r_srcPath(srcPath), m_destPath(destPath), m_enableLog(disableLog), @@ -27,7 +30,10 @@ DBS::CopyManagerOptions::CopyManagerOptions( m_logLevel(logLevel), m_flowControlThreshold(flowCtrlThreshold), m_numCopyThreads(numProcThreads), - m_copyTopLevel(copyTopLevel) + m_copyTopLevel(copyTopLevel), + m_preservePerms(preservePerms), + m_preserveOwner(preserveOwner), + m_preserveGroup(preserveGroup) { } diff --git a/src/CopyManagerOptions.h b/src/CopyManagerOptions.h index acb73d5..2450472 100644 --- a/src/CopyManagerOptions.h +++ b/src/CopyManagerOptions.h @@ -23,15 +23,18 @@ public: // Log option defaults static const bool DEFAULT_ENABLE_LOG{false}; - static const bool DEFAULT_LOG_TO_STDERR{true}; + static const bool DEFAULT_LOG_TO_STDERR{false}; static const int DEFAULT_LOG_LEVEL{3}; static inline const std::string DEFAULT_LOG_FILE_PATH{"./copyfile.log"}; - static const uint32_t DEFAULT_FC_THRESHOLD{1000}; static const uint32_t DEFAULT_NUM_COPY_THREADS{4}; static const bool DEFAULT_COPY_TOPLEVEL{false}; + static const bool DEFAULT_PRESERVE_PERMS{false}; + static const bool DEFAULT_PRESERVE_OWNER{false}; + static const bool DEFAULT_PRESERVE_GROUP{false}; + CopyManagerOptions( const std::string& srcPath, const std::string& destPath, @@ -41,7 +44,10 @@ public: int logLevel, uint32_t flowCtrlThreshold, uint32_t numProcThreads, - bool copyTopLevel); + bool copyTopLevel, + bool preservePerms, + bool preserveOwner, + bool preserveGroup); ~CopyManagerOptions(); @@ -91,6 +97,21 @@ public: return m_copyTopLevel; } + inline bool getPreservePerms() const + { + return m_preservePerms; + } + + inline bool getPreserveOwner() const + { + return m_preserveOwner; + } + + inline bool getPreserveGroup() const + { + return m_preserveGroup; + } + //--- Setters ------------------------------------------------------------- inline void setDestPath(const std::string& destPath) { @@ -119,6 +140,10 @@ private: uint32_t m_numCopyThreads; // Number of copy threads to spawn bool m_copyTopLevel; // Copy top level source dir to dest dir + bool m_preservePerms; // Preserve directory/file permissions + bool m_preserveOwner; // Preserve directory/file owner + bool m_preserveGroup; // Preserve directory/file group + }; } // namespace DBS diff --git a/src/copytool_main.cc b/src/copytool_main.cc index 6de445f..2a0884b 100644 --- a/src/copytool_main.cc +++ b/src/copytool_main.cc @@ -34,8 +34,7 @@ const std::string LBLUE{"\033[1;34m"}; const std::string UP_ONE = "\033[1A"; -int main(int argc, char** argv) -{ +int main(int argc, char** argv) { std::cout << "copytool -- " << VERSION << "\n" << std::endl; CLI::App app{"copytool"}; @@ -63,26 +62,37 @@ int main(int argc, char** argv) app.add_option("-l,--log-file-path", logFilePath, "log file path"); app.add_option("--log-level", logLevel, "default log message level (error=0, warn=1, info=2, debug=3, trace=4)") - ->check(CLI::Range(0, 4)); + ->check(CLI::Range(0, 4)); // Other options uint32_t flowControlThreshold = DBS::CopyManagerOptions::DEFAULT_FC_THRESHOLD; uint32_t numCopyThreads = DBS::CopyManagerOptions::DEFAULT_NUM_COPY_THREADS; bool copyTopLevel = DBS::CopyManagerOptions::DEFAULT_COPY_TOPLEVEL; + bool preservePerms = DBS::CopyManagerOptions::DEFAULT_PRESERVE_PERMS; + bool preserveOwner = DBS::CopyManagerOptions::DEFAULT_PRESERVE_OWNER; + bool preserveGroup = DBS::CopyManagerOptions::DEFAULT_PRESERVE_GROUP; app.add_option("--flow-control-threshold", flowControlThreshold, "flow control threshold (default: " + std::to_string(flowControlThreshold) + ")") - ->check(CLI::Range(100,65535)); + ->check(CLI::Range(100, 65535)); app.add_option("-n,--num-copy-threads", numCopyThreads, "number of copy threads to spawn (default: " + std::to_string(numCopyThreads) + ")") - ->check(CLI::Range(1, 512)); + ->check(CLI::Range(1, 512)); app.add_flag("--copy-top-level", copyTopLevel, "copy top level source directory to destination rather then " "the contents of the top level source directory (which is " "the default behavior)"); + app.add_flag("-p,--preserve-perms", preservePerms, + "preserve file/directory permissions"); + app.add_flag("-o,--preserve-owner", preserveOwner, + "preserve file/directory owner"); + app.add_flag("-g,--preserve-group", preserveGroup, + "preserve file/directory group"); + + // Parse options CLI11_PARSE(app, argc, argv); @@ -95,10 +105,23 @@ int main(int argc, char** argv) logLevel, flowControlThreshold, numCopyThreads, - copyTopLevel); + copyTopLevel, + preservePerms, + preserveOwner, + preserveGroup); // Prime the logger - DBS::Logger::getInstance(DBS::off, &options); + try + { + DBS::Logger::getInstance(DBS::off, &options); + } + catch (const std::runtime_error& excep) + { + std::cerr << BOLD << RED << "ERROR: " << ENDC << excep.what() + << "\n" << std::endl; + return -1; + } + LOG(DBS::info) << "copytool -- DBS"; // Display a warning if the specified number of copy threads exceeds the @@ -149,14 +172,23 @@ int main(int argc, char** argv) int rc = manager.waitForComplete(); - pBarEta.set_progress(100.0); - pBarEta.set_option(indicators::option::PostfixText{" -- " + - std::to_string(manager.getProcessedEntries()) + "/" + - std::to_string(manager.getTotalEntries()) - }); - pBarEta.mark_as_completed(); - indicators::show_console_cursor(true); - std::cout << std::endl; + if (manager.iterationError()) + { + std::cerr << BOLD << RED << "ERROR: " << ENDC << "An iteration exception " + << "occurred that was likely caused by a permissions error. " + << "Please rerun as root (using sudo)." << std::endl; + } + else + { + pBarEta.set_progress(100.0); + pBarEta.set_option(indicators::option::PostfixText{" -- " + + std::to_string(manager.getProcessedEntries()) + "/" + + std::to_string(manager.getTotalEntries()) + }); + pBarEta.mark_as_completed(); + indicators::show_console_cursor(true); + std::cout << std::endl; + } // Display warning to user if any invalid/unsupported directory entries were // found while copying @@ -164,9 +196,11 @@ int main(int argc, char** argv) { std::cout << BOLD << YELLOW << "WARNING: " << ENDC << manager.getInvalidEntries() << " unsupported directory " - << "entries (symlinks, FIFOs, etc.) were NOT copied.\n" + << "entries (symlinks, FIFOs, etc.) were NOT copied." << std::endl; } + std::cout << std::endl; + return rc; }