Project

General

Profile

« Previous | Next » 

Revision dbbd2afa

Added by david.sorber over 2 years ago

  • ID dbbd2afaf6410ef19291da7d0d18b16407571023
  • Parent 4fd46efa

WIP; adding options to preserve file/directory permissions, owner, and
group.

View differences:

src/CopyManager.cc
: r_options(options),
m_terminate(false),
m_running(false),
m_iterationError(false),
p_mainThread(nullptr),
p_counterThread(nullptr),
m_totalEntries(0),
......
// Recursively iterate over the source directory and add each entry to the
// queue
const std::string& srcPath(r_options.getSrcPath());
for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath))
try
{
// Flow control -- delay until the entry queue is below the specified
// threshold size before proceeding
while (m_entryQueue.size() >= r_options.getFlowControlThreshold())
{
std::this_thread::sleep_for(std::chrono::milliseconds(2));
}
for (const auto &dirEntry: sfs::recursive_directory_iterator(srcPath)) {
// Flow control -- delay until the entry queue is below the specified
// threshold size before proceeding
while (m_entryQueue.size() >= r_options.getFlowControlThreshold()) {
std::this_thread::sleep_for(std::chrono::milliseconds(2));
}
// std::cout << "PATH: " << dirEntry << std::endl;
m_entryQueue.push_back(dirEntry);
// std::cout << "PATH: " << dirEntry << std::endl;
m_entryQueue.push_back(dirEntry);
}
}
catch (const std::filesystem::filesystem_error& exp)
{
LOG(error) << "Exception while iterating... terminating";
m_iterationError = true;
terminate();
}
// Add copy thread terminators
......
std::error_code errorCode;
int fromFd = 0;
int toFd = 0;
std::string srcPath;
uid_t eUid = geteuid();
gid_t eGid = getegid();
while (true)
{
......
{
break;
}
if (m_terminate)
{
continue;
}
// TODO: restructure this
if (entry.is_directory())
{
// Directory entry
// const char* destDirPath = entry.path().string().c_str();
// rc = stat(destDirPath, &statBuf);
//--- Directory entry
// Stat directory if preserving perms, owner, or group
if (r_options.getPreservePerms() || r_options.getPreserveOwner() ||
r_options.getPreserveGroup())
{
srcPath.assign(entry.path().string());
rc = stat(srcPath.c_str(), &statBuf);
if (rc != 0)
{
LOG(error) << "Unable to stat(): " << srcPath << " -- "
<< errno << " -- " << std::strerror(errno);
}
}
// Create destination path
auto destDirPath = sfs::path{r_options.getDestPath()} /
......
}
}
// Preserve permissions
if (r_options.getPreservePerms() && rc == 0)
{
rc = chmod(destDirPath.string().c_str(), statBuf.st_mode);
if (rc != 0)
{
LOG(error) << "Unable to chmod(): " << destDirPath << " -- "
<< errno << " -- " << std::strerror(errno);
}
}
// Preserve owner / group
if (r_options.getPreserveOwner() || r_options.getPreserveGroup())
{
uid_t newUid = r_options.getPreserveOwner() ? statBuf.st_uid : eUid;
gid_t newGid = r_options.getPreserveGroup() ? statBuf.st_gid : eGid;
rc = chown(srcPath.c_str(), newUid, newGid);
if (rc != 0)
{
LOG(error) << "Unable to chown(): " << srcPath << " -- "
<< errno << " -- " << std::strerror(errno);
}
}
incProcessedEntries();
LOG(debug) << "Created directory: " << entry.path();
}
else if (entry.is_regular_file())
{
// Regular file entry
//--- Regular file entry
// Remove filename from file path
auto fileDirPath = entry.path();
......
while (rc > 0)
{
rc = sendfile(toFd, fromFd, 0, count);
if (m_terminate)
{
break;
}
}
if (rc < 0)
{
......
close(toFd);
incProcessedEntries();
LOG(debug) << "Copy: " << entry.path().string()
<< " to: " << destFilePath.string();
if (! m_terminate)
{
LOG(debug) << "Copy: " << entry.path().string()
<< " to: " << destFilePath.string();
}
}
else
{
......
void DBS::CopyManager::counterThreadBody()
{
const std::string& srcPath(r_options.getSrcPath());
for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath))
try
{
if (dirEntry.is_directory() || dirEntry.is_regular_file())
{
incTotalEntries();
}
else
for (const auto& dirEntry : sfs::recursive_directory_iterator(srcPath))
{
incInvalidEntries();
if (m_terminate)
{
break;
}
if (dirEntry.is_directory() || dirEntry.is_regular_file())
{
incTotalEntries();
}
else
{
incInvalidEntries();
}
}
}
catch (const std::filesystem::filesystem_error& exp)
{
LOG(error) << "Exception while iterating... terminating";
m_iterationError = true;
terminate();
}
}
src/CopyManager.h
*/
void run();
inline void terminate()
{
// TODO: something else???
m_terminate = true;
}
/**
* Wait for the main thread to complete. Should be called after run().
* @return
......
return m_running;
}
inline bool iterationError() const
{
return m_iterationError;
}
inline uint64_t getTotalEntries()
{
return m_totalEntries;
......
ProtectedQueue<sfs::directory_entry> m_entryQueue;
std::atomic_bool m_terminate;
std::atomic_bool m_running;
std::atomic_bool m_iterationError;
std::thread* p_mainThread;
std::vector<std::thread*> m_copyThreads;
src/CopyManagerOptions.cc
int logLevel,
uint32_t flowCtrlThreshold,
uint32_t numProcThreads,
bool copyTopLevel)
bool copyTopLevel,
bool preservePerms,
bool preserveOwner,
bool preserveGroup)
: r_srcPath(srcPath),
m_destPath(destPath),
m_enableLog(disableLog),
......
m_logLevel(logLevel),
m_flowControlThreshold(flowCtrlThreshold),
m_numCopyThreads(numProcThreads),
m_copyTopLevel(copyTopLevel)
m_copyTopLevel(copyTopLevel),
m_preservePerms(preservePerms),
m_preserveOwner(preserveOwner),
m_preserveGroup(preserveGroup)
{
}
src/CopyManagerOptions.h
// Log option defaults
static const bool DEFAULT_ENABLE_LOG{false};
static const bool DEFAULT_LOG_TO_STDERR{true};
static const bool DEFAULT_LOG_TO_STDERR{false};
static const int DEFAULT_LOG_LEVEL{3};
static inline const std::string DEFAULT_LOG_FILE_PATH{"./copyfile.log"};
static const uint32_t DEFAULT_FC_THRESHOLD{1000};
static const uint32_t DEFAULT_NUM_COPY_THREADS{4};
static const bool DEFAULT_COPY_TOPLEVEL{false};
static const bool DEFAULT_PRESERVE_PERMS{false};
static const bool DEFAULT_PRESERVE_OWNER{false};
static const bool DEFAULT_PRESERVE_GROUP{false};
CopyManagerOptions(
const std::string& srcPath,
const std::string& destPath,
......
int logLevel,
uint32_t flowCtrlThreshold,
uint32_t numProcThreads,
bool copyTopLevel);
bool copyTopLevel,
bool preservePerms,
bool preserveOwner,
bool preserveGroup);
~CopyManagerOptions();
......
return m_copyTopLevel;
}
inline bool getPreservePerms() const
{
return m_preservePerms;
}
inline bool getPreserveOwner() const
{
return m_preserveOwner;
}
inline bool getPreserveGroup() const
{
return m_preserveGroup;
}
//--- Setters -------------------------------------------------------------
inline void setDestPath(const std::string& destPath)
{
......
uint32_t m_numCopyThreads; // Number of copy threads to spawn
bool m_copyTopLevel; // Copy top level source dir to dest dir
bool m_preservePerms; // Preserve directory/file permissions
bool m_preserveOwner; // Preserve directory/file owner
bool m_preserveGroup; // Preserve directory/file group
};
} // namespace DBS
src/copytool_main.cc
const std::string UP_ONE = "\033[1A";
int main(int argc, char** argv)
{
int main(int argc, char** argv) {
std::cout << "copytool -- " << VERSION << "\n" << std::endl;
CLI::App app{"copytool"};
......
app.add_option("-l,--log-file-path", logFilePath, "log file path");
app.add_option("--log-level", logLevel,
"default log message level (error=0, warn=1, info=2, debug=3, trace=4)")
->check(CLI::Range(0, 4));
->check(CLI::Range(0, 4));
// Other options
uint32_t flowControlThreshold = DBS::CopyManagerOptions::DEFAULT_FC_THRESHOLD;
uint32_t numCopyThreads = DBS::CopyManagerOptions::DEFAULT_NUM_COPY_THREADS;
bool copyTopLevel = DBS::CopyManagerOptions::DEFAULT_COPY_TOPLEVEL;
bool preservePerms = DBS::CopyManagerOptions::DEFAULT_PRESERVE_PERMS;
bool preserveOwner = DBS::CopyManagerOptions::DEFAULT_PRESERVE_OWNER;
bool preserveGroup = DBS::CopyManagerOptions::DEFAULT_PRESERVE_GROUP;
app.add_option("--flow-control-threshold", flowControlThreshold,
"flow control threshold (default: "
+ std::to_string(flowControlThreshold) + ")")
->check(CLI::Range(100,65535));
->check(CLI::Range(100, 65535));
app.add_option("-n,--num-copy-threads", numCopyThreads,
"number of copy threads to spawn (default: "
+ std::to_string(numCopyThreads) + ")")
->check(CLI::Range(1, 512));
->check(CLI::Range(1, 512));
app.add_flag("--copy-top-level", copyTopLevel,
"copy top level source directory to destination rather then "
"the contents of the top level source directory (which is "
"the default behavior)");
app.add_flag("-p,--preserve-perms", preservePerms,
"preserve file/directory permissions");
app.add_flag("-o,--preserve-owner", preserveOwner,
"preserve file/directory owner");
app.add_flag("-g,--preserve-group", preserveGroup,
"preserve file/directory group");
// Parse options
CLI11_PARSE(app, argc, argv);
......
logLevel,
flowControlThreshold,
numCopyThreads,
copyTopLevel);
copyTopLevel,
preservePerms,
preserveOwner,
preserveGroup);
// Prime the logger
DBS::Logger::getInstance(DBS::off, &options);
try
{
DBS::Logger::getInstance(DBS::off, &options);
}
catch (const std::runtime_error& excep)
{
std::cerr << BOLD << RED << "ERROR: " << ENDC << excep.what()
<< "\n" << std::endl;
return -1;
}
LOG(DBS::info) << "copytool -- DBS";
// Display a warning if the specified number of copy threads exceeds the
......
int rc = manager.waitForComplete();
pBarEta.set_progress(100.0);
pBarEta.set_option(indicators::option::PostfixText{" -- " +
std::to_string(manager.getProcessedEntries()) + "/" +
std::to_string(manager.getTotalEntries())
});
pBarEta.mark_as_completed();
indicators::show_console_cursor(true);
std::cout << std::endl;
if (manager.iterationError())
{
std::cerr << BOLD << RED << "ERROR: " << ENDC << "An iteration exception "
<< "occurred that was likely caused by a permissions error. "
<< "Please rerun as root (using sudo)." << std::endl;
}
else
{
pBarEta.set_progress(100.0);
pBarEta.set_option(indicators::option::PostfixText{" -- " +
std::to_string(manager.getProcessedEntries()) + "/" +
std::to_string(manager.getTotalEntries())
});
pBarEta.mark_as_completed();
indicators::show_console_cursor(true);
std::cout << std::endl;
}
// Display warning to user if any invalid/unsupported directory entries were
// found while copying
......
{
std::cout << BOLD << YELLOW << "WARNING: " << ENDC
<< manager.getInvalidEntries() << " unsupported directory "
<< "entries (symlinks, FIFOs, etc.) were NOT copied.\n"
<< "entries (symlinks, FIFOs, etc.) were NOT copied."
<< std::endl;
}
std::cout << std::endl;
return rc;
}

Also available in: Unified diff