Revision f8e600bc
Added by David Sorber over 9 years ago
| software/photo_compress_archiver/PhotoCompressArchiver.hh | ||
|---|---|---|
|
* Fork and wait
|
||
|
*/
|
||
|
void fork_worker(uint32_t tid, std::vector<bfs::path>* file_sublist);
|
||
|
|
||
|
/**
|
||
|
* Calculate the return the "global" percent done as determined by the
|
||
|
* m_files_processed and m_num_input_files instance variables.
|
||
|
*/
|
||
|
double get_global_percent_done();
|
||
|
|
||
|
private:
|
||
|
|
||
|
const bfs::path m_path; // top level path
|
||
|
const bfs::path m_path; // top level path where to look for input files
|
||
|
uint32_t m_num_cores; // number of cores on the machine
|
||
|
std::mutex m_output_mutex; // output mutex
|
||
|
|
||
|
uint32_t m_num_input_files; // total number of input files found
|
||
|
std::atomic<uint64_t> m_total_uncompressed_size; // total uncompressed size of all images
|
||
|
std::atomic<uint64_t> m_total_compressed_size; // total compressed size of all images
|
||
|
std::atomic<uint32_t> m_files_processed; // number of input files processed
|
||
|
uint32_t m_num_input_files; // total number of input files found
|
||
|
std::atomic<uint64_t> m_total_uncompressed_size;// total uncompressed size of all images
|
||
|
std::atomic<uint64_t> m_total_compressed_size; // total compressed size of all images
|
||
|
|
||
|
std::vector<bfs::path> m_file_list; // input file list (either images or compressed images)
|
||
|
std::vector<bfs::path> m_file_list; // input file list (either images or compressed images)
|
||
|
|
||
|
std::thread* p_finder_thread; // file finder thread
|
||
|
std::vector<std::thread*> m_worker_threads; // worker threads
|
||
|
std::thread* p_finder_thread; // file finder thread
|
||
|
std::vector<std::thread*> m_worker_threads; // worker threads
|
||
|
};
|
||
|
|
||
|
#endif // PHOTOCOMPRESSARCHIVER_H
|
||
I figured out a somewhat silly, but highly effective way do progress reporting. The basic idea is that we know the output file names and the exact order in which they will appear. Therefore if output X+1 exists on the file system it means that output X is already complete. Now the worker thread polls the file system waiting for the output files to appear and then processses them instead of waiting until the forked subprocess terminates. In basic testing it appears that this mechanism is about the same speed as the previous version that does not include progress reporting. This implementation is cross platform and doesn't suffer from the problems of inotify (Linux only) or kqueues (BSD/Mac OS equivalent of inotify that requires an open file descriptor for each monitored file). I still need to implement input option parsing and a decompress mode so I can do proper testing.