Project

General

Profile

« Previous | Next » 

Revision 537f40e6

Added by David Sorber almost 8 years ago

Continued improvements to PCA; handle errors better, make number of
processing threads configurable, and reversed the "keep-orig" option to
"delete-orig" which makes more sense.

View differences:

software/photo_compress_archiver/PhotoCompressArchiver.cc
#include "PhotoCompressArchiver.hh"
#define OUT(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << msg << std::endl; \
}
#define ERROR(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << BOLD << RED << "ERROR: " << ENDC << msg << std::endl; \
}
#define ERROR std::cerr << BOLD << RED << "ERROR: " << ENDC
#define WRKR_OUT_REG(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
......
#define WRKR_OUT_ERR(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
std::cerr << "T[" << std::setw(2) << tid << "] " << BOLD << RED \
<< "ERROR: " << msg << std::endl; \
}
PhotoCompressArchiver::PhotoCompressArchiver(
const std::string& path,
uint32_t num_cores,
uint32_t num_threads,
bool decompress,
bool keep_orig,
bool delete_orig,
boost::regex& filter_regex)
: m_path(path),
m_num_cores(num_cores),
m_num_threads(num_threads),
m_decompress(decompress),
m_keep_orig(keep_orig),
m_delete_orig(delete_orig),
m_files_processed(0),
m_num_input_files(0),
m_total_uncompressed_size(0),
......
auto start = std::chrono::system_clock::now();
// Start the file finder thread
OUT("Starting file finder..." << std::flush);
std::cout << "Starting file finder..." << std::flush;
p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
OUT("DONE");
std::cout << "DONE" << std::endl;
// Start worker threads
OUT("Starting worker threads..." << std::flush);
uint32_t num_threads = m_num_cores;
for (uint32_t idx = 0; idx < num_threads; ++idx)
std::cout << "Starting worker threads..." << std::flush;
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
// Spawn worker thread and add to list for bookkeeping
std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
this, idx);
m_worker_threads.push_back(worker);
}
OUT("DONE\n");
std::cout << "DONE\n" << std::endl;
// Join the file finder thread
p_finder_thread->join();
OUT("File finder completed:");
std::cout << "File finder completed:" << std::endl;
// Error out of no input files were found
if (m_num_input_files == 0)
{
ERROR("No matching input files found. Please check the patch and try "
"again.");
ERROR << "No matching input files found. Please check the patch and "
<< "try again." << std::endl;
return -1;
}
OUT(" Found " << m_num_input_files << " files");
std::cout << " Found " << m_num_input_files << " files" << std::endl;
if (! m_decompress)
{
OUT(" Total uncompressed bytes: " << m_total_uncompressed_size << "\n");
std::cout << " Total uncompressed bytes: "
<< m_total_uncompressed_size << "\n" << std::endl;
}
// Add terminator for each worker thread
for (uint32_t idx = 0; idx < num_threads; ++idx)
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
m_file_list.push_back(nullptr);
}
......
// Grab start time and calculate duration
auto end = std::chrono::system_clock::now();
auto diff = end - start;
double duration = std::chrono::duration<double, std::milli>(diff).count();
double duration = std::chrono::duration<double>(diff).count();
// Print out message if any errors were found
if (m_errors.size() > 0)
{
std::cout << BOLD << RED << "NOTE: errors were encountered processing "
<< "the following " << m_errors.size() << " files:\n" << ENDC
<< std::endl;
WorkUnit* work_unit = nullptr;
for (uint32_t idx = 0; idx < m_errors.size(); ++idx)
{
work_unit = m_errors.pop_front();
std::cout << " " << work_unit->m_path << std::endl;
delete work_unit;
}
std::cout << std::endl;
}
// Print out compression information if compressing
if (! m_decompress)
{
OUT("\n Total uncompressed bytes: " << m_total_uncompressed_size);
OUT(" Total compressed bytes: " << m_total_compressed_size);
std::cout << "\n Total uncompressed bytes: "
<< m_total_uncompressed_size << std::endl;
std::cout << " Total compressed bytes: " << m_total_compressed_size
<< std::endl;
// Calculate and display ratio
double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
OUT(" Compression ratio: " << std::fixed << std::setprecision(4)
<< (ratio * 100) << "%");
// Display execution duration
OUT(" Total time: " << duration << " ms\n");
std::cout << " Compression ratio: " << std::fixed
<< std::setprecision(4) << (ratio * 100) << "%" << std::endl;
}
else
{
std::cout << std::endl;
}
// Display execution duration
std::cout << " Total time: " << duration << " s\n" << std::endl;
return 0;
}
......
std::ios::binary | std::ios::in);
if (! input_stream)
{
WRKR_OUT_ERR("ERROR: unable to read from \"" << work_unit->m_path
WRKR_OUT_ERR("unable to read from \"" << work_unit->m_path
<< "\"\n")
// TODO: what now?
m_errors.push_back(work_unit);
continue;
}
input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
input_stream.close();
// TODO: revise this
// Determine input, and therefore output, filetype / extension
// Determine input, and therefore output, filetype / extension based
// on mode
const std::string* output_file_extension = nullptr;
if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
{
output_file_extension = &PJG_EXTENSION;
}
else if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
if (m_decompress)
{
output_file_extension = &JPG_EXTENSION;
// We are decompressing so the file we found had better be a PJG
if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
{
output_file_extension = &JPG_EXTENSION;
}
else
{
WRKR_OUT_ERR("the input file does not appear to be a valid "
"packjpg (.pjg) file even though its filename "
"suggests it is!\n")
m_errors.push_back(work_unit);
continue;
}
}
else
{
WRKR_OUT_ERR("ERROR: Input file does not appear to be valid\n")
// TODO: what now?
// We are compressing so the file we found had better be a JPG
if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
{
output_file_extension = &PJG_EXTENSION;
}
else
{
WRKR_OUT_ERR("the input file does not appear to be a valid "
"JPEG (.jpg) file even though its filename "
"suggests it is!\n")
m_errors.push_back(work_unit);
continue;
}
}
// Do the thing!
instance->pjglib_init_streams(file_buffer->data(), 1,
work_unit->m_file_size, nullptr, 1);
// Do the thing!
bool rc = instance->pjglib_convert_stream2mem(&out_buffer,
&out_size, message);
if (!rc)
{
WRKR_OUT_ERR("An error occurred during the compression"
<< "/decompression operation: " << message << "\n")
// TODO: what now?
m_errors.push_back(work_unit);
continue;
}
//~ WRKR_OUT_REG("Status message: " << message)
......
++m_files_processed;
// Create output file path be replacing extension of input file
bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(*output_file_extension);
bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(
*output_file_extension);
// Write output file
std::ofstream output_stream(out_path.string(),
std::ios::binary | std::ios::out);
if (! output_stream)
{
WRKR_OUT_ERR("ERROR: unable to read from \""
<< work_unit->m_path << "\"")
// TODO: what now?
WRKR_OUT_ERR("unable to write to \"" << out_path << "\"")
m_errors.push_back(work_unit);
continue;
}
output_stream.write((const char*)out_buffer, out_size);
output_stream.close();
// Delete original file if so instructed
if (m_delete_orig)
{
bfs::remove(work_unit->m_path);
}
// Print status
if (m_decompress)
{
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
......
<< percent << "%")
}
delete work_unit;
}
WRKR_OUT_REG(" -- " << PURPLE << BOLD << "[[exiting]]" << ENDC)
// Clean up
std::free(out_buffer);
delete instance;

Also available in: Unified diff