Project

General

Profile

Download (16.1 KB) Statistics
| Branch: | Tag: | Revision:

root/software/photo_compress_archiver/PhotoCompressArchiver.cc @ ebc2733a

b216a5b0 David Sorber
#include <chrono>
ebc2733a David Sorber
#include <cmath>
328d0fa5 David Sorber
#include <cstdio>
4f6113f8 David Sorber
#include <fstream>
f8e600bc David Sorber
#include <iomanip>
b216a5b0 David Sorber
#include <iostream>
e0b7f8b9 David Sorber
#include <limits.h>
ebc2733a David Sorber
#include <thread>
b216a5b0 David Sorber
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
4f6113f8 David Sorber
#include <vector>

#include <packjpg.h>
b216a5b0 David Sorber
#include "PhotoCompressArchiver.hh"
ebc2733a David Sorber
#include "ProgressBar.h"
b216a5b0 David Sorber
537f40e6 David Sorber
#define ERROR std::cerr << BOLD << RED << "ERROR: " << ENDC
3eeb1485 David Sorber
4f6113f8 David Sorber
#define WRKR_OUT_REG(msg) { \
ebc2733a David Sorber
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
} \
4f6113f8 David Sorber
}
b216a5b0 David Sorber
4f6113f8 David Sorber
#define WRKR_OUT_ERR(msg) { \
ebc2733a David Sorber
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << "T[" << std::setw(2) << tid << "] " << BOLD << RED \
<< "ERROR: " << ENDC << msg << std::endl; \
} \
4f6113f8 David Sorber
}
e0b7f8b9 David Sorber
ebc2733a David Sorber
// Helper function
ddd2cb4a David Sorber
std::string _format_num_bytes(uint64_t num_bytes)
{
64e2408d David Sorber
const uint64_t tera = (1024 * 1024 * 1024 * 1024L);
ddd2cb4a David Sorber
const uint64_t giga = (1024 * 1024 * 1024L);
const uint64_t mega = (1024 * 1024L);
const uint64_t kilo = 1024L;
double fractional;
std::ostringstream output;
if (num_bytes >= tera)
{
// Tera
fractional = ((double)num_bytes) / tera;
output << std::fixed << std::setprecision(2) << fractional;
output << " TB";
}
else if (num_bytes >= giga)
{
// Giga
fractional = ((double)num_bytes) / giga;
output << std::fixed << std::setprecision(2) << fractional;
output << " GB";
}
else if (num_bytes >= mega)
{
// Mega
fractional = ((double)num_bytes) / mega;
output << std::fixed << std::setprecision(2) << fractional;
output << " MB";
}
else if (num_bytes >= kilo)
{
// Kilo
fractional = ((double)num_bytes) / kilo;
output << std::fixed << std::setprecision(2) << fractional;
output << " KB";
}
else
{
// Bytes
output << num_bytes << " b";
}
return output.str();
}

b216a5b0 David Sorber
PhotoCompressArchiver::PhotoCompressArchiver(
328d0fa5 David Sorber
const std::string& path,
537f40e6 David Sorber
uint32_t num_threads,
328d0fa5 David Sorber
bool decompress,
537f40e6 David Sorber
bool delete_orig,
ebc2733a David Sorber
bool verbose,
e0b7f8b9 David Sorber
boost::regex& filter_regex)
b216a5b0 David Sorber
: m_path(path),
537f40e6 David Sorber
m_num_threads(num_threads),
328d0fa5 David Sorber
m_decompress(decompress),
537f40e6 David Sorber
m_delete_orig(delete_orig),
ebc2733a David Sorber
m_verbose(verbose),
m_terminate_flag(false),
f8e600bc David Sorber
m_files_processed(0),
b216a5b0 David Sorber
m_num_input_files(0),
m_total_uncompressed_size(0),
m_total_compressed_size(0),
ebc2733a David Sorber
m_running_compressed_size(0),
e0b7f8b9 David Sorber
p_finder_thread(nullptr),
m_filter_empty(true),
m_filter_regex(filter_regex)
{
// This is a somewhat crude way to determining if the filter regex is empty
m_filter_empty = std::string("").compare(filter_regex.str()) == 0;
}
b216a5b0 David Sorber
PhotoCompressArchiver::~PhotoCompressArchiver()
{}

int PhotoCompressArchiver::execute()
{
4f6113f8 David Sorber
// Grab start time
auto start = std::chrono::system_clock::now();
b216a5b0 David Sorber
// Start the file finder thread
537f40e6 David Sorber
std::cout << "Starting file finder..." << std::flush;
b216a5b0 David Sorber
p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
4f6113f8 David Sorber
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
537f40e6 David Sorber
std::cout << "DONE" << std::endl;
3eeb1485 David Sorber
// Start worker threads
537f40e6 David Sorber
std::cout << "Starting worker threads..." << std::flush;
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
3eeb1485 David Sorber
{
// Spawn worker thread and add to list for bookkeeping
4f6113f8 David Sorber
std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
3eeb1485 David Sorber
this, idx);
m_worker_threads.push_back(worker);
}
537f40e6 David Sorber
std::cout << "DONE\n" << std::endl;
3eeb1485 David Sorber
// Join the file finder thread
b216a5b0 David Sorber
p_finder_thread->join();
57e6c22f David Sorber
// Error out of no input files were found
if (m_num_input_files == 0)
{
537f40e6 David Sorber
ERROR << "No matching input files found. Please check the patch and "
<< "try again." << std::endl;
57e6c22f David Sorber
return -1;
}
ebc2733a David Sorber
std::cout << "File finder completed: " << m_num_input_files << " files";
328d0fa5 David Sorber
if (! m_decompress)
{
ebc2733a David Sorber
std::cout << "; "
ddd2cb4a David Sorber
<< _format_num_bytes(m_total_uncompressed_size) << "\n"
<< std::endl;
328d0fa5 David Sorber
}
ebc2733a David Sorber
else
{
std::cout << std::endl;
}
b216a5b0 David Sorber
4f6113f8 David Sorber
// Add terminator for each worker thread
537f40e6 David Sorber
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
3eeb1485 David Sorber
{
4f6113f8 David Sorber
m_file_list.push_back(nullptr);
3eeb1485 David Sorber
}
b216a5b0 David Sorber
// Clean up the finder thread
delete p_finder_thread;
p_finder_thread = nullptr;
ebc2733a David Sorber
// Display status
std::cout << BOLD << "\n>>> Status:" << ENDC << std::endl;
std::string line(100, '-');
std::cout << line << std::endl;
if (! m_verbose)
{
// If not in verbose mode display running status
ProgressBar pbar = ProgressBar("Progress: ", PROGRESS_BAR_WIDTH);
double percent_done = 0.0;
while ((percent_done < 100.0) && (! m_terminate_flag))
{
// Grab current percent done
percent_done = get_global_percent_done();
// Calculate current elapsed time
auto current = std::chrono::system_clock::now();
auto diff = current - start;
double duration = std::chrono::duration<double>(diff).count();
double ratio = ((double)m_total_compressed_size /
m_running_compressed_size) * 100;
uint32_t num_errors = m_errors.size();
std::cout << "Elapsed time: " << std::fixed
<< std::setprecision(3) << duration << " s "
<< "Files processed: "
<< m_files_processed << " / "
<< m_num_input_files << " "
<< "Errors: " << num_errors << " "
<< std::endl;
pbar.update(std::lround(percent_done));
std::cout << "Compressed/uncompressed size: "
<< _format_num_bytes(m_total_compressed_size) << " / "
<< _format_num_bytes(m_running_compressed_size)
<< " Ratio: " << std::fixed
<< std::setprecision(2) << ratio << "% "
<< std::endl;
std::cout << line << std::endl;

// Delay slightly to prevent flickering
if (percent_done < 100.0)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
std::cout << UP_ONE << UP_ONE << UP_ONE << UP_ONE;
}
}
}
b216a5b0 David Sorber
// Now join the worker threads
for (auto worker : m_worker_threads)
{
worker->join();
delete worker;
}
4f6113f8 David Sorber
// Grab start time and calculate duration
auto end = std::chrono::system_clock::now();
auto diff = end - start;
537f40e6 David Sorber
double duration = std::chrono::duration<double>(diff).count();
// Print out message if any errors were found
if (m_errors.size() > 0)
{
64e2408d David Sorber
std::cout << BOLD << RED << "\n\nNOTE: errors were encountered "
<< "processing the following " << m_errors.size()
<< " files:\n" << ENDC << std::endl;
537f40e6 David Sorber
WorkUnit* work_unit = nullptr;
4126aaa3 David Sorber
uint32_t size = m_errors.size();
for (uint32_t idx = 0; idx < size; ++idx)
537f40e6 David Sorber
{
work_unit = m_errors.pop_front();
4126aaa3 David Sorber
std::cout << " " << work_unit->m_path.string() << std::endl;
537f40e6 David Sorber
delete work_unit;
}
std::cout << std::endl;
}
4f6113f8 David Sorber
cec188e4 David Sorber
// Print out compression information if compressing
328d0fa5 David Sorber
if (! m_decompress)
{
537f40e6 David Sorber
std::cout << "\n Total uncompressed bytes: "
ddd2cb4a David Sorber
<< _format_num_bytes(m_total_uncompressed_size) << std::endl;
std::cout << " Total compressed bytes: "
<< _format_num_bytes(m_total_compressed_size) << std::endl;
f6c21781 David Sorber
328d0fa5 David Sorber
// Calculate and display ratio
double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
ddd2cb4a David Sorber
std::cout << " Compression ratio: " << std::fixed
<< std::setprecision(2) << (ratio * 100) << "%" << std::endl;
537f40e6 David Sorber
}
else
{
std::cout << std::endl;
328d0fa5 David Sorber
}
b216a5b0 David Sorber
537f40e6 David Sorber
// Display execution duration
ddd2cb4a David Sorber
std::cout << " Total time: " << duration << " s\n" << std::endl;
537f40e6 David Sorber
b216a5b0 David Sorber
return 0;
}

void PhotoCompressArchiver::find_files(
const bfs::path& dir_path,
const boost::regex& file_regex)
{
ebc2733a David Sorber
// Check the terminate flag before proceeding
if (m_terminate_flag)
{
return;
}
b216a5b0 David Sorber
bfs::directory_iterator end_iter;
for (bfs::directory_iterator dir_iter(dir_path); dir_iter != end_iter; ++dir_iter)
{
if (bfs::is_directory(dir_iter->status()))
{
// Found directory recurse
find_files(dir_iter->path(), file_regex);
}
7b498bcd David Sorber
else if (boost::regex_match(dir_iter->path().filename().string(), file_regex) &&
bfs::is_regular_file(dir_iter->path()))
b216a5b0 David Sorber
{
// Found file match
e0b7f8b9 David Sorber
// If the filter is not empty, then check against it before proceeding
if (! m_filter_empty)
{
if (boost::regex_match(dir_iter->path().filename().string(),
m_filter_regex))
{
// The filter regex matched, skip this file
continue;
}
}
4f6113f8 David Sorber
// Create a WorkUnit and add it to the queue
WorkUnit* work_unit = new WorkUnit(dir_iter->path());
b216a5b0 David Sorber
++m_num_input_files;
4f6113f8 David Sorber
m_total_uncompressed_size += work_unit->m_file_size;
m_file_list.push_back(work_unit);
b216a5b0 David Sorber
}
}
}

4f6113f8 David Sorber
void PhotoCompressArchiver::worker_thread_body(uint32_t tid)
b216a5b0 David Sorber
{
4f6113f8 David Sorber
WorkUnit* work_unit = nullptr;
auto file_buffer = new std::vector<unsigned char>(4 * 1024 * 1024);
// Setup packJPG instance for this worker
packJPG* instance = new packJPG();
// (De/)compress worker variables
unsigned char* out_buffer = nullptr;
uint32_t out_size = 0;
char message[MSG_SIZE];
e0b7f8b9 David Sorber
3eeb1485 David Sorber
while (true)
b216a5b0 David Sorber
{
4f6113f8 David Sorber
// Reset worker vars
out_size = 0;
std::memset(message, 0, MSG_SIZE);
57e6c22f David Sorber
4f6113f8 David Sorber
// Blocking wait for next work unit
work_unit = m_file_list.pop_front();
ebc2733a David Sorber
if (m_terminate_flag)
{
break;
}
3eeb1485 David Sorber
if (work_unit == nullptr)
e0b7f8b9 David Sorber
{
3eeb1485 David Sorber
break;
e0b7f8b9 David Sorber
}
57e6c22f David Sorber
4f6113f8 David Sorber
// Resize file buffer if needed
if (work_unit->m_file_size > file_buffer->size())
c8352650 David Sorber
{
4f6113f8 David Sorber
file_buffer->resize(work_unit->m_file_size);
c8352650 David Sorber
}
4f6113f8 David Sorber
// Read in the input file
std::ifstream input_stream(work_unit->m_path.string(),
std::ios::binary | std::ios::in);
if (! input_stream)
b216a5b0 David Sorber
{
7b498bcd David Sorber
WRKR_OUT_ERR("unable to read from " << work_unit->m_path << "\n")
537f40e6 David Sorber
m_errors.push_back(work_unit);
continue;
f8e600bc David Sorber
}
4f6113f8 David Sorber
input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
input_stream.close();
537f40e6 David Sorber
// Determine input, and therefore output, filetype / extension based
// on mode
4f6113f8 David Sorber
const std::string* output_file_extension = nullptr;
537f40e6 David Sorber
if (m_decompress)
57e6c22f David Sorber
{
537f40e6 David Sorber
// We are decompressing so the file we found had better be a PJG
if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
{
output_file_extension = &JPG_EXTENSION;
}
else
{
7b498bcd David Sorber
WRKR_OUT_ERR("the input file " << work_unit->m_path
<< " does not appear to be a valid "
537f40e6 David Sorber
"packjpg (.pjg) file even though its filename "
4126aaa3 David Sorber
"suggests it is!")
537f40e6 David Sorber
m_errors.push_back(work_unit);
continue;
}
f8e600bc David Sorber
}
4f6113f8 David Sorber
else
57e6c22f David Sorber
{
537f40e6 David Sorber
// We are compressing so the file we found had better be a JPG
if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
{
output_file_extension = &PJG_EXTENSION;
}
else
{
7b498bcd David Sorber
WRKR_OUT_ERR("the input file " << work_unit->m_path
<< " does not appear to be a valid "
537f40e6 David Sorber
"JPEG (.jpg) file even though its filename "
4126aaa3 David Sorber
"suggests it is!")
537f40e6 David Sorber
m_errors.push_back(work_unit);
continue;
}
57e6c22f David Sorber
}
537f40e6 David Sorber
// Do the thing!
4f6113f8 David Sorber
instance->pjglib_init_streams(file_buffer->data(), 1,
work_unit->m_file_size, nullptr, 1);
57e6c22f David Sorber
4f6113f8 David Sorber
bool rc = instance->pjglib_convert_stream2mem(&out_buffer,
&out_size, message);
if (!rc)
f8e600bc David Sorber
{
64e2408d David Sorber
WRKR_OUT_ERR("An error occurred during the "
<< (m_decompress ? "decompression" : "compression")
<< " operation on " << work_unit->m_path
4126aaa3 David Sorber
<< ": " << message)
537f40e6 David Sorber
m_errors.push_back(work_unit);
continue;
f8e600bc David Sorber
}
4f6113f8 David Sorber
//~ WRKR_OUT_REG("Status message: " << message)
//~ WRKR_OUT_REG("Output size: " << out_size)
f8e600bc David Sorber
// Increment counts
ebc2733a David Sorber
m_running_compressed_size += work_unit->m_file_size;
4f6113f8 David Sorber
m_total_compressed_size += out_size;
f8e600bc David Sorber
++m_files_processed;
4f6113f8 David Sorber
// Create output file path be replacing extension of input file
537f40e6 David Sorber
bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(
*output_file_extension);
4f6113f8 David Sorber
// Write output file
std::ofstream output_stream(out_path.string(),
std::ios::binary | std::ios::out);
if (! output_stream)
328d0fa5 David Sorber
{
537f40e6 David Sorber
WRKR_OUT_ERR("unable to write to \"" << out_path << "\"")
m_errors.push_back(work_unit);
continue;
328d0fa5 David Sorber
}
4f6113f8 David Sorber
output_stream.write((const char*)out_buffer, out_size);
output_stream.close();
328d0fa5 David Sorber
537f40e6 David Sorber
// Delete original file if so instructed
if (m_delete_orig)
{
bfs::remove(work_unit->m_path);
}
// Print status
4f6113f8 David Sorber
if (m_decompress)
57e6c22f David Sorber
{
4f6113f8 David Sorber
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path)
57e6c22f David Sorber
}
4f6113f8 David Sorber
else
{
double percent = ((double)out_size) / work_unit->m_file_size;
percent *= 100;
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path << " -- "
<< std::fixed << std::setprecision(2)
<< percent << "%")
}
564d0193 David Sorber
std::free(out_buffer);
3eeb1485 David Sorber
delete work_unit;
328d0fa5 David Sorber
}
4f6113f8 David Sorber
WRKR_OUT_REG(" -- " << PURPLE << BOLD << "[[exiting]]" << ENDC)
537f40e6 David Sorber
564d0193 David Sorber
// Clean up
4f6113f8 David Sorber
delete instance;
delete file_buffer;
f8e600bc David Sorber
}

double PhotoCompressArchiver::get_global_percent_done()
{
ebc2733a David Sorber
return ((double)(m_files_processed + m_errors.size())
/ m_num_input_files) * 100;
}

void PhotoCompressArchiver::terminate()
{
m_terminate_flag = true;
m_file_list.terminate();
b216a5b0 David Sorber
}