Project

General

Profile

« Previous | Next » 

Revision 191d89ea

Added by david.sorber 8 months ago

Use computed library path for install.

View differences:

software/libpackjpg/lib_src/CMakeLists.txt
INSTALL(TARGETS packjpg
#~ LIBRARY DESTINATION "/usr/lib/x86_64-linux-gnu/")
LIBRARY DESTINATION "/opt/local/lib/")
LIBRARY DESTINATION ${LIBRARY_INSTALL_PATH})
INSTALL(FILES "packjpg.h"
DESTINATION "/opt/local/include/")
DESTINATION ${LIBRARY_INSTALL_PATH})
################################################################################
......
ADD_LIBRARY(packjpg_static STATIC ${packjpg_sources})
SET_TARGET_PROPERTIES(packjpg_static PROPERTIES OUTPUT_NAME "packjpg")
INSTALL(TARGETS packjpg
#~ LIBRARY DESTINATION "/usr/lib/x86_64-linux-gnu/")
LIBRARY DESTINATION ${LIBRARY_INSTALL_PATH})
INSTALL(FILES "packjpg.h"
DESTINATION ${LIBRARY_INSTALL_PATH})
################################################################################
# Target: simple utility
software/photo_compress_archiver/PhotoCompressArchiver.cc
#include <chrono>
#include <cmath>
#include <cstdio>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits.h>
#include <thread>
#include <sstream>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <vector>
#include <packjpg.h>
#include "PhotoCompressArchiver.hh"
#include "ProgressBar.h"
#define ERROR std::cerr << BOLD << RED << "ERROR: " << ENDC
#define WRKR_OUT_REG(msg) { \
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
} \
}
#define WRKR_OUT_ERR(msg) { \
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << "T[" << std::setw(2) << tid << "] " << BOLD << RED \
<< "ERROR: " << ENDC << msg << std::endl; \
} \
}
// Helper function
std::string _format_num_bytes(uint64_t num_bytes)
{
const uint64_t tera = (1024 * 1024 * 1024 * 1024L);
const uint64_t giga = (1024 * 1024 * 1024L);
const uint64_t mega = (1024 * 1024L);
const uint64_t kilo = 1024L;
double fractional;
std::ostringstream output;
if (num_bytes >= tera)
{
// Tera
fractional = ((double)num_bytes) / tera;
output << std::fixed << std::setprecision(2) << fractional;
output << " TB";
}
else if (num_bytes >= giga)
{
// Giga
fractional = ((double)num_bytes) / giga;
output << std::fixed << std::setprecision(2) << fractional;
output << " GB";
}
else if (num_bytes >= mega)
{
// Mega
fractional = ((double)num_bytes) / mega;
output << std::fixed << std::setprecision(2) << fractional;
output << " MB";
}
else if (num_bytes >= kilo)
{
// Kilo
fractional = ((double)num_bytes) / kilo;
output << std::fixed << std::setprecision(2) << fractional;
output << " KB";
}
else
{
// Bytes
output << num_bytes << " b";
}
return output.str();
}
PhotoCompressArchiver::PhotoCompressArchiver(
const std::string& path,
uint32_t num_threads,
bool decompress,
bool delete_orig,
bool verbose,
boost::regex& filter_regex)
: m_path(path),
m_num_threads(num_threads),
m_decompress(decompress),
m_delete_orig(delete_orig),
m_verbose(verbose),
m_terminate_flag(false),
m_files_processed(0),
m_num_input_files(0),
m_total_uncompressed_size(0),
m_total_compressed_size(0),
m_running_compressed_size(0),
p_finder_thread(nullptr),
m_filter_empty(true),
m_filter_regex(filter_regex)
{
// This is a somewhat crude way to determining if the filter regex is empty
m_filter_empty = std::string("") == filter_regex.str();
}
PhotoCompressArchiver::~PhotoCompressArchiver()
= default;
int PhotoCompressArchiver::execute()
{
// Grab start time
auto start = std::chrono::system_clock::now();
// Start the file finder thread
std::cout << "Starting file finder..." << std::flush;
p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
std::cout << "DONE" << std::endl;
// Start worker threads
std::cout << "Starting worker threads..." << std::flush;
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
// Spawn worker thread and add to list for bookkeeping
std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
this, idx);
m_worker_threads.push_back(worker);
}
std::cout << "DONE\n" << std::endl;
// Join the file finder thread
p_finder_thread->join();
// Error out of no input files were found
if (m_num_input_files == 0)
{
ERROR << "No matching input files found. Please check the patch and "
<< "try again." << std::endl;
return -1;
}
std::cout << "File finder completed: " << m_num_input_files << " files";
if (! m_decompress)
{
std::cout << "; "
<< _format_num_bytes(m_total_uncompressed_size) << "\n"
<< std::endl;
}
else
{
std::cout << std::endl;
}
// Add terminator for each worker thread
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
m_file_list.push_back(nullptr);
}
// Clean up the finder thread
delete p_finder_thread;
p_finder_thread = nullptr;
// Display status
std::cout << BOLD << "\n>>> Status:" << ENDC << std::endl;
std::cout << LINE << std::endl;
if (! m_verbose)
{
// If not in verbose mode display running status
ProgressBar pbar = ProgressBar("Progress: ", PROGRESS_BAR_WIDTH);
double percent_done = 0.0;
while ((percent_done < 100.0) && (! m_terminate_flag))
{
// Grab current percent done
percent_done = get_global_percent_done();
// Calculate current elapsed time
auto current = std::chrono::system_clock::now();
auto diff = current - start;
double duration = std::chrono::duration<double>(diff).count();
double ratio = ((double)m_total_compressed_size /
m_running_compressed_size) * 100;
// This looks odd but in fact checks for NaN which we don't want to
// display in the status
if (ratio != ratio)
{
ratio = 0.0;
}
// Highlight and non-zero number of errors in bold red for emphasis
uint32_t num_errors = m_errors.size();
std::ostringstream formatted_errors;
if (num_errors > 0)
{
formatted_errors << "Errors: " << BOLD << RED << num_errors
<< ENDC;
}
else
{
formatted_errors << "Errors: " << num_errors ;
}
std::cout << "Elapsed time: " << std::fixed
<< std::setprecision(3) << duration << " s "
<< "Files processed: "
<< m_files_processed << " / "
<< m_num_input_files << " "
<< formatted_errors.str() << " "
<< std::endl;
pbar.update(std::lround(percent_done));
std::cout << "Compressed/uncompressed size: "
<< _format_num_bytes(m_total_compressed_size) << " / "
<< _format_num_bytes(m_running_compressed_size)
<< " Ratio: " << std::fixed
<< std::setprecision(2) << ratio << "% "
<< std::endl;
std::cout << LINE << std::endl;
// Delay slightly to prevent cursor flickering
if (percent_done < 100.0)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
std::cout << UP_ONE << UP_ONE << UP_ONE << UP_ONE;
}
}
}
// Now join the worker threads
for (auto worker : m_worker_threads)
{
worker->join();
delete worker;
}
// Grab start time and calculate duration
auto end = std::chrono::system_clock::now();
auto diff = end - start;
double duration = std::chrono::duration<double>(diff).count();
// Print out message if any errors were found
if (m_errors.size() > 0)
{
std::cout << BOLD << RED << "\n\nNOTE: errors were encountered "
<< "processing the following " << m_errors.size()
<< " files:\n" << ENDC << std::endl;
WorkUnit* work_unit = nullptr;
uint32_t size = m_errors.size();
for (uint32_t idx = 0; idx < size; ++idx)
{
work_unit = m_errors.pop_front();
std::cout << " " << work_unit->m_path.string() << std::endl;
delete work_unit;
}
std::cout << std::endl;
}
// Print out compression information if compressing
if (! m_decompress)
{
std::cout << "\n Total uncompressed bytes: "
<< _format_num_bytes(m_total_uncompressed_size) << std::endl;
std::cout << " Total compressed bytes: "
<< _format_num_bytes(m_total_compressed_size) << std::endl;
// Calculate and display ratio
double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
std::cout << " Compression ratio: " << std::fixed
<< std::setprecision(2) << (ratio * 100) << "%" << std::endl;
}
else
{
std::cout << std::endl;
}
// Display execution duration
std::cout << " Total time: " << duration << " s\n" << std::endl;
return 0;
}
void PhotoCompressArchiver::find_files(
const bfs::path& dir_path,
const boost::regex& file_regex)
{
// Check the terminate flag before proceeding
if (m_terminate_flag)
{
return;
}
bfs::directory_iterator end_iter;
for (bfs::directory_iterator dir_iter(dir_path); dir_iter != end_iter; ++dir_iter)
{
if (bfs::is_directory(dir_iter->status()))
{
// Found directory recurse
find_files(dir_iter->path(), file_regex);
}
else if (boost::regex_match(dir_iter->path().filename().string(), file_regex) &&
bfs::is_regular_file(dir_iter->path()))
{
// Found file match
// If the filter is not empty, then check against it before proceeding
if (! m_filter_empty)
{
if (boost::regex_match(dir_iter->path().filename().string(),
m_filter_regex))
{
// The filter regex matched, skip this file
continue;
}
}
// Create a WorkUnit and add it to the queue
WorkUnit* work_unit = new WorkUnit(dir_iter->path());
++m_num_input_files;
m_total_uncompressed_size += work_unit->m_file_size;
m_file_list.push_back(work_unit);
}
}
}
void PhotoCompressArchiver::worker_thread_body(uint32_t tid)
{
WorkUnit* work_unit = nullptr;
auto file_buffer = new std::vector<unsigned char>(4 * 1024 * 1024);
// Setup packJPG instance for this worker
packJPG* instance = new packJPG();
// (De/)compress worker variables
unsigned char* out_buffer = nullptr;
uint32_t out_size = 0;
char message[MSG_SIZE];
while (true)
{
// Reset worker vars
out_size = 0;
std::memset(message, 0, MSG_SIZE);
// Blocking wait for next work unit
work_unit = m_file_list.pop_front();
if (m_terminate_flag)
{
break;
}
if (work_unit == nullptr)
{
break;
}
// Resize file buffer if needed
if (work_unit->m_file_size > file_buffer->size())
{
file_buffer->resize(work_unit->m_file_size);
}
// Read in the input file
std::ifstream input_stream(work_unit->m_path.string(),
std::ios::binary | std::ios::in);
if (! input_stream)
{
WRKR_OUT_ERR("unable to read from " << work_unit->m_path << "\n")
m_errors.push_back(work_unit);
continue;
}
input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
input_stream.close();
// Determine input, and therefore output, filetype / extension based
// on mode
const std::string* output_file_extension = nullptr;
if (m_decompress)
{
// We are decompressing so the file we found had better be a PJG
if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
{
output_file_extension = &JPG_EXTENSION;
}
else
{
WRKR_OUT_ERR("the input file " << work_unit->m_path
<< " does not appear to be a valid "
"packjpg (.pjg) file even though its filename "
"suggests it is!")
m_errors.push_back(work_unit);
continue;
}
}
else
{
// We are compressing so the file we found had better be a JPG
if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
{
output_file_extension = &PJG_EXTENSION;
}
else
{
WRKR_OUT_ERR("the input file " << work_unit->m_path
<< " does not appear to be a valid "
"JPEG (.jpg) file even though its filename "
"suggests it is!")
m_errors.push_back(work_unit);
continue;
}
}
// Do the thing!
instance->pjglib_init_streams(file_buffer->data(), 1,
work_unit->m_file_size, nullptr, 1);
bool rc = instance->pjglib_convert_stream2mem(&out_buffer,
&out_size, message);
if (!rc)
{
WRKR_OUT_ERR("An error occurred during the "
<< (m_decompress ? "decompression" : "compression")
<< " operation on " << work_unit->m_path
<< ": " << message)
m_errors.push_back(work_unit);
continue;
}
//~ WRKR_OUT_REG("Status message: " << message)
//~ WRKR_OUT_REG("Output size: " << out_size)
// Increment counts
m_running_compressed_size += work_unit->m_file_size;
m_total_compressed_size += out_size;
++m_files_processed;
// Create output file path be replacing extension of input file
bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(
*output_file_extension);
// Write output file
std::ofstream output_stream(out_path.string(),
std::ios::binary | std::ios::out);
if (! output_stream)
{
WRKR_OUT_ERR("unable to write to \"" << out_path << "\"")
m_errors.push_back(work_unit);
continue;
}
output_stream.write((const char*)out_buffer, out_size);
output_stream.close();
// Delete original file if so instructed
if (m_delete_orig)
{
bfs::remove(work_unit->m_path);
}
// Print status
if (m_decompress)
{
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path)
}
else
{
double percent = ((double)out_size) / work_unit->m_file_size;
percent *= 100;
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path << " -- "
<< std::fixed << std::setprecision(2)
<< percent << "%")
}
std::free(out_buffer);
delete work_unit;
}
WRKR_OUT_REG(" -- " << PURPLE << BOLD << "[[exiting]]" << ENDC)
// Clean up
delete instance;
delete file_buffer;
}
double PhotoCompressArchiver::get_global_percent_done()
{
return ((double)(m_files_processed + m_errors.size())
/ m_num_input_files) * 100;
}
void PhotoCompressArchiver::terminate()
{
m_terminate_flag = true;
m_file_list.terminate();
}
software/photo_compress_archiver/PhotoCompressArchiver.hh
#ifndef PHOTOCOMPRESSARCHIVER_H
#define PHOTOCOMPRESSARCHIVER_H
#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <deque>
#include <mutex>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include "ProtectedAndSynchronizedQueue.h"
// Alias the boost fs namespace to keep the code readable
namespace bfs = boost::filesystem;
const std::string BOLD("\033[1m");
const std::string ENDC("\033[0m");
const std::string RED("\033[31m");
const std::string YELLOW("\033[33m");
const std::string PURPLE("\033[35m");
const std::string UP_ONE = "\033[1A";
const std::string DOWN_ONE = "\033[1B";
const std::string version("v0.4.1");
const boost::regex JPEG_REGEX("^.+\\.(jpg)|(jpeg)$", boost::regex::icase);
const boost::regex PJG_REGEX("^.+\\.pjg$", boost::regex::icase);
const std::string JPG_EXTENSION(".jpg");
const std::string PJG_EXTENSION(".pjg");
const uint32_t PROGRESS_BAR_WIDTH = 80;
const std::string LINE(100, '-');
class WorkUnit
{
public:
bfs::path m_path;
uint32_t m_file_size;
explicit WorkUnit(const bfs::path& path)
: m_path(path),
m_file_size(0)
{
// Automagically determine the file size
m_file_size = bfs::file_size(path);
};
};
class PhotoCompressArchiver
{
public:
PhotoCompressArchiver(
const std::string& path,
uint32_t num_threads,
bool decompress,
bool delete_orig,
bool verbose,
boost::regex& filter_regex);
~PhotoCompressArchiver();
/**
* Start processing
*/
int execute();
/**
* Recursively find input files based on passed regex
*/
void find_files(
const bfs::path& dir_path,
const boost::regex& file_regex);
/**
* Compress/decompress worker thread body
*/
void worker_thread_body(uint32_t tid);
/**
* Calculate the return the "global" percent done as determined by the
* m_files_processed and m_num_input_files instance variables.
*/
double get_global_percent_done();
/**
* Asynchronously terminate processing.
*/
void terminate();
private:
const bfs::path m_path; // top level path where to look for input files
uint32_t m_num_threads; // number of processing threads to use
bool m_decompress; // decompress files instead of compress them
bool m_delete_orig; // delete original files after de/compression
bool m_verbose; // verbose processing output
std::mutex m_output_mutex; // output mutex
bool m_terminate_flag; // terminate flag
std::atomic<uint32_t> m_files_processed; // number of input files processed
uint32_t m_num_input_files; // total number of input files found
uint64_t m_total_uncompressed_size; // total uncompressed size of all images
uint64_t m_total_compressed_size; // total compressed size of all images
uint64_t m_running_compressed_size; // running compressed size (for status)
// input file list (either images or compressed images)
ProtectedAndSynchronizedQueue<WorkUnit*> m_file_list;
// list of any errors
ProtectedAndSynchronizedQueue<WorkUnit*> m_errors;
std::thread* p_finder_thread; // file finder thread
std::vector<std::thread*> m_worker_threads; // worker threads
bool m_filter_empty; // is the filter regex empty
boost::regex& m_filter_regex; // filter regular expression
};
#endif // PHOTOCOMPRESSARCHIVER_H
software/photo_compress_archiver/ProgressBar.h
#ifndef __PROGRESS_BAR_H__
#define __PROGRESS_BAR_H__
#include <cmath>
#include <cstdint>
#include <iostream>
#include <string>
class ProgressBar
{
public:
ProgressBar(
const std::string& label,
uint32_t bar_length,
uint8_t progress=0)
: label(label),
bar_length(bar_length),
progress(progress)
{ }
~ProgressBar() = default;
void update(uint8_t new_progress)
{
// Progress is defined between 0 and 100 inclusive
if (this->progress >= 100)
{
return;
}
this->progress = new_progress;
std::cout << this->label;
std::cout << " [";
uint32_t pos = std::lround((static_cast<double>(this->progress) / 100)
* this->bar_length);
for (uint32_t idx = 0; idx < this->bar_length; ++idx)
{
if (idx < pos)
{
std::cout << "=";
}
else if ((idx == pos) && (idx == this->bar_length))
{
std::cout << "=";
}
else if (idx == pos)
{
std::cout << ">";
}
else
{
std::cout << " ";
}
}
std::cout << "] " << int(this->progress) << " %\r";
// NOTE: normally newline would only be output when progress reaches
// 100% but here we redrawing an entire section of lines so
// we want a newline for each update() call
std::cout << std::endl;
}
ProgressBar& operator++()
{
this->update(this->progress + 1);
return *this;
}
private:
std::string label;
uint32_t bar_length;
uint8_t progress;
};
#endif // __PROGRESS_BAR_H__
software/photo_compress_archiver/ProtectedAndSynchronizedQueue.h
#ifndef __PROTECTEDANDSYNCHRONIZEDQUEUE_H__
#define __PROTECTEDANDSYNCHRONIZEDQUEUE_H__
#include <queue>
#include <mutex>
#include <condition_variable>
#include <iostream>
template<class T>
class ProtectedAndSynchronizedQueue
{
public:
ProtectedAndSynchronizedQueue()
{
m_terminateFlag = false;
}
virtual ~ProtectedAndSynchronizedQueue() {};
// add a new element to the back of the queue
void push_back(T newElement)
{
{
std::lock_guard<std::mutex> lock(m_mutex);
m_queue.push(newElement);
}
m_CV.notify_one();
}
// retrieve the first element of the queue and remove it
// from the queue
T pop_front()
{
// Wait on queue blocking
std::unique_lock<std::mutex> muxLock(m_mutex);
while (m_queue.empty())
{
m_CV.wait(muxLock);
// If we've been asked to terminate, break out and return
// NOTE: the return value is garbage if terminate has been
// called prior this call returning
if (m_terminateFlag)
{
T elementToReturn;
return elementToReturn;
}
}
T elementToReturn = m_queue.front();
m_queue.pop();
muxLock.unlock();
return elementToReturn;
}
// retrieve the first element of the queue and remove it
// from the queue
T pop_front(bool waitOnEmptyQueue, bool& elementReturned)
{
// Wait on queue blocking
std::unique_lock<std::mutex> muxLock(m_mutex);
if (waitOnEmptyQueue)
{
while (m_queue.empty())
{
m_CV.wait(muxLock);
// If we've been asked to terminate, break out and return
// NOTE: the return value is garbage if terminate has been
// called prior this call returning
if (m_terminateFlag)
{
break;
}
}
}
T elementToReturn;
if (m_queue.size() > 0)
{
elementReturned = true;
elementToReturn = m_queue.front();
m_queue.pop();
}
else
{
elementReturned = false;
}
muxLock.unlock();
return elementToReturn;
}
// terminate any waiting pop_front() requests
void terminate()
{
m_terminateFlag = true;
m_CV.notify_all();
}
bool empty()
{
std::lock_guard<std::mutex> lock(m_mutex);
return m_queue.empty();
}
uint32_t size()
{
std::lock_guard<std::mutex> lock(m_mutex);
return m_queue.size();
}
private:
bool m_terminateFlag;
std::queue<T> m_queue;
std::mutex m_mutex;
std::condition_variable m_CV;
};
#endif
software/photo_compress_archiver/main.cc
#include <iostream>
#include <string>
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
#include <boost/regex.hpp>
#include "PhotoCompressArchiver.hh"
// Alias the namespace for readability
namespace bpo = boost::program_options;
void header()
{
std::cout << BOLD << "Photo Compress Archiver -- " << version << "\n" << ENDC;
std::cout << " Copyright 2017-2018 -- David Sorber\n";
std::cout << " >>> Utilizing packJPG by Matthias Stirner\n" << std::endl;
}
void usage(char** argv)
{
// Remove leading "./" if it exits
char* app_name = argv[0];
if (argv[0][0] == '.' && argv[0][1] == '/')
{
app_name = &argv[0][2];
}
header();
std::cout << "USAGE: " << app_name << " [-hdk] [-t <num threads>] "
<< "[-f <filter regex>] <file path>\n\n";
std::cout << "Positional arguments:\n";
std::cout << " <file path> path in which to recursively look "
<< "for input files\n\n";
std::cout << "General options:\n";
std::cout << " -h [ --help ] display this help message\n";
std::cout << " -d [ --decompress ] find '*.pjg' files and decompress them\n";
std::cout << " -k [ --delete-orig ] delete original files\n";
std::cout << " -v [ --verbose ] verbose output while processing\n";
std::cout << " -t [ --num-threads ] number of processing threads to use (max 2x system cores)\n";
std::cout << " -f [ --filter ] regex with which to filter out input files\n";
std::cout << std::endl;
}
// NOTE: useful filter: -f ^\\._.+$
int main(int argc, char** argv)
{
uint32_t num_cores = std::thread::hardware_concurrency();
uint32_t num_threads = 0;
std::string search_path;
std::string filter_str;
bpo::variables_map general_opts_vm;
try
{
// General options
bpo::options_description general_opts("General options");
general_opts.add_options()
("help,h", "display this help message")
("decompress,d", "find '*.pjg' files and decompress them")
("delete-orig,k", "delete orignal files")
("verbose,v", "verbose output while processing")
("num-threads,t",
bpo::value<uint32_t>(&num_threads)->default_value(num_cores),
"number of processing threads to use")
("filter,f",
bpo::value<std::string>(&filter_str),
"regex with which to filter out input files")
// Note this has to be here even though it's a positional argument
("file_path", bpo::value<std::string>(&search_path)->required(),
"path in which to recursively look for input files")
;
// Positional arguments
bpo::positional_options_description pos_opts;
pos_opts.add("file_path", 1);
// Parse general/positional options
bpo::store(bpo::command_line_parser(argc, argv)
.options(general_opts)
.positional(pos_opts).run(),
general_opts_vm);
// Display help if the option is listed or if no arguments are provided
if (general_opts_vm.count("help") || (argc == 1))
{
usage(argv);
return 0;
}
bpo::notify(general_opts_vm);
}
catch (std::exception& e)
{
std::cerr << BOLD << RED << "\nERROR: " << ENDC << e.what() << "\n"
<< std::endl;
return -1;
}
// Make sure that path specified actually exists before proceeding
if (! bfs::exists(search_path))
{
std::cerr << BOLD << RED << "\nERROR: " << ENDC << "the path \""
<< search_path << "\" does not exist\n" << std::endl;
return -2;
}
// Validate number of threads
if (num_threads > (2 * std::thread::hardware_concurrency()))
{
std::cerr << BOLD << RED << "\nERROR: " << ENDC << "the maximum number "
<< "of threads is: " << (2 * num_cores) << "(2 * " << num_cores
<< ")\n" << std::endl;
return -2;
}
// Check for, and if present validate the filter regex
boost::regex filter_regex("");
if (general_opts_vm.count("filter"))
{
try
{
filter_regex = filter_str;
}
catch (std::exception& e)
{
std::cerr << BOLD << RED << "ERROR: " << ENDC << "the filter \""
<< filter_str << "\" is not a valid regular expression.\n";
std::cerr << " " << e.what() << std::endl;
return -1;
}
}
// Display header and options
header();
std::cout << " [" << num_cores << " cores detected]\n" << std::endl;
std::cout << BOLD << ">>> Options:" << ENDC << std::endl;
// Decode/store decompress option
bool decompress = false;
if (general_opts_vm.count("decompress"))
{
decompress = true;
}
std::cout << " decompress mode => " << (decompress ? "ON" : "OFF")
<< std::endl;
// Decode/store delete-orig option
bool delete_orig = false;
if (general_opts_vm.count("delete-orig"))
{
delete_orig = true;
}
std::cout << " delete original files => " << (delete_orig ? "ON" : "OFF")
<< std::endl;
// Verbose output option
bool verbose = false;
if (general_opts_vm.count("verbose"))
{
verbose = true;
}
std::cout << " verbose => " << (verbose ? "ON" : "OFF")
<< std::endl;
// Number of processing threads option
std::cout << " num processing threads => " << num_threads << std::endl;
// Filter regex option
if (general_opts_vm.count("filter"))
{
std::cout << " filter regex => ON (" << filter_str << ")\n";
}
std::cout << std::endl;
// Create the PCA object and let 'er rip!
auto pca = new PhotoCompressArchiver(search_path, num_threads, decompress,
delete_orig, verbose, filter_regex);
pca->execute();
delete pca;
return 0;
}
software/photo_compress_archiver/src/PhotoCompressArchiver.cc
#include <chrono>
#include <cmath>
#include <cstdio>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits.h>
#include <thread>
#include <sstream>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <vector>
#include <packjpg.h>
#include "PhotoCompressArchiver.hh"
#include "ProgressBar.h"
#define ERROR std::cerr << BOLD << RED << "ERROR: " << ENDC
#define WRKR_OUT_REG(msg) { \
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
} \
}
#define WRKR_OUT_ERR(msg) { \
if (m_verbose) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << "T[" << std::setw(2) << tid << "] " << BOLD << RED \
<< "ERROR: " << ENDC << msg << std::endl; \
} \
}
// Helper function
std::string _format_num_bytes(uint64_t num_bytes)
{
const uint64_t tera = (1024 * 1024 * 1024 * 1024L);
const uint64_t giga = (1024 * 1024 * 1024L);
const uint64_t mega = (1024 * 1024L);
const uint64_t kilo = 1024L;
double fractional;
std::ostringstream output;
if (num_bytes >= tera)
{
// Tera
fractional = ((double)num_bytes) / tera;
output << std::fixed << std::setprecision(2) << fractional;
output << " TB";
}
else if (num_bytes >= giga)
{
// Giga
fractional = ((double)num_bytes) / giga;
output << std::fixed << std::setprecision(2) << fractional;
output << " GB";
}
else if (num_bytes >= mega)
{
// Mega
fractional = ((double)num_bytes) / mega;
output << std::fixed << std::setprecision(2) << fractional;
output << " MB";
}
else if (num_bytes >= kilo)
{
// Kilo
fractional = ((double)num_bytes) / kilo;
output << std::fixed << std::setprecision(2) << fractional;
output << " KB";
}
else
{
// Bytes
output << num_bytes << " b";
}
return output.str();
}
PhotoCompressArchiver::PhotoCompressArchiver(
const std::string& path,
uint32_t num_threads,
bool decompress,
bool delete_orig,
bool verbose,
boost::regex& filter_regex)
: m_path(path),
m_num_threads(num_threads),
m_decompress(decompress),
m_delete_orig(delete_orig),
m_verbose(verbose),
m_terminate_flag(false),
m_files_processed(0),
m_num_input_files(0),
m_total_uncompressed_size(0),
m_total_compressed_size(0),
m_running_compressed_size(0),
p_finder_thread(nullptr),
m_filter_empty(true),
m_filter_regex(filter_regex)
{
// This is a somewhat crude way to determining if the filter regex is empty
m_filter_empty = std::string("") == filter_regex.str();
}
PhotoCompressArchiver::~PhotoCompressArchiver()
= default;
int PhotoCompressArchiver::execute()
{
// Grab start time
auto start = std::chrono::system_clock::now();
// Start the file finder thread
std::cout << "Starting file finder..." << std::flush;
p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
std::cout << "DONE" << std::endl;
// Start worker threads
std::cout << "Starting worker threads..." << std::flush;
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
// Spawn worker thread and add to list for bookkeeping
std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
this, idx);
m_worker_threads.push_back(worker);
}
std::cout << "DONE\n" << std::endl;
// Join the file finder thread
p_finder_thread->join();
// Error out of no input files were found
if (m_num_input_files == 0)
{
ERROR << "No matching input files found. Please check the patch and "
<< "try again." << std::endl;
return -1;
}
std::cout << "File finder completed: " << m_num_input_files << " files";
if (! m_decompress)
{
std::cout << "; "
<< _format_num_bytes(m_total_uncompressed_size) << "\n"
<< std::endl;
}
else
{
std::cout << std::endl;
}
// Add terminator for each worker thread
for (uint32_t idx = 0; idx < m_num_threads; ++idx)
{
m_file_list.push_back(nullptr);
}
// Clean up the finder thread
delete p_finder_thread;
p_finder_thread = nullptr;
// Display status
std::cout << BOLD << "\n>>> Status:" << ENDC << std::endl;
std::cout << LINE << std::endl;
if (! m_verbose)
{
// If not in verbose mode display running status
ProgressBar pbar = ProgressBar("Progress: ", PROGRESS_BAR_WIDTH);
double percent_done = 0.0;
while ((percent_done < 100.0) && (! m_terminate_flag))
{
// Grab current percent done
percent_done = get_global_percent_done();
// Calculate current elapsed time
auto current = std::chrono::system_clock::now();
auto diff = current - start;
double duration = std::chrono::duration<double>(diff).count();
double ratio = ((double)m_total_compressed_size /
m_running_compressed_size) * 100;
// This looks odd but in fact checks for NaN which we don't want to
// display in the status
if (ratio != ratio)
{
ratio = 0.0;
}
// Highlight and non-zero number of errors in bold red for emphasis
uint32_t num_errors = m_errors.size();
std::ostringstream formatted_errors;
if (num_errors > 0)
{
formatted_errors << "Errors: " << BOLD << RED << num_errors
<< ENDC;
}
else
{
formatted_errors << "Errors: " << num_errors ;
}
std::cout << "Elapsed time: " << std::fixed
<< std::setprecision(3) << duration << " s "
<< "Files processed: "
<< m_files_processed << " / "
<< m_num_input_files << " "
<< formatted_errors.str() << " "
<< std::endl;
pbar.update(std::lround(percent_done));
std::cout << "Compressed/uncompressed size: "
<< _format_num_bytes(m_total_compressed_size) << " / "
<< _format_num_bytes(m_running_compressed_size)
<< " Ratio: " << std::fixed
<< std::setprecision(2) << ratio << "% "
<< std::endl;
std::cout << LINE << std::endl;
// Delay slightly to prevent cursor flickering
if (percent_done < 100.0)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
std::cout << UP_ONE << UP_ONE << UP_ONE << UP_ONE;
}
}
}
// Now join the worker threads
for (auto worker : m_worker_threads)
{
worker->join();
delete worker;
}
// Grab start time and calculate duration
auto end = std::chrono::system_clock::now();
auto diff = end - start;
double duration = std::chrono::duration<double>(diff).count();
// Print out message if any errors were found
if (m_errors.size() > 0)
{
std::cout << BOLD << RED << "\n\nNOTE: errors were encountered "
<< "processing the following " << m_errors.size()
<< " files:\n" << ENDC << std::endl;
WorkUnit* work_unit = nullptr;
uint32_t size = m_errors.size();
for (uint32_t idx = 0; idx < size; ++idx)
{
work_unit = m_errors.pop_front();
std::cout << " " << work_unit->m_path.string() << std::endl;
delete work_unit;
}
std::cout << std::endl;
}
// Print out compression information if compressing
if (! m_decompress)
{
std::cout << "\n Total uncompressed bytes: "
<< _format_num_bytes(m_total_uncompressed_size) << std::endl;
std::cout << " Total compressed bytes: "
<< _format_num_bytes(m_total_compressed_size) << std::endl;
// Calculate and display ratio
double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
std::cout << " Compression ratio: " << std::fixed
<< std::setprecision(2) << (ratio * 100) << "%" << std::endl;
}
else
{
std::cout << std::endl;
}
// Display execution duration
std::cout << " Total time: " << duration << " s\n" << std::endl;
return 0;
}
void PhotoCompressArchiver::find_files(
const bfs::path& dir_path,
const boost::regex& file_regex)
{
// Check the terminate flag before proceeding
if (m_terminate_flag)
{
return;
}
bfs::directory_iterator end_iter;
for (bfs::directory_iterator dir_iter(dir_path); dir_iter != end_iter; ++dir_iter)
{
if (bfs::is_directory(dir_iter->status()))
{
// Found directory recurse
find_files(dir_iter->path(), file_regex);
}
else if (boost::regex_match(dir_iter->path().filename().string(), file_regex) &&
bfs::is_regular_file(dir_iter->path()))
{
// Found file match
// If the filter is not empty, then check against it before proceeding
if (! m_filter_empty)
{
if (boost::regex_match(dir_iter->path().filename().string(),
m_filter_regex))
{
// The filter regex matched, skip this file
continue;
}
}
// Create a WorkUnit and add it to the queue
WorkUnit* work_unit = new WorkUnit(dir_iter->path());
++m_num_input_files;
m_total_uncompressed_size += work_unit->m_file_size;
m_file_list.push_back(work_unit);
}
}
}
void PhotoCompressArchiver::worker_thread_body(uint32_t tid)
{
WorkUnit* work_unit = nullptr;
auto file_buffer = new std::vector<unsigned char>(4 * 1024 * 1024);
// Setup packJPG instance for this worker
packJPG* instance = new packJPG();
// (De/)compress worker variables
unsigned char* out_buffer = nullptr;
uint32_t out_size = 0;
char message[MSG_SIZE];
while (true)
{
// Reset worker vars
out_size = 0;
std::memset(message, 0, MSG_SIZE);
// Blocking wait for next work unit
work_unit = m_file_list.pop_front();
if (m_terminate_flag)
{
break;
}
if (work_unit == nullptr)
{
break;
}
// Resize file buffer if needed
if (work_unit->m_file_size > file_buffer->size())
{
file_buffer->resize(work_unit->m_file_size);
}
// Read in the input file
std::ifstream input_stream(work_unit->m_path.string(),
std::ios::binary | std::ios::in);
if (! input_stream)
{
WRKR_OUT_ERR("unable to read from " << work_unit->m_path << "\n")
m_errors.push_back(work_unit);
continue;
}
input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
input_stream.close();
// Determine input, and therefore output, filetype / extension based
// on mode
const std::string* output_file_extension = nullptr;
if (m_decompress)
{
// We are decompressing so the file we found had better be a PJG
if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
{
output_file_extension = &JPG_EXTENSION;
}
else
{
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff