Project

General

Profile

« Previous | Next » 

Revision 4f6113f8

Added by David Sorber almost 8 years ago

Converted PCA to be multithreaded and use libpackjpg instead of calling
the old packjpg executable. This works much better! Need to handle a few
error conditions and do some more testing with decompress.

View differences:

software/photo_compress_archiver/CMakeLists.txt
project("Photo Compress Archiver")
cmake_minimum_required(VERSION 2.8)
cmake_minimum_required(VERSION 3.2)
set(CMAKE_CXX_STANDARD 11)
include_directories(.)
include_directories(/usr/include/)
set(EXTERNAL_LIBS)
......
set(EXTERNAL_LIBS ${EXTERNAL_LIBS} ${Boost_LIBRARIES})
endif()
# build packJPG as an external project
include(ExternalProject)
ExternalProject_Add(packjpg
DOWNLOAD_COMMAND pwd # fake this out since we already have the source
SOURCE_DIR ../packJPG
BINARY_DIR ../packJPG/build
INSTALL_DIR ../
INSTALL_COMMAND pwd # also fake this out, we don't want to install either
)
# NOTE: PCA now uses the libpackjpg that I created instead of the executable
# Not sure why this is required... but doesn't seem to work otherwise
SET(LIBPACKJPG)
find_library(LIBPACKJPG packjpg REQUIRED)
#~ message(STATUS LIBPACKJPG: ${LIBPACKJPG})
set(EXTERNAL_LIBS ${EXTERNAL_LIBS} ${LIBPACKJPG})
#~ message(STATUS LIBS: ${EXTERNAL_LIBS})
# PCA sources
set(pca_sources
......
add_executable(pca
${pca_sources}
../main.cc)
add_dependencies(pca packjpg)
target_link_libraries(pca ${CMAKE_THREAD_LIBS_INIT} ${EXTERNAL_LIBS})
target_link_libraries(pca ${CMAKE_THREAD_LIBS_INIT} ${EXTERNAL_LIBS} stdc++fs)
software/photo_compress_archiver/PhotoCompressArchiver.cc
#include <chrono>
#include <cstdio>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <limits.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <vector>
#include <packjpg.h>
#include "PhotoCompressArchiver.hh"
#define OUT(x) {\
std::lock_guard<std::mutex> lock(m_output_mutex);\
x\
#define OUT(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << msg << std::endl; \
}
const uint32_t PTR_LEN = sizeof(char*);
const uint32_t BAILOUT_MAX = 600; // 30s in 50ms increments
const uint32_t FILES_PER_WORK_UNIT = 40;
#define ERROR(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << BOLD << RED << "ERROR: " << ENDC << msg << std::endl; \
}
const char *PROG_NAME = "./packjpg";
const char *PROG_OPT1 = "-np";
const char *PROG_OPT2 = "-o";
const char *PROG_OPT3 = "-p";
#define WRKR_OUT_REG(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cout << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
}
const uint32_t ARGV_MAX_SIZE(sysconf(_SC_ARG_MAX));
const uint32_t ARGV_EXTRA(4096);
#define WRKR_OUT_ERR(msg) { \
std::lock_guard<std::mutex> lock(m_output_mutex); \
std::cerr << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
}
PhotoCompressArchiver::PhotoCompressArchiver(
const std::string& path,
......
{
// This is a somewhat crude way to determining if the filter regex is empty
m_filter_empty = std::string("").compare(filter_regex.str()) == 0;
// Remove this...
std::cout << "argv size: " << sysconf(_SC_ARG_MAX) << std::endl;
}
PhotoCompressArchiver::~PhotoCompressArchiver()
......
int PhotoCompressArchiver::execute()
{
// Grab start time
auto start = std::chrono::system_clock::now();
// Start the file finder thread
OUT(std::cout << "Starting file finder..." << std::flush;);
OUT("Starting file finder..." << std::flush);
p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
OUT(std::cout << "DONE" << std::endl;)
m_path,
(m_decompress ? PJG_REGEX : JPEG_REGEX));
OUT("DONE");
// Start worker threads
OUT(std::cout << "Starting worker threads..." << std::flush;)
OUT("Starting worker threads..." << std::flush);
uint32_t num_threads = m_num_cores;
for (uint32_t idx = 0; idx < num_threads; ++idx)
{
// Spawn worker thread and add to list for bookkeeping
std::thread* worker = new std::thread(&PhotoCompressArchiver::fork_worker,
std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
this, idx);
m_worker_threads.push_back(worker);
}
OUT(std::cout << "DONE\n" << std::endl;)
OUT("DONE\n");
// Join the file finder thread
p_finder_thread->join();
OUT(std::cout << "File finder completed:" << std::endl;)
OUT("File finder completed:");
// Error out of no input files were found
if (m_num_input_files == 0)
{
OUT(std::cerr << BOLD << RED << "ERROR: " << ENDC << "No matching "
<< "input files found. Please check the patch and try "
<< "again. " << std::endl;)
ERROR("No matching input files found. Please check the patch and try "
"again.");
return -1;
}
OUT(std::cout << " Found " << m_num_input_files << " files" << std::endl;)
OUT(" Found " << m_num_input_files << " files");
if (! m_decompress)
{
OUT(std::cout << " Total uncompressed bytes: "
<< m_total_uncompressed_size << "\n" << std::endl;)
OUT(" Total uncompressed bytes: " << m_total_uncompressed_size << "\n");
}
// Build work unit queue
uint32_t input_list_idx = 0;
uint32_t queue_size = 0;
while (input_list_idx < m_file_list.size())
{
// Safely read the size of the queue
{
std::lock_guard<std::mutex> lock(m_work_unit_mtx);
queue_size = m_work_unit_queue.size();
}
// Wait until the queue has decreased in the size before proceeding,
// otherwise we could chew up a bunch of memory
if (queue_size > num_threads)
{
std::this_thread::sleep_for(std::chrono::milliseconds(250));
continue;
}
WorkUnit* work_unit = new WorkUnit();
work_unit->exec_argv.push_back(&PROG_NAME[2]);
work_unit->exec_argv.push_back(PROG_OPT1);
work_unit->exec_argv.push_back(PROG_OPT2);
work_unit->exec_argv.push_back(PROG_OPT3);
uint32_t argv_size = sizeof(PROG_NAME) + sizeof(PROG_OPT1) +
sizeof(PROG_OPT2) + sizeof(PROG_OPT3) + ARGV_EXTRA;
uint32_t local_count = 0;
while ((input_list_idx < m_file_list.size()) &&
(local_count < FILES_PER_WORK_UNIT))
{
bfs::path& file_path = m_file_list[input_list_idx];
uint32_t path_len = file_path.string().size();
// Check if adding this next file would make the argv too big; if
// so bail out
if (argv_size + path_len + PTR_LEN > ARGV_MAX_SIZE)
{
break;
}
// Add the file to the argv array
work_unit->exec_argv.push_back(file_path.string().c_str());
argv_size += path_len + PTR_LEN;
// Create new filename for expected
uint32_t end_pos = file_path.string().rfind(".");
std::string* filename = new std::string(file_path.string().begin(),
file_path.string().begin()
+ end_pos);
// Add extension depending on if decompress mode is enabled or not
(*filename) += (m_decompress ? ".jpg" : ".pjg");
work_unit->output_filenames.push_back(filename);
// Increment our index to the next file
++input_list_idx;
++local_count;
}
work_unit->exec_argv.push_back(nullptr);
// Add the create work unit to the queue; and notify the worker threads
// that it has been handed off
{
std::lock_guard<std::mutex> lock(m_work_unit_mtx);
m_work_unit_queue.push_back(work_unit);
}
m_work_unit_cv.notify_all();
}
// Add null terminate indicator for each thread
// Add terminator for each worker thread
for (uint32_t idx = 0; idx < num_threads; ++idx)
{
std::lock_guard<std::mutex> lock(m_work_unit_mtx);
for (uint32_t idx = 0; idx < num_threads; ++idx)
{
m_work_unit_queue.push_back(nullptr);
}
m_file_list.push_back(nullptr);
}
m_work_unit_cv.notify_all();
// Clean up the finder thread
delete p_finder_thread;
......
delete worker;
}
// Grab start time and calculate duration
auto end = std::chrono::system_clock::now();
auto diff = end - start;
double duration = std::chrono::duration<double, std::milli>(diff).count();
// Print out compression information if compressing
if (! m_decompress)
{
OUT(std::cout << "\n Total compressed bytes: "
<< m_total_compressed_size
<< std::endl;)
OUT("\n Total uncompressed bytes: " << m_total_uncompressed_size);
OUT(" Total compressed bytes: " << m_total_compressed_size);
// Calculate and display ratio
double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
OUT(std::cout << " Compression ratio: " << std::fixed
<< std::setprecision(4) << (ratio * 100) << "%\n"
<< std::endl;)
OUT(" Compression ratio: " << std::fixed << std::setprecision(4)
<< (ratio * 100) << "%");
// Display execution duration
OUT(" Total time: " << duration << " ms\n");
}
return 0;
......
}
}
m_file_list.push_back(dir_iter->path());
// Create a WorkUnit and add it to the queue
WorkUnit* work_unit = new WorkUnit(dir_iter->path());
++m_num_input_files;
// Stat the file to get its size
struct stat statbuf;
int rc = stat(dir_iter->path().string().c_str(), &statbuf);
if (rc)
{
OUT(std::cerr << BOLD << RED << "ERROR: " << ENDC
<< "unable to stat file " << dir_iter->path()
<< std::endl;)
}
m_total_uncompressed_size += statbuf.st_size;
m_total_uncompressed_size += work_unit->m_file_size;
m_file_list.push_back(work_unit);
}
}
}
void PhotoCompressArchiver::fork_worker(
uint32_t tid)
void PhotoCompressArchiver::worker_thread_body(uint32_t tid)
{
// Loop vars
int status;
uint32_t work_unit_items = 0;
uint32_t bailout_ctr = 0;
uint32_t output_idx = 1;
int stat_rc = 0;
struct stat statbuf;
uint32_t files_processed = 0;
double complete_percent = 0.0;
WorkUnit* work_unit = nullptr;
auto file_buffer = new std::vector<unsigned char>(4 * 1024 * 1024);
// Setup packJPG instance for this worker
packJPG* instance = new packJPG();
// (De/)compress worker variables
unsigned char* out_buffer = nullptr;
uint32_t out_size = 0;
char message[MSG_SIZE];
while (true)
{
// Reset worker vars
out_size = 0;
std::memset(message, 0, MSG_SIZE);
std::unique_lock<std::mutex> mux_lock(m_work_unit_mtx);
while (m_work_unit_queue.empty())
{
m_work_unit_cv.wait_for(mux_lock, std::chrono::milliseconds(20));
}
// While holding the mutex grab the next chunk off of the add chunk
// queue
WorkUnit* work_unit = m_work_unit_queue.front();
m_work_unit_queue.pop_front();
mux_lock.unlock();
// Blocking wait for next work unit
work_unit = m_file_list.pop_front();
if (work_unit == nullptr)
{
break;
}
work_unit_items = work_unit->output_filenames.size();
#if 0
// DEBUGGING
uint32_t idx = 4;
OUT(
for (auto outfile : work_unit->output_filenames)
// Resize file buffer if needed
if (work_unit->m_file_size > file_buffer->size())
{
std::cout << "T[" << tid << "] output file: " << *outfile
<< "\n input file: "
<< static_cast<const char*>(work_unit->exec_argv[idx++])
<< std::endl;
file_buffer->resize(work_unit->m_file_size);
}
)
continue;
#endif
// Create a pipe to hold stdout from child process
int filedes[2];
if (pipe(filedes) == -1)
// Read in the input file
std::ifstream input_stream(work_unit->m_path.string(),
std::ios::binary | std::ios::in);
if (! input_stream)
{
perror("pipe");
exit(1);
WRKR_OUT_ERR("ERROR: unable to read from \"" << work_unit->m_path
<< "\"\n")
// TODO: what now?
}
// Fork off the childprocess
//~ pid_t parent = getpid();
pid_t pid = vfork();
if (pid == -1)
input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
input_stream.close();
// TODO: revise this
// Determine input, and therefore output, filetype / extension
const std::string* output_file_extension = nullptr;
if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
{
// error, failed to fork()
OUT(std::cout << "T[" << tid << "] fork failed!" << std::endl;)
}
else if (pid == 0)
output_file_extension = &PJG_EXTENSION;
}
else if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
(file_buffer->at(1) == packJPG::pjg_magic[1]))
{
// This is the child process...
// Set child process's stdout to the pipe entry
while ((dup2(filedes[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {}
close(filedes[0]);
close(filedes[1]);
// Child after fork
int rc = execv(PROG_NAME, (char **)work_unit->exec_argv.data());
if (rc)
{
std::cerr << "execv failed: " << errno << std::endl;
}
_exit(EXIT_FAILURE); // exec never returns
output_file_extension = &JPG_EXTENSION;
}
// This is the parent process...
// This is a little silly... but it works. We know the order in which
// the output files will be created so we wait until output files X + 1
// exists (i.e. we can stat() it) and then process file X.
bailout_ctr = 0;
output_idx = 1;
stat_rc = 0;
files_processed = 0;
complete_percent = 0.0;
while (output_idx < work_unit_items)
else
{
// Poll, waiting for output file X + 1 (aka "z") to exist
bailout_ctr = 0;
const char* filename_z = work_unit->output_filenames[output_idx]->c_str();
while ((stat(filename_z, &statbuf) != 0) && (++bailout_ctr < BAILOUT_MAX))
{
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
// Check if timeout has occurred
if (bailout_ctr == BAILOUT_MAX)
{
OUT(std::cerr << "T[" << tid << "] " << BOLD << RED << "ERROR: "
<< ENDC << "timed out while waiting "
<< "for output file: "
<< *work_unit->output_filenames[output_idx]
<< "; aborting" << std::endl;)
// The subprocess timed out... get its status
waitpid(pid, &status, 0);
OUT(std::cerr << "T[" << tid << "] " << BOLD << RED << "ERROR: "
<< ENDC << PURPLE << " RC => " << status << ENDC
<< std::endl;)
return;
}
// Now stat output file X (aka "y") which should exist
const char* filename_y = work_unit->output_filenames[output_idx - 1]->c_str();
stat_rc = stat(filename_y, &statbuf);
if (stat_rc)
{
OUT(std::cerr << "T[" << tid << "] " << BOLD << RED << "ERROR "
<< ENDC << "while stating: " << filename_y
<< std::endl;)
}
// Increment counts
m_total_compressed_size += statbuf.st_size;
++m_files_processed;
++files_processed;
// Unless keeping original files, remove the source file
if (! m_keep_orig)
{
std::remove(work_unit->exec_argv[4 + output_idx - 1]);
}
// Calculate the local percent done
complete_percent = (double)files_processed / work_unit_items;
complete_percent *= 100;
OUT(std::cout << "T[" << tid << "] " << std::setw(6)
<< std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "% -- (" << std::setw(2) << files_processed << "/"
<< work_unit_items << " -- " << std::setw(6)
<< complete_percent
<< "%) file: " << filename_y
<< " -- " << statbuf.st_size << std::endl;)
++output_idx;
WRKR_OUT_ERR("ERROR: Input file does not appear to be valid\n")
// TODO: what now?
}
// Wait for forked child process to terminate
waitpid(pid, &status, 0);
close(filedes[0]);
close(filedes[1]);
instance->pjglib_init_streams(file_buffer->data(), 1,
work_unit->m_file_size, nullptr, 1);
// Now that the forked process has completed, we can handle the last
// output file
const char* filename_y = work_unit->output_filenames[output_idx - 1]->c_str();
stat_rc = stat(filename_y, &statbuf);
if (stat_rc)
// Do the thing!
bool rc = instance->pjglib_convert_stream2mem(&out_buffer,
&out_size, message);
if (!rc)
{
OUT(std::cerr << "T[" << tid << "] " << BOLD << RED << "ERROR "
<< ENDC << "while stating: " << filename_y << std::endl;)
WRKR_OUT_ERR("An error occurred during the compression"
<< "/decompression operation: " << message << "\n")
// TODO: what now?
}
//~ WRKR_OUT_REG("Status message: " << message)
//~ WRKR_OUT_REG("Output size: " << out_size)
// Increment counts
m_total_compressed_size += statbuf.st_size;
m_total_compressed_size += out_size;
++m_files_processed;
++files_processed;
// Unless keeping original files, remove the source file
if (! m_keep_orig)
// Create output file path be replacing extension of input file
bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(*output_file_extension);
// Write output file
std::ofstream output_stream(out_path.string(),
std::ios::binary | std::ios::out);
if (! output_stream)
{
std::remove(work_unit->exec_argv[4 + output_idx - 1]);
WRKR_OUT_ERR("ERROR: unable to read from \""
<< work_unit->m_path << "\"")
// TODO: what now?
}
output_stream.write((const char*)out_buffer, out_size);
output_stream.close();
// Calculate the local percent done
complete_percent = (double)files_processed / work_unit_items;
complete_percent *= 100;
OUT(std::cout << "T[" << tid << "] " << std::setw(6) << std::fixed
<< std::setprecision(2) << get_global_percent_done()
<< "% -- (" << std::setw(2) << files_processed
<< "/" << work_unit_items << " -- " << std::setw(5)
<< complete_percent << "%) file: " << filename_y
<< " -- " << statbuf.st_size << std::endl;)
// Free up the output filename list
for (auto output_filename : work_unit->output_filenames)
if (m_decompress)
{
delete output_filename;
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path)
}
else
{
double percent = ((double)out_size) / work_unit->m_file_size;
percent *= 100;
WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
<< get_global_percent_done()
<< "%) Output file: " << out_path << " -- "
<< std::fixed << std::setprecision(2)
<< percent << "%")
}
delete work_unit;
}
OUT(std::cout << "T[" << tid << "] " << PURPLE << "complete ==> RC: "
<< status << ENDC << std::endl;)
WRKR_OUT_REG(" -- " << PURPLE << BOLD << "[[exiting]]" << ENDC)
// Clean up
std::free(out_buffer);
delete instance;
delete file_buffer;
}
double PhotoCompressArchiver::get_global_percent_done()
software/photo_compress_archiver/PhotoCompressArchiver.hh
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include "ProtectedAndSynchronizedQueue.h"
// Alias the boost fs namespace to keep the code readable
namespace bfs = boost::filesystem;
......
const std::string YELLOW("\033[33m");
const std::string PURPLE("\033[35m");
const std::string version("v0.1.0");
const std::string version("v0.2.0");
const boost::regex JPEG_REGEX("^.+\\.(jpg)|(jpeg)$", boost::regex::icase);
const boost::regex PJG_REGEX("^.+\\.pjg$", boost::regex::icase);
struct work_unit_t {
std::vector<const char*> exec_argv;
std::vector<std::string*> output_filenames;
const std::string JPG_EXTENSION(".jpg");
const std::string PJG_EXTENSION(".pjg");
class WorkUnit
{
public:
bfs::path m_path;
uint32_t m_file_size;
WorkUnit(const bfs::path& path)
: m_path(path),
m_file_size(0)
{
m_file_size = bfs::file_size(path);
};
};
typedef struct work_unit_t WorkUnit;
class PhotoCompressArchiver
{
......
const boost::regex& file_regex);
/**
* Fork and wait
* Compress/decompress worker thread body
*/
void fork_worker(uint32_t tid);
void worker_thread_body(uint32_t tid);
/**
* Calculate the return the "global" percent done as determined by the
......
std::atomic<uint64_t> m_total_uncompressed_size;// total uncompressed size of all images
std::atomic<uint64_t> m_total_compressed_size; // total compressed size of all images
std::vector<bfs::path> m_file_list; // input file list (either images or compressed images)
// input file list (either images or compressed images)
ProtectedAndSynchronizedQueue<WorkUnit*> m_file_list;
std::thread* p_finder_thread; // file finder thread
std::vector<std::thread*> m_worker_threads; // worker threads
std::deque<WorkUnit*> m_work_unit_queue;
std::mutex m_work_unit_mtx;
std::condition_variable m_work_unit_cv;
bool m_filter_empty; // is the filter regex empty
boost::regex& m_filter_regex; // filter regular expression
};
software/photo_compress_archiver/ProtectedAndSynchronizedQueue.h
#include <condition_variable>
#include <iostream>
template<class T>
class ProtectedAndSynchronizedQueue
{
......
virtual ~ProtectedAndSynchronizedQueue() {};
/*
* add a new element to the back of the queue
*/
// add a new element to the back of the queue
void push_back(T newElement)
{
{
......
m_CV.notify_one();
}
/*
* retrieve the first element of the queue and remove it
* from the queue
*/
// retrieve the first element of the queue and remove it
// from the queue
T pop_front()
{
// Wait on queue blocking
......
return elementToReturn;
}
/*
* retrieve the first element of the queue and remove it
* from the queue
*/
// retrieve the first element of the queue and remove it
// from the queue
T pop_front(bool waitOnEmptyQueue, bool& elementReturned)
{
// Wait on queue blocking
......
else
{
elementReturned = false;
elementToReturn = m_queue.front();
}
muxLock.unlock();
return elementToReturn;
}
/*
* terminate any waiting pop_front() requests
*/
// terminate any waiting pop_front() requests
void terminate()
{
m_terminateFlag = true;
......
return m_queue.empty();
}
uint64_t size()
uint32_t size()
{
std::lock_guard<std::mutex> lock(m_mutex);
return m_queue.size();
......
};
#endif
software/photo_compress_archiver/main.cc
void header()
{
std::cout << BOLD << "Photo Compress Archiver -- " << version << "\n" << ENDC;
std::cout << " Copyright 2017 - David Sorber\n";
std::cout << " Copyright 2018 - David Sorber\n";
std::cout << " >>> Utilizing packJPG by Matthias Stirner\n" << std::endl;
}
software/photo_compress_archiver/packJPG/CMakeLists.txt
project("packJPG")
cmake_minimum_required(VERSION 2.8)
include_directories(.)
set(packjpg_sources
../aricoder.cpp
../bitops.cpp
)
add_definitions("-std=c++14 -O3 -Wall -pedantic -DUNIX")
add_definitions("-funroll-loops -ffast-math -fomit-frame-pointer")
add_definitions("-march=native")
#~ -fsched-spec-load
add_executable(packjpg
${packjpg_sources}
../packjpg.cpp)
install(TARGETS packjpg DESTINATION .)
software/photo_compress_archiver/packJPG/README
===============================================================================
DBS - 20170525
packJPG source is current as of today.
https://github.com/packjpg/packJPG
===============================================================================
packJPG v2.5k (01/22/2016)
~~~~~~~~~~~~~~~~~~~~~~~~~~
packJPG is a compression program specially designed for further
compression of JPEG images without causing any further loss. Typically
it reduces the file size of a JPEG file by 20%.
LGPL v3 license and special permissions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
All programs in this package are free software; you can redistribute
them and/or modify them under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either version 3
of the License, or (at your option) any later version.
The package is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
General Public License for more details at
http://www.gnu.org/copyleft/lgpl.html.
If the LGPL v3 license is not compatible with your software project you
might contact us and ask for a special permission to use the packJPG
library under different conditions. In any case, usage of the packJPG
algorithm under the LGPL v3 or above is highly advised and special
permissions will only be given where necessary on a case by case basis.
This offer is aimed mainly at closed source freeware developers seeking
to add PJG support to their software projects.
Copyright 2006...2014 by HTW Aalen University and Matthias Stirner.
Usage of packJPG
~~~~~~~~~~~~~~~~
JPEG files are compressed and PJG files are decompressed using this
command:
"packJPG [file(s)]"
packJPG recognizes file types on its own and decides whether to compress
(JPG) or decompress (PJG). For unrecognized file types no action is
taken. Files are recognized by content, not by extension.
packJPG supports wildcards like "*.*" and drag and drop of multiple
files. Filenames for output files are created automatically. In default
mode, files are never overwritten. If a filename is already in use,
packJPG creates a new filename by adding underscores.
If "-" is used as a filename input from stdin is assumed and output is
written to stdout. This can be useful for example if jpegtran is to be
used as a preprocessor.
Usage examples:
"packJPG *.pjg"
"packJPG lena.jpg"
"packJPG kodim??.jpg"
"packJPG - < sail.pjg > sail.jpg"
Command line switches
~~~~~~~~~~~~~~~~~~~~~
-ver verify files after processing
-v? level of verbosity; 0,1 or 2 is allowed (default 0)
-np no pause after processing files
-o overwrite existing files
-p proceed on warnings
-d discard meta-info
By default, compression is cancelled on warnings. If warnings are
skipped by using "-p", most files with warnings can also be compressed,
but JPEG files reconstructed from PJG files might not be bitwise
identical with the original JPEG files. There won't be any loss to
image data or quality however.
Unnecessary meta information can be discarded using "-d". This reduces
compressed files' sizes. Be warned though, reconstructed files won't be
bitwise identical with the original files and meta information will be
lost forever. As with "-p" there won't be any loss to image data or
quality.
There is no known case in which a file compressed by packJPG (without
the "-p" option, see above) couldn't be reconstructed to exactly the
state it was before. If you want an additional layer of safety you can
also use the verify option "-ver". In this mode, files are compressed,
then decompressed and the decompressed file compared to the original
file. If this test doesn't pass there will be an error message and the
compressed file won't be written to the drive.
Please note that the "-ver" option should never be used in conjunction
with the "-d" and/or "-p" options. As stated above, the "-p" and "-d"
options will most likely lead to reconstructed JPG files not being
bitwise identical to the original JPG files. In turn, the verification
process may fail on various files although nothing actually went wrong.
Usage examples:
"packJPG -v1 -o baboon.pjg"
"packJPG -ver lena.jpg"
"packJPG -d tiffany.jpg"
"packJPG -p *.jpg"
Known Limitations
~~~~~~~~~~~~~~~~~
packJPG is a compression program specially for JPEG files, so it doesn't
compress other file types.
packJPG has low error tolerance. JPEG files might not work with packJPG
even if they work perfectly with other image processing software. The
command line switch "-p" can be used to increase error tolerance and
compatibility.
If you try to drag and drop to many files at once, there might be a
windowed error message about missing privileges. In that case you can
try it again with less files or consider using the command prompt.
packJPG has been tested to work perfectly with thousands of files from
the command line. This issue also happens with drag and drop in other
applications, so it might not be a limitation of packJPG but a
limitation of Windows.
Compressed PJG files are not compatible between different packJPG
versions. You will get an error message if you try to decompress PJG
files with a different version than the one used for compression. You
may download older versions of packJPG from:
http://www.elektronik.htw-aalen.de/packJPG/binaries/old/
Open source release / developer info
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The packJPG source codes is found inside the "source" subdirectory.
Additional documents aimed to developers, containing detailed
instructions on compiling the source code and using special
functionality, are included in the "packJPG" subdirectory.
History
~~~~~~~
v1.9a (04/20/2007) (non public)
- first released version
- only for testing purposes
v2.0 (05/28/2007) (public)
- first public version of packJPG
- minor improvements to overall compression
- minor bugfixes
v2.2 (08/05/2007) (public)
- around 40% faster compression & decompression
- major improvements to overall compression (around 2% on average)
- reading from stdin, writing to stdout
- smaller executable
- minor bugfixes
- various minor improvements
v2.3 (09/18/2007) (public)
- compatibility with JPEG progressive mode
- compatibility with JPEG extended sequential mode
- compatibility with the CMYK color space
- compatibility with older CPUs
- around 15% faster compression & decompression
- new switch: [-d] (discard meta-info)
- various bugfixes
v2.3a (11/21/2007) (public)
- crash issue with certain images fixed
- compatibility with packJPG v2.3 maintained
v2.3b (12/20/2007) (public)
- some minor errors in the packJPG library fixed
- compatibility with packJPG v2.3 maintained
v2.4 (03/24/2010) (public)
- major improvements (1%...2%) to overall compression
- around 10% faster compression & decompression
- major improvements to JPG compatibility
- size of executable reduced to ~33%
- new switch: [-ver] (verify file after processing)
- new switch: [-np] (no pause after processing)
- new progress bar output mode
- arithmetic coding routines rewritten from scratch
- various smaller improvements to numerous to list here
- new SFX (self extracting) archive format
v2.5 (11/11/2011) (public)
- improvements (~0.5%) to overall compression
- several minor bugfixes
- major code cleanup
- removed packJPX from the package
- added packARC to the package
- packJPG is now open source!
v2.5a (11/21/11) (public)
- source code compatibility improvements (Gerhard Seelmann)
- avoid some compiler warnings (Gerhard Seelmann)
- source code clean up (Gerhard Seelmann)
v2.5b (01/27/12) (public)
- further removal of redundant code
- some fixes for the packJPG static library
- compiler fix for Mac OS (thanks to Sergio Lopez)
- improved compression ratio calculation
- eliminated the need for temp files
v2.5c (04/13/12) (public)
- various source code optimizations
v2.5d (07/03/12) (public)
- fixed a rare bug with progressive JPEG
v2.5e (07/03/12) (public)
- some minor source code optimizations
- changed packJPG licensing to LGPL
- moved packARC to a separate package
v2.5f (02/24/13) (public)
- fixed a minor bug in the JPG parser (thanks to Stephan Busch)
v2.5g (09/14/13) (public)
- fixed a rare crash bug with manipulated JPEG files
v2.5h (12/07/13) (public)
- added a warning for inefficient huffman coding (thanks to Moinak Ghosh)
v2.5i (12/26/13) (public)
- fixed possible crash with malformed JPEG (thanks to Moinak Ghosh)
v2.5j (01/15/14) (public)
- various source code optimizations (using cppcheck)
v2.5k (01/22/16) (public)
- Updated contact info
- fixed a minor bug
Acknowledgements
~~~~~~~~~~~~~~~~
packJPG is the result of countless hours of research and development. It
is part of my final year project for Hochschule Aalen.
Prof. Dr. Gerhard Seelmann from Hochschule Aalen supported my
development of packJPG with his extensive knowledge in the field of data
compression. Without his advice, packJPG would not be possible.
The official developer blog for packJPG is hosted by encode.ru.
packJPG logo and icon are designed by Michael Kaufmann.
Contact
~~~~~~~
The official developer blog for packJPG:
http://packjpg.encode.ru/
For questions and bug reports:
packjpg (at) matthiasstirner.com
____________________________________
packJPG by Matthias Stirner, 01/2016
software/photo_compress_archiver/packJPG/aricoder.cpp
#include "aricoder.h"
#include "bitops.h"
#include <algorithm>
#include <functional>
#include <stdlib.h>
/* -----------------------------------------------
constructor for aricoder class
----------------------------------------------- */
aricoder::aricoder( iostream* stream, StreamMode iomode ) : sptr(stream), mode(iomode)
{
if ( mode == StreamMode::kRead) { // mode is reading / decoding
// code buffer has to be filled before starting decoding
for (uint32_t i = 0; i < CODER_USE_BITS; i++ )
ccode = ( ccode << 1 ) | read_bit();
} // mode is writing / encoding otherwise
}
/* -----------------------------------------------
destructor for aricoder class
----------------------------------------------- */
aricoder::~aricoder()
{
if ( mode == StreamMode::kWrite) { // mode is writing / encoding
// due to clow < CODER_LIMIT050, and chigh >= CODER_LIMIT050
// there are only two possible cases
if ( clow < CODER_LIMIT025 ) { // case a.)
write_bit<0>();
// write remaining bits
write_bit<1>();
writeNrbitsAsOne();
}
else { // case b.), clow >= CODER_LIMIT025
write_bit<1>();
} // done, zeroes are auto-read by the decoder
// pad code with zeroes
while (cbit > 0) {
write_bit<0>();
}
}
}
/* -----------------------------------------------
arithmetic encoder function
----------------------------------------------- */
void aricoder::encode( symbol* s )
{
// Make local copies of clow_ and chigh_ for cache performance:
uint32_t clow_local = clow;
uint32_t chigh_local = chigh;
// update steps, low count, high count
cstep = (chigh_local - clow_local + 1) / s->scale;
chigh_local = clow_local + (cstep * s->high_count) - 1;
clow_local = clow_local + (cstep * s->low_count);
// e3 scaling is performed for speed and to avoid underflows
// if both, low and high are either in the lower half or in the higher half
// one bit can be safely shifted out
while ( clow_local >= CODER_LIMIT050 || chigh_local < CODER_LIMIT050 ) {
if (chigh_local < CODER_LIMIT050 ) { // this means both, high and low are below, and 0 can be safely shifted out
// write 0 bit
write_bit<0>();
// shift out remaing e3 bits
writeNrbitsAsOne();
}
else { // if the first wasn't the case, it's clow >= CODER_LIMIT050
// write 1 bit
write_bit<1>();
clow_local &= CODER_LIMIT050 - 1;
chigh_local &= CODER_LIMIT050 - 1;
// shift out remaing e3 bits
writeNrbitsAsZero();
}
clow_local <<= 1;
chigh_local <<= 1;
chigh_local++;
}
// e3 scaling, to make sure that theres enough space between low and high
while ( (clow_local >= CODER_LIMIT025 ) && (chigh_local < CODER_LIMIT075 ) ) {
nrbits++;
clow_local &= CODER_LIMIT025 - 1;
chigh_local ^= CODER_LIMIT025 + CODER_LIMIT050;
// clow -= CODER_LIMIT025;
// chigh -= CODER_LIMIT025;
clow_local <<= 1;
chigh_local <<= 1;
chigh_local++;
}
clow = clow_local;
chigh = chigh_local;
}
void aricoder::writeNrbitsAsZero() {
if (nrbits + cbit >= 8) {
int remainingBits = 8 - cbit;
nrbits -= remainingBits;
bbyte <<= remainingBits;
sptr->write_byte(bbyte);
cbit = 0;
}
constexpr uint8_t zero = 0;
while (nrbits >= 8) {
sptr->write_byte(zero);
nrbits -= 8;
}
/*
No need to check if cbits is 8, since nrbits is strictly less than 8
and cbit is initially 0 here:
*/
bbyte <<= nrbits;
cbit += nrbits;
nrbits = 0;
}
void aricoder::writeNrbitsAsOne() {
if (nrbits + cbit >= 8) {
int remainingBits = 8 - cbit;
nrbits -= remainingBits;
bbyte <<= remainingBits;
bbyte |= std::numeric_limits<uint8_t>::max() >> (8 - remainingBits);
sptr->write_byte(bbyte);
cbit = 0;
}
constexpr uint8_t all_ones = std::numeric_limits<uint8_t>::max();
while (nrbits >= 8) {
sptr->write_byte(all_ones);
nrbits -= 8;
}
/*
No need to check if cbits is 8, since nrbits is strictly less than 8
and cbit is initially 0 here:
*/
bbyte = (bbyte << nrbits) | (std::numeric_limits<uint8_t>::max() >> (8 - nrbits));
cbit += nrbits;
nrbits = 0;
}
/* -----------------------------------------------
arithmetic decoder get count function
----------------------------------------------- */
unsigned int aricoder::decode_count( symbol* s )
{
// update cstep, which is needed to remove the symbol from the stream later
cstep = ( ( chigh - clow ) + 1 ) / s->scale;
// return counts, needed to decode the symbol from the statistical model
return ( ccode - clow ) / cstep;
}
/* -----------------------------------------------
arithmetic decoder function
----------------------------------------------- */
void aricoder::decode( symbol* s )
{
// no actual decoding takes place, as this has to happen in the statistical model
// the symbol has to be removed from the stream, though
// alread have steps updated from decoder_count
// update low count and high count
uint32_t ccode_local = ccode;
uint32_t clow_local = clow;
uint32_t chigh_local = clow_local + (cstep * s->high_count) - 1;
clow_local = clow_local + (cstep * s->low_count);
// e3 scaling is performed for speed and to avoid underflows
// if both, low and high are either in the lower half or in the higher half
// one bit can be safely shifted out
while ( (clow_local >= CODER_LIMIT050 ) || (chigh_local < CODER_LIMIT050 ) ) {
if (clow_local >= CODER_LIMIT050 ) {
clow_local &= CODER_LIMIT050 - 1;
chigh_local &= CODER_LIMIT050 - 1;
ccode_local &= CODER_LIMIT050 - 1;
} // if the first wasn't the case, it's chigh < CODER_LIMIT050
clow_local <<= 1;
chigh_local <<= 1;
chigh_local++;
ccode_local <<= 1;
ccode_local |= read_bit();
}
// e3 scaling, to make sure that theres enough space between low and high
while ( (clow_local >= CODER_LIMIT025 ) && (chigh_local < CODER_LIMIT075 ) ) {
clow_local &= CODER_LIMIT025 - 1;
chigh_local ^= CODER_LIMIT025 + CODER_LIMIT050;
// clow -= CODER_LIMIT025;
// chigh -= CODER_LIMIT025;
ccode_local -= CODER_LIMIT025;
clow_local <<= 1;
chigh_local <<= 1;
chigh_local++;
ccode_local <<= 1;
ccode_local |= read_bit();
}
chigh = chigh_local;
clow = clow_local;
ccode = ccode_local;
}
/* -----------------------------------------------
bit reader function
----------------------------------------------- */
unsigned char aricoder::read_bit()
{
// read in new byte if needed
if ( cbit == 0 ) {
if ( !sptr->read_byte(&bbyte)) // read next byte if available
bbyte = 0; // if no more data is left in the stream
cbit = 8;
}
// decrement current bit position
cbit--;
// return bit at cbit position
return BITN( bbyte, cbit );
}
/* -----------------------------------------------
universal statistical model for arithmetic coding
boundaries of this model:
max_s (maximum symbol) -> 1 <= max_s <= 1024 (???)
max_c (maximum context) -> 1 <= max_c <= 1024 (???)
max_o (maximum order) -> -1 <= max_o <= 4
c_lim (maximum count) -> 2 <= c_lim <= 4096 (???)
WARNING: this can be memory intensive, so don't overdo it
max_s == 256; max_c == 256; max_o == 4 would be way too much
----------------------------------------------- */
model_s::model_s( int max_s, int max_c, int max_o, int c_lim ) :
// Copy settings into the model:
max_symbol(max_s),
max_context(max_c),
max_order(max_o + 1),
max_count(c_lim),
current_order(max_o + 1),
sb0_count(max_s),
totals(max_s + 2),
scoreboard(new bool[max_s]),
contexts(max_o + 3)
{
std::fill(scoreboard, scoreboard + max_symbol, false);
// set up null table
table_s* null_table = new table_s;
null_table->counts = std::vector<uint16_t>(max_symbol, uint16_t(1)); // Set all probabilities to 1.
// set up internal counts
null_table->max_count = 1;
null_table->max_symbol = max_symbol;
// set up start table
table_s* start_table = new table_s;
start_table->links = std::vector<table_s*>(max_context);
// integrate tables into contexts
contexts[ 0 ] = null_table;
contexts[ 1 ] = start_table;
// build initial 'normal' tables
for (int i = 2; i <= max_order; i++ ) {
// set up current order table
contexts[i] = new table_s;
// build forward links
if ( i < max_order ) {
contexts[i]->links = std::vector<table_s*>(max_context);
}
contexts[ i - 1 ]->links[ 0 ] = contexts[ i ];
}
}
/* -----------------------------------------------
model class destructor - recursive cleanup of memory is done here
----------------------------------------------- */
model_s::~model_s()
{
// clean up each 'normal' table
delete contexts[1];
// clean up null table
delete contexts[0];
// free everything else
delete[] scoreboard;
}
/* -----------------------------------------------
Updates statistics for a specific symbol / resets to highest order.
Use -1 if you just want to reset without updating statistics.
----------------------------------------------- */
void model_s::update_model( int symbol )
{
// only contexts, that were actually used to encode
// the symbol get its count updated
if ( symbol >= 0 ) {
for (int local_order = ( current_order < 1 ) ? 1 : current_order;
local_order <= max_order; local_order++ ) {
table_s* context = contexts[ local_order ];
auto& count = context->counts[symbol];
// update count for specific symbol & scale
count++;
// store side information for totalize_table
context->max_count = std::max(count, context->max_count);
context->max_symbol = std::max(uint16_t(symbol + 1), context->max_symbol);
// if count for that symbol have gone above the maximum count
// the table has to be resized (scale factor 2)
if (count == max_count) {
context->rescale_table();
}
}
}
// reset scoreboard and current order
current_order = max_order;
std::fill(scoreboard, scoreboard + max_symbol, false);
sb0_count = max_symbol;
}
/* -----------------------------------------------
shift in one context (max no of contexts is max_c)
----------------------------------------------- */
void model_s::shift_context( int c )
{
// shifting is not possible if max_order is below 1
// or context index is negative
if ( ( max_order < 2 ) || ( c < 0 ) ) return;
// shift each orders' context
for (int i = max_order; i > 1; i-- ) {
// this is the new current order context
table_s* context = contexts[ i - 1 ]->links[ c ];
// check if context exists, build if needed
if ( context == nullptr ) {
// reserve memory for next table_s
context = new table_s;
// finished here if this is a max order context
if ( i < max_order ) {
// build links to higher order tables otherwise
context->links.resize(max_context);
}
// put context to its right place
contexts[ i - 1 ]->links[ c ] = context;
}
// switch context
contexts[ i ] = context;
}
}
/* -----------------------------------------------
Flushes the entire model by calling rescale_table on all contexts.
----------------------------------------------- */
void model_s::flush_model()
{
contexts[1]->recursive_flush();
}
/* -----------------------------------------------
Excludes every symbol above c.
----------------------------------------------- */
void model_s::exclude_symbols(int c)
{
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff