commit b216a5b02ba0a55dbf02196ca86810f9d859255e
Author: David Sorber <dsorber@prometheus.fios-router.home>
Date:   Sun Feb 5 12:42:00 2017 -0500

    Adding my initial PCA work. This project depends on packJPG. I see intermittent issues and I'm not sure if it is caused by my code or by the packJPG code.

diff --git a/software/photo_compress_archiver/Makefile b/software/photo_compress_archiver/Makefile
new file mode 100644
index 0000000..9423c77
--- /dev/null
+++ b/software/photo_compress_archiver/Makefile
@@ -0,0 +1,18 @@
+
+CXX=clang++
+CXXFLAGS=-std=c++11 -g
+OBJECTS = main.o PhotoCompressArchiver.o
+
+LDFLAGS=-L. -l packjpg -L/opt/local/lib -lboost_system-mt -lboost_regex-mt -lboost_filesystem-mt
+
+all: main
+	
+main: $(OBJECTS)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o pca $(OBJECTS) 
+
+%.o : %.cc
+	$(CXX) $(CXXFLAGS) -I/opt/local/include -c $< 
+
+clean:
+	rm -f *.o
+	rm -f pca
diff --git a/software/photo_compress_archiver/PhotoCompressArchiver.cc b/software/photo_compress_archiver/PhotoCompressArchiver.cc
new file mode 100644
index 0000000..4e330c6
--- /dev/null
+++ b/software/photo_compress_archiver/PhotoCompressArchiver.cc
@@ -0,0 +1,232 @@
+#include <chrono>
+#include <iostream>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "PhotoCompressArchiver.hh"
+
+#include "packjpglib.h"
+
+#define OUT(x) {\
+    std::lock_guard<std::mutex> lock(m_output_mutex);\
+    x\
+}
+
+const char *PROG_NAME = "./packJPG";
+const char *PROG_OPT1 = "-np";
+const char *PROG_OPT2 = "-o";
+const char *PROG_OPT3 = "-p";
+
+PhotoCompressArchiver::PhotoCompressArchiver(
+    const char* path, 
+    uint32_t num_cores)
+    : m_path(path),
+      m_num_cores(num_cores),
+      m_num_input_files(0),
+      m_total_uncompressed_size(0),
+      m_total_compressed_size(0),
+      p_finder_thread(nullptr)
+{}
+
+PhotoCompressArchiver::~PhotoCompressArchiver()
+{}
+
+int PhotoCompressArchiver::execute()
+{
+    // Start the file finder thread
+    OUT(std::cout << "Starting file finder..." << std::flush;);
+    p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this, 
+                                    m_path, JPEG_REGEX);
+    OUT(std::cout << "DONE" << std::endl;)
+    p_finder_thread->join();
+    OUT(std::cout << "File finder completed:" << std::endl;)
+    
+    OUT(std::cout << "  Found " << m_num_input_files << " images" << std::endl;)
+    OUT(std::cout << "  Total uncompressed bytes: " << m_total_uncompressed_size 
+                  << "\n" << std::endl;)
+    
+    // Start worker threads
+    OUT(std::cout << "Starting worker threads..." << std::endl;)
+    
+    // TODO: need to handle the cause were number of input files is less than
+    //       the number of threads
+    uint32_t num_threads = m_num_cores;
+    uint32_t sublist_len = m_num_input_files / num_threads;
+ 
+    for (uint32_t idx = 0; idx < num_threads; ++idx)
+    {
+        // Subdivide the input file list (using the vector copy constructor 
+        // and start/end iterators)
+        decltype(m_file_list.begin()) start_iter;
+        decltype(m_file_list.begin()) end_iter;
+        if (idx == (num_threads - 1))
+        {
+            // Make sure the last sublist goes all the way to the end of the 
+            // input file list (if number of images is not an even multiple of
+            // number of threads)
+            start_iter = m_file_list.begin() + (idx * sublist_len);
+            end_iter   = m_file_list.end();
+        }
+        else
+        {
+            start_iter = m_file_list.begin() + (idx * sublist_len);
+            end_iter   = m_file_list.begin() + ((idx + 1) * sublist_len);
+        }
+        
+        auto sublist = new std::vector<bfs::path>(start_iter, end_iter);
+        
+        // Spawn worker thread and add to list for bookkeeping
+        std::thread* worker = new std::thread(&PhotoCompressArchiver::fork_worker, 
+                                              this, idx, sublist);
+        m_worker_threads.push_back(worker);        
+    }
+    OUT(std::cout << "All worker threads started" << std::endl;)
+    
+    // Clean up the finder thread
+    delete p_finder_thread;
+    p_finder_thread = nullptr;
+    
+    // Now join the worker threads
+    for (auto worker : m_worker_threads)
+    {
+        worker->join();
+        delete worker;
+    }
+    
+    OUT(std::cout << "  Total compressed bytes: " << m_total_compressed_size 
+                  << "\n" << std::endl;)
+    
+    
+    OUT(std::cout << "execute terminating" << std::endl;)
+    
+    return 0;
+}
+
+void PhotoCompressArchiver::find_files(
+    const bfs::path& dir_path, 
+    const boost::regex& file_regex)
+{
+    bfs::directory_iterator end_iter;
+    for (bfs::directory_iterator dir_iter(dir_path); dir_iter != end_iter; ++dir_iter)
+    {
+        if (bfs::is_directory(dir_iter->status()))
+        {
+            // Found directory recurse
+            find_files(dir_iter->path(), file_regex);
+        }
+        else if (boost::regex_match(dir_iter->path().filename().string(), file_regex))
+        {
+            // Found file match
+            m_file_list.push_back(dir_iter->path());
+            ++m_num_input_files;
+            
+            // Stat the file to get its size
+            struct stat statbuf;
+            int rc = stat(dir_iter->path().string().c_str(), &statbuf);
+            if (rc)
+            {
+                OUT(std::cerr << "ERROR: unable to stat file " 
+                              << dir_iter->path() << std::endl;)
+            }     
+            m_total_uncompressed_size += statbuf.st_size;
+        }
+    }
+}
+
+void PhotoCompressArchiver::fork_worker(
+    uint32_t tid, 
+    std::vector<bfs::path>* file_sublist)
+{
+    OUT(std::cout << "T[" << tid << "] top of the morning to ya" << std::endl;)
+    
+    // Print out the input files given to this worker
+#if 0
+    for (auto& filepath : *file_sublist)
+    {
+        OUT(std::cout << "T[" << tid << "] file: " << filepath.string() 
+                      << std::endl;)
+    }
+#endif
+        
+    uint32_t argv_size = file_sublist->size() + 5;
+    const char **exec_argv = new const char* [argv_size];
+    
+    // Build the argv array; pass in default program options
+    exec_argv[0] = &PROG_NAME[2]; // discard leading "./"
+    exec_argv[1] = PROG_OPT1;
+    exec_argv[2] = PROG_OPT2;
+    exec_argv[3] = PROG_OPT3;
+    
+    // Add the sublist files to the argv array
+    uint32_t idx = 3;
+    for (auto& image_path : m_file_list)
+    {
+        exec_argv[idx++] = image_path.string().c_str();
+    }
+    exec_argv[argv_size - 1] = nullptr;
+    
+    // Create a pipe to hold stdout from child process
+    int filedes[2];
+    if (pipe(filedes) == -1) 
+    {
+        perror("pipe");
+        exit(1);
+    }
+    
+    // Fork off the childprocess
+    //~ pid_t parent = getpid();
+    pid_t pid = vfork();
+
+    if (pid == -1)
+    {
+        // error, failed to fork()
+        OUT(std::cout << "T[" << tid << "] fork failed!" << std::endl;)
+    } 
+    else if (pid > 0)
+    {
+        // Wait for forked child process to terminate
+        int status;
+        waitpid(pid, &status, 0);
+        OUT(std::cout << "T[" << tid << "] RC: " << status << std::endl;)
+    }
+    else 
+    {
+        // Set child process's stdout to the pipe entry
+        while ((dup2(filedes[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {}
+        close(filedes[1]);
+        close(filedes[0]);
+        
+        // Child after fork
+        execv(PROG_NAME, (char **)exec_argv);
+        _exit(EXIT_FAILURE);   // exec never returns
+    }
+    
+    // Free up the argv array
+    delete[] exec_argv;
+    
+    OUT(std::cout << "T[" << tid << "] compression complete" << std::endl;)
+    
+    // Iterate over the files and find their compressed size
+    std::string new_file;
+    for (auto& filepath : *file_sublist)
+    {
+        // Create new filename that contains the ".jpg" replaced with ".pjg"
+        uint32_t end_pos = filepath.string().find(".");
+        new_file.assign(filepath.string().begin(), 
+                        filepath.string().begin() + end_pos);
+        new_file += ".pjg";
+                      
+        // Stat the file to get its size
+        struct stat statbuf;
+        int rc = stat(new_file.c_str(), &statbuf);
+        if (rc)
+        {
+            OUT(std::cerr << "ERROR: unable to stat file " << new_file << std::endl;)
+        }     
+        m_total_compressed_size += statbuf.st_size;
+    }
+
+    
+    OUT(std::cout << "T[" << tid << "] terminating" << std::endl;)
+}
diff --git a/software/photo_compress_archiver/PhotoCompressArchiver.hh b/software/photo_compress_archiver/PhotoCompressArchiver.hh
new file mode 100644
index 0000000..290ea83
--- /dev/null
+++ b/software/photo_compress_archiver/PhotoCompressArchiver.hh
@@ -0,0 +1,61 @@
+#ifndef PHOTOCOMPRESSARCHIVER_H
+#define PHOTOCOMPRESSARCHIVER_H
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <boost/filesystem.hpp>
+#include <boost/regex.hpp>
+
+// Alias the boost fs namespace to keep the code readable
+namespace bfs = boost::filesystem;
+
+const boost::regex JPEG_REGEX("^.+\\.(jpg)|(jpeg)$", boost::regex::icase);
+
+class PhotoCompressArchiver
+{
+    public:
+    
+        PhotoCompressArchiver(const char* path, uint32_t num_cores);
+        
+        ~PhotoCompressArchiver();
+        
+        /**
+         * Start processing
+         */ 
+        int execute();
+        
+        /**
+         * Recursively find input files based on passed regex
+         */ 
+        void find_files(
+            const bfs::path& dir_path, 
+            const boost::regex& file_regex);
+        
+        /**
+         * Fork and wait 
+         */ 
+        void fork_worker(uint32_t tid, std::vector<bfs::path>* file_sublist);
+    
+    private:
+    
+        const bfs::path m_path;             // top level path
+        uint32_t m_num_cores;               // number of cores on the machine
+        std::mutex m_output_mutex;          // output mutex
+        
+        uint32_t m_num_input_files;         // total number of input files found
+        std::atomic<uint64_t> m_total_uncompressed_size;    // total uncompressed size of all images
+        std::atomic<uint64_t> m_total_compressed_size;      // total compressed size of all images
+        
+        std::vector<bfs::path> m_file_list; // input file list (either images or compressed images)
+        
+        std::thread* p_finder_thread;             // file finder thread
+        std::vector<std::thread*> m_worker_threads; // worker threads
+};
+
+#endif // PHOTOCOMPRESSARCHIVER_H
diff --git a/software/photo_compress_archiver/main.cc b/software/photo_compress_archiver/main.cc
new file mode 100644
index 0000000..dd346f3
--- /dev/null
+++ b/software/photo_compress_archiver/main.cc
@@ -0,0 +1,41 @@
+#include <iostream>
+#include <string>
+
+#include <boost/filesystem.hpp>
+#include <boost/regex.hpp>
+
+#include "packjpglib.h"
+
+#include "PhotoCompressArchiver.hh"
+
+// clang++ -std=c++11 -L. -l packjpg -I/opt/local/include -L/opt/local/lib 
+//  -lboost_system-mt -lboost_regex-mt -lboost_filesystem-mt -o pca main.cc
+int main(int argc, char** argv)
+{
+    std::cout << "Photo Compress Archiver -- v0.1.0" << std::endl;
+    
+    uint32_t num_cores = std::thread::hardware_concurrency();
+    std::cout << "    [" << num_cores << " cores detected]\n" << std::endl;
+    
+    // Make sure the correct number of arguments were specified
+    if (argc < 2)
+    {
+        std::cerr << "ERROR: Not enough arguments" << std::endl;
+        return -1;
+    }
+    
+    // Make sure that path specified actually exists
+    if (! bfs::exists(argv[1]))
+    {
+        std::cerr << "ERROR: the path \"" << argv[1] << "\" does not exist" 
+                  << std::endl;
+        return -2;
+    }
+    
+    auto pca = new PhotoCompressArchiver(argv[1], num_cores);
+    pca->execute();
+    
+    //~ delete pca;
+    
+    return 0;
+}