commit 2423ae90b65199ecc711ab5104858de3bbfeb10f
Author: David Sorber <dsorber@prometheus.fios-router.home>
Date:   Sun Feb 26 20:58:02 2017 -0500

    I finally found the mystery error that I've been trying to figure out for a while now. The issue was the dreaded too many file descriptors open.  This was caused by not closing the pipe() file descriptors from the parent.  I also noticed that handing each instance of packjpg the maximum number of files it can take leads to problems. I guess this is also a cause of "too many file descriptors open", so I have limited the number of files handed to each instance of packjpg to 40. The entire program now runs to completion using my Photos library as a (quite large) test data set!

diff --git a/software/photo_compress_archiver/PhotoCompressArchiver.cc b/software/photo_compress_archiver/PhotoCompressArchiver.cc
index 2298024..a08206b 100644
--- a/software/photo_compress_archiver/PhotoCompressArchiver.cc
+++ b/software/photo_compress_archiver/PhotoCompressArchiver.cc
@@ -235,7 +235,8 @@ void PhotoCompressArchiver::fork_worker(
         
         output_filenames.clear();
         
-        while (sublist_idx < sublist_file_count)
+        uint32_t local_count = 0;
+        while ((sublist_idx < sublist_file_count) && (local_count < 40))
         {
             bfs::path& file_path = file_sublist->at(sublist_idx);
             uint32_t path_len = file_path.string().size();
@@ -264,18 +265,21 @@ void PhotoCompressArchiver::fork_worker(
             
             // Increment our index to the next file
             ++sublist_idx;
+            ++local_count;
         }
         
 #if 0
         // DEBUGGING
         uint32_t idx = 4;
+        OUT(
         for (auto outfile : output_filenames)
         {
-            OUT(std::cout << "T[" << tid << "] output file: " << *outfile 
+            std::cout << "T[" << tid << "] output file: " << *outfile 
                           << "\n     input file:  " 
-                          << static_cast<const char*>(exec_argv[idx++]) << std::endl;)
+                          << static_cast<const char*>(exec_argv[idx++]) << std::endl;
         }
-        continue;
+        )
+        //~ continue;
 #endif 
         
         // Add a nullptr to the end of the argv array as a terminator
@@ -301,9 +305,9 @@ void PhotoCompressArchiver::fork_worker(
         {
             // This is the child process...
             // Set child process's stdout to the pipe entry
-            //~ while ((dup2(filedes[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {}
-            //~ close(filedes[1]);
-            //~ close(filedes[0]);
+            while ((dup2(filedes[1], STDOUT_FILENO) == -1) && (errno == EINTR)) {}
+            close(filedes[0]);
+            close(filedes[1]);
             
             // Child after fork
             int rc = execv(PROG_NAME, (char **)exec_argv.data());
@@ -325,7 +329,7 @@ void PhotoCompressArchiver::fork_worker(
         int stat_rc = 0;
         struct stat statbuf;
         double complete_percent = 0.0;
-        while (output_idx < (output_filenames.size() - 1))
+        while (output_idx < (output_filenames.size()))
         {
             // Poll, waiting for output file X + 1 to exist
             bailout_ctr = 0;
@@ -391,6 +395,8 @@ void PhotoCompressArchiver::fork_worker(
         
         // Wait for forked child process to terminate
         waitpid(pid, &status, 0);
+        close(filedes[0]);
+        close(filedes[1]);
         
         // Now that the forked process has completed, we can handle the last 
         // output file