/software/photo_compress_archiver/PhotoCompressArchiver.cc - Annotate - Main - Redmine

root/software/photo_compress_archiver/PhotoCompressArchiver.cc @ 4f6113f8

b216a5b0	David Sorber	#include <chrono>
328d0fa5	David Sorber	#include <cstdio>
4f6113f8	David Sorber	#include <fstream>
f8e600bc	David Sorber	#include <iomanip>
b216a5b0	David Sorber	#include <iostream>
e0b7f8b9	David Sorber	#include <limits.h>
b216a5b0	David Sorber	#include <sys/stat.h>
		#include <sys/wait.h>
		#include <unistd.h>
4f6113f8	David Sorber	#include <vector>

		#include <packjpg.h>
b216a5b0	David Sorber
		#include "PhotoCompressArchiver.hh"

4f6113f8	David Sorber	#define OUT(msg) { \
		std::lock_guard<std::mutex> lock(m_output_mutex); \
		std::cout << msg << std::endl; \
b216a5b0	David Sorber	}

4f6113f8	David Sorber	#define ERROR(msg) { \
		std::lock_guard<std::mutex> lock(m_output_mutex); \
		std::cerr << BOLD << RED << "ERROR: " << ENDC << msg << std::endl; \
		}
3eeb1485	David Sorber
4f6113f8	David Sorber	#define WRKR_OUT_REG(msg) { \
		std::lock_guard<std::mutex> lock(m_output_mutex); \
		std::cout << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
		}
b216a5b0	David Sorber
4f6113f8	David Sorber	#define WRKR_OUT_ERR(msg) { \
		std::lock_guard<std::mutex> lock(m_output_mutex); \
		std::cerr << "T[" << std::setw(2) << tid << "] " << msg << std::endl; \
		}
e0b7f8b9	David Sorber
b216a5b0	David Sorber	PhotoCompressArchiver::PhotoCompressArchiver(
328d0fa5	David Sorber	const std::string& path,
		uint32_t num_cores,
		bool decompress,
e0b7f8b9	David Sorber	bool keep_orig,
		boost::regex& filter_regex)
b216a5b0	David Sorber	: m_path(path),
		m_num_cores(num_cores),
328d0fa5	David Sorber	m_decompress(decompress),
		m_keep_orig(keep_orig),
f8e600bc	David Sorber	m_files_processed(0),
b216a5b0	David Sorber	m_num_input_files(0),
		m_total_uncompressed_size(0),
		m_total_compressed_size(0),
e0b7f8b9	David Sorber	p_finder_thread(nullptr),
		m_filter_empty(true),
		m_filter_regex(filter_regex)
		{
		// This is a somewhat crude way to determining if the filter regex is empty
		m_filter_empty = std::string("").compare(filter_regex.str()) == 0;
		}
b216a5b0	David Sorber
		PhotoCompressArchiver::~PhotoCompressArchiver()
		{}

		int PhotoCompressArchiver::execute()
		{
4f6113f8	David Sorber	// Grab start time
		auto start = std::chrono::system_clock::now();

b216a5b0	David Sorber	// Start the file finder thread
4f6113f8	David Sorber	OUT("Starting file finder..." << std::flush);
b216a5b0	David Sorber	p_finder_thread = new std::thread(&PhotoCompressArchiver::find_files, this,
4f6113f8	David Sorber	m_path,
		(m_decompress ? PJG_REGEX : JPEG_REGEX));
		OUT("DONE");
3eeb1485	David Sorber
		// Start worker threads
4f6113f8	David Sorber	OUT("Starting worker threads..." << std::flush);
3eeb1485	David Sorber
		uint32_t num_threads = m_num_cores;
		for (uint32_t idx = 0; idx < num_threads; ++idx)
		{
		// Spawn worker thread and add to list for bookkeeping
4f6113f8	David Sorber	std::thread* worker = new std::thread(&PhotoCompressArchiver::worker_thread_body,
3eeb1485	David Sorber	this, idx);
		m_worker_threads.push_back(worker);
		}
4f6113f8	David Sorber	OUT("DONE\n");
3eeb1485	David Sorber
		// Join the file finder thread
b216a5b0	David Sorber	p_finder_thread->join();
4f6113f8	David Sorber	OUT("File finder completed:");
b216a5b0	David Sorber
57e6c22f	David Sorber	// Error out of no input files were found
		if (m_num_input_files == 0)
		{
4f6113f8	David Sorber	ERROR("No matching input files found. Please check the patch and try "
		"again.");
57e6c22f	David Sorber	return -1;
		}

4f6113f8	David Sorber	OUT(" Found " << m_num_input_files << " files");
328d0fa5	David Sorber
		if (! m_decompress)
		{
4f6113f8	David Sorber	OUT(" Total uncompressed bytes: " << m_total_uncompressed_size << "\n");
328d0fa5	David Sorber	}
b216a5b0	David Sorber
4f6113f8	David Sorber	// Add terminator for each worker thread
		for (uint32_t idx = 0; idx < num_threads; ++idx)
3eeb1485	David Sorber	{
4f6113f8	David Sorber	m_file_list.push_back(nullptr);
3eeb1485	David Sorber	}
b216a5b0	David Sorber
		// Clean up the finder thread
		delete p_finder_thread;
		p_finder_thread = nullptr;

		// Now join the worker threads
		for (auto worker : m_worker_threads)
		{
		worker->join();
		delete worker;
		}

4f6113f8	David Sorber	// Grab start time and calculate duration
		auto end = std::chrono::system_clock::now();
		auto diff = end - start;
		double duration = std::chrono::duration<double, std::milli>(diff).count();

cec188e4	David Sorber	// Print out compression information if compressing
328d0fa5	David Sorber	if (! m_decompress)
		{
4f6113f8	David Sorber	OUT("\n Total uncompressed bytes: " << m_total_uncompressed_size);
		OUT(" Total compressed bytes: " << m_total_compressed_size);
f6c21781	David Sorber
328d0fa5	David Sorber	// Calculate and display ratio
		double ratio = (double)m_total_compressed_size / m_total_uncompressed_size;
4f6113f8	David Sorber	OUT(" Compression ratio: " << std::fixed << std::setprecision(4)
		<< (ratio * 100) << "%");

		// Display execution duration
		OUT(" Total time: " << duration << " ms\n");
328d0fa5	David Sorber	}
b216a5b0	David Sorber
		return 0;
		}

		void PhotoCompressArchiver::find_files(
		const bfs::path& dir_path,
		const boost::regex& file_regex)
		{
		bfs::directory_iterator end_iter;
		for (bfs::directory_iterator dir_iter(dir_path); dir_iter != end_iter; ++dir_iter)
		{
		if (bfs::is_directory(dir_iter->status()))
		{
		// Found directory recurse
		find_files(dir_iter->path(), file_regex);
		}
		else if (boost::regex_match(dir_iter->path().filename().string(), file_regex))
		{
		// Found file match
e0b7f8b9	David Sorber
		// If the filter is not empty, then check against it before proceeding
		if (! m_filter_empty)
		{
		if (boost::regex_match(dir_iter->path().filename().string(),
		m_filter_regex))
		{
		// The filter regex matched, skip this file
		continue;
		}
		}

4f6113f8	David Sorber	// Create a WorkUnit and add it to the queue
		WorkUnit* work_unit = new WorkUnit(dir_iter->path());
b216a5b0	David Sorber	++m_num_input_files;
4f6113f8	David Sorber	m_total_uncompressed_size += work_unit->m_file_size;
		m_file_list.push_back(work_unit);
b216a5b0	David Sorber	}
		}
		}

4f6113f8	David Sorber	void PhotoCompressArchiver::worker_thread_body(uint32_t tid)
b216a5b0	David Sorber	{
4f6113f8	David Sorber	WorkUnit* work_unit = nullptr;
		auto file_buffer = new std::vector<unsigned char>(4 * 1024 * 1024);

		// Setup packJPG instance for this worker
		packJPG* instance = new packJPG();

		// (De/)compress worker variables
		unsigned char* out_buffer = nullptr;
		uint32_t out_size = 0;
		char message[MSG_SIZE];
e0b7f8b9	David Sorber
3eeb1485	David Sorber	while (true)
b216a5b0	David Sorber	{
4f6113f8	David Sorber	// Reset worker vars
		out_size = 0;
		std::memset(message, 0, MSG_SIZE);
57e6c22f	David Sorber
4f6113f8	David Sorber	// Blocking wait for next work unit
		work_unit = m_file_list.pop_front();
3eeb1485	David Sorber	if (work_unit == nullptr)
e0b7f8b9	David Sorber	{
3eeb1485	David Sorber	break;
e0b7f8b9	David Sorber	}
57e6c22f	David Sorber
4f6113f8	David Sorber	// Resize file buffer if needed
		if (work_unit->m_file_size > file_buffer->size())
c8352650	David Sorber	{
4f6113f8	David Sorber	file_buffer->resize(work_unit->m_file_size);
c8352650	David Sorber	}

4f6113f8	David Sorber	// Read in the input file
		std::ifstream input_stream(work_unit->m_path.string(),
		std::ios::binary \| std::ios::in);
		if (! input_stream)
b216a5b0	David Sorber	{
4f6113f8	David Sorber	WRKR_OUT_ERR("ERROR: unable to read from \"" << work_unit->m_path
		<< "\"\n")
		// TODO: what now?
f8e600bc	David Sorber	}

4f6113f8	David Sorber	input_stream.read((char*)file_buffer->data(), work_unit->m_file_size);
		input_stream.close();

		// TODO: revise this
		// Determine input, and therefore output, filetype / extension
		const std::string* output_file_extension = nullptr;
		if ((file_buffer->at(0) == 0xFF) && (file_buffer->at(1) == 0xD8))
f8e600bc	David Sorber	{
4f6113f8	David Sorber	output_file_extension = &PJG_EXTENSION;
		}
		else if ((file_buffer->at(0) == packJPG::pjg_magic[0]) &&
		(file_buffer->at(1) == packJPG::pjg_magic[1]))
57e6c22f	David Sorber	{
4f6113f8	David Sorber	output_file_extension = &JPG_EXTENSION;
f8e600bc	David Sorber	}
4f6113f8	David Sorber	else
57e6c22f	David Sorber	{
4f6113f8	David Sorber	WRKR_OUT_ERR("ERROR: Input file does not appear to be valid\n")
		// TODO: what now?
57e6c22f	David Sorber	}

4f6113f8	David Sorber	instance->pjglib_init_streams(file_buffer->data(), 1,
		work_unit->m_file_size, nullptr, 1);
57e6c22f	David Sorber
4f6113f8	David Sorber	// Do the thing!
		bool rc = instance->pjglib_convert_stream2mem(&out_buffer,
		&out_size, message);
		if (!rc)
f8e600bc	David Sorber	{
4f6113f8	David Sorber	WRKR_OUT_ERR("An error occurred during the compression"
		<< "/decompression operation: " << message << "\n")
		// TODO: what now?
f8e600bc	David Sorber	}

4f6113f8	David Sorber	//~ WRKR_OUT_REG("Status message: " << message)
		//~ WRKR_OUT_REG("Output size: " << out_size)

f8e600bc	David Sorber	// Increment counts
4f6113f8	David Sorber	m_total_compressed_size += out_size;
f8e600bc	David Sorber	++m_files_processed;

4f6113f8	David Sorber	// Create output file path be replacing extension of input file
		bfs::path out_path = bfs::path(work_unit->m_path).replace_extension(*output_file_extension);

		// Write output file
		std::ofstream output_stream(out_path.string(),
		std::ios::binary \| std::ios::out);
		if (! output_stream)
328d0fa5	David Sorber	{
4f6113f8	David Sorber	WRKR_OUT_ERR("ERROR: unable to read from \""
		<< work_unit->m_path << "\"")

		// TODO: what now?
328d0fa5	David Sorber	}
4f6113f8	David Sorber	output_stream.write((const char*)out_buffer, out_size);
		output_stream.close();
328d0fa5	David Sorber
4f6113f8	David Sorber	if (m_decompress)
57e6c22f	David Sorber	{
4f6113f8	David Sorber	WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
		<< get_global_percent_done()
		<< "%) Output file: " << out_path)
57e6c22f	David Sorber	}
4f6113f8	David Sorber	else
		{
		double percent = ((double)out_size) / work_unit->m_file_size;
		percent *= 100;
		WRKR_OUT_REG(" -- (" << std::fixed << std::setprecision(2)
		<< get_global_percent_done()
		<< "%) Output file: " << out_path << " -- "
		<< std::fixed << std::setprecision(2)
		<< percent << "%")
		}


3eeb1485	David Sorber	delete work_unit;
328d0fa5	David Sorber	}

4f6113f8	David Sorber	WRKR_OUT_REG(" -- " << PURPLE << BOLD << "[[exiting]]" << ENDC)

		// Clean up
		std::free(out_buffer);
		delete instance;
		delete file_buffer;
f8e600bc	David Sorber	}

		double PhotoCompressArchiver::get_global_percent_done()
		{
		return ((double)m_files_processed / m_num_input_files) * 100;
b216a5b0	David Sorber	}

Project

General

Profile

Main