commit 3dfe71315cb18e6189887c1db93bc94d1857d621 Author: David Sorber Date: Tue Feb 28 16:37:11 2017 -0500 Adding simple program that computes the cosine similarity of two ClusterNode structures. diff --git a/software/clustering_proto/compute_cosine_sim.cc b/software/clustering_proto/compute_cosine_sim.cc new file mode 100644 index 0000000..3d3c728 --- /dev/null +++ b/software/clustering_proto/compute_cosine_sim.cc @@ -0,0 +1,128 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "cluster.h" + + +// g++ -std=c++11 -O3 -funroll-loops -ffast-math -fomit-frame-pointer -o compute_cosine_sim compute_cosine_sim.cc +int main(int argc, char** argv) +{ + auto start = std::chrono::high_resolution_clock::now(); + + if (argc < 4) + { + std::cerr << "ERROR: not enough arguments!" << std::endl; + //~ usage(argv); + return -1; + } + + // Parse command line arguments + std::string base_path(argv[1]); + uint32_t file1_idx = std::strtoul(argv[2], nullptr, 0); + uint32_t file2_idx = std::strtoul(argv[3], nullptr, 0); + + // Build file path for file1 + std::ostringstream fname; + fname << base_path << "/node_" << std::setw(10) << std::setfill('0') + << file1_idx << ".bin"; + + std::string file1_path(fname.str()); + + // Build file path for file2 + fname.str(""); + fname << base_path << "/node_" << std::setw(10) << std::setfill('0') + << file2_idx << ".bin"; + + std::string file2_path(fname.str()); + + // Open and map file1 + int file1_fd = ::open(file1_path.c_str(), O_RDWR); + void* file1_buf = ::mmap(nullptr, sizeof(ClusterNode), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, file1_fd, 0); + if (file1_buf == MAP_FAILED) + { + std::cerr << "mmap failed" << std::endl; + return -1; + } + ClusterNode* file1_overlay = reinterpret_cast(file1_buf); + + // Open and map file2 + int file2_fd = ::open(file2_path.c_str(), O_RDWR); + void* file2_buf = ::mmap(nullptr, sizeof(ClusterNode), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, file2_fd, 0); + if (file2_buf == MAP_FAILED) + { + std::cerr << "mmap failed" << std::endl; + return -1; + } + ClusterNode* file2_overlay = reinterpret_cast(file2_buf); + + + // Let's do some maths!!! + //========================================================================= + auto math_start = std::chrono::high_resolution_clock::now(); + + double dot_product = 0.0; + double vec1_sq_product = 0.0; + double vec2_sq_product = 0.0; + + for (uint32_t idx = 0; idx < VECTOR_LEN; ++idx) + { + dot_product += file1_overlay->tf_vector[idx] * file2_overlay->tf_vector[idx]; + vec1_sq_product += file1_overlay->tf_vector[idx] * file1_overlay->tf_vector[idx]; + vec2_sq_product += file2_overlay->tf_vector[idx] * file2_overlay->tf_vector[idx]; + } + + double cosine_sim = dot_product / (std::sqrt(vec1_sq_product) * std::sqrt(vec2_sq_product)); + + auto math_end = std::chrono::high_resolution_clock::now(); + auto math_diff = math_end - math_start; + std::cout << "Calculation Duration: " + << std::chrono::duration (math_diff).count() + << " us" << std::endl; + //========================================================================= + + // Output cosine similarity + std::cout << "Cosine similarity: " + << std::setprecision(std::numeric_limits::digits10) + << cosine_sim << std::endl; + +#if 0 + // Print out the frequencies from the file for debugging purposes + for (auto freq : file2_overlay->tf_vector) + { + std::cout << "Freq: " + << std::setprecision(std::numeric_limits::digits10) + << freq << std::endl; + } +#endif + + // Unmap and close file 1 and file 2 + ::munmap(file1_buf, sizeof(ClusterNode)); + ::munmap(file2_buf, sizeof(ClusterNode)); + ::close(file1_fd); + ::close(file2_fd); + + auto end = std::chrono::high_resolution_clock::now(); + auto diff = end - start; + std::cout << "\nTotal Duration: " + << std::chrono::duration (diff).count() + << " us" << std::endl; + + + return 0; +} diff --git a/software/clustering_proto/make_random_nodes.cc b/software/clustering_proto/make_random_nodes.cc index 5d0779b..33dcb2a 100644 --- a/software/clustering_proto/make_random_nodes.cc +++ b/software/clustering_proto/make_random_nodes.cc @@ -15,9 +15,9 @@ #include #include -#include #include -#include +#include +#include #include "cluster.h" @@ -65,7 +65,7 @@ void create_nodes_thread( for (uint32_t node_id = start_id; node_id < (start_id + num_nodes); ++node_id) { // Create a new clutser node - ClusterNode new_node; + ClusterNode new_node{0}; new_node.id = node_id; // Prime out random number generation @@ -86,29 +86,28 @@ void create_nodes_thread( } // Create the TF vector be assigning frequencies - std::vector tf_freqs(VECTOR_LEN, 0.0); double total = 0.0; for (auto idx : indicies) { - tf_freqs[idx] = -std::log(tf_dist(generator)); - total += tf_freqs[idx]; + new_node.tf_vector[idx] = -std::log(tf_dist(generator)); + total += new_node.tf_vector[idx]; } // Now normalize the frequencies double scaled_total = 0.0; for (auto idx : indicies) { - tf_freqs[idx] /= total; - scaled_total += tf_freqs[idx]; + new_node.tf_vector[idx] /= total; + scaled_total += new_node.tf_vector[idx]; } #if 0 // Print out the frequencies for (uint32_t idx = 0; idx < VECTOR_LEN; ++idx) - { + { std::cout << std::setw(3) << idx << " -- " << std::setprecision(std::numeric_limits::digits10) - << tf_freqs[idx] << std::endl; + << new_node.tf_vector[idx] << std::endl; } #endif @@ -121,7 +120,7 @@ void create_nodes_thread( // Build filename std::ostringstream fname; - fname << base_path << "/node_0" << std::setw(10) << std::setfill('0') + fname << base_path << "/node_" << std::setw(10) << std::setfill('0') << node_id << ".bin"; // Write out the cluster node file