Project

General

Profile

« Previous | Next » 

Revision 3dfe7131

Added by David Sorber over 9 years ago

Adding simple program that computes the cosine similarity of two ClusterNode structures.

View differences:

software/clustering_proto/compute_cosine_sim.cc
#include <chrono>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <limits>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include "cluster.h"
// g++ -std=c++11 -O3 -funroll-loops -ffast-math -fomit-frame-pointer -o compute_cosine_sim compute_cosine_sim.cc
int main(int argc, char** argv)
{
auto start = std::chrono::high_resolution_clock::now();
if (argc < 4)
{
std::cerr << "ERROR: not enough arguments!" << std::endl;
//~ usage(argv);
return -1;
}
// Parse command line arguments
std::string base_path(argv[1]);
uint32_t file1_idx = std::strtoul(argv[2], nullptr, 0);
uint32_t file2_idx = std::strtoul(argv[3], nullptr, 0);
// Build file path for file1
std::ostringstream fname;
fname << base_path << "/node_" << std::setw(10) << std::setfill('0')
<< file1_idx << ".bin";
std::string file1_path(fname.str());
// Build file path for file2
fname.str("");
fname << base_path << "/node_" << std::setw(10) << std::setfill('0')
<< file2_idx << ".bin";
std::string file2_path(fname.str());
// Open and map file1
int file1_fd = ::open(file1_path.c_str(), O_RDWR);
void* file1_buf = ::mmap(nullptr, sizeof(ClusterNode),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, file1_fd, 0);
if (file1_buf == MAP_FAILED)
{
std::cerr << "mmap failed" << std::endl;
return -1;
}
ClusterNode* file1_overlay = reinterpret_cast<ClusterNode*>(file1_buf);
// Open and map file2
int file2_fd = ::open(file2_path.c_str(), O_RDWR);
void* file2_buf = ::mmap(nullptr, sizeof(ClusterNode),
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, file2_fd, 0);
if (file2_buf == MAP_FAILED)
{
std::cerr << "mmap failed" << std::endl;
return -1;
}
ClusterNode* file2_overlay = reinterpret_cast<ClusterNode*>(file2_buf);
// Let's do some maths!!!
//=========================================================================
auto math_start = std::chrono::high_resolution_clock::now();
double dot_product = 0.0;
double vec1_sq_product = 0.0;
double vec2_sq_product = 0.0;
for (uint32_t idx = 0; idx < VECTOR_LEN; ++idx)
{
dot_product += file1_overlay->tf_vector[idx] * file2_overlay->tf_vector[idx];
vec1_sq_product += file1_overlay->tf_vector[idx] * file1_overlay->tf_vector[idx];
vec2_sq_product += file2_overlay->tf_vector[idx] * file2_overlay->tf_vector[idx];
}
double cosine_sim = dot_product / (std::sqrt(vec1_sq_product) * std::sqrt(vec2_sq_product));
auto math_end = std::chrono::high_resolution_clock::now();
auto math_diff = math_end - math_start;
std::cout << "Calculation Duration: "
<< std::chrono::duration <double, std::micro>(math_diff).count()
<< " us" << std::endl;
//=========================================================================
// Output cosine similarity
std::cout << "Cosine similarity: "
<< std::setprecision(std::numeric_limits<double>::digits10)
<< cosine_sim << std::endl;
#if 0
// Print out the frequencies from the file for debugging purposes
for (auto freq : file2_overlay->tf_vector)
{
std::cout << "Freq: "
<< std::setprecision(std::numeric_limits<double>::digits10)
<< freq << std::endl;
}
#endif
// Unmap and close file 1 and file 2
::munmap(file1_buf, sizeof(ClusterNode));
::munmap(file2_buf, sizeof(ClusterNode));
::close(file1_fd);
::close(file2_fd);
auto end = std::chrono::high_resolution_clock::now();
auto diff = end - start;
std::cout << "\nTotal Duration: "
<< std::chrono::duration <double, std::micro>(diff).count()
<< " us" << std::endl;
return 0;
}
software/clustering_proto/make_random_nodes.cc
#include <vector>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/stat.h>
#include <unistd.h>
#include "cluster.h"
......
for (uint32_t node_id = start_id; node_id < (start_id + num_nodes); ++node_id)
{
// Create a new clutser node
ClusterNode new_node;
ClusterNode new_node{0};
new_node.id = node_id;
// Prime out random number generation
......
}
// Create the TF vector be assigning frequencies
std::vector<double> tf_freqs(VECTOR_LEN, 0.0);
double total = 0.0;
for (auto idx : indicies)
{
tf_freqs[idx] = -std::log(tf_dist(generator));
total += tf_freqs[idx];
new_node.tf_vector[idx] = -std::log(tf_dist(generator));
total += new_node.tf_vector[idx];
}
// Now normalize the frequencies
double scaled_total = 0.0;
for (auto idx : indicies)
{
tf_freqs[idx] /= total;
scaled_total += tf_freqs[idx];
new_node.tf_vector[idx] /= total;
scaled_total += new_node.tf_vector[idx];
}
#if 0
// Print out the frequencies
for (uint32_t idx = 0; idx < VECTOR_LEN; ++idx)
{
{
std::cout << std::setw(3) << idx << " -- "
<< std::setprecision(std::numeric_limits<double>::digits10)
<< tf_freqs[idx] << std::endl;
<< new_node.tf_vector[idx] << std::endl;
}
#endif
......
// Build filename
std::ostringstream fname;
fname << base_path << "/node_0" << std::setw(10) << std::setfill('0')
fname << base_path << "/node_" << std::setw(10) << std::setfill('0')
<< node_id << ".bin";
// Write out the cluster node file

Also available in: Unified diff