commit e28ae23afded81208b1cfd1ade980661067091ea Author: dsorber Date: Sat Jul 6 17:07:29 2013 -0400 I finally got around to playing with pHash today. I also found py-phash which are the Python binds for pHash. Here are a couple of scripts I created to build an image index and then search it to find duplicates. This is good starting point for now, I'll play with these more as I have time. diff --git a/similar images/build_index.py b/similar images/build_index.py new file mode 100755 index 0000000..9c6cb0b --- /dev/null +++ b/similar images/build_index.py @@ -0,0 +1,62 @@ +#!/opt/local/bin/python2.7 +import bz2 +import cPickle +import imghdr +import os +import sys + +import pHash + +from image import Image + +# Build index +# TODO: write more schtuff here + +def main(): + + image_list = [] + + # Make sure at least one command line argument was provided + if len(sys.argv) < 2: + print 'Usage: build_index.py ' + return -1; + + # Print out the path we're walking + path = sys.argv[1] + print 'Building image index from path: {:s}\n'.format(path) + + # Walk the root path and count all images + img_counter = 0 + for root, dirs, files in os.walk(path): + for file in files: + img_path = os.path.join(root, file) + # Check the make sure the file is a JPEG + if not imghdr.what(img_path) == 'jpeg': + continue + img_counter += 1 + print 'Done counting, {:d} images total.'.format(img_counter) + + # Walk the root path + ctr = 0 + for root, dirs, files in os.walk(path): + for file in files: + img_path = os.path.join(root, file) + # Check the make sure the file is a JPEG + if not imghdr.what(img_path) == 'jpeg': + continue + ctr += 1 + percent_done = int((float(ctr) / img_counter) * 100) + print '{:3d}% -- {:s}'.format(percent_done, img_path) + img_obj = Image(img_path) + # print '{:s} -> {:016X}'.format(img_path, img_obj.hash) + image_list.append(img_obj) + + # Write index as a bzip2'd file to save space + index_file = bz2.BZ2File('images_index.bz2', 'w') + cPickle.dump(image_list, index_file) + index_file.close() + + print '\nIndex file images_index.bz2 created. Done.' + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/similar images/find_duplicates_simple.py b/similar images/find_duplicates_simple.py new file mode 100755 index 0000000..c29e68f --- /dev/null +++ b/similar images/find_duplicates_simple.py @@ -0,0 +1,50 @@ +#!/opt/local/bin/python2.7 +from collections import defaultdict +import imghdr +import os +import sys + +import pHash + +# Find Duplicates Simple +# Attempt to find images by looking for exact hash collisions. +# This method is not very sophisticated and does not work all +# that well. + +def main(): + + img_map = defaultdict(list) + + # Make sure at least one command line argument was provided + if len(sys.argv) < 2: + print 'Usage: find_duplicate_simple ' + return -1; + + # Print out the path we're walking + path = sys.argv[1] + print 'Step 1: Walk path: {:s}\n'.format(path) + + # Walk the root path + for root, dirs, files in os.walk(path): + for file in files: + img_path = os.path.join(root, file) + # Check the make sure the file is an actual image + if not imghdr.what(img_path): + continue + img_hash = pHash.imagehash(img_path) + print '{:s} -> {:016X}'.format(img_path, img_hash) + img_map[img_hash].append(img_path) + + + print '\n\nStep 2: find duplicates\n' + + # Now look through the image hash dictionary for matches + for img_hash, files in img_map.items(): + if len(files) > 2: + print '{:016X}'.format(img_hash) + for file_path in files: + print '\t{:s}'.format(file_path) + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/similar images/find_similar.py b/similar images/find_similar.py new file mode 100755 index 0000000..15ec184 --- /dev/null +++ b/similar images/find_similar.py @@ -0,0 +1,98 @@ +#!/opt/local/bin/python2.7 +import bz2 +from collections import defaultdict +import cPickle +import imghdr +import os +import sys + +import pHash + +from image import Image + +HAMMING_THRESHOLD = 10 +CORRELATION_THRESHOLD = 70.0 + +# Find duplicates by looking over index +# write some description + +def find_similar(total_done, index, image): + index_length = len(index) + image_digest = image.make_digest() + + # Search the index + ctr = 0 + for index_image in index: + ctr += 1 + percent_done = int((float(ctr) / index_length) * 100) + sys.stdout.write('\r\tTotal: {:3d}% --- This image: {:3d}%'.format(total_done, percent_done)) + sys.stdout.flush() + # Original might be in index, skip it + if image.hash == index_image.hash and image.path == index_image.path: + continue + + # Hamming distance hash compare + hamming_distance = pHash.hamming_distance(image.hash, index_image.hash) + + # Cross correlation compare + correlation = pHash.crosscorr(image_digest, index_image.make_digest())[1] + + # Determine if comparison criteria are within the thresholds + if hamming_distance <= HAMMING_THRESHOLD or \ + correlation >= CORRELATION_THRESHOLD: + sys.stdout.write('\r\t--- SIMILAR IMAGE FOUND --- \n') + print '\tHamming distance: {:d}'.format(hamming_distance) + print '\tCross correlation: {:f}'.format(correlation) + print '\tImage path: {:s}\n'.format(index_image.path) + print + + +def main(): + + # Make sure at least one command line argument was provided + if len(sys.argv) < 3: + print 'Usage: find_similar.py ' + return -1; + + # Print out the path we're walking + root_path = sys.argv[1] + index_path = sys.argv[2] + + # Load image index + index_file = bz2.BZ2File(index_path, 'r') + image_index = cPickle.load(index_file) + index_file.close() + + print 'Looking for similar images from from path: {:s}'.format(root_path) + + # Walk the root path and count all images + # (is there a better way to do this?) + image_counter = 0 + for root, dirs, files in os.walk(root_path): + for file in files: + image_path = os.path.join(root, file) + # Check the make sure the file is a JPEG + if not imghdr.what(image_path) == 'jpeg': + continue + image_counter += 1 + print 'Done counting, {:d} images total.'.format(image_counter) + + # Walk the root path + ctr = 0 + for root, dirs, files in os.walk(root_path): + for file in files: + image_path = os.path.join(root, file) + # Check the make sure the file is a JPEG + if not imghdr.what(image_path) == 'jpeg': + continue + ctr += 1 + percent_done = int((float(ctr) / image_counter) * 100) + print '\n\nLooking for similar -- {:s}'.format(image_path) + image_obj = Image(image_path) + # Find similar images by searching the index + find_similar(percent_done, image_index, image_obj) + + print '\nDone.' + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/similar images/image.py b/similar images/image.py new file mode 100644 index 0000000..692a898 --- /dev/null +++ b/similar images/image.py @@ -0,0 +1,29 @@ +import copy + +import pHash + +class Image(object): + + def __init__(self, path, hash=None, coefficients=None): + self.path = path + + if not hash: + self.hash = pHash.imagehash(self.path) + else: + self.hash = hash + + if not coefficients: + self.coefficients = copy.copy(pHash.image_digest(self.path, 1.0, 1.0, 180).coeffs) + else: + self.coefficients = coefficients + + def make_digest(self): + digest = pHash.Digest() + digest.coeffs = copy.copy(self.coefficients) + digest.size = len(self.coefficients) + return digest + + def __repr__(self): + print 'Image: {:s}'.format(self.path) + print '\tHash: {:016X}'.format(self.hash) + print '\tDigest coefficients: {:s}'.format(str(self.coefficients))