Revision e28ae23a
Added by dsorber about 13 years ago
| similar images/build_index.py | ||
|---|---|---|
|
#!/opt/local/bin/python2.7
|
||
|
import bz2
|
||
|
import cPickle
|
||
|
import imghdr
|
||
|
import os
|
||
|
import sys
|
||
|
|
||
|
import pHash
|
||
|
|
||
|
from image import Image
|
||
|
|
||
|
# Build index
|
||
|
# TODO: write more schtuff here
|
||
|
|
||
|
def main():
|
||
|
|
||
|
image_list = []
|
||
|
|
||
|
# Make sure at least one command line argument was provided
|
||
|
if len(sys.argv) < 2:
|
||
|
print 'Usage: build_index.py <path>'
|
||
|
return -1;
|
||
|
|
||
|
# Print out the path we're walking
|
||
|
path = sys.argv[1]
|
||
|
print 'Building image index from path: {:s}\n'.format(path)
|
||
|
|
||
|
# Walk the root path and count all images
|
||
|
img_counter = 0
|
||
|
for root, dirs, files in os.walk(path):
|
||
|
for file in files:
|
||
|
img_path = os.path.join(root, file)
|
||
|
# Check the make sure the file is a JPEG
|
||
|
if not imghdr.what(img_path) == 'jpeg':
|
||
|
continue
|
||
|
img_counter += 1
|
||
|
print 'Done counting, {:d} images total.'.format(img_counter)
|
||
|
|
||
|
# Walk the root path
|
||
|
ctr = 0
|
||
|
for root, dirs, files in os.walk(path):
|
||
|
for file in files:
|
||
|
img_path = os.path.join(root, file)
|
||
|
# Check the make sure the file is a JPEG
|
||
|
if not imghdr.what(img_path) == 'jpeg':
|
||
|
continue
|
||
|
ctr += 1
|
||
|
percent_done = int((float(ctr) / img_counter) * 100)
|
||
|
print '{:3d}% -- {:s}'.format(percent_done, img_path)
|
||
|
img_obj = Image(img_path)
|
||
|
# print '{:s} -> {:016X}'.format(img_path, img_obj.hash)
|
||
|
image_list.append(img_obj)
|
||
|
|
||
|
# Write index as a bzip2'd file to save space
|
||
|
index_file = bz2.BZ2File('images_index.bz2', 'w')
|
||
|
cPickle.dump(image_list, index_file)
|
||
|
index_file.close()
|
||
|
|
||
|
print '\nIndex file images_index.bz2 created. Done.'
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
| similar images/find_duplicates_simple.py | ||
|---|---|---|
|
#!/opt/local/bin/python2.7
|
||
|
from collections import defaultdict
|
||
|
import imghdr
|
||
|
import os
|
||
|
import sys
|
||
|
|
||
|
import pHash
|
||
|
|
||
|
# Find Duplicates Simple
|
||
|
# Attempt to find images by looking for exact hash collisions.
|
||
|
# This method is not very sophisticated and does not work all
|
||
|
# that well.
|
||
|
|
||
|
def main():
|
||
|
|
||
|
img_map = defaultdict(list)
|
||
|
|
||
|
# Make sure at least one command line argument was provided
|
||
|
if len(sys.argv) < 2:
|
||
|
print 'Usage: find_duplicate_simple <path>'
|
||
|
return -1;
|
||
|
|
||
|
# Print out the path we're walking
|
||
|
path = sys.argv[1]
|
||
|
print 'Step 1: Walk path: {:s}\n'.format(path)
|
||
|
|
||
|
# Walk the root path
|
||
|
for root, dirs, files in os.walk(path):
|
||
|
for file in files:
|
||
|
img_path = os.path.join(root, file)
|
||
|
# Check the make sure the file is an actual image
|
||
|
if not imghdr.what(img_path):
|
||
|
continue
|
||
|
img_hash = pHash.imagehash(img_path)
|
||
|
print '{:s} -> {:016X}'.format(img_path, img_hash)
|
||
|
img_map[img_hash].append(img_path)
|
||
|
|
||
|
|
||
|
print '\n\nStep 2: find duplicates\n'
|
||
|
|
||
|
# Now look through the image hash dictionary for matches
|
||
|
for img_hash, files in img_map.items():
|
||
|
if len(files) > 2:
|
||
|
print '{:016X}'.format(img_hash)
|
||
|
for file_path in files:
|
||
|
print '\t{:s}'.format(file_path)
|
||
|
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
| similar images/find_similar.py | ||
|---|---|---|
|
#!/opt/local/bin/python2.7
|
||
|
import bz2
|
||
|
from collections import defaultdict
|
||
|
import cPickle
|
||
|
import imghdr
|
||
|
import os
|
||
|
import sys
|
||
|
|
||
|
import pHash
|
||
|
|
||
|
from image import Image
|
||
|
|
||
|
HAMMING_THRESHOLD = 10
|
||
|
CORRELATION_THRESHOLD = 70.0
|
||
|
|
||
|
# Find duplicates by looking over index
|
||
|
# write some description
|
||
|
|
||
|
def find_similar(total_done, index, image):
|
||
|
index_length = len(index)
|
||
|
image_digest = image.make_digest()
|
||
|
|
||
|
# Search the index
|
||
|
ctr = 0
|
||
|
for index_image in index:
|
||
|
ctr += 1
|
||
|
percent_done = int((float(ctr) / index_length) * 100)
|
||
|
sys.stdout.write('\r\tTotal: {:3d}% --- This image: {:3d}%'.format(total_done, percent_done))
|
||
|
sys.stdout.flush()
|
||
|
# Original might be in index, skip it
|
||
|
if image.hash == index_image.hash and image.path == index_image.path:
|
||
|
continue
|
||
|
|
||
|
# Hamming distance hash compare
|
||
|
hamming_distance = pHash.hamming_distance(image.hash, index_image.hash)
|
||
|
|
||
|
# Cross correlation compare
|
||
|
correlation = pHash.crosscorr(image_digest, index_image.make_digest())[1]
|
||
|
|
||
|
# Determine if comparison criteria are within the thresholds
|
||
|
if hamming_distance <= HAMMING_THRESHOLD or \
|
||
|
correlation >= CORRELATION_THRESHOLD:
|
||
|
sys.stdout.write('\r\t--- SIMILAR IMAGE FOUND --- \n')
|
||
|
print '\tHamming distance: {:d}'.format(hamming_distance)
|
||
|
print '\tCross correlation: {:f}'.format(correlation)
|
||
|
print '\tImage path: {:s}\n'.format(index_image.path)
|
||
|
print
|
||
|
|
||
|
|
||
|
def main():
|
||
|
|
||
|
# Make sure at least one command line argument was provided
|
||
|
if len(sys.argv) < 3:
|
||
|
print 'Usage: find_similar.py <image dir root path> <index path>'
|
||
|
return -1;
|
||
|
|
||
|
# Print out the path we're walking
|
||
|
root_path = sys.argv[1]
|
||
|
index_path = sys.argv[2]
|
||
|
|
||
|
# Load image index
|
||
|
index_file = bz2.BZ2File(index_path, 'r')
|
||
|
image_index = cPickle.load(index_file)
|
||
|
index_file.close()
|
||
|
|
||
|
print 'Looking for similar images from from path: {:s}'.format(root_path)
|
||
|
|
||
|
# Walk the root path and count all images
|
||
|
# (is there a better way to do this?)
|
||
|
image_counter = 0
|
||
|
for root, dirs, files in os.walk(root_path):
|
||
|
for file in files:
|
||
|
image_path = os.path.join(root, file)
|
||
|
# Check the make sure the file is a JPEG
|
||
|
if not imghdr.what(image_path) == 'jpeg':
|
||
|
continue
|
||
|
image_counter += 1
|
||
|
print 'Done counting, {:d} images total.'.format(image_counter)
|
||
|
|
||
|
# Walk the root path
|
||
|
ctr = 0
|
||
|
for root, dirs, files in os.walk(root_path):
|
||
|
for file in files:
|
||
|
image_path = os.path.join(root, file)
|
||
|
# Check the make sure the file is a JPEG
|
||
|
if not imghdr.what(image_path) == 'jpeg':
|
||
|
continue
|
||
|
ctr += 1
|
||
|
percent_done = int((float(ctr) / image_counter) * 100)
|
||
|
print '\n\nLooking for similar -- {:s}'.format(image_path)
|
||
|
image_obj = Image(image_path)
|
||
|
# Find similar images by searching the index
|
||
|
find_similar(percent_done, image_index, image_obj)
|
||
|
|
||
|
print '\nDone.'
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
| similar images/image.py | ||
|---|---|---|
|
import copy
|
||
|
|
||
|
import pHash
|
||
|
|
||
|
class Image(object):
|
||
|
|
||
|
def __init__(self, path, hash=None, coefficients=None):
|
||
|
self.path = path
|
||
|
|
||
|
if not hash:
|
||
|
self.hash = pHash.imagehash(self.path)
|
||
|
else:
|
||
|
self.hash = hash
|
||
|
|
||
|
if not coefficients:
|
||
|
self.coefficients = copy.copy(pHash.image_digest(self.path, 1.0, 1.0, 180).coeffs)
|
||
|
else:
|
||
|
self.coefficients = coefficients
|
||
|
|
||
|
def make_digest(self):
|
||
|
digest = pHash.Digest()
|
||
|
digest.coeffs = copy.copy(self.coefficients)
|
||
|
digest.size = len(self.coefficients)
|
||
|
return digest
|
||
|
|
||
|
def __repr__(self):
|
||
|
print 'Image: {:s}'.format(self.path)
|
||
|
print '\tHash: {:016X}'.format(self.hash)
|
||
|
print '\tDigest coefficients: {:s}'.format(str(self.coefficients))
|
||
I finally got around to playing with pHash today. I also found py-phash which are the Python binds for pHash. Here are a couple of scripts I created to build an image index and then search it to find duplicates. This is good starting point for now, I'll play with these more as I have time.