root/similar images/build_index_faster.py @ 2e26af92
| 8dea60e7 | dsorber | #!/opt/local/bin/python2.7
|
|
import bz2
|
|||
import cPickle
|
|||
import imghdr
|
|||
import math
|
|||
import multiprocessing
|
|||
import os
|
|||
import Queue
|
|||
import random
|
|||
import sys
|
|||
import pHash
|
|||
from image import Image
|
|||
NUM_WORKERS = 7
|
|||
# Build index
|
|||
# TODO: write more schtuff here
|
|||
class IndexWorker(multiprocessing.Process):
|
|||
def __init__(self, file_list, queue):
|
|||
multiprocessing.Process.__init__(self)
|
|||
self.file_list = file_list
|
|||
self.queue = queue
|
|||
def run(self):
|
|||
list_size = len(self.file_list)
|
|||
ctr = 0
|
|||
for file_path in self.file_list:
|
|||
ctr += 1
|
|||
percent_done = int((float(ctr) / list_size) * 100)
|
|||
# We can only index jpegs right now
|
|||
if not imghdr.what(file_path) == 'jpeg':
|
|||
continue
|
|||
print '{:d} -- {:3d}% -- {:s}'.format(self.pid, percent_done,
|
|||
file_path)
|
|||
image_obj = Image(file_path)
|
|||
self.queue.put(image_obj, True)
|
|||
def main():
|
|||
# Make sure at least one command line argument was provided
|
|||
if len(sys.argv) < 2:
|
|||
print 'Usage: build_index_faster.py <path>'
|
|||
return -1;
|
|||
# Print out the path we're walking
|
|||
path = sys.argv[1]
|
|||
print 'Building image index from path: {:s}\n'.format(path)
|
|||
master_file_list = []
|
|||
image_index = []
|
|||
# Walk the root path and create a list of all files in the root path
|
|||
for root, dirs, files in os.walk(path):
|
|||
for file in files:
|
|||
master_file_list.append(os.path.join(root, file))
|
|||
random.shuffle(master_file_list)
|
|||
print 'Done counting, {:d} files total.'.format(len(master_file_list))
|
|||
workers = []
|
|||
queue = multiprocessing.Queue(64)
|
|||
def chunker(seq, size):
|
|||
return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
|
|||
chunk_size = int(math.ceil(len(master_file_list) / NUM_WORKERS))
|
|||
# Create worker processes
|
|||
for work_chunk in chunker(master_file_list, chunk_size):
|
|||
workers.append(IndexWorker(work_chunk, queue))
|
|||
# Start workers
|
|||
for worker in workers:
|
|||
worker.start()
|
|||
print 'Waiting for workers to complete'
|
|||
# Loop until all workers are finished
|
|||
while any([w.is_alive() for w in workers]):
|
|||
try:
|
|||
image_obj = queue.get(True, 2)
|
|||
image_index.append(image_obj)
|
|||
except Queue.Empty:
|
|||
print 'Queue was empty'
|
|||
continue
|
|||
print 'Workers are complete: {:d} images found'.format(len(image_index))
|
|||
# Write index as a bzip2'd file to save space
|
|||
index_file = bz2.BZ2File('images_index.bz2', 'w')
|
|||
cPickle.dump(image_index, index_file)
|
|||
index_file.close()
|
|||
print '\nIndex file images_index.bz2 created. Done.'
|
|||
if __name__ == '__main__':
|
|||
sys.exit(main())
|