commit be16b11bf21031dbfabfc422a235e38144bd2003 Author: dsorber Date: Thu Nov 13 20:38:38 2014 -0500 Adding fiemap testing scripts which includes the very handy python fiemap bindings I found on the intarwebz. diff --git a/software/fiemap_testing/analyze.py b/software/fiemap_testing/analyze.py new file mode 100644 index 0000000..c1d1b6d --- /dev/null +++ b/software/fiemap_testing/analyze.py @@ -0,0 +1,65 @@ +import os +import sys + +import fiemap + +def main(): + + if len(sys.argv) < 2: + print 'ERROR: not enough arguments provided' + return -1 + + base_path = sys.argv[1] + + # Statistics variables + smallest_extent = 2**20 + largest_extent = 0 + number_extents = 0 + number_chunks = 0 + total_extent_size = 0 + + directory_count = 0 + + # Iterate over all of the "file" directories + directories = [d for d in os.listdir(base_path) + if os.path.isdir(os.path.join(base_path, d))] + dir_total = len(directories) + + for directory in directories: + path = os.path.join(base_path, directory) + directory_count += 1 + sys.stdout.write('(%d of %d) -- file: %s --- analyzing ... ' % + (directory_count, dir_total, directory)) + sys.stdout.flush() + + # Iterate over all of the chunk files + for file_ in os.listdir(path): + file_path = os.path.join(path, file_) + + number_chunks += 1 + + # Open a chunk file to perform the fiemap call + with open(file_path, 'r') as fd: + map_ = fiemap.get_all_mappings(fd) + + # Update statistics variables + number_extents += len(map_.extents) + for extent in map_.extents: + if extent.length < smallest_extent: + smallest_extent = extent.length + if extent.length > largest_extent: + largest_extent = extent.length + total_extent_size += extent.length + + sys.stdout.write('DONE\n') + sys.stdout.flush() + + # Print out final statistics + print '-' * 80 + print 'Smallest extent size: %8d' % smallest_extent + print 'Largest extent size: %8d' % largest_extent + print 'Average extents per chunk: %8d' % (number_extents / number_chunks) + print 'Average extent size: %8d' % (total_extent_size / number_extents) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/software/fiemap_testing/fiemap.py b/software/fiemap_testing/fiemap.py new file mode 100644 index 0000000..8a65c18 --- /dev/null +++ b/software/fiemap_testing/fiemap.py @@ -0,0 +1,175 @@ +import array +import fcntl +import struct +import collections + +# Public API +# From linux/fiemap.h +FIEMAP_FLAG_SYNC = 0x0001 +FIEMAP_FLAG_XATTR = 0x0002 +FIEMAP_FLAGS_COMPAT = FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR + +FIEMAP_EXTENT_LAST = 0x0001 +FIEMAP_EXTENT_UNKNOWN = 0x0002 +FIEMAP_EXTENT_DELALLOC = 0x0004 +FIEMAP_EXTENT_ENCODED = 0x0008 +FIEMAP_EXTENT_DATA_ENCRYPTED = 0x0080 +FIEMAP_EXTENT_NOT_ALIGNED = 0x0100 +FIEMAP_EXTENT_DATA_INLINE = 0x0200 +FIEMAP_EXTENT_DATA_TAIL = 0x0400 +FIEMAP_EXTENT_UNWRITTEN = 0x0800 +FIEMAP_EXTENT_MERGED = 0x1000 +FIEMAP_EXTENT_SHARED = 0x2000 + +# Internals and plumbing +# From asm-generic/ioctl.h +_IOC_NRBITS = 8 +_IOC_TYPEBITS = 8 +_IOC_SIZEBITS = 14 + +_IOC_NRSHIFT = 0 +_IOC_TYPESHIFT = _IOC_NRSHIFT + _IOC_NRBITS +_IOC_SIZESHIFT = _IOC_TYPESHIFT + _IOC_TYPEBITS +_IOC_DIRSHIFT = _IOC_SIZESHIFT + _IOC_SIZEBITS + +_IOC_TYPECHECK = lambda struct: struct.size + +_IOC = lambda dir_, type_, nr, size: \ + (dir_ << _IOC_DIRSHIFT) | (type_ << _IOC_TYPESHIFT) \ + | (nr << _IOC_NRSHIFT) | (size << _IOC_SIZESHIFT) +_IOC_WRITE = 1 +_IOC_READ = 2 +_IOWR = lambda type_, nr, size: \ + _IOC(_IOC_READ | _IOC_WRITE, type_, nr, _IOC_TYPECHECK(size)) + +# Derived from linux/fiemap.h +_struct_fiemap = struct.Struct('=QQLLLL') +_struct_fiemap_extent = struct.Struct('=QQQQQLLLL') + +# From linux/fs.h +_FS_IOC_FIEMAP = _IOWR(ord('f'), 11, _struct_fiemap) + +_UINT64_MAX = (2 ** 64) - 1 + +_fiemap = collections.namedtuple('fiemap', + 'start length flags mapped_extents extent_count extents') +_fiemap_extent = collections.namedtuple('fiemap_extent', + 'logical physical length flags') + + +# Public API, part 2 +def fiemap(fd, start=0, length=_UINT64_MAX, flags=0, count=0): + '''Retrieve extent mappings of a file using a given file descriptor + + This uses the *fiemap* ioctl implemented in the Linux kernel. See + `Documentation/fiemap.txt`_ for more information. + + This procedure returns a named tuple containing all non-reserved fields as + exposed by the fiemap and fiemap_extent structs. See + :py:func:get_all_mappings if you want to retrieve all mappings of a given + file. + + Note the attributes of the result don't have an *fm_* or *fe_* prefix. + + .. _Documentation/fiemap.txt: http://www.mjmwired.net/kernel/Documentation/filesystems/fiemap.txt + + :param fd: File descriptor to use + :type fd: File-like object or `int` + :param start: Start offset + :type start: `int` + :param length: Query length + :type length: `int` + :param flags: Flags + :type flags: `int` + :param count: Number of extents to request + :type count: `int` + + :return: Mapping information + :rtype: `_fiemap` + ''' + + fiemap_buffer = '%s%s' % ( + _struct_fiemap.pack(start, length, flags, 0, count, 0), + '\0' * (_struct_fiemap_extent.size * count)) + + # Turn into mutable C-level array of chars + buffer_ = array.array('c', fiemap_buffer) + + # Syscall + ret = fcntl.ioctl(fd, _FS_IOC_FIEMAP, buffer_) + + if ret < 0: + raise IOError('ioctl') + + # Read out fiemap struct + fm_start, fm_length, fm_flags, fm_mapped_extents, fm_extent_count, \ + fm_reserved = _struct_fiemap.unpack_from(buffer_) + + # Read out fiemap_extent structs + fm_extents = [] + + offset = _struct_fiemap.size + for i in xrange(fm_extent_count): + fe_logical, fe_physical, fe_length, _1, _2, fe_flags, _3, _4, _5 = \ + _struct_fiemap_extent.unpack_from( + buffer_[offset:offset + _struct_fiemap_extent.size]) + + fm_extents.append( + _fiemap_extent(fe_logical, fe_physical, fe_length, fe_flags)) + + del fe_logical, fe_physical, fe_length, fe_flags + + offset += _struct_fiemap_extent.size + + if fm_extents: + assert fm_extents[-1].flags | FIEMAP_EXTENT_LAST == FIEMAP_EXTENT_LAST + + return _fiemap( + fm_start, fm_length, fm_flags, fm_mapped_extents, fm_extent_count, + fm_extents) + + +def get_all_mappings(fd, start=0, length=_UINT64_MAX, flags=0): + '''Retrieve all extent mappings of a file using a given file descriptor + + This uses :py:func:`fiemap` to retrieve the mappings, twice. + + This procedure returns a named tuple containing all non-reserved fields as + exposed by the fiemap and fiemap_extent structs. + + Note the attributes of the result don't have an *fm_* or *fe_* prefix. + + :param fd: File descriptor to use + :type fd: File-like object or `int` + :param start: Start offset + :type start: `int` + :param length: Query length + :type length: `int` + :param flags: Flags + :type flags: `int` + + :return: Mapping information + :rtype: `_fiemap` + ''' + + map1 = fiemap(fd, start=start, length=length, flags=flags, count=0) + map2 = fiemap(fd, start=start, length=length, flags=flags, + count=map1.mapped_extents) + return map2 + + +if __name__ == '__main__': + import sys + import pprint + + if len(sys.argv) < 2: + sys.stderr.write('No filename(s) given') + sys.exit(1) + + for file_ in sys.argv[1:]: + with open(file_, 'r') as fd: + print file_ + print '-' * len(file_) + map_ = get_all_mappings(fd) + pprint.pprint(map_) + print \ No newline at end of file diff --git a/software/fiemap_testing/filler.py b/software/fiemap_testing/filler.py new file mode 100644 index 0000000..042851c --- /dev/null +++ b/software/fiemap_testing/filler.py @@ -0,0 +1,74 @@ +import os +import random +import subprocess +import sys + +MEGA = 2**20 + +def main(): + + if len(sys.argv) < 6: + print 'ERROR: not enough arguments provided' + return -1 + + # Assign input arguments + write_path = sys.argv[1] + min_chunks_per_file = int(sys.argv[2]) + max_chunks_per_file = int(sys.argv[3]) + chunk_size = int(sys.argv[4]) + max_write_amount = int(sys.argv[5]) + + # Do some basic input validation for S&G + if min_chunks_per_file > max_chunks_per_file: + print 'ERROR: specified minimum chunks per file is larger than '\ + 'specified maximum chunks per file' + return -1 + + if chunk_size > max_write_amount: + print 'ERROR: specified chunk size is larger than specified maximum '\ + 'write amount' + + amount_written = 0 + file_counter = 0 + chunk_idx_counter = 0 + total_written = 0 + keep_running = True + + while keep_running: + + # Create the file name and randomly determine the number of blocks + file_name = '%08X' % file_counter + file_counter += 1 + chunks_per_file = random.randint(min_chunks_per_file, max_chunks_per_file) + file_size = chunks_per_file * chunk_size + + # Make sure we don't write more than intended + if (total_written + file_size) > max_write_amount: + chunks_per_file = (max_write_amount - total_written) / chunk_size + file_size = chunks_per_file * chunk_size + keep_running = False + + # Print out status + sys.stdout.write('Creating file "%s" with %5d chunks - %7d MB ... ' % + (file_name, chunks_per_file, file_size)) + sys.stdout.flush() + + # Make the "file" directory + path = os.path.join(write_path, file_name) + os.mkdir(path) + + # Create each of the actual chunk files + for chunk in xrange(chunks_per_file): + chunk_name = '%016X' % chunk_idx_counter + chunk_idx_counter += 1 + chunk_path = os.path.join(path, chunk_name) + command = 'head -c %d /dev/zero > %s' % (chunk_size * MEGA, chunk_path) + subprocess.call(command, shell=True) + + sys.stdout.write('DONE\n') + + total_written += file_size + print ' Total written: %d MB' % total_written + +if __name__ == '__main__': + sys.exit(main())