Revision be16b11b
Added by dsorber over 11 years ago
| software/fiemap_testing/analyze.py | ||
|---|---|---|
|
import os
|
||
|
import sys
|
||
|
|
||
|
import fiemap
|
||
|
|
||
|
def main():
|
||
|
|
||
|
if len(sys.argv) < 2:
|
||
|
print 'ERROR: not enough arguments provided'
|
||
|
return -1
|
||
|
|
||
|
base_path = sys.argv[1]
|
||
|
|
||
|
# Statistics variables
|
||
|
smallest_extent = 2**20
|
||
|
largest_extent = 0
|
||
|
number_extents = 0
|
||
|
number_chunks = 0
|
||
|
total_extent_size = 0
|
||
|
|
||
|
directory_count = 0
|
||
|
|
||
|
# Iterate over all of the "file" directories
|
||
|
directories = [d for d in os.listdir(base_path)
|
||
|
if os.path.isdir(os.path.join(base_path, d))]
|
||
|
dir_total = len(directories)
|
||
|
|
||
|
for directory in directories:
|
||
|
path = os.path.join(base_path, directory)
|
||
|
directory_count += 1
|
||
|
sys.stdout.write('(%d of %d) -- file: %s --- analyzing ... ' %
|
||
|
(directory_count, dir_total, directory))
|
||
|
sys.stdout.flush()
|
||
|
|
||
|
# Iterate over all of the chunk files
|
||
|
for file_ in os.listdir(path):
|
||
|
file_path = os.path.join(path, file_)
|
||
|
|
||
|
number_chunks += 1
|
||
|
|
||
|
# Open a chunk file to perform the fiemap call
|
||
|
with open(file_path, 'r') as fd:
|
||
|
map_ = fiemap.get_all_mappings(fd)
|
||
|
|
||
|
# Update statistics variables
|
||
|
number_extents += len(map_.extents)
|
||
|
for extent in map_.extents:
|
||
|
if extent.length < smallest_extent:
|
||
|
smallest_extent = extent.length
|
||
|
if extent.length > largest_extent:
|
||
|
largest_extent = extent.length
|
||
|
total_extent_size += extent.length
|
||
|
|
||
|
sys.stdout.write('DONE\n')
|
||
|
sys.stdout.flush()
|
||
|
|
||
|
# Print out final statistics
|
||
|
print '-' * 80
|
||
|
print 'Smallest extent size: %8d' % smallest_extent
|
||
|
print 'Largest extent size: %8d' % largest_extent
|
||
|
print 'Average extents per chunk: %8d' % (number_extents / number_chunks)
|
||
|
print 'Average extent size: %8d' % (total_extent_size / number_extents)
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
| software/fiemap_testing/fiemap.py | ||
|---|---|---|
|
import array
|
||
|
import fcntl
|
||
|
import struct
|
||
|
import collections
|
||
|
|
||
|
# Public API
|
||
|
# From linux/fiemap.h
|
||
|
FIEMAP_FLAG_SYNC = 0x0001
|
||
|
FIEMAP_FLAG_XATTR = 0x0002
|
||
|
FIEMAP_FLAGS_COMPAT = FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR
|
||
|
|
||
|
FIEMAP_EXTENT_LAST = 0x0001
|
||
|
FIEMAP_EXTENT_UNKNOWN = 0x0002
|
||
|
FIEMAP_EXTENT_DELALLOC = 0x0004
|
||
|
FIEMAP_EXTENT_ENCODED = 0x0008
|
||
|
FIEMAP_EXTENT_DATA_ENCRYPTED = 0x0080
|
||
|
FIEMAP_EXTENT_NOT_ALIGNED = 0x0100
|
||
|
FIEMAP_EXTENT_DATA_INLINE = 0x0200
|
||
|
FIEMAP_EXTENT_DATA_TAIL = 0x0400
|
||
|
FIEMAP_EXTENT_UNWRITTEN = 0x0800
|
||
|
FIEMAP_EXTENT_MERGED = 0x1000
|
||
|
FIEMAP_EXTENT_SHARED = 0x2000
|
||
|
|
||
|
# Internals and plumbing
|
||
|
# From asm-generic/ioctl.h
|
||
|
_IOC_NRBITS = 8
|
||
|
_IOC_TYPEBITS = 8
|
||
|
_IOC_SIZEBITS = 14
|
||
|
|
||
|
_IOC_NRSHIFT = 0
|
||
|
_IOC_TYPESHIFT = _IOC_NRSHIFT + _IOC_NRBITS
|
||
|
_IOC_SIZESHIFT = _IOC_TYPESHIFT + _IOC_TYPEBITS
|
||
|
_IOC_DIRSHIFT = _IOC_SIZESHIFT + _IOC_SIZEBITS
|
||
|
|
||
|
_IOC_TYPECHECK = lambda struct: struct.size
|
||
|
|
||
|
_IOC = lambda dir_, type_, nr, size: \
|
||
|
(dir_ << _IOC_DIRSHIFT) | (type_ << _IOC_TYPESHIFT) \
|
||
|
| (nr << _IOC_NRSHIFT) | (size << _IOC_SIZESHIFT)
|
||
|
_IOC_WRITE = 1
|
||
|
_IOC_READ = 2
|
||
|
_IOWR = lambda type_, nr, size: \
|
||
|
_IOC(_IOC_READ | _IOC_WRITE, type_, nr, _IOC_TYPECHECK(size))
|
||
|
|
||
|
# Derived from linux/fiemap.h
|
||
|
_struct_fiemap = struct.Struct('=QQLLLL')
|
||
|
_struct_fiemap_extent = struct.Struct('=QQQQQLLLL')
|
||
|
|
||
|
# From linux/fs.h
|
||
|
_FS_IOC_FIEMAP = _IOWR(ord('f'), 11, _struct_fiemap)
|
||
|
|
||
|
_UINT64_MAX = (2 ** 64) - 1
|
||
|
|
||
|
_fiemap = collections.namedtuple('fiemap',
|
||
|
'start length flags mapped_extents extent_count extents')
|
||
|
_fiemap_extent = collections.namedtuple('fiemap_extent',
|
||
|
'logical physical length flags')
|
||
|
|
||
|
|
||
|
# Public API, part 2
|
||
|
def fiemap(fd, start=0, length=_UINT64_MAX, flags=0, count=0):
|
||
|
'''Retrieve extent mappings of a file using a given file descriptor
|
||
|
|
||
|
This uses the *fiemap* ioctl implemented in the Linux kernel. See
|
||
|
`Documentation/fiemap.txt`_ for more information.
|
||
|
|
||
|
This procedure returns a named tuple containing all non-reserved fields as
|
||
|
exposed by the fiemap and fiemap_extent structs. See
|
||
|
:py:func:get_all_mappings if you want to retrieve all mappings of a given
|
||
|
file.
|
||
|
|
||
|
Note the attributes of the result don't have an *fm_* or *fe_* prefix.
|
||
|
|
||
|
.. _Documentation/fiemap.txt: http://www.mjmwired.net/kernel/Documentation/filesystems/fiemap.txt
|
||
|
|
||
|
:param fd: File descriptor to use
|
||
|
:type fd: File-like object or `int`
|
||
|
:param start: Start offset
|
||
|
:type start: `int`
|
||
|
:param length: Query length
|
||
|
:type length: `int`
|
||
|
:param flags: Flags
|
||
|
:type flags: `int`
|
||
|
:param count: Number of extents to request
|
||
|
:type count: `int`
|
||
|
|
||
|
:return: Mapping information
|
||
|
:rtype: `_fiemap`
|
||
|
'''
|
||
|
|
||
|
fiemap_buffer = '%s%s' % (
|
||
|
_struct_fiemap.pack(start, length, flags, 0, count, 0),
|
||
|
'\0' * (_struct_fiemap_extent.size * count))
|
||
|
|
||
|
# Turn into mutable C-level array of chars
|
||
|
buffer_ = array.array('c', fiemap_buffer)
|
||
|
|
||
|
# Syscall
|
||
|
ret = fcntl.ioctl(fd, _FS_IOC_FIEMAP, buffer_)
|
||
|
|
||
|
if ret < 0:
|
||
|
raise IOError('ioctl')
|
||
|
|
||
|
# Read out fiemap struct
|
||
|
fm_start, fm_length, fm_flags, fm_mapped_extents, fm_extent_count, \
|
||
|
fm_reserved = _struct_fiemap.unpack_from(buffer_)
|
||
|
|
||
|
# Read out fiemap_extent structs
|
||
|
fm_extents = []
|
||
|
|
||
|
offset = _struct_fiemap.size
|
||
|
for i in xrange(fm_extent_count):
|
||
|
fe_logical, fe_physical, fe_length, _1, _2, fe_flags, _3, _4, _5 = \
|
||
|
_struct_fiemap_extent.unpack_from(
|
||
|
buffer_[offset:offset + _struct_fiemap_extent.size])
|
||
|
|
||
|
fm_extents.append(
|
||
|
_fiemap_extent(fe_logical, fe_physical, fe_length, fe_flags))
|
||
|
|
||
|
del fe_logical, fe_physical, fe_length, fe_flags
|
||
|
|
||
|
offset += _struct_fiemap_extent.size
|
||
|
|
||
|
if fm_extents:
|
||
|
assert fm_extents[-1].flags | FIEMAP_EXTENT_LAST == FIEMAP_EXTENT_LAST
|
||
|
|
||
|
return _fiemap(
|
||
|
fm_start, fm_length, fm_flags, fm_mapped_extents, fm_extent_count,
|
||
|
fm_extents)
|
||
|
|
||
|
|
||
|
def get_all_mappings(fd, start=0, length=_UINT64_MAX, flags=0):
|
||
|
'''Retrieve all extent mappings of a file using a given file descriptor
|
||
|
|
||
|
This uses :py:func:`fiemap` to retrieve the mappings, twice.
|
||
|
|
||
|
This procedure returns a named tuple containing all non-reserved fields as
|
||
|
exposed by the fiemap and fiemap_extent structs.
|
||
|
|
||
|
Note the attributes of the result don't have an *fm_* or *fe_* prefix.
|
||
|
|
||
|
:param fd: File descriptor to use
|
||
|
:type fd: File-like object or `int`
|
||
|
:param start: Start offset
|
||
|
:type start: `int`
|
||
|
:param length: Query length
|
||
|
:type length: `int`
|
||
|
:param flags: Flags
|
||
|
:type flags: `int`
|
||
|
|
||
|
:return: Mapping information
|
||
|
:rtype: `_fiemap`
|
||
|
'''
|
||
|
|
||
|
map1 = fiemap(fd, start=start, length=length, flags=flags, count=0)
|
||
|
map2 = fiemap(fd, start=start, length=length, flags=flags,
|
||
|
count=map1.mapped_extents)
|
||
|
return map2
|
||
|
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
import sys
|
||
|
import pprint
|
||
|
|
||
|
if len(sys.argv) < 2:
|
||
|
sys.stderr.write('No filename(s) given')
|
||
|
sys.exit(1)
|
||
|
|
||
|
for file_ in sys.argv[1:]:
|
||
|
with open(file_, 'r') as fd:
|
||
|
print file_
|
||
|
print '-' * len(file_)
|
||
|
map_ = get_all_mappings(fd)
|
||
|
pprint.pprint(map_)
|
||
|
print
|
||
| software/fiemap_testing/filler.py | ||
|---|---|---|
|
import os
|
||
|
import random
|
||
|
import subprocess
|
||
|
import sys
|
||
|
|
||
|
MEGA = 2**20
|
||
|
|
||
|
def main():
|
||
|
|
||
|
if len(sys.argv) < 6:
|
||
|
print 'ERROR: not enough arguments provided'
|
||
|
return -1
|
||
|
|
||
|
# Assign input arguments
|
||
|
write_path = sys.argv[1]
|
||
|
min_chunks_per_file = int(sys.argv[2])
|
||
|
max_chunks_per_file = int(sys.argv[3])
|
||
|
chunk_size = int(sys.argv[4])
|
||
|
max_write_amount = int(sys.argv[5])
|
||
|
|
||
|
# Do some basic input validation for S&G
|
||
|
if min_chunks_per_file > max_chunks_per_file:
|
||
|
print 'ERROR: specified minimum chunks per file is larger than '\
|
||
|
'specified maximum chunks per file'
|
||
|
return -1
|
||
|
|
||
|
if chunk_size > max_write_amount:
|
||
|
print 'ERROR: specified chunk size is larger than specified maximum '\
|
||
|
'write amount'
|
||
|
|
||
|
amount_written = 0
|
||
|
file_counter = 0
|
||
|
chunk_idx_counter = 0
|
||
|
total_written = 0
|
||
|
keep_running = True
|
||
|
|
||
|
while keep_running:
|
||
|
|
||
|
# Create the file name and randomly determine the number of blocks
|
||
|
file_name = '%08X' % file_counter
|
||
|
file_counter += 1
|
||
|
chunks_per_file = random.randint(min_chunks_per_file, max_chunks_per_file)
|
||
|
file_size = chunks_per_file * chunk_size
|
||
|
|
||
|
# Make sure we don't write more than intended
|
||
|
if (total_written + file_size) > max_write_amount:
|
||
|
chunks_per_file = (max_write_amount - total_written) / chunk_size
|
||
|
file_size = chunks_per_file * chunk_size
|
||
|
keep_running = False
|
||
|
|
||
|
# Print out status
|
||
|
sys.stdout.write('Creating file "%s" with %5d chunks - %7d MB ... ' %
|
||
|
(file_name, chunks_per_file, file_size))
|
||
|
sys.stdout.flush()
|
||
|
|
||
|
# Make the "file" directory
|
||
|
path = os.path.join(write_path, file_name)
|
||
|
os.mkdir(path)
|
||
|
|
||
|
# Create each of the actual chunk files
|
||
|
for chunk in xrange(chunks_per_file):
|
||
|
chunk_name = '%016X' % chunk_idx_counter
|
||
|
chunk_idx_counter += 1
|
||
|
chunk_path = os.path.join(path, chunk_name)
|
||
|
command = 'head -c %d /dev/zero > %s' % (chunk_size * MEGA, chunk_path)
|
||
|
subprocess.call(command, shell=True)
|
||
|
|
||
|
sys.stdout.write('DONE\n')
|
||
|
|
||
|
total_written += file_size
|
||
|
print ' Total written: %d MB' % total_written
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
Adding fiemap testing scripts which includes the very handy python fiemap bindings I found on the intarwebz.