commit 1136d40653c3743c9f5d04f613c519584cbc9dfa Author: David Sorber Date: Mon Feb 20 10:48:01 2023 -0500 Update checker.py script with a more object oriented design. Google SMTP relay no longer works without 2 factor authentication so email notificaiton does not currently work. diff --git a/software/misc/build_ffmpeg_shared.sh b/software/misc/build_ffmpeg_shared.sh index cea9a4a..098e840 100755 --- a/software/misc/build_ffmpeg_shared.sh +++ b/software/misc/build_ffmpeg_shared.sh @@ -34,7 +34,7 @@ BIN_DIR="$INSTALL_DIR/bin" LIB_DIR="$INSTALL_DIR/lib" # Set to a string containing the version you wish to install -FFMPEG_VERSION="5.0" # <--- set this to latest version +FFMPEG_VERSION="5.1" # <--- set this to latest version NUM_CORES=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l) diff --git a/software/misc/checker.py b/software/misc/checker.py index 6d56d65..69414f2 100755 --- a/software/misc/checker.py +++ b/software/misc/checker.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 import datetime from email.mime.text import MIMEText +import os +import re from smtplib import SMTP_SSL import socket import subprocess @@ -33,118 +35,165 @@ def send_email(subject, msg_text): print(exec) sys.exit("Mail failed: {}".format(exc)) - -def check_smart(devices): - """ Check the SMART status of the passed list of block devices - """ - if type(devices) is not list: - devices = [devices] - - errors = [] - output = [] +class DiskDrive(object): - for device in devices: - # Get information about this device and SMART health check; prune the - # smartmontools banner from the output - cmd = '/usr/sbin/smartctl -iH /dev/{:s}'.format(device) - cmd_out = subprocess.check_output(cmd, shell=True) - raw_out = cmd_out.decode('utf-8').strip().split('\n')[4:] + def __init__(self, name, raid_status): + self.name = name + self.raid_status = raid_status + self.exists = os.path.exists(f'/dev/{self.name}') + self.serial_num = '' - # Parse out the SMART "overall-health" line and status - smart_output = raw_out[len(raw_out) - 1] - smart_status = smart_output[50:] + self.model_family = '' + self.device_model = '' + self.capacity = '' + self.rotation = '' + self.smart_status = 'unknown' - # Now check smart status - if smart_status != 'PASSED': - errors.append('/dev/{:s} SMART check failed'.format(device)) + self.error = not self.exists - raw_out.insert(0, '/dev/{:s}'.format(device)) - raw_out.append('{:s}\n'.format(SEP)) - output.extend(raw_out) + if self.exists: + # Set serial number: + cmd = f'hdparm -I /dev/{self.name} | grep "Serial\ Number"' + cmd_out = subprocess.check_output(cmd, shell=True) + cmd_out = cmd_out.decode('utf-8').strip() + self.serial_num = cmd_out.split(':')[1].strip() + + # Check/set status: + self.check_status() + + def check_status(self): + cmd = f'/usr/sbin/smartctl -iH /dev/{self.name}' + cmd_out = subprocess.check_output(cmd, shell=True) + out_list = cmd_out.decode('utf-8').strip().split('\n') - #~ print('\n'.join(raw_out)) - - return (errors, output) + # Parse info out of command output + self.model_family = out_list[4].split(':')[1].strip() + self.device_model = out_list[5].split(':')[1].strip() + self.capacity = out_list[9].split(':')[1].strip().split('[')[1][:-1] + self.rotation = out_list[11].split(':')[1].strip() + self.smart_status = out_list[21].split(':')[1].strip() + + def okay(self): + if self.exists: + return (self.smart_status == 'PASSED') + else: + return False -def check_raid(devices): - """ Check the status of the passed list of software RAID devices - """ - if type(devices) is not list: - devices = [devices] - - errors = [] - output = [] - - for device in devices: - # Get information about this device - cmd = '/sbin/mdadm --detail /dev/{:s}'.format(device) - cmd_out = subprocess.check_output(cmd, shell=True) - raw_out = cmd_out.decode('utf-8').strip().split('\n') + def output(self): + output = [] + output.append(f'===[/dev/{self.name}]==================================================================') + output.append(f'Family : {self.model_family}') + output.append(f'Device : {self.device_model}') + output.append(f'Serial : {self.serial_num}') + output.append(f'Capacity : {self.capacity}') + output.append(f'Rotation : {self.rotation}') + output.append(f'SMART status : {self.smart_status}') + output.append('') + return '\n'.join(output) - # Parse out the 'state' line - raid_state = raw_out[11][20:].strip() - if raid_state != 'clean': - errors.append('/dev/{:s} not in "clean" state ({:s})' - .format(device, raid_state)) - output.extend(raw_out) +class RaidDevice(object): - return (errors, output) - + def __init__(self, name, num_drives_expected): + self.name = name + self.expected_num_drives = num_drives_expected + self.exists = os.path.exists(f'/dev/{self.name}') + self.raid_level = '' + self.status = 'unkown' + self.drives = [] + self.mdadm_output = '' + self.error = not self.exists + self.error_msg = '' + + def _refresh_mdadm_out(self): + if self.exists: + # Get information about this device + cmd = f'/sbin/mdadm --detail /dev/{self.name}' + cmd_out = subprocess.check_output(cmd, shell=True) + self.mdadm_output = cmd_out.decode('utf-8').strip().split('\n') + + def check_status(self): + if self.exists: + # Refresh internal buffer containing "mdadm --detail" output + self._refresh_mdadm_out() + + # Now parse the command output + self.raid_level = self.mdadm_output[3].split(':')[1].strip() + self.status = self.mdadm_output[11].split(':')[1].strip() + num_devices = int(self.mdadm_output[6].split(':')[1].strip()) + + if self.status != 'clean': + self.error = True + self.error_msg = f'RAID status not "clean": {self.status}' + + if num_devices != self.expected_num_drives: + self.error = True + self.error_msg = f'{self.expected_num_drives} drives expected '\ + f'but only {len(self.drives)} found' + + # Now build DiskDevice instances + base_offset = 27 + for idx in range (num_devices): + # This parsing is a little hairy... + line = self.mdadm_output[base_offset + idx] + tokens = re.sub(' +', ' ', line).strip().split(' ')[4:] + dev_name = tokens[-1].split('/')[2] + dev_status = ' '.join(tokens[:-1]) + drive = DiskDrive(dev_name, dev_status) + self.drives.append(drive) + if not drive.okay(): + self.error = True + self.error_msg = 'One or more drives failed SMART check!' + else: + self.status = 'device does not exist!' + + def okay(self): + return (not self.error) + + def output(self, hostname, timestamp): + output = [] + output.append(f'Hostname : {hostname}') + output.append(f'Timestamp : {timestamp}') + output.append(f'RAID device : /dev/{self.name}') + output.append(f'Status : {self.error and "ERROR" or "OKAY"} '\ + f'{self.error and " -- " + self.error_msg or ""}') + output.append(f'Level : {self.raid_level}') + output.append(f'Drives : {len(self.drives)}/{self.expected_num_drives}') + output.append('') + + for drive in sorted(self.drives, key=lambda x: x.name): + output.append(drive.output()) + return '\n'.join(output) + def main(): # Grab timestamp timestamp = datetime.datetime.now() ts_str = timestamp.strftime('%Y%m%d_%H%M') - hostname = socket.gethostname() - print('Checker for {:s} - {:s}\n\n'.format(hostname, ts_str)) - - errors = [] - output = [] - output.append('Report for {:s} - {:s}\n'.format(hostname, ts_str)) - output.append('\n') - - # Report uptime - output.append('Uptime:') - cmd_out = subprocess.check_output('uptime', shell=True) - output.append(cmd_out.decode('utf-8').strip()) - output.append('{:s}\n'.format(SEP)) - - # Check software RAID devices - raid_errors, raid_output = check_raid(RAID_DEVS) - errors.extend(raid_errors) - output.extend(raid_output) + raid_dev = RaidDevice('md0', 5) + raid_dev.check_status() + + output = raid_dev.output(hostname, ts_str) + print(output) - # Report df -h (disk utilization) - cmd_out = subprocess.check_output('df -h', shell=True) - output.append(cmd_out.decode('utf-8').strip()) - output.append('{:s}\n'.format(SEP)) + # Send an email if: there are any errors OR + # every Thursday at 3 AM even if status is okay + should_send = ((not raid_dev.okay()) or + (timestamp.weekday() == 3 and timestamp.hour == 3)) - # Check block devices - smart_errors, smart_output = check_smart(BLOCK_DEVS) - errors.extend(smart_errors) - output.extend(smart_output) - - # If no errors prepend "all okay" message - should_send = False - if not errors: - output.insert(2, 'All OKAY.\n') - - # Send an email every Thursday at 3 AM even if status is okay - if timestamp.weekday() == 3 and timestamp.hour == 3: - should_send = True + if should_send: + print('Sending notification email...') else: - # Send email if there were any errors - output.insert(2, '\n'.join(errors)) - should_send = True - + print('Not sending notification email.') + # Send email if should_send: - send_email('Report: {:s} - {:s}\n'.format(hostname, ts_str), - '\n'.join(output)) + send_email(f'Report: {hostname} - {ts_str}\n', output) + + return 0 if __name__ == '__main__': sys.exit(main())