Revision 1136d406
Added by David Sorber over 3 years ago
| software/misc/build_ffmpeg_shared.sh | ||
|---|---|---|
|
LIB_DIR="$INSTALL_DIR/lib"
|
||
|
|
||
|
# Set to a string containing the version you wish to install
|
||
|
FFMPEG_VERSION="5.0" # <--- set this to latest version
|
||
|
FFMPEG_VERSION="5.1" # <--- set this to latest version
|
||
|
|
||
|
NUM_CORES=$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | wc -l)
|
||
|
|
||
| software/misc/checker.py | ||
|---|---|---|
|
#!/usr/bin/python3
|
||
|
import datetime
|
||
|
from email.mime.text import MIMEText
|
||
|
import os
|
||
|
import re
|
||
|
from smtplib import SMTP_SSL
|
||
|
import socket
|
||
|
import subprocess
|
||
| ... | ... | |
|
print(exec)
|
||
|
sys.exit("Mail failed: {}".format(exc))
|
||
|
|
||
|
|
||
|
def check_smart(devices):
|
||
|
""" Check the SMART status of the passed list of block devices
|
||
|
"""
|
||
|
if type(devices) is not list:
|
||
|
devices = [devices]
|
||
|
|
||
|
errors = []
|
||
|
output = []
|
||
|
class DiskDrive(object):
|
||
|
|
||
|
for device in devices:
|
||
|
# Get information about this device and SMART health check; prune the
|
||
|
# smartmontools banner from the output
|
||
|
cmd = '/usr/sbin/smartctl -iH /dev/{:s}'.format(device)
|
||
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
||
|
raw_out = cmd_out.decode('utf-8').strip().split('\n')[4:]
|
||
|
def __init__(self, name, raid_status):
|
||
|
self.name = name
|
||
|
self.raid_status = raid_status
|
||
|
self.exists = os.path.exists(f'/dev/{self.name}')
|
||
|
self.serial_num = ''
|
||
|
|
||
|
# Parse out the SMART "overall-health" line and status
|
||
|
smart_output = raw_out[len(raw_out) - 1]
|
||
|
smart_status = smart_output[50:]
|
||
|
self.model_family = ''
|
||
|
self.device_model = ''
|
||
|
self.capacity = ''
|
||
|
self.rotation = ''
|
||
|
self.smart_status = 'unknown'
|
||
|
|
||
|
# Now check smart status
|
||
|
if smart_status != 'PASSED':
|
||
|
errors.append('/dev/{:s} SMART check failed'.format(device))
|
||
|
self.error = not self.exists
|
||
|
|
||
|
raw_out.insert(0, '/dev/{:s}'.format(device))
|
||
|
raw_out.append('{:s}\n'.format(SEP))
|
||
|
output.extend(raw_out)
|
||
|
if self.exists:
|
||
|
# Set serial number:
|
||
|
cmd = f'hdparm -I /dev/{self.name} | grep "Serial\ Number"'
|
||
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
||
|
cmd_out = cmd_out.decode('utf-8').strip()
|
||
|
self.serial_num = cmd_out.split(':')[1].strip()
|
||
|
|
||
|
# Check/set status:
|
||
|
self.check_status()
|
||
|
|
||
|
def check_status(self):
|
||
|
cmd = f'/usr/sbin/smartctl -iH /dev/{self.name}'
|
||
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
||
|
out_list = cmd_out.decode('utf-8').strip().split('\n')
|
||
|
|
||
|
#~ print('\n'.join(raw_out))
|
||
|
|
||
|
return (errors, output)
|
||
|
# Parse info out of command output
|
||
|
self.model_family = out_list[4].split(':')[1].strip()
|
||
|
self.device_model = out_list[5].split(':')[1].strip()
|
||
|
self.capacity = out_list[9].split(':')[1].strip().split('[')[1][:-1]
|
||
|
self.rotation = out_list[11].split(':')[1].strip()
|
||
|
self.smart_status = out_list[21].split(':')[1].strip()
|
||
|
|
||
|
def okay(self):
|
||
|
if self.exists:
|
||
|
return (self.smart_status == 'PASSED')
|
||
|
else:
|
||
|
return False
|
||
|
|
||
|
def check_raid(devices):
|
||
|
""" Check the status of the passed list of software RAID devices
|
||
|
"""
|
||
|
if type(devices) is not list:
|
||
|
devices = [devices]
|
||
|
|
||
|
errors = []
|
||
|
output = []
|
||
|
|
||
|
for device in devices:
|
||
|
# Get information about this device
|
||
|
cmd = '/sbin/mdadm --detail /dev/{:s}'.format(device)
|
||
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
||
|
raw_out = cmd_out.decode('utf-8').strip().split('\n')
|
||
|
def output(self):
|
||
|
output = []
|
||
|
output.append(f'===[/dev/{self.name}]==================================================================')
|
||
|
output.append(f'Family : {self.model_family}')
|
||
|
output.append(f'Device : {self.device_model}')
|
||
|
output.append(f'Serial : {self.serial_num}')
|
||
|
output.append(f'Capacity : {self.capacity}')
|
||
|
output.append(f'Rotation : {self.rotation}')
|
||
|
output.append(f'SMART status : {self.smart_status}')
|
||
|
output.append('')
|
||
|
return '\n'.join(output)
|
||
|
|
||
|
# Parse out the 'state' line
|
||
|
raid_state = raw_out[11][20:].strip()
|
||
|
if raid_state != 'clean':
|
||
|
errors.append('/dev/{:s} not in "clean" state ({:s})'
|
||
|
.format(device, raid_state))
|
||
|
|
||
|
output.extend(raw_out)
|
||
|
class RaidDevice(object):
|
||
|
|
||
|
return (errors, output)
|
||
|
|
||
|
def __init__(self, name, num_drives_expected):
|
||
|
self.name = name
|
||
|
self.expected_num_drives = num_drives_expected
|
||
|
self.exists = os.path.exists(f'/dev/{self.name}')
|
||
|
self.raid_level = ''
|
||
|
self.status = 'unkown'
|
||
|
self.drives = []
|
||
|
self.mdadm_output = ''
|
||
|
self.error = not self.exists
|
||
|
self.error_msg = ''
|
||
|
|
||
|
def _refresh_mdadm_out(self):
|
||
|
if self.exists:
|
||
|
# Get information about this device
|
||
|
cmd = f'/sbin/mdadm --detail /dev/{self.name}'
|
||
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
||
|
self.mdadm_output = cmd_out.decode('utf-8').strip().split('\n')
|
||
|
|
||
|
def check_status(self):
|
||
|
if self.exists:
|
||
|
# Refresh internal buffer containing "mdadm --detail" output
|
||
|
self._refresh_mdadm_out()
|
||
|
|
||
|
# Now parse the command output
|
||
|
self.raid_level = self.mdadm_output[3].split(':')[1].strip()
|
||
|
self.status = self.mdadm_output[11].split(':')[1].strip()
|
||
|
num_devices = int(self.mdadm_output[6].split(':')[1].strip())
|
||
|
|
||
|
if self.status != 'clean':
|
||
|
self.error = True
|
||
|
self.error_msg = f'RAID status not "clean": {self.status}'
|
||
|
|
||
|
if num_devices != self.expected_num_drives:
|
||
|
self.error = True
|
||
|
self.error_msg = f'{self.expected_num_drives} drives expected '\
|
||
|
f'but only {len(self.drives)} found'
|
||
|
|
||
|
# Now build DiskDevice instances
|
||
|
base_offset = 27
|
||
|
for idx in range (num_devices):
|
||
|
# This parsing is a little hairy...
|
||
|
line = self.mdadm_output[base_offset + idx]
|
||
|
tokens = re.sub(' +', ' ', line).strip().split(' ')[4:]
|
||
|
dev_name = tokens[-1].split('/')[2]
|
||
|
dev_status = ' '.join(tokens[:-1])
|
||
|
drive = DiskDrive(dev_name, dev_status)
|
||
|
self.drives.append(drive)
|
||
|
if not drive.okay():
|
||
|
self.error = True
|
||
|
self.error_msg = 'One or more drives failed SMART check!'
|
||
|
else:
|
||
|
self.status = 'device does not exist!'
|
||
|
|
||
|
def okay(self):
|
||
|
return (not self.error)
|
||
|
|
||
|
def output(self, hostname, timestamp):
|
||
|
output = []
|
||
|
output.append(f'Hostname : {hostname}')
|
||
|
output.append(f'Timestamp : {timestamp}')
|
||
|
output.append(f'RAID device : /dev/{self.name}')
|
||
|
output.append(f'Status : {self.error and "ERROR" or "OKAY"} '\
|
||
|
f'{self.error and " -- " + self.error_msg or ""}')
|
||
|
output.append(f'Level : {self.raid_level}')
|
||
|
output.append(f'Drives : {len(self.drives)}/{self.expected_num_drives}')
|
||
|
output.append('')
|
||
|
|
||
|
for drive in sorted(self.drives, key=lambda x: x.name):
|
||
|
output.append(drive.output())
|
||
|
return '\n'.join(output)
|
||
|
|
||
|
|
||
|
def main():
|
||
|
|
||
|
# Grab timestamp
|
||
|
timestamp = datetime.datetime.now()
|
||
|
ts_str = timestamp.strftime('%Y%m%d_%H%M')
|
||
|
|
||
|
hostname = socket.gethostname()
|
||
|
|
||
|
print('Checker for {:s} - {:s}\n\n'.format(hostname, ts_str))
|
||
|
|
||
|
errors = []
|
||
|
output = []
|
||
|
output.append('Report for {:s} - {:s}\n'.format(hostname, ts_str))
|
||
|
output.append('\n')
|
||
|
|
||
|
# Report uptime
|
||
|
output.append('Uptime:')
|
||
|
cmd_out = subprocess.check_output('uptime', shell=True)
|
||
|
output.append(cmd_out.decode('utf-8').strip())
|
||
|
output.append('{:s}\n'.format(SEP))
|
||
|
|
||
|
# Check software RAID devices
|
||
|
raid_errors, raid_output = check_raid(RAID_DEVS)
|
||
|
errors.extend(raid_errors)
|
||
|
output.extend(raid_output)
|
||
|
raid_dev = RaidDevice('md0', 5)
|
||
|
raid_dev.check_status()
|
||
|
|
||
|
output = raid_dev.output(hostname, ts_str)
|
||
|
print(output)
|
||
|
|
||
|
# Report df -h (disk utilization)
|
||
|
cmd_out = subprocess.check_output('df -h', shell=True)
|
||
|
output.append(cmd_out.decode('utf-8').strip())
|
||
|
output.append('{:s}\n'.format(SEP))
|
||
|
# Send an email if: there are any errors OR
|
||
|
# every Thursday at 3 AM even if status is okay
|
||
|
should_send = ((not raid_dev.okay()) or
|
||
|
(timestamp.weekday() == 3 and timestamp.hour == 3))
|
||
|
|
||
|
# Check block devices
|
||
|
smart_errors, smart_output = check_smart(BLOCK_DEVS)
|
||
|
errors.extend(smart_errors)
|
||
|
output.extend(smart_output)
|
||
|
|
||
|
# If no errors prepend "all okay" message
|
||
|
should_send = False
|
||
|
if not errors:
|
||
|
output.insert(2, 'All OKAY.\n')
|
||
|
|
||
|
# Send an email every Thursday at 3 AM even if status is okay
|
||
|
if timestamp.weekday() == 3 and timestamp.hour == 3:
|
||
|
should_send = True
|
||
|
if should_send:
|
||
|
print('Sending notification email...')
|
||
|
else:
|
||
|
# Send email if there were any errors
|
||
|
output.insert(2, '\n'.join(errors))
|
||
|
should_send = True
|
||
|
|
||
|
print('Not sending notification email.')
|
||
|
|
||
|
# Send email
|
||
|
if should_send:
|
||
|
send_email('Report: {:s} - {:s}\n'.format(hostname, ts_str),
|
||
|
'\n'.join(output))
|
||
|
send_email(f'Report: {hostname} - {ts_str}\n', output)
|
||
|
|
||
|
return 0
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
sys.exit(main())
|
||
Update checker.py script with a more object oriented design. Google SMTP
relay no longer works without 2 factor authentication so email
notificaiton does not currently work.