root/software/misc/checker.py @ 0fc29199
| 0fc29199 | David Sorber | #!/usr/bin/python3
|
|
import datetime
|
|||
from email.mime.text import MIMEText
|
|||
from smtplib import SMTP_SSL
|
|||
import socket
|
|||
import subprocess
|
|||
import sys
|
|||
RELAY_EMAIL = 'dsorber.phalanx@gmail.com'
|
|||
RELAY_PASSWD = "You'reNotMySupervisor"
|
|||
TO_EMAIL = 'baranovich@gmail.com'
|
|||
RAID_DEVS = ['md0']
|
|||
BLOCK_DEVS = ['sda', 'sdb', 'sdc', 'sdd']
|
|||
SEP = '-' * 80
|
|||
def send_email(subject, msg_text):
|
|||
msg = MIMEText(msg_text, 'plain')
|
|||
msg['Subject'] = subject
|
|||
msg['To'] = TO_EMAIL
|
|||
try:
|
|||
conn = SMTP_SSL('smtp.gmail.com')
|
|||
conn.set_debuglevel(True)
|
|||
conn.login(RELAY_EMAIL, RELAY_PASSWD)
|
|||
try:
|
|||
conn.sendmail(RELAY_EMAIL, TO_EMAIL, msg.as_string())
|
|||
finally:
|
|||
conn.close()
|
|||
except Exception as exc:
|
|||
print('ERROR')
|
|||
print(exec)
|
|||
sys.exit("Mail failed: {}".format(exc))
|
|||
def check_smart(devices):
|
|||
""" Check the SMART status of the passed list of block devices
|
|||
"""
|
|||
if type(devices) is not list:
|
|||
devices = [devices]
|
|||
errors = []
|
|||
output = []
|
|||
for device in devices:
|
|||
# Get information about this device and SMART health check; prune the
|
|||
# smartmontools banner from the output
|
|||
cmd = 'smartctl -iH /dev/{:s}'.format(device)
|
|||
cmd_out = subprocess.check_output(cmd, shell=True)
|
|||
raw_out = cmd_out.decode('utf-8').strip().split('\n')[4:]
|
|||
# Parse out the SMART "overall-health" line and status
|
|||
smart_output = raw_out[len(raw_out) - 1]
|
|||
smart_status = smart_output[50:]
|
|||
# Now check smart status
|
|||
if smart_status != 'PASSED':
|
|||
errors.append('/dev/{:s} SMART check failed'.format(device))
|
|||
raw_out.insert(0, '/dev/{:s}'.format(device))
|
|||
raw_out.append('{:s}\n'.format(SEP))
|
|||
output.extend(raw_out)
|
|||
#~ print('\n'.join(raw_out))
|
|||
return (errors, output)
|
|||
def check_raid(devices):
|
|||
""" Check the status of the passed list of software RAID devices
|
|||
"""
|
|||
if type(devices) is not list:
|
|||
devices = [devices]
|
|||
errors = []
|
|||
output = []
|
|||
for device in devices:
|
|||
# Get information about this device
|
|||
cmd = 'mdadm --detail /dev/{:s}'.format(device)
|
|||
cmd_out = subprocess.check_output(cmd, shell=True)
|
|||
raw_out = cmd_out.decode('utf-8').strip().split('\n')
|
|||
# Parse out the 'state' line
|
|||
raid_state = raw_out[11][18:].strip()
|
|||
if raid_state != 'clean':
|
|||
errors.append('/dev/{:s} not in "clean" state'.format(device))
|
|||
output.extend(raw_out)
|
|||
return (errors, output)
|
|||
def main():
|
|||
# Grab timestamp
|
|||
timestamp = datetime.datetime.now()
|
|||
ts_str = timestamp.strftime('%Y%m%d_%H%M')
|
|||
hostname = socket.gethostname()
|
|||
print('Checker for {:s} - {:s}\n\n'.format(hostname, ts_str))
|
|||
errors = []
|
|||
output = []
|
|||
output.append('Report for {:s} - {:s}\n'.format(hostname, ts_str))
|
|||
output.append('\n')
|
|||
# Report uptime
|
|||
output.append('Uptime:')
|
|||
cmd_out = subprocess.check_output('uptime', shell=True)
|
|||
output.append(cmd_out.decode('utf-8').strip())
|
|||
output.append('{:s}\n'.format(SEP))
|
|||
# Report df -h (disk utilization)
|
|||
cmd_out = subprocess.check_output('df -h', shell=True)
|
|||
output.append(cmd_out.decode('utf-8').strip())
|
|||
output.append('{:s}\n'.format(SEP))
|
|||
# Check block devices
|
|||
smart_errors, smart_output = check_smart(BLOCK_DEVS)
|
|||
errors.extend(smart_errors)
|
|||
output.extend(smart_output)
|
|||
# Check software RAID devices
|
|||
raid_errors, raid_output = check_raid(RAID_DEVS)
|
|||
errors.extend(raid_errors)
|
|||
output.extend(raid_output)
|
|||
# If no errors prepend "all okay" message
|
|||
send_email = False
|
|||
if not errors:
|
|||
output.insert(2, 'All OKAY.\n')
|
|||
# Send an email every Thursday even if status is okay
|
|||
if timestamp.weekday() == 3:
|
|||
send_email = True
|
|||
else:
|
|||
# Send email if there were any errors
|
|||
output.insert(2, '\n'.join(errors))
|
|||
send_email = True
|
|||
# Send email
|
|||
if send_email:
|
|||
send_email('Report: {:s} - {:s}\n'.format(hostname, ts_str),
|
|||
'\n'.join(output))
|
|||
if __name__ == '__main__':
|
|||
sys.exit(main())
|