|
#!/usr/bin/python3
|
|
import datetime
|
|
from email.mime.text import MIMEText
|
|
from smtplib import SMTP_SSL
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
|
|
RELAY_EMAIL = 'dsorber.phalanx@gmail.com'
|
|
RELAY_PASSWD = "You'reNotMySupervisor"
|
|
TO_EMAIL = 'baranovich@gmail.com'
|
|
|
|
RAID_DEVS = ['md0']
|
|
BLOCK_DEVS = ['sda', 'sdb', 'sdc', 'sdd', 'sde', 'sdf']
|
|
|
|
SEP = '-' * 80
|
|
|
|
def send_email(subject, msg_text):
|
|
msg = MIMEText(msg_text, 'plain')
|
|
msg['Subject'] = subject
|
|
msg['To'] = TO_EMAIL
|
|
try:
|
|
conn = SMTP_SSL('smtp.gmail.com')
|
|
conn.set_debuglevel(True)
|
|
conn.login(RELAY_EMAIL, RELAY_PASSWD)
|
|
try:
|
|
conn.sendmail(RELAY_EMAIL, TO_EMAIL, msg.as_string())
|
|
finally:
|
|
conn.close()
|
|
|
|
except Exception as exc:
|
|
print('ERROR')
|
|
print(exec)
|
|
sys.exit("Mail failed: {}".format(exc))
|
|
|
|
|
|
def check_smart(devices):
|
|
""" Check the SMART status of the passed list of block devices
|
|
"""
|
|
if type(devices) is not list:
|
|
devices = [devices]
|
|
|
|
errors = []
|
|
output = []
|
|
|
|
for device in devices:
|
|
# Get information about this device and SMART health check; prune the
|
|
# smartmontools banner from the output
|
|
cmd = '/usr/sbin/smartctl -iH /dev/{:s}'.format(device)
|
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
|
raw_out = cmd_out.decode('utf-8').strip().split('\n')[4:]
|
|
|
|
# Parse out the SMART "overall-health" line and status
|
|
smart_output = raw_out[len(raw_out) - 1]
|
|
smart_status = smart_output[50:]
|
|
|
|
# Now check smart status
|
|
if smart_status != 'PASSED':
|
|
errors.append('/dev/{:s} SMART check failed'.format(device))
|
|
|
|
raw_out.insert(0, '/dev/{:s}'.format(device))
|
|
raw_out.append('{:s}\n'.format(SEP))
|
|
output.extend(raw_out)
|
|
|
|
#~ print('\n'.join(raw_out))
|
|
|
|
return (errors, output)
|
|
|
|
def check_raid(devices):
|
|
""" Check the status of the passed list of software RAID devices
|
|
"""
|
|
if type(devices) is not list:
|
|
devices = [devices]
|
|
|
|
errors = []
|
|
output = []
|
|
|
|
for device in devices:
|
|
# Get information about this device
|
|
cmd = '/sbin/mdadm --detail /dev/{:s}'.format(device)
|
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
|
raw_out = cmd_out.decode('utf-8').strip().split('\n')
|
|
|
|
# Parse out the 'state' line
|
|
raid_state = raw_out[11][20:].strip()
|
|
if raid_state != 'clean':
|
|
errors.append('/dev/{:s} not in "clean" state ({:s})'
|
|
.format(device, raid_state))
|
|
|
|
output.extend(raw_out)
|
|
|
|
return (errors, output)
|
|
|
|
|
|
def main():
|
|
|
|
# Grab timestamp
|
|
timestamp = datetime.datetime.now()
|
|
ts_str = timestamp.strftime('%Y%m%d_%H%M')
|
|
|
|
hostname = socket.gethostname()
|
|
|
|
print('Checker for {:s} - {:s}\n\n'.format(hostname, ts_str))
|
|
|
|
errors = []
|
|
output = []
|
|
output.append('Report for {:s} - {:s}\n'.format(hostname, ts_str))
|
|
output.append('\n')
|
|
|
|
# Report uptime
|
|
output.append('Uptime:')
|
|
cmd_out = subprocess.check_output('uptime', shell=True)
|
|
output.append(cmd_out.decode('utf-8').strip())
|
|
output.append('{:s}\n'.format(SEP))
|
|
|
|
# Check software RAID devices
|
|
raid_errors, raid_output = check_raid(RAID_DEVS)
|
|
errors.extend(raid_errors)
|
|
output.extend(raid_output)
|
|
|
|
# Report df -h (disk utilization)
|
|
cmd_out = subprocess.check_output('df -h', shell=True)
|
|
output.append(cmd_out.decode('utf-8').strip())
|
|
output.append('{:s}\n'.format(SEP))
|
|
|
|
# Check block devices
|
|
smart_errors, smart_output = check_smart(BLOCK_DEVS)
|
|
errors.extend(smart_errors)
|
|
output.extend(smart_output)
|
|
|
|
# If no errors prepend "all okay" message
|
|
should_send = False
|
|
if not errors:
|
|
output.insert(2, 'All OKAY.\n')
|
|
|
|
# Send an email every Thursday at 3 AM even if status is okay
|
|
if timestamp.weekday() == 3 and timestamp.hour == 3:
|
|
should_send = True
|
|
else:
|
|
# Send email if there were any errors
|
|
output.insert(2, '\n'.join(errors))
|
|
should_send = True
|
|
|
|
# Send email
|
|
if should_send:
|
|
send_email('Report: {:s} - {:s}\n'.format(hostname, ts_str),
|
|
'\n'.join(output))
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|