root/software/misc/checker.py @ 5e80c35d
| 0fc29199 | David Sorber | #!/usr/bin/python3
|
||
import datetime
|
||||
from email.mime.text import MIMEText
|
||||
from smtplib import SMTP_SSL
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
RELAY_EMAIL = 'dsorber.phalanx@gmail.com'
|
||||
RELAY_PASSWD = "You'reNotMySupervisor"
|
||||
TO_EMAIL = 'baranovich@gmail.com'
|
||||
RAID_DEVS = ['md0']
|
||||
| 6dbacc3b | David Sorber | BLOCK_DEVS = ['sda', 'sdb', 'sdc', 'sdd', 'sde', 'sdf']
|
||
| 0fc29199 | David Sorber | |||
SEP = '-' * 80
|
||||
def send_email(subject, msg_text):
|
||||
msg = MIMEText(msg_text, 'plain')
|
||||
msg['Subject'] = subject
|
||||
msg['To'] = TO_EMAIL
|
||||
try:
|
||||
conn = SMTP_SSL('smtp.gmail.com')
|
||||
conn.set_debuglevel(True)
|
||||
conn.login(RELAY_EMAIL, RELAY_PASSWD)
|
||||
try:
|
||||
conn.sendmail(RELAY_EMAIL, TO_EMAIL, msg.as_string())
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
print('ERROR')
|
||||
print(exec)
|
||||
sys.exit("Mail failed: {}".format(exc))
|
||||
def check_smart(devices):
|
||||
""" Check the SMART status of the passed list of block devices
|
||||
"""
|
||||
if type(devices) is not list:
|
||||
devices = [devices]
|
||||
errors = []
|
||||
output = []
|
||||
for device in devices:
|
||||
# Get information about this device and SMART health check; prune the
|
||||
# smartmontools banner from the output
|
||||
| d7fc31b1 | David Sorber | cmd = '/usr/sbin/smartctl -iH /dev/{:s}'.format(device)
|
||
| 0fc29199 | David Sorber | cmd_out = subprocess.check_output(cmd, shell=True)
|
||
raw_out = cmd_out.decode('utf-8').strip().split('\n')[4:]
|
||||
# Parse out the SMART "overall-health" line and status
|
||||
smart_output = raw_out[len(raw_out) - 1]
|
||||
smart_status = smart_output[50:]
|
||||
# Now check smart status
|
||||
if smart_status != 'PASSED':
|
||||
errors.append('/dev/{:s} SMART check failed'.format(device))
|
||||
raw_out.insert(0, '/dev/{:s}'.format(device))
|
||||
raw_out.append('{:s}\n'.format(SEP))
|
||||
output.extend(raw_out)
|
||||
#~ print('\n'.join(raw_out))
|
||||
return (errors, output)
|
||||
def check_raid(devices):
|
||||
""" Check the status of the passed list of software RAID devices
|
||||
"""
|
||||
if type(devices) is not list:
|
||||
devices = [devices]
|
||||
errors = []
|
||||
output = []
|
||||
for device in devices:
|
||||
# Get information about this device
|
||||
| d7fc31b1 | David Sorber | cmd = '/sbin/mdadm --detail /dev/{:s}'.format(device)
|
||
| 0fc29199 | David Sorber | cmd_out = subprocess.check_output(cmd, shell=True)
|
||
raw_out = cmd_out.decode('utf-8').strip().split('\n')
|
||||
# Parse out the 'state' line
|
||||
raid_state = raw_out[11][18:].strip()
|
||||
if raid_state != 'clean':
|
||||
errors.append('/dev/{:s} not in "clean" state'.format(device))
|
||||
output.extend(raw_out)
|
||||
return (errors, output)
|
||||
def main():
|
||||
# Grab timestamp
|
||||
timestamp = datetime.datetime.now()
|
||||
ts_str = timestamp.strftime('%Y%m%d_%H%M')
|
||||
hostname = socket.gethostname()
|
||||
print('Checker for {:s} - {:s}\n\n'.format(hostname, ts_str))
|
||||
errors = []
|
||||
output = []
|
||||
output.append('Report for {:s} - {:s}\n'.format(hostname, ts_str))
|
||||
output.append('\n')
|
||||
# Report uptime
|
||||
output.append('Uptime:')
|
||||
cmd_out = subprocess.check_output('uptime', shell=True)
|
||||
output.append(cmd_out.decode('utf-8').strip())
|
||||
output.append('{:s}\n'.format(SEP))
|
||||
| 6dbacc3b | David Sorber | # Check software RAID devices
|
||
raid_errors, raid_output = check_raid(RAID_DEVS)
|
||||
errors.extend(raid_errors)
|
||||
output.extend(raid_output)
|
||||
| 0fc29199 | David Sorber | # Report df -h (disk utilization)
|
||
cmd_out = subprocess.check_output('df -h', shell=True)
|
||||
output.append(cmd_out.decode('utf-8').strip())
|
||||
output.append('{:s}\n'.format(SEP))
|
||||
# Check block devices
|
||||
smart_errors, smart_output = check_smart(BLOCK_DEVS)
|
||||
errors.extend(smart_errors)
|
||||
output.extend(smart_output)
|
||||
# If no errors prepend "all okay" message
|
||||
| c5211c85 | David Sorber | should_send = False
|
||
| 0fc29199 | David Sorber | if not errors:
|
||
output.insert(2, 'All OKAY.\n')
|
||||
| d7fc31b1 | David Sorber | # Send an email every Thursday at 3 AM even if status is okay
|
||
if timestamp.weekday() == 3 and timestamp.hour == 3:
|
||||
| c5211c85 | David Sorber | should_send = True
|
||
| 0fc29199 | David Sorber | else:
|
||
# Send email if there were any errors
|
||||
output.insert(2, '\n'.join(errors))
|
||||
| c5211c85 | David Sorber | should_send = True
|
||
| 0fc29199 | David Sorber | |||
# Send email
|
||||
| c5211c85 | David Sorber | if should_send:
|
||
| 0fc29199 | David Sorber | send_email('Report: {:s} - {:s}\n'.format(hostname, ts_str),
|
||
'\n'.join(output))
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|