|
#!/usr/bin/python3
|
|
import datetime
|
|
from email.mime.text import MIMEText
|
|
import os
|
|
import re
|
|
from smtplib import SMTP_SSL
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
|
|
RELAY_EMAIL = 'dsorber.phalanx@gmail.com'
|
|
RELAY_PASSWD = "You'reNotMySupervisor"
|
|
TO_EMAIL = 'baranovich@gmail.com'
|
|
|
|
RAID_DEVS = ['md0']
|
|
BLOCK_DEVS = ['sda', 'sdb', 'sdc', 'sdd', 'sde', 'sdf']
|
|
|
|
SEP = '-' * 80
|
|
|
|
def send_email(subject, msg_text):
|
|
msg = MIMEText(msg_text, 'plain')
|
|
msg['Subject'] = subject
|
|
msg['To'] = TO_EMAIL
|
|
try:
|
|
conn = SMTP_SSL('smtp.gmail.com')
|
|
conn.set_debuglevel(True)
|
|
conn.login(RELAY_EMAIL, RELAY_PASSWD)
|
|
try:
|
|
conn.sendmail(RELAY_EMAIL, TO_EMAIL, msg.as_string())
|
|
finally:
|
|
conn.close()
|
|
|
|
except Exception as exc:
|
|
print('ERROR')
|
|
print(exec)
|
|
sys.exit("Mail failed: {}".format(exc))
|
|
|
|
class DiskDrive(object):
|
|
|
|
def __init__(self, name, raid_status):
|
|
self.name = name
|
|
self.raid_status = raid_status
|
|
self.exists = os.path.exists(f'/dev/{self.name}')
|
|
self.serial_num = ''
|
|
|
|
self.model_family = ''
|
|
self.device_model = ''
|
|
self.capacity = ''
|
|
self.rotation = ''
|
|
self.smart_status = 'unknown'
|
|
|
|
self.error = not self.exists
|
|
|
|
if self.exists:
|
|
# Set serial number:
|
|
cmd = f'hdparm -I /dev/{self.name} | grep "Serial\ Number"'
|
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
|
cmd_out = cmd_out.decode('utf-8').strip()
|
|
self.serial_num = cmd_out.split(':')[1].strip()
|
|
|
|
# Check/set status:
|
|
self.check_status()
|
|
|
|
def check_status(self):
|
|
cmd = f'/usr/sbin/smartctl -iH /dev/{self.name}'
|
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
|
out_list = cmd_out.decode('utf-8').strip().split('\n')
|
|
|
|
# Parse info out of command output
|
|
self.model_family = out_list[4].split(':')[1].strip()
|
|
self.device_model = out_list[5].split(':')[1].strip()
|
|
self.capacity = out_list[9].split(':')[1].strip().split('[')[1][:-1]
|
|
self.rotation = out_list[11].split(':')[1].strip()
|
|
self.smart_status = out_list[21].split(':')[1].strip()
|
|
|
|
def okay(self):
|
|
if self.exists:
|
|
return (self.smart_status == 'PASSED')
|
|
else:
|
|
return False
|
|
|
|
def output(self):
|
|
output = []
|
|
output.append(f'===[/dev/{self.name}]==================================================================')
|
|
output.append(f'Family : {self.model_family}')
|
|
output.append(f'Device : {self.device_model}')
|
|
output.append(f'Serial : {self.serial_num}')
|
|
output.append(f'Capacity : {self.capacity}')
|
|
output.append(f'Rotation : {self.rotation}')
|
|
output.append(f'SMART status : {self.smart_status}')
|
|
output.append('')
|
|
return '\n'.join(output)
|
|
|
|
|
|
class RaidDevice(object):
|
|
|
|
def __init__(self, name, num_drives_expected):
|
|
self.name = name
|
|
self.expected_num_drives = num_drives_expected
|
|
self.exists = os.path.exists(f'/dev/{self.name}')
|
|
self.raid_level = ''
|
|
self.status = 'unkown'
|
|
self.drives = []
|
|
self.mdadm_output = ''
|
|
self.error = not self.exists
|
|
self.error_msg = ''
|
|
|
|
def _refresh_mdadm_out(self):
|
|
if self.exists:
|
|
# Get information about this device
|
|
cmd = f'/sbin/mdadm --detail /dev/{self.name}'
|
|
cmd_out = subprocess.check_output(cmd, shell=True)
|
|
self.mdadm_output = cmd_out.decode('utf-8').strip().split('\n')
|
|
|
|
def check_status(self):
|
|
if self.exists:
|
|
# Refresh internal buffer containing "mdadm --detail" output
|
|
self._refresh_mdadm_out()
|
|
|
|
# Now parse the command output
|
|
self.raid_level = self.mdadm_output[3].split(':')[1].strip()
|
|
self.status = self.mdadm_output[11].split(':')[1].strip()
|
|
num_devices = int(self.mdadm_output[6].split(':')[1].strip())
|
|
|
|
if self.status != 'clean':
|
|
self.error = True
|
|
self.error_msg = f'RAID status not "clean": {self.status}'
|
|
|
|
if num_devices != self.expected_num_drives:
|
|
self.error = True
|
|
self.error_msg = f'{self.expected_num_drives} drives expected '\
|
|
f'but only {len(self.drives)} found'
|
|
|
|
# Now build DiskDevice instances
|
|
base_offset = 27
|
|
for idx in range (num_devices):
|
|
# This parsing is a little hairy...
|
|
line = self.mdadm_output[base_offset + idx]
|
|
tokens = re.sub(' +', ' ', line).strip().split(' ')[4:]
|
|
dev_name = tokens[-1].split('/')[2]
|
|
dev_status = ' '.join(tokens[:-1])
|
|
drive = DiskDrive(dev_name, dev_status)
|
|
self.drives.append(drive)
|
|
if not drive.okay():
|
|
self.error = True
|
|
self.error_msg = 'One or more drives failed SMART check!'
|
|
else:
|
|
self.status = 'device does not exist!'
|
|
|
|
def okay(self):
|
|
return (not self.error)
|
|
|
|
def output(self, hostname, timestamp):
|
|
output = []
|
|
output.append(f'Hostname : {hostname}')
|
|
output.append(f'Timestamp : {timestamp}')
|
|
output.append(f'RAID device : /dev/{self.name}')
|
|
output.append(f'Status : {self.error and "ERROR" or "OKAY"} '\
|
|
f'{self.error and " -- " + self.error_msg or ""}')
|
|
output.append(f'Level : {self.raid_level}')
|
|
output.append(f'Drives : {len(self.drives)}/{self.expected_num_drives}')
|
|
output.append('')
|
|
|
|
for drive in sorted(self.drives, key=lambda x: x.name):
|
|
output.append(drive.output())
|
|
return '\n'.join(output)
|
|
|
|
|
|
def main():
|
|
|
|
# Grab timestamp
|
|
timestamp = datetime.datetime.now()
|
|
ts_str = timestamp.strftime('%Y%m%d_%H%M')
|
|
hostname = socket.gethostname()
|
|
|
|
raid_dev = RaidDevice('md0', 5)
|
|
raid_dev.check_status()
|
|
|
|
output = raid_dev.output(hostname, ts_str)
|
|
print(output)
|
|
|
|
# Send an email if: there are any errors OR
|
|
# every Thursday at 3 AM even if status is okay
|
|
should_send = ((not raid_dev.okay()) or
|
|
(timestamp.weekday() == 3 and timestamp.hour == 3))
|
|
|
|
if should_send:
|
|
print('Sending notification email...')
|
|
else:
|
|
print('Not sending notification email.')
|
|
|
|
# Send email
|
|
if should_send:
|
|
send_email(f'Report: {hostname} - {ts_str}\n', output)
|
|
|
|
return 0
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|