Project

General

Profile

Download (6.8 KB) Statistics
| Branch: | Tag: | Revision:

root/software/misc/checker.py @ 3dda5d70

0fc29199 David Sorber
#!/usr/bin/python3
import datetime
from email.mime.text import MIMEText
1136d406 David Sorber
import os
import re
0fc29199 David Sorber
from smtplib import SMTP_SSL
import socket
import subprocess
import sys

RELAY_EMAIL = 'dsorber.phalanx@gmail.com'
RELAY_PASSWD = "You'reNotMySupervisor"
TO_EMAIL = 'baranovich@gmail.com'

RAID_DEVS = ['md0']
6dbacc3b David Sorber
BLOCK_DEVS = ['sda', 'sdb', 'sdc', 'sdd', 'sde', 'sdf']
0fc29199 David Sorber
SEP = '-' * 80

def send_email(subject, msg_text):
msg = MIMEText(msg_text, 'plain')
msg['Subject'] = subject
msg['To'] = TO_EMAIL
try:
conn = SMTP_SSL('smtp.gmail.com')
conn.set_debuglevel(True)
conn.login(RELAY_EMAIL, RELAY_PASSWD)
try:
conn.sendmail(RELAY_EMAIL, TO_EMAIL, msg.as_string())
finally:
conn.close()

except Exception as exc:
print('ERROR')
print(exec)
sys.exit("Mail failed: {}".format(exc))

1136d406 David Sorber
class DiskDrive(object):
0fc29199 David Sorber
1136d406 David Sorber
def __init__(self, name, raid_status):
self.name = name
self.raid_status = raid_status
self.exists = os.path.exists(f'/dev/{self.name}')
self.serial_num = ''
0fc29199 David Sorber
1136d406 David Sorber
self.model_family = ''
self.device_model = ''
self.capacity = ''
self.rotation = ''
self.smart_status = 'unknown'
0fc29199 David Sorber
1136d406 David Sorber
self.error = not self.exists
0fc29199 David Sorber
1136d406 David Sorber
if self.exists:
# Set serial number:
cmd = f'hdparm -I /dev/{self.name} | grep "Serial\ Number"'
cmd_out = subprocess.check_output(cmd, shell=True)
cmd_out = cmd_out.decode('utf-8').strip()
self.serial_num = cmd_out.split(':')[1].strip()
# Check/set status:
self.check_status()
def check_status(self):
cmd = f'/usr/sbin/smartctl -iH /dev/{self.name}'
cmd_out = subprocess.check_output(cmd, shell=True)
out_list = cmd_out.decode('utf-8').strip().split('\n')
0fc29199 David Sorber
1136d406 David Sorber
# Parse info out of command output
self.model_family = out_list[4].split(':')[1].strip()
self.device_model = out_list[5].split(':')[1].strip()
self.capacity = out_list[9].split(':')[1].strip().split('[')[1][:-1]
self.rotation = out_list[11].split(':')[1].strip()
self.smart_status = out_list[21].split(':')[1].strip()
def okay(self):
if self.exists:
return (self.smart_status == 'PASSED')
else:
return False
0fc29199 David Sorber
1136d406 David Sorber
def output(self):
output = []
output.append(f'===[/dev/{self.name}]==================================================================')
output.append(f'Family : {self.model_family}')
output.append(f'Device : {self.device_model}')
output.append(f'Serial : {self.serial_num}')
output.append(f'Capacity : {self.capacity}')
output.append(f'Rotation : {self.rotation}')
output.append(f'SMART status : {self.smart_status}')
output.append('')
return '\n'.join(output)
0fc29199 David Sorber
1136d406 David Sorber
class RaidDevice(object):
0fc29199 David Sorber
1136d406 David Sorber
def __init__(self, name, num_drives_expected):
self.name = name
self.expected_num_drives = num_drives_expected
self.exists = os.path.exists(f'/dev/{self.name}')
self.raid_level = ''
self.status = 'unkown'
self.drives = []
self.mdadm_output = ''
self.error = not self.exists
self.error_msg = ''
def _refresh_mdadm_out(self):
if self.exists:
# Get information about this device
cmd = f'/sbin/mdadm --detail /dev/{self.name}'
cmd_out = subprocess.check_output(cmd, shell=True)
self.mdadm_output = cmd_out.decode('utf-8').strip().split('\n')
def check_status(self):
if self.exists:
# Refresh internal buffer containing "mdadm --detail" output
self._refresh_mdadm_out()
# Now parse the command output
self.raid_level = self.mdadm_output[3].split(':')[1].strip()
self.status = self.mdadm_output[11].split(':')[1].strip()
num_devices = int(self.mdadm_output[6].split(':')[1].strip())
if self.status != 'clean':
self.error = True
self.error_msg = f'RAID status not "clean": {self.status}'
if num_devices != self.expected_num_drives:
self.error = True
self.error_msg = f'{self.expected_num_drives} drives expected '\
f'but only {len(self.drives)} found'
# Now build DiskDevice instances
base_offset = 27
for idx in range (num_devices):
# This parsing is a little hairy...
line = self.mdadm_output[base_offset + idx]
tokens = re.sub(' +', ' ', line).strip().split(' ')[4:]
dev_name = tokens[-1].split('/')[2]
dev_status = ' '.join(tokens[:-1])
drive = DiskDrive(dev_name, dev_status)
self.drives.append(drive)
if not drive.okay():
self.error = True
self.error_msg = 'One or more drives failed SMART check!'
else:
self.status = 'device does not exist!'
def okay(self):
return (not self.error)
def output(self, hostname, timestamp):
output = []
output.append(f'Hostname : {hostname}')
output.append(f'Timestamp : {timestamp}')
output.append(f'RAID device : /dev/{self.name}')
output.append(f'Status : {self.error and "ERROR" or "OKAY"} '\
f'{self.error and " -- " + self.error_msg or ""}')
output.append(f'Level : {self.raid_level}')
output.append(f'Drives : {len(self.drives)}/{self.expected_num_drives}')
output.append('')
for drive in sorted(self.drives, key=lambda x: x.name):
output.append(drive.output())
return '\n'.join(output)
0fc29199 David Sorber
1136d406 David Sorber
0fc29199 David Sorber
def main():
# Grab timestamp
timestamp = datetime.datetime.now()
ts_str = timestamp.strftime('%Y%m%d_%H%M')
hostname = socket.gethostname()
1136d406 David Sorber
raid_dev = RaidDevice('md0', 5)
raid_dev.check_status()

output = raid_dev.output(hostname, ts_str)
print(output)
6dbacc3b David Sorber
1136d406 David Sorber
# Send an email if: there are any errors OR
# every Thursday at 3 AM even if status is okay
should_send = ((not raid_dev.okay()) or
(timestamp.weekday() == 3 and timestamp.hour == 3))
0fc29199 David Sorber
1136d406 David Sorber
if should_send:
print('Sending notification email...')
0fc29199 David Sorber
else:
1136d406 David Sorber
print('Not sending notification email.')
0fc29199 David Sorber
# Send email
c5211c85 David Sorber
if should_send:
1136d406 David Sorber
send_email(f'Report: {hostname} - {ts_str}\n', output)
return 0
0fc29199 David Sorber
if __name__ == '__main__':
sys.exit(main())