commit 4c6410dcc94aa83fb163e3cc8ed8d15bbb987d62 Author: dsorber Date: Tue Jul 22 17:37:09 2014 -0400 Finished up the bookmarks importer and made the whole thing a package that I'm calling bmklib. I'm also preparing to start working on the web interface by copying the existing .wsgi file I created to use as a template. diff --git a/software/bookmark_library/lib/__init__.py b/software/bookmark_library/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/software/bookmark_library/lib/bookmark.py b/software/bookmark_library/lib/bookmark.py index b6dcb03..19f1ca4 100644 --- a/software/bookmark_library/lib/bookmark.py +++ b/software/bookmark_library/lib/bookmark.py @@ -1,5 +1,6 @@ import datetime import re +from urllib.error import HTTPError, URLError from urllib.request import urlopen import dateutil.parser @@ -202,14 +203,14 @@ class Bookmark(object): values.append(_format_date(getattr(self, key))) else: values.append(getattr(self, key)) - + + db = self._db_conn.cursor() if values and sets: # Add the record id as the last value values.append(self.id) # Build full SQL statement then execute it sql = SQL_UPDATE_BOOKMARK.format(','.join(sets)) - db = self._db_conn.cursor() db.execute(sql, values) # Reset dirty flags @@ -217,7 +218,7 @@ class Bookmark(object): self._dirty[key] = False # Inner function FTW - def update_tags_lists(delete_sql, insert_sql, tags): + def update_tag_lists(delete_sql, insert_sql, tags): db.execute(delete_sql, (self.id,)) tag_updates = [] for tag in tags: @@ -256,9 +257,16 @@ class Bookmark(object): def _get_content(self): """ Read the page and get its raw page HTML content. """ if self.url: - response = urlopen(self.url) - if response.getcode() == HTTP_OKAY: - self._content = response.read().decode('utf-8') + try: + response = urlopen(self.url) + if response.getcode() == HTTP_OKAY: + content = response.read() + try: + self._content = content.decode('utf-8') + except UnicodeDecodeError: + self._content = content.decode('ISO-8859-1') + except (HTTPError, URLError): + return '' def suggest_title(self): """ Return the contents of the page's tag as a title suggestion @@ -268,17 +276,20 @@ class Bookmark(object): try: title = re.search('<title>(.+)', self._content).group(1) - except IndexError: + except (IndexError, AttributeError, TypeError): title = None return title def is_reachable(self): """ Check if the page is reachable. """ if self.url: - response = urlopen(self.url) - if response.getcode() == HTTP_OKAY: - self.last_reachable = datetime.datetime.now() - return True + try: + response = urlopen(self.url) + if response.getcode() == HTTP_OKAY: + self.last_reachable = datetime.datetime.now() + return True + except (HTTPError, URLError): + return False return False def visit(self): diff --git a/software/bookmark_library/setup.py b/software/bookmark_library/setup.py new file mode 100644 index 0000000..eff29cc --- /dev/null +++ b/software/bookmark_library/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup, find_packages + +setup( + name = 'bmklib', + version = '0.1', + packages = find_packages(), + install_requires = ['dateutils>=0.6.6'], +) diff --git a/software/bookmark_library/utils/__init__.py b/software/bookmark_library/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/software/bookmark_library/utils/bookmark_converter.py b/software/bookmark_library/utils/bookmark_converter.py index 6aad230..547d35b 100644 --- a/software/bookmark_library/utils/bookmark_converter.py +++ b/software/bookmark_library/utils/bookmark_converter.py @@ -8,6 +8,7 @@ def main(): """ if len(sys.argv) < 2: print('ERROR: not enough arguments provided') + return -1 file_contents = open(sys.argv[1]).read() matches = re.finditer(' ' + '[ ...]') + def main(): - pass + """ Import bookmarks into the database from a bookmarks export. + + NOTE: If using a Google Chrome export with "date added" information, you + need to comment out the date_added field protections in bookmark.py. + + """ + + # Input error checking + if len(sys.argv) < 3: + print('ERROR: not enough arguments specified') + usage() + return -1 + + if not os.path.isfile(sys.argv[1]): + print('ERROR: unable to open database "{:s}"'.format(sys.argv[1])) + return -1 + + try: + input_file = open(sys.argv[2], 'r') + except FileNotFoundError: + print('ERROR: unable to open file "{:s}"'.format(sys.argv[2])) + return -1 + + # Count the number of lines in the file for output + num_lines = sum(1 for line in open(sys.argv[2]) if line.strip()) + + # Open a connection to the database + conn = sqlite3.connect('bookmarks.db') + + # Build list of category tags if needed + cat_tags = set() + if len(sys.argv) > 3: + for tag in sys.argv[3:len(sys.argv)]: + cat_tags.add(Tag(conn, value=tag)) + + # Iterate over the input file and create new + bmark_counter = 1 + for line in input_file: + if line: + print('Adding bookmark {:d} of {:d}'.format(bmark_counter, num_lines)) + + # Parse input line and create new bookmark + splits = line.strip().split() + bookmark = Bookmark(conn) + bookmark.url = splits[0] + bookmark.title = bookmark.suggest_title() + bookmark.is_reachable() + bookmark.category_tags = bookmark.category_tags.union(cat_tags) + + # If the date added timestamp (unix epoch format) is present, set + # it. + if len(splits) == 2: + added = datetime.datetime.fromtimestamp(int(splits[1])) + bookmark.date_added = added + + bookmark.save() + bmark_counter += 1 + if __name__ == '__main__': main() diff --git a/software/bookmark_library/web/__init__.py b/software/bookmark_library/web/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/software/bookmark_library/web/bmklib.wsgi b/software/bookmark_library/web/bmklib.wsgi new file mode 100644 index 0000000..81b60e4 --- /dev/null +++ b/software/bookmark_library/web/bmklib.wsgi @@ -0,0 +1,178 @@ +import cgi +import datetime +import os +import sqlite3 +from genshi.template import TemplateLoader +from genshi.template.base import Context + +BASE_PATH = '/home/dsorber/bfcs' +DB_PATH = os.path.join(BASE_PATH, 'beer_temps.db') +TEMPLATE_PATH = os.path.join(BASE_PATH, 'templates') + +# It accepts two arguments: +# environ points to a dictionary containing CGI like environment variables +# which is filled by the server for each received request from the client +# start_response is a callback function supplied by the server +# which will be used to send the HTTP status and headers to the server +def application(environ, start_response): + loader = TemplateLoader(TEMPLATE_PATH, auto_reload=True) + if not environ['PATH_INFO'] or environ['PATH_INFO'] == '/': + # Load the index template + template = loader.load('index.html') + # Create the response args + response_args = {'title': 'Beer Fermentation Control System'} + # Open up the connection to the database, execute the query, + # grab the results, then close the DB connection. + db_conn = sqlite3.connect(DB_PATH) + db = db_conn.cursor() + db.execute("select strftime('%m/%d/%Y', date(ts, 'unixepoch')), " + " count(ts), avg(temp1), max(temp1), min(temp1)," + " avg(temp2), max(temp2), min(temp2), avg(temp3)," + " max(temp3), min(temp3), avg(temp4), max(temp4)," + " min(temp4)" + "from temp_data " + "group by date(ts, 'unixepoch')") + response_args['results'] = db.fetchall() + db_conn.close() + # Generate the stream and then render it to HTML + stream = template.generate(**response_args) + response_body = stream.render('html', doctype='html', encoding='utf-8') + # Set the appropriate response headers + response_headers = [('Content-Type', 'text/html; charset=utf-8'), + ('Content-Length', str(len(response_body)))] + elif environ['PATH_INFO'].startswith('/day/') and environ['PATH_INFO'].endswith('/all'): + # Load the all template + template = loader.load('all.html') + # Parse the date out of the URL (TODO: validate all this) + day = environ['PATH_INFO'][5:7] + month = environ['PATH_INFO'][7:9] + year = environ['PATH_INFO'][9:13] + date = '%s/%s/%s' % (month, day, year) + # Create the response args + response_args = {'title': 'Sensor Readings for %s' % date} + # Open up the connection to the database, execute the query + # (limit to today and the past 5 minutes), + # grab the results, then close the DB connection. + db_conn = sqlite3.connect(DB_PATH) + db = db_conn.cursor() + db.execute("select strftime('%%H:%%M:%%S', ts, 'unixepoch'), " + " temp1, temp2, temp3, temp4, relay_status " + "from temp_data " + "where date(ts, 'unixepoch') is date('%s-%s-%s')" + "order by ts desc" % (year, month, day)) + response_args['results'] = db.fetchall() + db_conn.close() + # Generate the stream and then render it to HTML + stream = template.generate(**response_args) + response_body = stream.render('html', doctype='html', encoding='utf-8') + # Set the appropriate response headers + response_headers = [('Content-Type', 'text/html; charset=utf-8'), + ('Content-Length', str(len(response_body)))] + elif environ['PATH_INFO'].startswith('/day/'): + # Load the day template + template = loader.load('day.html') + + # Parse the date out of the URL (TODO: validate all this) + day = environ['PATH_INFO'][5:7] + month = environ['PATH_INFO'][7:9] + year = environ['PATH_INFO'][9:13] + date = '%s/%s/%s' % (month, day, year) + # Create the response args + response_args = {'title': 'Sensor Readings for %s' % date, + 'url_date': environ['PATH_INFO'][5:13]} + # Open up the connection to the database, execute the query + # (limit to today and the past 5 minutes), + # grab the results, then close the DB connection. + db_conn = sqlite3.connect(DB_PATH) + db = db_conn.cursor() + db.execute("select strftime('%%H:%%M:%%S', ts, 'unixepoch'), " + " temp1, temp2, temp3, temp4, relay_status " + "from temp_data " + "where date(ts, 'unixepoch') is date('%s-%s-%s')" + "order by ts desc limit 300" % (year, month, day)) + response_args['results'] = db.fetchall() + # Chart query + # + db.execute("select strftime('%%H:%%M:%%S', ts, 'unixepoch'), ts, temp1, " + " temp2, temp3, temp4 from ( " + "select ts, temp1, temp2, temp3, temp4 " + "from temp_data " + "where date(ts, 'unixepoch') is date('%s-%s-%s')" + "order by ts desc limit 60) order by ts" + % (year, month, day)) + chart_results = db.fetchall() + # Process chart query results + sensor1_data = [] + sensor2_data = [] + sensor3_data = [] + sensor4_data = [] + for row in chart_results: + sensor1_data.append([int(row[1]) * 1000, row[2]]) + sensor2_data.append([int(row[1]) * 1000, row[3]]) + sensor3_data.append([int(row[1]) * 1000, row[4]]) + sensor4_data.append([int(row[1]) * 1000, row[5]]) + response_args['sensor1_data'] = str(sensor1_data) + response_args['sensor2_data'] = str(sensor2_data) + response_args['sensor3_data'] = str(sensor3_data) + response_args['sensor4_data'] = str(sensor4_data) + # This is a bit of hack to get around a javascript issue in + # the template + # response_args['chart_results'] = chart_results[:-1] + # response_args['chart_last_row'] = chart_results[-1] + db_conn.close() + # Generate the stream and then render it to HTML + stream = template.generate(**response_args) + response_body = stream.render('html', doctype='html', encoding='utf-8') + # Set the appropriate response headers + response_headers = [('Content-Type', 'text/html; charset=utf-8'), + ('Content-Length', str(len(response_body)))] + elif environ['PATH_INFO'] == ('/chart_update'): + # Update the self updating chart + response_body = '' + if environ['QUERY_STRING']: + # Parse the query string to get the last timestamp from the client + get_args = cgi.parse_qs(environ['QUERY_STRING']) + last_ts = int(get_args['last_ts'][0]) + date_from_ts = datetime.datetime.fromtimestamp(last_ts) + # Reconstruct portions of the date from the ts + year = date_from_ts.year + month = '%02d' % date_from_ts.month + day = '%02d' % date_from_ts.day + + # Grab any entries from the supplied date that are greater than + # the last timestamp sent from the client javascript + db_conn = sqlite3.connect(DB_PATH) + db = db_conn.cursor() + db.execute("select strftime('%%H:%%M:%%S', ts, 'unixepoch'), ts, " + " temp1, temp2, temp3, temp4, relay_status " + "from temp_data " + "where date(ts, 'unixepoch') is date('%s-%s-%s') and " + " ts > %s " + "order by ts asc limit 60" % (year, month, day, last_ts)) + db_rows = db.fetchall() + db_conn.close() + # Iterate over the rows fetched by the query and pretty them up + # in preparation of being sent back to the client javascript for + # display + result_rows = [] + for row in db_rows: + result_rows.append('%s %s %s %s %s %s %s' % + (row[0], (int(row[1]) * 1000), + row[2], row[3], row[4], row[5], + int(row[6]) and 'on' or 'off')) + response_body = ','.join(result_rows) + response_body = response_body.encode('utf-8') + response_headers = [('Content-Type', 'text/plain; charset=utf-8'), + ('Content-Length', str(len(response_body)))] + else: + # User supplied an invalid URL + response_body = 'Invalid URL "%s"' % environ['PATH_INFO'] + response_headers = [('Content-Type', 'text/plain'), + ('Content-Length', str(len(response_body)))] + # HTTP response code and message + status = '200 OK' + # Send the status and response headers to the server + start_response(status, response_headers) + # Return the response body. (Notice it is wrapped in a list although it + # could be any iterable.) + return [response_body]