root/software/bashquote/bashquote_populate @ ba1e6b20
| ba1e6b20 | dsorber | #!/usr/bin/env python
|
|
import cPickle
|
|||
import os
|
|||
import re
|
|||
import sys
|
|||
import urllib
|
|||
BQ_DIR = '/Users/dsorber/.bashquote'
|
|||
BQ_CACHE = '/Users/dsorber/.bashquote/bq_cache'
|
|||
def main():
|
|||
# Create a list to hold the quotes
|
|||
quotes = []
|
|||
# Create the bashquote directory if it does not exist
|
|||
if not os.path.exists(BQ_DIR):
|
|||
print 'Creating bashquote directory...',
|
|||
os.mkdir(BQ_DIR)
|
|||
print 'Done'
|
|||
# Create the cache file, or open it if it already exists
|
|||
if not os.path.exists(BQ_CACHE):
|
|||
print 'Creating bashquote cache...',
|
|||
cache = open(BQ_CACHE, 'w')
|
|||
cache.close()
|
|||
print 'Done'
|
|||
else:
|
|||
cache = open(BQ_CACHE, 'r')
|
|||
quotes = cPickle.load(cache)
|
|||
cache.close()
|
|||
# Only grab fresh quotes if there are less than 50 in the cache
|
|||
if not len(quotes) < 50:
|
|||
print 'plenty left'
|
|||
sys.exit()
|
|||
# Goto to the random >0 bash quotes page
|
|||
listpage = urllib.urlopen("http://www.bash.org/?random1")
|
|||
listpagetext = listpage.read()
|
|||
quote_regex = '<p class="quote".+<b>#(\d+)</b>.+</a>\((\d+)\)<a .+<p class="qt">([\s\S]+?)</p>'
|
|||
quotes = []
|
|||
for quote_match in re.finditer(quote_regex, listpagetext):
|
|||
quotes.append((quote_match.group(1),
|
|||
quote_match.group(2),
|
|||
unhtmlify(quote_match.group(3))))
|
|||
# Store the list of quotes in the cache file, using the highest protocol
|
|||
cache = open(BQ_CACHE, 'w')
|
|||
cPickle.dump(quotes, cache, -1)
|
|||
cache.close()
|
|||
sys.exit()
|
|||
def unhtmlify(quote):
|
|||
# Filter out that HTML bullshit
|
|||
return quote.replace("<", "<").replace("\'<", "<")\
|
|||
.replace(">", ">").replace(""", "\"")\
|
|||
.replace(" ", " ").replace("\\'", "'")\
|
|||
.replace("\\x92", "'").replace("<br />\\r\\n', '", "\n")\
|
|||
.replace("<br />\\r\\n\", '", "\n").replace("&", "&")\
|
|||
.replace("<br />\\r\\n', \"", "\n")\
|
|||
.replace("<br />\\r\\n\", \"", "\n").replace("<br />", "")
|
|||
if __name__=="__main__":
|
|||
main()
|