2015-04-13 13:39:23 +00:00
|
|
|
'''
|
|
|
|
filecache
|
|
|
|
|
|
|
|
filecache is a decorator which saves the return value of functions even
|
|
|
|
after the interpreter dies. For example this is useful on functions that download
|
|
|
|
and parse webpages. All you need to do is specify how long
|
|
|
|
the return values should be cached (use seconds, like time.sleep).
|
|
|
|
|
|
|
|
USAGE:
|
|
|
|
|
|
|
|
from filecache import filecache
|
|
|
|
|
|
|
|
@filecache(24 * 60 * 60)
|
|
|
|
def time_consuming_function(args):
|
|
|
|
# etc
|
|
|
|
|
|
|
|
@filecache(filecache.YEAR)
|
|
|
|
def another_function(args):
|
|
|
|
# etc
|
|
|
|
|
|
|
|
|
|
|
|
NOTE: All arguments of the decorated function and the return value need to be
|
|
|
|
picklable for this to work.
|
|
|
|
|
|
|
|
NOTE: The cache isn't automatically cleaned, it is only overwritten. If your
|
|
|
|
function can receive many different arguments that rarely repeat, your
|
|
|
|
cache may forever grow. One day I might add a feature that once in every
|
|
|
|
100 calls scans the db for outdated stuff and erases.
|
|
|
|
|
|
|
|
NOTE: This is less useful on methods of a class because the instance (self)
|
|
|
|
is cached, and if the instance isn't the same, the cache isn't used. This
|
|
|
|
makes sense because class methods are affected by changes in whatever
|
|
|
|
is attached to self.
|
|
|
|
|
|
|
|
Tested on python 2.7 and 3.1
|
|
|
|
|
|
|
|
License: BSD, do what you wish with this. Could be awesome to hear if you found
|
|
|
|
it useful and/or you have suggestions. ubershmekel at gmail
|
|
|
|
|
|
|
|
|
|
|
|
A trick to invalidate a single value:
|
|
|
|
|
|
|
|
@filecache.filecache
|
|
|
|
def somefunc(x, y, z):
|
|
|
|
return x * y * z
|
|
|
|
|
|
|
|
del somefunc._db[filecache._args_key(somefunc, (1,2,3), {})]
|
|
|
|
# or just iterate of somefunc._db (it's a shelve, like a dict) to find the right key.
|
|
|
|
|
|
|
|
|
|
|
|
'''
|
|
|
|
|
|
|
|
|
|
|
|
import collections as _collections
|
|
|
|
import datetime as _datetime
|
|
|
|
import functools as _functools
|
|
|
|
import inspect as _inspect
|
|
|
|
import os as _os
|
|
|
|
import pickle as _pickle
|
|
|
|
import shelve as _shelve
|
|
|
|
import sys as _sys
|
|
|
|
import time as _time
|
|
|
|
import traceback as _traceback
|
|
|
|
import types
|
2016-05-19 09:50:33 +00:00
|
|
|
import atexit
|
2015-04-13 13:39:23 +00:00
|
|
|
|
|
|
|
_retval = _collections.namedtuple('_retval', 'timesig data')
|
|
|
|
_SRC_DIR = _os.path.dirname(_os.path.abspath(__file__))
|
|
|
|
|
|
|
|
SECOND = 1
|
|
|
|
MINUTE = 60 * SECOND
|
|
|
|
HOUR = 60 * MINUTE
|
|
|
|
DAY = 24 * HOUR
|
|
|
|
WEEK = 7 * DAY
|
|
|
|
MONTH = 30 * DAY
|
|
|
|
YEAR = 365 * DAY
|
|
|
|
FOREVER = None
|
|
|
|
|
|
|
|
OPEN_DBS = dict()
|
|
|
|
|
|
|
|
def _get_cache_name(function):
|
|
|
|
"""
|
|
|
|
returns a name for the module's cache db.
|
|
|
|
"""
|
|
|
|
module_name = _inspect.getfile(function)
|
|
|
|
module_name = _os.path.abspath(module_name)
|
|
|
|
cache_name = module_name
|
|
|
|
|
|
|
|
# fix for '<string>' or '<stdin>' in exec or interpreter usage.
|
|
|
|
cache_name = cache_name.replace('<', '_lt_')
|
|
|
|
cache_name = cache_name.replace('>', '_gt_')
|
|
|
|
|
|
|
|
tmpdir = _os.getenv('TMPDIR') or _os.getenv('TEMP') or _os.getenv('TMP')
|
|
|
|
if tmpdir:
|
|
|
|
cache_name = tmpdir + '/filecache_' + cache_name.replace(_os.sep, '@')
|
|
|
|
|
|
|
|
cache_name += '.cache'
|
|
|
|
return cache_name
|
|
|
|
|
|
|
|
|
|
|
|
def _log_error(error_str):
|
|
|
|
try:
|
|
|
|
error_log_fname = _os.path.join(_SRC_DIR, 'filecache.err.log')
|
|
|
|
if _os.path.isfile(error_log_fname):
|
|
|
|
fhand = open(error_log_fname, 'a')
|
|
|
|
else:
|
|
|
|
fhand = open(error_log_fname, 'w')
|
|
|
|
fhand.write('[%s] %s\r\n' % (_datetime.datetime.now().isoformat(), error_str))
|
|
|
|
fhand.close()
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
|
|
|
|
def _args_key(function, args, kwargs):
|
|
|
|
arguments = (args, kwargs)
|
|
|
|
# Check if you have a valid, cached answer, and return it.
|
|
|
|
# Sadly this is python version dependant
|
|
|
|
if _sys.version_info[0] == 2:
|
|
|
|
arguments_pickle = _pickle.dumps(arguments)
|
|
|
|
else:
|
|
|
|
# NOTE: protocol=0 so it's ascii, this is crucial for py3k
|
|
|
|
# because shelve only works with proper strings.
|
|
|
|
# Otherwise, we'd get an exception because
|
|
|
|
# function.__name__ is str but dumps returns bytes.
|
|
|
|
arguments_pickle = _pickle.dumps(arguments, protocol=0).decode('ascii')
|
|
|
|
|
|
|
|
key = function.__name__ + arguments_pickle
|
|
|
|
return key
|
|
|
|
|
|
|
|
def filecache(seconds_of_validity=None, fail_silently=False):
|
|
|
|
'''
|
|
|
|
filecache is called and the decorator should be returned.
|
|
|
|
'''
|
|
|
|
def filecache_decorator(function):
|
|
|
|
@_functools.wraps(function)
|
|
|
|
def function_with_cache(*args, **kwargs):
|
|
|
|
try:
|
|
|
|
key = _args_key(function, args, kwargs)
|
|
|
|
|
|
|
|
if key in function._db:
|
|
|
|
rv = function._db[key]
|
|
|
|
if seconds_of_validity is None or _time.time() - rv.timesig < seconds_of_validity:
|
|
|
|
return rv.data
|
|
|
|
except Exception:
|
|
|
|
# in any case of failure, don't let filecache break the program
|
|
|
|
error_str = _traceback.format_exc()
|
|
|
|
_log_error(error_str)
|
|
|
|
if not fail_silently:
|
|
|
|
raise
|
|
|
|
|
|
|
|
retval = function(*args, **kwargs)
|
|
|
|
|
|
|
|
# store in cache
|
|
|
|
# NOTE: no need to _db.sync() because there was no mutation
|
|
|
|
# NOTE: it's importatnt to do _db.sync() because otherwise the cache doesn't survive Ctrl-Break!
|
|
|
|
try:
|
|
|
|
function._db[key] = _retval(_time.time(), retval)
|
|
|
|
function._db.sync()
|
|
|
|
except Exception:
|
|
|
|
# in any case of failure, don't let filecache break the program
|
|
|
|
error_str = _traceback.format_exc()
|
|
|
|
_log_error(error_str)
|
|
|
|
if not fail_silently:
|
|
|
|
raise
|
|
|
|
|
|
|
|
return retval
|
|
|
|
|
|
|
|
# make sure cache is loaded
|
|
|
|
if not hasattr(function, '_db'):
|
|
|
|
cache_name = _get_cache_name(function)
|
|
|
|
if cache_name in OPEN_DBS:
|
|
|
|
function._db = OPEN_DBS[cache_name]
|
|
|
|
else:
|
|
|
|
function._db = _shelve.open(cache_name)
|
|
|
|
OPEN_DBS[cache_name] = function._db
|
2016-05-19 09:50:33 +00:00
|
|
|
atexit.register(function._db.close)
|
2015-04-13 13:39:23 +00:00
|
|
|
|
|
|
|
function_with_cache._db = function._db
|
|
|
|
|
|
|
|
return function_with_cache
|
|
|
|
|
|
|
|
if type(seconds_of_validity) == types.FunctionType:
|
|
|
|
# support for when people use '@filecache.filecache' instead of '@filecache.filecache()'
|
|
|
|
func = seconds_of_validity
|
2016-05-19 09:50:33 +00:00
|
|
|
seconds_of_validity = None
|
2015-04-13 13:39:23 +00:00
|
|
|
return filecache_decorator(func)
|
|
|
|
|
|
|
|
return filecache_decorator
|