add local version of filecache (used just for unittests), cache globally (to tmpdir)
This commit is contained in:
parent
17f3942f77
commit
e2a8f08581
184
trezorlib/filecache.py
Normal file
184
trezorlib/filecache.py
Normal file
@ -0,0 +1,184 @@
|
||||
'''
|
||||
filecache
|
||||
|
||||
filecache is a decorator which saves the return value of functions even
|
||||
after the interpreter dies. For example this is useful on functions that download
|
||||
and parse webpages. All you need to do is specify how long
|
||||
the return values should be cached (use seconds, like time.sleep).
|
||||
|
||||
USAGE:
|
||||
|
||||
from filecache import filecache
|
||||
|
||||
@filecache(24 * 60 * 60)
|
||||
def time_consuming_function(args):
|
||||
# etc
|
||||
|
||||
@filecache(filecache.YEAR)
|
||||
def another_function(args):
|
||||
# etc
|
||||
|
||||
|
||||
NOTE: All arguments of the decorated function and the return value need to be
|
||||
picklable for this to work.
|
||||
|
||||
NOTE: The cache isn't automatically cleaned, it is only overwritten. If your
|
||||
function can receive many different arguments that rarely repeat, your
|
||||
cache may forever grow. One day I might add a feature that once in every
|
||||
100 calls scans the db for outdated stuff and erases.
|
||||
|
||||
NOTE: This is less useful on methods of a class because the instance (self)
|
||||
is cached, and if the instance isn't the same, the cache isn't used. This
|
||||
makes sense because class methods are affected by changes in whatever
|
||||
is attached to self.
|
||||
|
||||
Tested on python 2.7 and 3.1
|
||||
|
||||
License: BSD, do what you wish with this. Could be awesome to hear if you found
|
||||
it useful and/or you have suggestions. ubershmekel at gmail
|
||||
|
||||
|
||||
A trick to invalidate a single value:
|
||||
|
||||
@filecache.filecache
|
||||
def somefunc(x, y, z):
|
||||
return x * y * z
|
||||
|
||||
del somefunc._db[filecache._args_key(somefunc, (1,2,3), {})]
|
||||
# or just iterate of somefunc._db (it's a shelve, like a dict) to find the right key.
|
||||
|
||||
|
||||
'''
|
||||
|
||||
|
||||
import collections as _collections
|
||||
import datetime as _datetime
|
||||
import functools as _functools
|
||||
import inspect as _inspect
|
||||
import os as _os
|
||||
import pickle as _pickle
|
||||
import shelve as _shelve
|
||||
import sys as _sys
|
||||
import time as _time
|
||||
import traceback as _traceback
|
||||
import types
|
||||
|
||||
_retval = _collections.namedtuple('_retval', 'timesig data')
|
||||
_SRC_DIR = _os.path.dirname(_os.path.abspath(__file__))
|
||||
|
||||
SECOND = 1
|
||||
MINUTE = 60 * SECOND
|
||||
HOUR = 60 * MINUTE
|
||||
DAY = 24 * HOUR
|
||||
WEEK = 7 * DAY
|
||||
MONTH = 30 * DAY
|
||||
YEAR = 365 * DAY
|
||||
FOREVER = None
|
||||
|
||||
OPEN_DBS = dict()
|
||||
|
||||
def _get_cache_name(function):
|
||||
"""
|
||||
returns a name for the module's cache db.
|
||||
"""
|
||||
module_name = _inspect.getfile(function)
|
||||
module_name = _os.path.abspath(module_name)
|
||||
cache_name = module_name
|
||||
|
||||
# fix for '<string>' or '<stdin>' in exec or interpreter usage.
|
||||
cache_name = cache_name.replace('<', '_lt_')
|
||||
cache_name = cache_name.replace('>', '_gt_')
|
||||
|
||||
tmpdir = _os.getenv('TMPDIR') or _os.getenv('TEMP') or _os.getenv('TMP')
|
||||
if tmpdir:
|
||||
cache_name = tmpdir + '/filecache_' + cache_name.replace(_os.sep, '@')
|
||||
|
||||
cache_name += '.cache'
|
||||
return cache_name
|
||||
|
||||
|
||||
def _log_error(error_str):
|
||||
try:
|
||||
error_log_fname = _os.path.join(_SRC_DIR, 'filecache.err.log')
|
||||
if _os.path.isfile(error_log_fname):
|
||||
fhand = open(error_log_fname, 'a')
|
||||
else:
|
||||
fhand = open(error_log_fname, 'w')
|
||||
fhand.write('[%s] %s\r\n' % (_datetime.datetime.now().isoformat(), error_str))
|
||||
fhand.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _args_key(function, args, kwargs):
|
||||
arguments = (args, kwargs)
|
||||
# Check if you have a valid, cached answer, and return it.
|
||||
# Sadly this is python version dependant
|
||||
if _sys.version_info[0] == 2:
|
||||
arguments_pickle = _pickle.dumps(arguments)
|
||||
else:
|
||||
# NOTE: protocol=0 so it's ascii, this is crucial for py3k
|
||||
# because shelve only works with proper strings.
|
||||
# Otherwise, we'd get an exception because
|
||||
# function.__name__ is str but dumps returns bytes.
|
||||
arguments_pickle = _pickle.dumps(arguments, protocol=0).decode('ascii')
|
||||
|
||||
key = function.__name__ + arguments_pickle
|
||||
return key
|
||||
|
||||
def filecache(seconds_of_validity=None, fail_silently=False):
|
||||
'''
|
||||
filecache is called and the decorator should be returned.
|
||||
'''
|
||||
def filecache_decorator(function):
|
||||
@_functools.wraps(function)
|
||||
def function_with_cache(*args, **kwargs):
|
||||
try:
|
||||
key = _args_key(function, args, kwargs)
|
||||
|
||||
if key in function._db:
|
||||
rv = function._db[key]
|
||||
if seconds_of_validity is None or _time.time() - rv.timesig < seconds_of_validity:
|
||||
return rv.data
|
||||
except Exception:
|
||||
# in any case of failure, don't let filecache break the program
|
||||
error_str = _traceback.format_exc()
|
||||
_log_error(error_str)
|
||||
if not fail_silently:
|
||||
raise
|
||||
|
||||
retval = function(*args, **kwargs)
|
||||
|
||||
# store in cache
|
||||
# NOTE: no need to _db.sync() because there was no mutation
|
||||
# NOTE: it's importatnt to do _db.sync() because otherwise the cache doesn't survive Ctrl-Break!
|
||||
try:
|
||||
function._db[key] = _retval(_time.time(), retval)
|
||||
function._db.sync()
|
||||
except Exception:
|
||||
# in any case of failure, don't let filecache break the program
|
||||
error_str = _traceback.format_exc()
|
||||
_log_error(error_str)
|
||||
if not fail_silently:
|
||||
raise
|
||||
|
||||
return retval
|
||||
|
||||
# make sure cache is loaded
|
||||
if not hasattr(function, '_db'):
|
||||
cache_name = _get_cache_name(function)
|
||||
if cache_name in OPEN_DBS:
|
||||
function._db = OPEN_DBS[cache_name]
|
||||
else:
|
||||
function._db = _shelve.open(cache_name)
|
||||
OPEN_DBS[cache_name] = function._db
|
||||
|
||||
function_with_cache._db = function._db
|
||||
|
||||
return function_with_cache
|
||||
|
||||
if type(seconds_of_validity) == types.FunctionType:
|
||||
# support for when people use '@filecache.filecache' instead of '@filecache.filecache()'
|
||||
func = seconds_of_validity
|
||||
return filecache_decorator(func)
|
||||
|
||||
return filecache_decorator
|
@ -2,16 +2,7 @@ import binascii
|
||||
import urllib2
|
||||
import json
|
||||
from decimal import Decimal
|
||||
try:
|
||||
raise Exception() # remove this line to enable caching
|
||||
from filecache import filecache, DAY
|
||||
except:
|
||||
def filecache(x):
|
||||
def _inner(y):
|
||||
return y
|
||||
return _inner
|
||||
DAY = None
|
||||
|
||||
from filecache import filecache, DAY
|
||||
import types_pb2 as proto_types
|
||||
|
||||
def op_push_data(data):
|
||||
|
Loading…
Reference in New Issue
Block a user