2020-09-03 21:11:58 +00:00
|
|
|
"""
|
|
|
|
Storage for both keys and data
|
|
|
|
"""
|
2020-09-06 17:59:38 +00:00
|
|
|
import os,sys; sys.path.append(os.path.abspath(os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__),'..')),'..')))
|
2020-09-29 13:09:56 +00:00
|
|
|
from comrad import *
|
2020-09-25 03:13:14 +00:00
|
|
|
# from simplekv.fs import FilesystemStore
|
2020-09-24 15:46:34 +00:00
|
|
|
# from simplekv.memory.redisstore import RedisStore
|
|
|
|
# import redis
|
2020-09-03 21:11:58 +00:00
|
|
|
import hashlib,os
|
|
|
|
import zlib
|
2020-09-17 14:36:10 +00:00
|
|
|
from pythemis.exception import ThemisError
|
2020-09-25 03:13:14 +00:00
|
|
|
# from vedis import Vedis
|
|
|
|
# from walrus.tusks.rlite import WalrusLite
|
|
|
|
import hirlite
|
2020-09-06 17:59:38 +00:00
|
|
|
|
2020-09-26 12:11:56 +00:00
|
|
|
LOG_GET_SET = 0
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
2020-09-17 08:16:52 +00:00
|
|
|
|
|
|
|
|
2020-09-04 12:46:22 +00:00
|
|
|
class Crypt(Logger):
|
2020-09-17 14:22:41 +00:00
|
|
|
def __init__(self,
|
|
|
|
name=None,
|
|
|
|
fn=None,
|
|
|
|
use_secret=CRYPT_USE_SECRET,
|
|
|
|
path_secret=PATH_CRYPT_SECRET,
|
2020-09-17 15:15:15 +00:00
|
|
|
encrypt_values=False,
|
2020-09-17 14:56:56 +00:00
|
|
|
encryptor_func=None,
|
|
|
|
decryptor_func=None):
|
2020-09-17 14:22:41 +00:00
|
|
|
|
|
|
|
# defaults
|
2020-09-03 21:11:58 +00:00
|
|
|
if not name and fn: name=os.path.basename(fn).replace('.','_')
|
2020-09-17 14:22:41 +00:00
|
|
|
self.name,self.fn=name,fn
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-17 14:56:56 +00:00
|
|
|
|
2020-09-17 14:22:41 +00:00
|
|
|
# use secret? for salting
|
2020-09-10 09:00:11 +00:00
|
|
|
if use_secret and path_secret:
|
|
|
|
if not os.path.exists(path_secret):
|
|
|
|
self.secret = get_random_binary_id()
|
2020-09-29 13:09:56 +00:00
|
|
|
from comrad.backend.keymaker import make_key_discreet
|
2020-09-12 15:02:29 +00:00
|
|
|
self.log('shhh! creating secret:',make_key_discreet(self.secret))
|
2020-09-10 09:00:11 +00:00
|
|
|
with open(path_secret,'wb') as of:
|
|
|
|
of.write(self.secret)
|
|
|
|
else:
|
|
|
|
with open(path_secret,'rb') as f:
|
|
|
|
self.secret = f.read()
|
|
|
|
else:
|
|
|
|
self.secret = b''
|
2020-09-12 20:46:49 +00:00
|
|
|
self.encrypt_values = encrypt_values
|
2020-09-17 14:56:56 +00:00
|
|
|
if self.secret and encrypt_values and (not encryptor_func or not decryptor_func):
|
2020-09-29 13:09:56 +00:00
|
|
|
from comrad.backend.keymaker import ComradSymmetricKeyWithPassphrase
|
|
|
|
self.key = ComradSymmetricKeyWithPassphrase(
|
2020-09-17 14:56:56 +00:00
|
|
|
passphrase=self.secret
|
|
|
|
)
|
|
|
|
encryptor_func = self.key.encrypt
|
|
|
|
decryptor_func = self.key.decrypt
|
2020-09-17 14:22:41 +00:00
|
|
|
self.encryptor_func=encryptor_func
|
|
|
|
self.decryptor_func=decryptor_func
|
2020-09-18 15:52:19 +00:00
|
|
|
|
2020-09-25 03:13:14 +00:00
|
|
|
# self.store = FilesystemStore(self.fn)
|
2020-09-24 15:46:34 +00:00
|
|
|
# self.store = RedisStore(redis.StrictRedis())
|
2020-09-25 03:13:14 +00:00
|
|
|
# self.db = Vedis(self.fn)
|
|
|
|
# self.db = WalrusLite(self.fn)
|
|
|
|
self.db = hirlite.Rlite(path=self.fn)
|
2020-09-17 14:22:41 +00:00
|
|
|
|
2020-09-04 12:46:22 +00:00
|
|
|
|
2020-09-29 08:58:24 +00:00
|
|
|
def log(self,*x,**y):
|
2020-09-04 12:46:22 +00:00
|
|
|
if LOG_GET_SET:
|
|
|
|
super().log(*x)
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
def hash(self,binary_data):
|
2020-09-25 03:13:14 +00:00
|
|
|
# return binary_data
|
2020-09-12 14:32:03 +00:00
|
|
|
return hasher(binary_data,self.secret)
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
def force_binary(self,k_b):
|
2020-09-06 18:40:56 +00:00
|
|
|
if k_b is None: return None
|
2020-09-03 21:11:58 +00:00
|
|
|
if type(k_b)==str: k_b=k_b.encode()
|
2020-09-06 14:45:40 +00:00
|
|
|
if type(k_b)!=bytes: k_b=k_b.decode()
|
2020-09-03 21:11:58 +00:00
|
|
|
return k_b
|
|
|
|
|
2020-09-04 00:06:47 +00:00
|
|
|
def package_key(self,k,prefix=''):
|
2020-09-06 18:43:18 +00:00
|
|
|
if not k: return b''
|
2020-09-28 19:31:36 +00:00
|
|
|
|
2020-09-03 21:11:58 +00:00
|
|
|
k_b = self.force_binary(k)
|
2020-09-06 18:12:18 +00:00
|
|
|
k_b2 = self.force_binary(prefix) + k_b
|
2020-09-06 18:24:51 +00:00
|
|
|
return k_b2
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-17 14:36:10 +00:00
|
|
|
def package_val(self,k,encrypt=None):
|
|
|
|
if encrypt is None: encrypt=self.encrypt_values
|
2020-09-03 21:11:58 +00:00
|
|
|
k_b = self.force_binary(k)
|
2020-09-17 14:36:10 +00:00
|
|
|
if encrypt:
|
|
|
|
try:
|
|
|
|
k_b = self.encryptor_func(k_b)
|
|
|
|
except ThemisError as e:
|
|
|
|
self.log('!! ENCRYPTION ERROR:',e)
|
2020-09-12 20:19:53 +00:00
|
|
|
return k_b
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-17 14:36:10 +00:00
|
|
|
def unpackage_val(self,k_b,encrypt=None):
|
|
|
|
if encrypt is None: encrypt=self.encrypt_values
|
2020-09-17 14:43:31 +00:00
|
|
|
if encrypt:
|
2020-09-17 14:36:10 +00:00
|
|
|
try:
|
|
|
|
k_b = self.decryptor_func(k_b)
|
2020-09-17 14:40:44 +00:00
|
|
|
except ThemisError as e:
|
2020-09-17 14:36:10 +00:00
|
|
|
self.log('!! DECRYPTION ERROR:',e)
|
2020-09-12 20:19:53 +00:00
|
|
|
return k_b
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-10 09:00:11 +00:00
|
|
|
def has(self,k,prefix=''):
|
2020-09-25 03:13:14 +00:00
|
|
|
got=self.get(k,prefix=prefix)
|
|
|
|
# self.log('has got',got)
|
|
|
|
return bool(got)
|
2020-09-10 09:00:11 +00:00
|
|
|
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-17 14:36:10 +00:00
|
|
|
def set(self,k,v,prefix='',override=False,encrypt=True):
|
2020-09-15 06:16:27 +00:00
|
|
|
if self.has(k,prefix=prefix) and not override:
|
2020-09-17 14:22:41 +00:00
|
|
|
self.log(f"I'm afraid I can't let you do that, overwrite someone's data!\n\nat {prefix}{k} = {v}")
|
2020-09-13 10:53:17 +00:00
|
|
|
return False #(False,None,None)
|
2020-09-10 09:00:11 +00:00
|
|
|
|
2020-09-04 00:06:47 +00:00
|
|
|
k_b=self.package_key(k,prefix=prefix)
|
2020-09-06 18:24:51 +00:00
|
|
|
k_b_hash = self.hash(k_b)
|
2020-09-17 14:36:10 +00:00
|
|
|
v_b=self.package_val(v,encrypt = (self.encrypt_values and encrypt))
|
2020-09-15 06:42:42 +00:00
|
|
|
if not override:
|
|
|
|
self.log(f'''Crypt.set(\n\t{k_b}\n\n\t{k_b_hash}\n\n\t{v_b}\n)''')
|
2020-09-25 03:13:14 +00:00
|
|
|
|
|
|
|
#self.store.put(k_b_hash,v_b)
|
|
|
|
#with self.db.transaction():
|
|
|
|
# self.db[k_b_hash]=v_b
|
|
|
|
return self.db.command('set',k_b_hash,v_b)
|
|
|
|
# return True
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-05 14:09:31 +00:00
|
|
|
def exists(self,k,prefix=''):
|
2020-09-10 09:00:11 +00:00
|
|
|
return self.has(k,prefix=prefix)
|
2020-09-05 14:09:31 +00:00
|
|
|
|
2020-09-10 12:52:07 +00:00
|
|
|
def key2hash(self,k,prefix=''):
|
|
|
|
return self.hash(
|
|
|
|
self.package_key(
|
|
|
|
prefix + k
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
2020-09-14 04:43:59 +00:00
|
|
|
def delete(self,k,prefix=''):
|
|
|
|
k_b=self.package_key(k,prefix=prefix)
|
|
|
|
k_b_hash = self.hash(k_b)
|
2020-09-25 03:13:14 +00:00
|
|
|
|
|
|
|
with self.db.transaction():
|
|
|
|
del self.db[k_b_hash]
|
|
|
|
|
|
|
|
return True
|
2020-09-14 04:43:59 +00:00
|
|
|
|
|
|
|
|
2020-09-04 00:06:47 +00:00
|
|
|
def get(self,k,prefix=''):
|
|
|
|
k_b=self.package_key(k,prefix=prefix)
|
2020-09-06 18:32:03 +00:00
|
|
|
k_b_hash = self.hash(k_b)
|
2020-09-25 03:13:14 +00:00
|
|
|
# v=self.db.get(k_b_hash)
|
|
|
|
self.log('getting k',k,'with prefix',prefix)
|
|
|
|
self.log('getting k_b',k_b)
|
|
|
|
self.log('getting k_b_hash',k_b_hash)
|
|
|
|
|
|
|
|
v = self.db.command('get',k_b_hash)
|
|
|
|
self.log('<--',v)
|
|
|
|
|
2020-09-03 21:11:58 +00:00
|
|
|
v_b=self.unpackage_val(v)
|
|
|
|
return v_b
|
|
|
|
|
|
|
|
|
|
|
|
class KeyCrypt(Crypt):
|
|
|
|
def __init__(self):
|
2020-09-06 09:33:46 +00:00
|
|
|
return super().__init__(name=PATH_CRYPT_CA_KEYS.replace('.','_'))
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
|
|
|
|
class DataCrypt(Crypt):
|
|
|
|
def __init__(self):
|
2020-09-06 09:33:46 +00:00
|
|
|
return super().__init__(name=PATH_CRYPT_CA_DATA.replace('.','_'))
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
|
2020-09-17 08:16:52 +00:00
|
|
|
|
2020-09-18 16:21:32 +00:00
|
|
|
class CryptList(Crypt): # like inbox
|
|
|
|
def __init__(self,
|
|
|
|
crypt,
|
|
|
|
keyname,
|
|
|
|
prefix='',
|
|
|
|
encryptor_func=lambda x: x,
|
|
|
|
decryptor_func=lambda x: x):
|
2020-09-17 08:16:52 +00:00
|
|
|
|
2020-09-18 16:21:32 +00:00
|
|
|
self.crypt=crypt
|
2020-09-25 03:13:14 +00:00
|
|
|
self.db=self.crypt.db
|
|
|
|
self.keyname=self.crypt.package_key(keyname,prefix)
|
2020-09-18 16:21:32 +00:00
|
|
|
|
|
|
|
@property
|
2020-09-25 03:13:14 +00:00
|
|
|
def values(self): return list(self.l)
|
2020-09-18 15:52:19 +00:00
|
|
|
|
|
|
|
def package_val(self,val):
|
2020-09-25 03:13:14 +00:00
|
|
|
# if type(val)!=bytes: val=val.encode()
|
2020-09-18 16:06:01 +00:00
|
|
|
return val
|
2020-09-18 15:52:19 +00:00
|
|
|
|
|
|
|
def unpackage_val(self,val):
|
2020-09-25 03:13:14 +00:00
|
|
|
# if type(val)==str: val=val.encode()
|
2020-09-18 16:06:01 +00:00
|
|
|
return val
|
2020-09-18 15:52:19 +00:00
|
|
|
|
|
|
|
def append(self,val):
|
2020-09-18 16:03:43 +00:00
|
|
|
self.log('<--val',val)
|
2020-09-18 15:59:36 +00:00
|
|
|
if type(val)==list: return [self.append(x) for x in val]
|
2020-09-18 15:52:19 +00:00
|
|
|
val_x = self.package_val(val)
|
2020-09-25 03:13:14 +00:00
|
|
|
# with self.db.transaction():
|
|
|
|
# res = self.db.lpush(self.keyname,val_x)
|
|
|
|
res = self.db.command('rpush',self.keyname,val_x)
|
2020-09-18 16:03:43 +00:00
|
|
|
self.log('-->',res)
|
|
|
|
return res
|
2020-09-18 15:52:19 +00:00
|
|
|
|
|
|
|
def prepend(self,val):
|
2020-09-18 16:03:43 +00:00
|
|
|
self.log('<--val',val)
|
2020-09-18 15:59:36 +00:00
|
|
|
if type(val)==list: return [self.prepend(x) for x in val]
|
2020-09-18 15:52:19 +00:00
|
|
|
val_x = self.package_val(val)
|
2020-09-25 03:13:14 +00:00
|
|
|
res = self.db.command('lpush',self.keyname,val_x)
|
2020-09-18 16:03:43 +00:00
|
|
|
self.log('-->',res)
|
|
|
|
return res
|
2020-09-18 15:52:19 +00:00
|
|
|
|
2020-09-17 08:16:52 +00:00
|
|
|
@property
|
2020-09-25 03:13:14 +00:00
|
|
|
def values(self):
|
|
|
|
l = self.db.command('lrange',self.keyname, '0', '-1')
|
|
|
|
self.log('<-- l',l)
|
|
|
|
if not l: return []
|
2020-09-18 16:03:43 +00:00
|
|
|
vals = [self.unpackage_val(x) for x in l]
|
|
|
|
self.log('-->',vals)
|
|
|
|
return vals
|
2020-09-18 15:52:19 +00:00
|
|
|
|
|
|
|
def remove(self,val):
|
2020-09-18 16:03:43 +00:00
|
|
|
self.log('<--',val)
|
2020-09-18 15:59:36 +00:00
|
|
|
if type(val)==list: return [self.remove(x) for x in val]
|
2020-09-18 15:52:19 +00:00
|
|
|
val_x = self.package_val(val)
|
2020-09-25 03:13:14 +00:00
|
|
|
self.db.command('lrem',self.keyname,'0',val_x)
|
|
|
|
|
2020-09-17 08:16:52 +00:00
|
|
|
|
2020-09-06 16:48:47 +00:00
|
|
|
|
2020-09-03 21:11:58 +00:00
|
|
|
|
|
|
|
if __name__=='__main__':
|
2020-09-25 03:13:14 +00:00
|
|
|
crypt = Crypt(fn='tes22t.db')
|
|
|
|
print(crypt.set(
|
|
|
|
'testing22',
|
|
|
|
b'wooooooboy',
|
|
|
|
prefix='/test/',
|
|
|
|
))
|
|
|
|
|
|
|
|
print('got back', crypt.get(
|
|
|
|
'testing22',
|
|
|
|
prefix='/test/'
|
|
|
|
))
|
2020-09-03 21:11:58 +00:00
|
|
|
|
2020-09-17 08:16:52 +00:00
|
|
|
|
|
|
|
|
|
|
|
crypt_list = CryptList(
|
2020-09-25 03:13:14 +00:00
|
|
|
keyname='MyInbosdx35',
|
|
|
|
crypt=crypt
|
2020-09-17 08:16:52 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
print(crypt_list.values)
|
|
|
|
|
2020-09-18 15:10:11 +00:00
|
|
|
# print(crypt_list.remove('cool thing 0'))
|
2020-09-17 08:16:52 +00:00
|
|
|
|
|
|
|
# print(crypt_list.append('cool thing 1'))
|
|
|
|
|
2020-09-25 03:13:14 +00:00
|
|
|
print(crypt_list.append('Appended'))
|
2020-09-18 15:52:19 +00:00
|
|
|
print(crypt_list.append('cool thing 0'))
|
2020-09-25 03:13:14 +00:00
|
|
|
print(crypt_list.prepend('Prepended'))
|
|
|
|
print()
|
|
|
|
print()
|
2020-09-18 15:52:19 +00:00
|
|
|
print(crypt_list.remove('cool thing 0'))
|
2020-09-25 03:13:14 +00:00
|
|
|
print()
|
|
|
|
print()
|
2020-09-17 08:16:52 +00:00
|
|
|
print(crypt_list.values)
|