Archives
/
catcli
mirror of https://github.com/deadc0de6/catcli
2
1
Fork 0
Code Issues Releases Wiki Activity

refactoring

Browse Source
pull/6/head
deadc0de6 6 years ago
parent 927d5e2773
commit 39efd9b8fe
7 changed files with 99 additions and 90 deletions
Show Stats Download Patch File Download Diff File

29
catcli/catalog.py
Unescape Escape View File

@ -22,16 +22,15 @@ class Catalog:
self.verbose = verbose # verbosity self.verbose = verbose # verbosity
self.force = force # force overwrite if exists self.force = force # force overwrite if exists
self.metanode = None self.metanode = None
# prefer json for git versioning
self.pickle = pickle self.pickle = pickle
def set_metanode(self, metanode): def set_metanode(self, metanode):
''' remove the metanode until tree is re-written ''' '''remove the metanode until tree is re-written'''
self.metanode = metanode self.metanode = metanode
self.metanode.parent = None self.metanode.parent = None
def restore(self): def restore(self):
''' restore the catalog ''' '''restore the catalog'''
if not self.path: if not self.path:
return None return None
if not os.path.exists(self.path): if not os.path.exists(self.path):
@ -41,7 +40,7 @@ class Catalog:
return self._restore_json(open(self.path, 'r').read()) return self._restore_json(open(self.path, 'r').read())
def save(self, node): def save(self, node):
''' save the catalog ''' '''save the catalog'''
if not self.path: if not self.path:
Logger.err('Path not defined') Logger.err('Path not defined')
return False return False
@ -62,14 +61,22 @@ class Catalog:
return self._save_json(node) return self._save_json(node)
def _save_pickle(self, node): def _save_pickle(self, node):
''' pickle the catalog''' '''pickle the catalog'''
pickle.dump(node, open(self.path, 'wb')) pickle.dump(node, open(self.path, 'wb'))
if self.verbose: if self.verbose:
Logger.info('Catalog saved to pickle \"{}\"'.format(self.path)) Logger.info('Catalog saved to pickle \"{}\"'.format(self.path))
return True return True
def _restore_pickle(self):
'''restore the pickled tree'''
root = pickle.load(open(self.path, 'rb'))
if self.verbose:
m = 'Catalog imported from pickle \"{}\"'.format(self.path)
Logger.info(m)
return root
def _save_json(self, node): def _save_json(self, node):
''' export the catalog in json ''' '''export the catalog in json'''
exp = JsonExporter(indent=2, sort_keys=True) exp = JsonExporter(indent=2, sort_keys=True)
with open(self.path, 'w') as f: with open(self.path, 'w') as f:
exp.write(node, f) exp.write(node, f)
@ -77,16 +84,8 @@ class Catalog:
Logger.info('Catalog saved to json \"{}\"'.format(self.path)) Logger.info('Catalog saved to json \"{}\"'.format(self.path))
return True return True
def _restore_pickle(self):
''' restore the pickled tree '''
root = pickle.load(open(self.path, 'rb'))
if self.verbose:
m = 'Catalog imported from pickle \"{}\"'.format(self.path)
Logger.info(m)
return root
def _restore_json(self, string): def _restore_json(self, string):
''' restore the tree from json ''' '''restore the tree from json'''
imp = JsonImporter() imp = JsonImporter()
root = imp.import_(string) root = imp.import_(string)
if self.verbose: if self.verbose:

4
catcli/catcli.py
Unescape Escape View File

@ -80,7 +80,7 @@ def cmd_index(args, noder, catalog, top):
node.parent = None node.parent = None
start = datetime.datetime.now() start = datetime.datetime.now()
walker = Walker(noder, nohash=nohash) walker = Walker(noder, nohash=nohash)
attr = noder.clean_storage_attr(args['--meta']) attr = noder.format_storage_attr(args['--meta'])
root = noder.storage_node(name, path, parent=top, attr=attr) root = noder.storage_node(name, path, parent=top, attr=attr)
_, cnt = walker.index(path, name, parent=root, parentpath=path) _, cnt = walker.index(path, name, parent=root, parentpath=path)
if subsize: if subsize:
@ -223,7 +223,7 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
''' entry point ''' '''entry point'''
if main(): if main():
sys.exit(0) sys.exit(0)
sys.exit(1) sys.exit(1)

5
catcli/decomp.py
Unescape Escape View File

@ -29,22 +29,25 @@ class Decomp:
'zip': self._zip} 'zip': self._zip}
def get_format(self): def get_format(self):
'''return list of supported extensions'''
return list(self.ext.keys()) return list(self.ext.keys())
def get_names(self, path): def get_names(self, path):
''' get tree of compressed archive ''' '''get tree of compressed archive'''
ext = os.path.splitext(path)[1][1:] ext = os.path.splitext(path)[1][1:]
if ext in list(self.ext.keys()): if ext in list(self.ext.keys()):
return self.ext[ext](path) return self.ext[ext](path)
return None return None
def _tar(self, path): def _tar(self, path):
'''return list of file names in tar'''
if not tarfile.is_tarfile(path): if not tarfile.is_tarfile(path):
return None return None
tar = tarfile.open(path, "r") tar = tarfile.open(path, "r")
return tar.getnames() return tar.getnames()
def _zip(self, path): def _zip(self, path):
'''return list of file names in zip'''
if not zipfile.is_zipfile(path): if not zipfile.is_zipfile(path):
return None return None
z = zipfile.ZipFile(path) z = zipfile.ZipFile(path)

17
catcli/logger.py
Unescape Escape View File

@ -29,7 +29,7 @@ class Logger:
# node specific output # node specific output
###################################################################### ######################################################################
def storage(pre, name, attr): def storage(pre, name, attr):
''' print a storage node ''' '''print a storage node'''
end = '' end = ''
if attr: if attr:
end = ' {}({}){}'.format(Logger.GRAY, attr, Logger.RESET) end = ' {}({}){}'.format(Logger.GRAY, attr, Logger.RESET)
@ -38,13 +38,13 @@ class Logger:
sys.stdout.write('{}\n'.format(s)) sys.stdout.write('{}\n'.format(s))
def file(pre, name, attr): def file(pre, name, attr):
''' print a file node ''' '''print a file node'''
s = '{}{}'.format(pre, name) s = '{}{}'.format(pre, name)
s += ' {}[{}]{}'.format(Logger.GRAY, attr, Logger.RESET) s += ' {}[{}]{}'.format(Logger.GRAY, attr, Logger.RESET)
sys.stdout.write('{}\n'.format(s)) sys.stdout.write('{}\n'.format(s))
def dir(pre, name, depth='', attr=None): def dir(pre, name, depth='', attr=None):
''' print a directory node ''' '''print a directory node'''
end = [] end = []
if depth != '': if depth != '':
end.append('nbfiles:{}'.format(depth)) end.append('nbfiles:{}'.format(depth))
@ -65,27 +65,28 @@ class Logger:
# generic output # generic output
###################################################################### ######################################################################
def out(string): def out(string):
''' to stdout ''' '''to stdout'''
sys.stdout.write('{}\n'.format(string)) sys.stdout.write('{}\n'.format(string))
def log(string): def log(string):
''' to stderr ''' '''to stderr'''
sys.stderr.write('{}\n'.format(string)) sys.stderr.write('{}\n'.format(string))
def info(string): def info(string):
''' to stderr in color ''' '''to stderr in color'''
s = '{}{}{}'.format(Logger.MAGENTA, string, Logger.RESET) s = '{}{}{}'.format(Logger.MAGENTA, string, Logger.RESET)
sys.stderr.write('{}\n'.format(s)) sys.stderr.write('{}\n'.format(s))
def err(string): def err(string):
''' to stderr in RED ''' '''to stderr in RED'''
s = '{}{}{}'.format(Logger.RED, string, Logger.RESET) s = '{}{}{}'.format(Logger.RED, string, Logger.RESET)
sys.stderr.write('{}\n'.format(s)) sys.stderr.write('{}\n'.format(s))
def progr(string): def progr(string):
''' print progress ''' '''print progress'''
sys.stderr.write('{}\r'.format(string)) sys.stderr.write('{}\r'.format(string))
sys.stderr.flush() sys.stderr.flush()
def bold(string): def bold(string):
'''make it bold'''
return '{}{}{}'.format(Logger.BOLD, string, Logger.RESET) return '{}{}{}'.format(Logger.BOLD, string, Logger.RESET)

124
catcli/noder.py
Unescape Escape View File

@ -44,28 +44,20 @@ class Noder:
if self.arc: if self.arc:
self.decomp = Decomp() self.decomp = Decomp()
def set_hashing(self, val):
self.hash = val
def get_storage_names(self, top): def get_storage_names(self, top):
''' return a list of all storage names ''' '''return a list of all storage names'''
return [x.name for x in list(top.children)] return [x.name for x in list(top.children)]
def get_storage_node(self, top, name): def get_storage_node(self, top, name):
''' return the storage node ''' '''return the storage node if any'''
for n in top.children: for n in top.children:
if n.type != self.TYPE_STORAGE: if n.type != self.TYPE_STORAGE:
continue continue
if n.name == name: if n.name == name:
return n return n
def clean_storage_attr(self, attr):
if not attr:
return ''
return ', '.join(attr)
def get_node(self, top, path): def get_node(self, top, path):
''' get the node by internal tree path ''' '''get the node by internal tree path'''
r = anytree.resolver.Resolver('name') r = anytree.resolver.Resolver('name')
try: try:
return r.get(top, path) return r.get(top, path)
@ -73,15 +65,53 @@ class Noder:
Logger.err('No node at path \"{}\"'.format(path)) Logger.err('No node at path \"{}\"'.format(path))
return None return None
def get_meta_node(self, top):
'''return the meta node if any'''
try:
return next(filter(lambda x: x.type == self.TYPE_META,
top.children))
except StopIteration:
return None
def rec_size(self, node):
'''recursively traverse tree and store dir size'''
if self.verbose:
Logger.info('getting folder size recursively')
if node.type == self.TYPE_FILE:
return node.size
size = 0
for i in node.children:
if node.type == self.TYPE_DIR:
size += self.rec_size(i)
if node.type == self.TYPE_STORAGE:
self.rec_size(i)
else:
continue
node.size = size
return size
###############################################################
# public helpers
###############################################################
def format_storage_attr(self, attr):
'''format the storage attr for saving'''
if not attr:
return ''
return ', '.join(attr)
def set_hashing(self, val):
'''hash files when indexing'''
self.hash = val
############################################################### ###############################################################
# node creationg # node creationg
############################################################### ###############################################################
def new_top_node(self): def new_top_node(self):
''' create a new top node''' '''create a new top node'''
return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP) return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP)
def update_metanode(self, meta): def update_metanode(self, meta):
''' create or update meta node information ''' '''create or update meta node information'''
epoch = int(time.time()) epoch = int(time.time())
if not meta: if not meta:
attr = {} attr = {}
@ -94,7 +124,7 @@ class Noder:
return meta return meta
def file_node(self, name, path, parent, storagepath): def file_node(self, name, path, parent, storagepath):
''' create a new node representing a file ''' '''create a new node representing a file'''
if not os.path.exists(path): if not os.path.exists(path):
Logger.err('File \"{}\" does not exist'.format(path)) Logger.err('File \"{}\" does not exist'.format(path))
return None return None
@ -120,13 +150,13 @@ class Noder:
return n return n
def dir_node(self, name, path, parent, storagepath): def dir_node(self, name, path, parent, storagepath):
''' create a new node representing a directory ''' '''create a new node representing a directory'''
path = os.path.abspath(path) path = os.path.abspath(path)
relpath = os.path.relpath(path, start=storagepath) relpath = os.path.relpath(path, start=storagepath)
return self._node(name, self.TYPE_DIR, relpath, parent) return self._node(name, self.TYPE_DIR, relpath, parent)
def storage_node(self, name, path, parent, attr=None): def storage_node(self, name, path, parent, attr=None):
''' create a new node representing a storage ''' '''create a new node representing a storage'''
path = os.path.abspath(path) path = os.path.abspath(path)
free = psutil.disk_usage(path).free free = psutil.disk_usage(path).free
total = psutil.disk_usage(path).total total = psutil.disk_usage(path).total
@ -135,12 +165,13 @@ class Noder:
total=total, parent=parent, attr=attr, ts=epoch) total=total, parent=parent, attr=attr, ts=epoch)
def archive_node(self, name, path, parent, archive): def archive_node(self, name, path, parent, archive):
'''crete a new node for archive data'''
return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path,
parent=parent, size=0, md5=None, parent=parent, size=0, md5=None,
archive=archive) archive=archive)
def _node(self, name, type, relpath, parent, size=None, md5=None): def _node(self, name, type, relpath, parent, size=None, md5=None):
''' generic node creation ''' '''generic node creation'''
return anytree.AnyNode(name=name, type=type, relpath=relpath, return anytree.AnyNode(name=name, type=type, relpath=relpath,
parent=parent, size=size, md5=md5) parent=parent, size=size, md5=md5)
@ -149,7 +180,7 @@ class Noder:
############################################################### ###############################################################
def _print_node(self, node, pre='', withpath=False, def _print_node(self, node, pre='', withpath=False,
withdepth=False, withstorage=False): withdepth=False, withstorage=False):
''' print a node ''' '''print a node'''
if node.type == self.TYPE_TOP: if node.type == self.TYPE_TOP:
Logger.out('{}{}'.format(pre, node.name)) Logger.out('{}{}'.format(pre, node.name))
elif node.type == self.TYPE_FILE: elif node.type == self.TYPE_FILE:
@ -193,16 +224,22 @@ class Noder:
# Logger.out('{}{}'.format(pre, node.name)) # Logger.out('{}{}'.format(pre, node.name))
def print_tree(self, node, style=anytree.ContRoundStyle()): def print_tree(self, node, style=anytree.ContRoundStyle()):
''' print the tree similar to unix tool "tree" ''' '''print the tree similar to unix tool "tree"'''
rend = anytree.RenderTree(node, childiter=self._sort_tree) rend = anytree.RenderTree(node, childiter=self._sort_tree)
for pre, fill, node in rend: for pre, fill, node in rend:
self._print_node(node, pre=pre, withdepth=True) self._print_node(node, pre=pre, withdepth=True)
def to_dot(self, node, path='tree.dot'):
'''export to dot for graphing'''
anytree.exporter.DotExporter(node).to_dotfile(path)
Logger.info('dot file created under \"{}\"'.format(path))
return 'dot {} -T png -o /tmp/tree.png'.format(path)
############################################################### ###############################################################
# searching # searching
############################################################### ###############################################################
def find_name(self, root, key, script=False): def find_name(self, root, key, script=False):
''' find files based on their names ''' '''find files based on their names'''
if self.verbose: if self.verbose:
Logger.info('searching for \"{}\"'.format(key)) Logger.info('searching for \"{}\"'.format(key))
self.term = key self.term = key
@ -222,7 +259,7 @@ class Noder:
return found return found
def _find_name(self, node): def _find_name(self, node):
''' callback for finding files ''' '''callback for finding files'''
if self.term.lower() in node.name.lower(): if self.term.lower() in node.name.lower():
return True return True
return False return False
@ -231,7 +268,7 @@ class Noder:
# climbing # climbing
############################################################### ###############################################################
def walk(self, root, path, rec=False): def walk(self, root, path, rec=False):
''' walk the tree for ls based on names ''' '''walk the tree for ls based on names'''
if self.verbose: if self.verbose:
Logger.info('walking path: \"{}\"'.format(path)) Logger.info('walking path: \"{}\"'.format(path))
r = anytree.resolver.Resolver('name') r = anytree.resolver.Resolver('name')
@ -256,7 +293,7 @@ class Noder:
# tree creationg # tree creationg
############################################################### ###############################################################
def _add_entry(self, name, top, resolv): def _add_entry(self, name, top, resolv):
''' add an entry to the tree ''' '''add an entry to the tree'''
entries = name.rstrip(os.sep).split(os.sep) entries = name.rstrip(os.sep).split(os.sep)
if len(entries) == 1: if len(entries) == 1:
self.archive_node(name, name, top, top.name) self.archive_node(name, name, top, top.name)
@ -270,7 +307,7 @@ class Noder:
self.archive_node(f, name, top, top.name) self.archive_node(f, name, top, top.name)
def list_to_tree(self, parent, names): def list_to_tree(self, parent, names):
''' convert list of files to a tree ''' '''convert list of files to a tree'''
if not names: if not names:
return return
r = anytree.resolver.Resolver('name') r = anytree.resolver.Resolver('name')
@ -282,7 +319,7 @@ class Noder:
# diverse # diverse
############################################################### ###############################################################
def _sort_tree(self, items): def _sort_tree(self, items):
''' sorting a list of items ''' '''sorting a list of items'''
return sorted(items, key=self._sort, reverse=self.sortsize) return sorted(items, key=self._sort, reverse=self.sortsize)
def _sort(self, x): def _sort(self, x):
@ -291,11 +328,11 @@ class Noder:
return self._sort_fs(x) return self._sort_fs(x)
def _sort_fs(self, n): def _sort_fs(self, n):
''' sorting nodes dir first and alpha ''' '''sorting nodes dir first and alpha'''
return (n.type, n.name.lstrip('\.').lower()) return (n.type, n.name.lstrip('\.').lower())
def _sort_size(self, n): def _sort_size(self, n):
''' sorting nodes by size ''' '''sorting nodes by size'''
try: try:
if not n.size: if not n.size:
return 0 return 0
@ -303,37 +340,6 @@ class Noder:
except AttributeError: except AttributeError:
return 0 return 0
def to_dot(self, node, path='tree.dot'):
''' export to dot for graphing '''
anytree.exporter.DotExporter(node).to_dotfile(path)
Logger.info('dot file created under \"{}\"'.format(path))
return 'dot {} -T png -o /tmp/tree.png'.format(path)
def _get_storage(self, node): def _get_storage(self, node):
''' recursively traverse up to find storage ''' '''recursively traverse up to find storage'''
return node.ancestors[1] return node.ancestors[1]
def get_meta_node(self, top):
''' return the meta node if any '''
try:
return next(filter(lambda x: x.type == self.TYPE_META,
top.children))
except StopIteration:
return None
def rec_size(self, node):
''' recursively traverse tree and store dir size '''
if self.verbose:
Logger.info('getting folder size recursively')
if node.type == self.TYPE_FILE:
return node.size
size = 0
for i in node.children:
if node.type == self.TYPE_DIR:
size += self.rec_size(i)
if node.type == self.TYPE_STORAGE:
self.rec_size(i)
else:
continue
node.size = size
return size

8
catcli/utils.py
Unescape Escape View File

@ -16,7 +16,7 @@ from catcli.logger import Logger
def md5sum(path): def md5sum(path):
''' calculate md5 sum of a file ''' '''calculate md5 sum of a file'''
p = os.path.realpath(path) p = os.path.realpath(path)
if not os.path.exists(p): if not os.path.exists(p):
Logger.err('\nunable to get md5sum on {}'.format(path)) Logger.err('\nunable to get md5sum on {}'.format(path))
@ -36,7 +36,7 @@ def md5sum(path):
def human(size): def human(size):
''' human readable size ''' '''human readable size'''
div = 1024. div = 1024.
suf = ['B', 'K', 'M', 'G', 'T', 'P'] suf = ['B', 'K', 'M', 'G', 'T', 'P']
if size < div: if size < div:
@ -49,13 +49,13 @@ def human(size):
def ask(question): def ask(question):
''' ask the user what to do ''' '''ask the user what to do'''
resp = input('{} [y|N] ? '.format(question)) resp = input('{} [y|N] ? '.format(question))
return resp.lower() == 'y' return resp.lower() == 'y'
def edit(string): def edit(string):
''' edit the information with the default EDITOR ''' '''edit the information with the default EDITOR'''
string = string.encode('utf-8') string = string.encode('utf-8')
EDITOR = os.environ.get('EDITOR', 'vim') EDITOR = os.environ.get('EDITOR', 'vim')
with tempfile.NamedTemporaryFile(prefix='catcli', suffix='.tmp') as f: with tempfile.NamedTemporaryFile(prefix='catcli', suffix='.tmp') as f:

2
catcli/walker.py
Unescape Escape View File

@ -22,7 +22,7 @@ class Walker:
self.noder.set_hashing(not nohash) self.noder.set_hashing(not nohash)
def index(self, path, name, parentpath=None, parent=None, isdir=False): def index(self, path, name, parentpath=None, parent=None, isdir=False):
''' index a folder and store in tree ''' '''index a folder and store in tree'''
if not parent: if not parent:
parent = noder.dir_node(name, path, parent) parent = noder.dir_node(name, path, parent)

Write Preview
Loading…
Cancel
Save