diff --git a/catcli/catalog.py b/catcli/catalog.py index 445f866..b5df73d 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -36,6 +36,14 @@ class Catalog: self.metanode = metanode self.metanode.parent = None + def exists(self): + """does catalog exist""" + if not self.path: + return False + if os.path.exists(self.path): + return True + return False + def restore(self): """restore the catalog""" if not self.path: diff --git a/catcli/catcli.py b/catcli/catcli.py index b3f95d9..4d6315b 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -101,10 +101,10 @@ def cmd_index(args, noder, catalog, top): start = datetime.datetime.now() walker = Walker(noder, usehash=usehash, debug=debug) attr = noder.format_storage_attr(args['--meta']) - root = noder.storage_node(name, path, parent=top, attr=attr) + root = noder.new_storage_node(name, path, parent=top, attr=attr) _, cnt = walker.index(path, root, name) if subsize: - noder.rec_size(root) + noder.rec_size(root, store=True) stop = datetime.datetime.now() diff = stop - start Logger.info(f'Indexed {cnt} file(s) in {diff}') @@ -132,7 +132,7 @@ def cmd_update(args, noder, catalog, top): logpath=logpath) cnt = walker.reindex(path, root, top) if subsize: - noder.rec_size(root) + noder.rec_size(root, store=True) stop = datetime.datetime.now() diff = stop - start Logger.info(f'updated {cnt} file(s) in {diff}') @@ -148,7 +148,7 @@ def cmd_ls(args, noder, top): if not path.startswith(SEPARATOR): path = SEPARATOR + path # prepend with top node path - pre = f'{SEPARATOR}{noder.TOPNAME}' + pre = f'{SEPARATOR}{noder.NAME_TOP}' if not path.startswith(pre): path = pre + path # ensure ends with a separator @@ -161,7 +161,7 @@ def cmd_ls(args, noder, top): fmt = args['--format'] if fmt.startswith('fzf'): raise BadFormatException('fzf is not supported in ls, use find') - found = noder.walk(top, path, + found = noder.list(top, path, rec=args['--recursive'], fmt=fmt, raw=args['--raw-size']) @@ -305,7 +305,8 @@ def main(): noder = Noder(debug=args['--verbose'], sortsize=args['--sortsize'], arc=args['--archive']) # init catalog - catalog = Catalog(args['--catalog'], debug=args['--verbose'], + catalog_path = args['--catalog'] + catalog = Catalog(catalog_path, debug=args['--verbose'], force=args['--force']) # init top node top = catalog.restore() @@ -321,20 +322,44 @@ def main(): if args['index']: cmd_index(args, noder, catalog, top) if args['update']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_update(args, noder, catalog, top) elif args['find']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_find(args, noder, top) elif args['tree']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_tree(args, noder, top) elif args['ls']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_ls(args, noder, top) elif args['rm']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_rm(args, noder, catalog, top) elif args['graph']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_graph(args, noder, top) elif args['rename']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_rename(args, catalog, top) elif args['edit']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False cmd_edit(args, noder, catalog, top) except CatcliException as exc: Logger.stderr_nocolor('ERROR ' + str(exc)) diff --git a/catcli/noder.py b/catcli/noder.py index fadfd79..ff629f0 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -30,14 +30,16 @@ class Noder: * "file" node representing a file """ - TOPNAME = 'top' - METANAME = 'meta' + NAME_TOP = 'top' + NAME_META = 'meta' + TYPE_TOP = 'top' TYPE_FILE = 'file' TYPE_DIR = 'dir' TYPE_ARC = 'arc' TYPE_STORAGE = 'storage' TYPE_META = 'meta' + CSV_HEADER = ('name,type,path,size,indexed_at,' 'maccess,md5,nbfiles,free_space,' 'total_space,meta') @@ -124,7 +126,7 @@ class Noder: self._debug(f'\tchange: no change for \"{path}\"') return node, False - def _rec_size(self, node, store=True): + def rec_size(self, node, store=True): """ recursively traverse tree and return size @store: store the size in the node @@ -137,12 +139,12 @@ class Noder: size = 0 for i in node.children: if node.type == self.TYPE_DIR: - size = self._rec_size(i, store=store) + size = self.rec_size(i, store=store) if store: i.size = size size += size if node.type == self.TYPE_STORAGE: - size = self._rec_size(i, store=store) + size = self.rec_size(i, store=store) if store: i.size = size size += size @@ -152,10 +154,6 @@ class Noder: node.size = size return size - def rec_size(self, node): - """recursively traverse tree and store dir size""" - return self._rec_size(node, store=True) - ############################################################### # public helpers ############################################################### @@ -173,35 +171,13 @@ class Noder: self.hash = val ############################################################### - # node creationg + # node creation ############################################################### def new_top_node(self): """create a new top node""" - return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP) + return anytree.AnyNode(name=self.NAME_TOP, type=self.TYPE_TOP) - def update_metanode(self, top): - """create or update meta node information""" - meta = self._get_meta_node(top) - epoch = int(time.time()) - if not meta: - attr = {} - attr['created'] = epoch - attr['created_version'] = VERSION - meta = anytree.AnyNode(name=self.METANAME, type=self.TYPE_META, - attr=attr) - meta.attr['access'] = epoch - meta.attr['access_version'] = VERSION - return meta - - def _get_meta_node(self, top): - """return the meta node if any""" - try: - return next(filter(lambda x: x.type == self.TYPE_META, - top.children)) - except StopIteration: - return None - - def file_node(self, name, path, parent, storagepath): + def new_file_node(self, name, path, parent, storagepath): """create a new node representing a file""" if not os.path.exists(path): Logger.err(f'File \"{path}\" does not exist') @@ -218,8 +194,9 @@ class Noder: relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - node = self._node(name, self.TYPE_FILE, relpath, parent, - size=stat.st_size, md5=md5, maccess=maccess) + node = self._new_generic_node(name, self.TYPE_FILE, relpath, parent, + size=stat.st_size, md5=md5, + maccess=maccess) if self.arc: ext = os.path.splitext(path)[1][1:] if ext.lower() in self.decomp.get_formats(): @@ -230,13 +207,60 @@ class Noder: self._debug(f'{path} is NOT an archive') return node - def dir_node(self, name, path, parent, storagepath): + def new_dir_node(self, name, path, parent, storagepath): """create a new node representing a directory""" path = os.path.abspath(path) relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - return self._node(name, self.TYPE_DIR, relpath, - parent, maccess=maccess) + return self._new_generic_node(name, self.TYPE_DIR, relpath, + parent, maccess=maccess) + + def new_storage_node(self, name, path, parent, attr=None): + """create a new node representing a storage""" + path = os.path.abspath(path) + free = shutil.disk_usage(path).free + total = shutil.disk_usage(path).total + epoch = int(time.time()) + return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free, + total=total, parent=parent, attr=attr, ts=epoch) + + def new_archive_node(self, name, path, parent, archive): + """create a new node for archive data""" + return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, + parent=parent, size=0, md5=None, + archive=archive) + + def _new_generic_node(self, name, nodetype, relpath, parent, + size=None, md5=None, maccess=None): + """generic node creation""" + return anytree.AnyNode(name=name, type=nodetype, relpath=relpath, + parent=parent, size=size, + md5=md5, maccess=maccess) + + ############################################################### + # node management + ############################################################### + def update_metanode(self, top): + """create or update meta node information""" + meta = self._get_meta_node(top) + epoch = int(time.time()) + if not meta: + attr = {} + attr['created'] = epoch + attr['created_version'] = VERSION + meta = anytree.AnyNode(name=self.NAME_META, type=self.TYPE_META, + attr=attr) + meta.attr['access'] = epoch + meta.attr['access_version'] = VERSION + return meta + + def _get_meta_node(self, top): + """return the meta node if any""" + try: + return next(filter(lambda x: x.type == self.TYPE_META, + top.children)) + except StopIteration: + return None def clean_not_flagged(self, top): """remove any node not flagged and clean flags""" @@ -261,28 +285,6 @@ class Noder: del node.flag return False - def storage_node(self, name, path, parent, attr=None): - """create a new node representing a storage""" - path = os.path.abspath(path) - free = shutil.disk_usage(path).free - total = shutil.disk_usage(path).total - epoch = int(time.time()) - return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free, - total=total, parent=parent, attr=attr, ts=epoch) - - def archive_node(self, name, path, parent, archive): - """crete a new node for archive data""" - return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, - parent=parent, size=0, md5=None, - archive=archive) - - def _node(self, name, nodetype, relpath, parent, - size=None, md5=None, maccess=None): - """generic node creation""" - return anytree.AnyNode(name=name, type=nodetype, relpath=relpath, - parent=parent, size=size, - md5=md5, maccess=maccess) - ############################################################### # printing ############################################################### @@ -304,7 +306,7 @@ class Noder: out.append(node.name) # name out.append(node.type) # type out.append('') # fake full path - size = self._rec_size(node, store=False) + size = self.rec_size(node, store=False) out.append(size_to_str(size, raw=raw)) # size out.append(epoch_to_str(node.ts)) # indexed_at out.append('') # fake maccess @@ -416,7 +418,7 @@ class Noder: timestamp += epoch_to_str(node.ts) disksize = '' # the children size - size = self._rec_size(node, store=False) + size = self.rec_size(node, store=False) size = size_to_str(size, raw=raw) disksize = 'totsize:' + f'{size}' # format the output @@ -477,7 +479,7 @@ class Noder: def _to_fzf(self, node, fmt): """ - print node to fzf + fzf prompt with list and print selected node(s) @node: node to start with @fmt: output format for selected nodes """ @@ -611,12 +613,12 @@ class Noder: ############################################################### # ls ############################################################### - def walk(self, top, path, + def list(self, top, path, rec=False, fmt='native', raw=False): """ - walk the tree for "ls" based on names + list nodes for "ls" @top: top node @path: path to search for @rec: recursive walk @@ -679,15 +681,15 @@ class Noder: """add an entry to the tree""" entries = name.rstrip(os.sep).split(os.sep) if len(entries) == 1: - self.archive_node(name, name, top, top.name) + self.new_archive_node(name, name, top, top.name) return sub = os.sep.join(entries[:-1]) nodename = entries[-1] try: parent = resolv.get(top, sub) - parent = self.archive_node(nodename, name, parent, top.name) + parent = self.new_archive_node(nodename, name, parent, top.name) except anytree.resolver.ChildResolverError: - self.archive_node(nodename, name, top, top.name) + self.new_archive_node(nodename, name, top, top.name) def list_to_tree(self, parent, names): """convert list of files to a tree""" diff --git a/catcli/walker.py b/catcli/walker.py index 7063698..4740faf 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -39,7 +39,7 @@ class Walker: """ self._debug(f'indexing starting at {path}') if not parent: - parent = self.noder.dir_node(name, path, parent) + parent = self.noder.new_dir_node(name, path, parent) if os.path.islink(path): rel = os.readlink(path) @@ -56,8 +56,8 @@ class Walker: continue self._progress(file) self._debug(f'index file {sub}') - node = self.noder.file_node(os.path.basename(file), sub, - parent, storagepath) + node = self.noder.new_file_node(os.path.basename(file), sub, + parent, storagepath) if node: cnt += 1 for adir in dirs: @@ -67,7 +67,7 @@ class Walker: self._debug(f'index directory {sub}') if not os.path.exists(sub): continue - dummy = self.noder.dir_node(base, sub, parent, storagepath) + dummy = self.noder.new_dir_node(base, sub, parent, storagepath) if not dummy: continue cnt += 1 @@ -106,8 +106,8 @@ class Walker: self.noder.flag(node) continue self._log2file(f'update catalog for \"{sub}\"') - node = self.noder.file_node(os.path.basename(file), sub, - parent, storagepath) + node = self.noder.new_file_node(os.path.basename(file), sub, + parent, storagepath) self.noder.flag(node) cnt += 1 for adir in dirs: @@ -118,7 +118,8 @@ class Walker: reindex, dummy = self._need_reindex(parent, sub, treepath) if reindex: self._log2file(f'update catalog for \"{sub}\"') - dummy = self.noder.dir_node(base, sub, parent, storagepath) + dummy = self.noder.new_dir_node(base, sub, + parent, storagepath) cnt += 1 self.noder.flag(dummy) self._debug(f'reindexing deeper under {sub}')