From ac4ba145fe84d3a8f5d6dee379863e1af7cb44ca Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 19:41:59 +0100 Subject: [PATCH 01/33] clear relpath and add full path --- catcli/catcli.py | 5 +--- catcli/fuser.py | 2 +- catcli/noder.py | 45 +++++++++-------------------- catcli/nodes.py | 6 ---- catcli/utils.py | 15 +++++----- tests-ng/assets/github.catalog.json | 4 --- tests/helpers.py | 8 ----- 7 files changed, 23 insertions(+), 62 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 1f2d066..cc88a68 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -41,7 +41,7 @@ USAGE = f""" Usage: {NAME} ls [--catalog=] [--format=] [-aBCrVSs] [] {NAME} find [--catalog=] [--format=] - [-aBCbdVsP] [--path=] [] + [-aBCbdVs] [--path=] [] {NAME} index [--catalog=] [--meta=...] [-aBCcfnV] {NAME} update [--catalog=] [-aBCcfnV] [--lpath=] @@ -68,7 +68,6 @@ Options: -f --force Do not ask when updating the catalog [default: False]. -l --lpath= Path where changes are logged [default: ] -n --no-subsize Do not store size of directories [default: False]. - -P --parent Ignore stored relpath [default: True]. -p --path= Start path. -r --recursive Recursive [default: False]. -s --raw-size Print raw size [default: False]. @@ -203,7 +202,6 @@ def cmd_find(args: Dict[str, Any], noder: Noder, top: NodeTop) -> List[NodeAny]: """find action""" - fromtree = args['--parent'] directory = args['--directory'] startpath = args['--path'] fmt = args['--format'] @@ -216,7 +214,6 @@ def cmd_find(args: Dict[str, Any], script=script, startnode=startpath, only_dir=directory, - parentfromtree=fromtree, fmt=fmt, raw=raw) return found diff --git a/catcli/fuser.py b/catcli/fuser.py index 4efc553..f015e36 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -107,7 +107,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def getattr(self, path: str, _fh: Any = None) -> Dict[str, Any]: """return attr of file pointed by path""" - if path == '/': + if path == os.path.sep: # mountpoint curt = time() meta = { diff --git a/catcli/noder.py b/catcli/noder.py index cfd21da..9a6c6bf 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -212,11 +212,9 @@ class Noder: md5 = '' if self.hash: md5 = self._get_hash(path) - relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) node = NodeFile(name, - relpath, stat.st_size, md5, maccess, @@ -235,10 +233,8 @@ class Noder: parent: NodeAny, storagepath: str) -> NodeDir: """create a new node representing a directory""" path = os.path.abspath(path) - relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) return NodeDir(name, - relpath, 0, maccess, parent=parent) @@ -264,7 +260,7 @@ class Noder: def new_archive_node(self, name: str, path: str, parent: str, archive: str) -> NodeArchived: """create a new node for archive data""" - return NodeArchived(name=name, relpath=path, + return NodeArchived(name=name, parent=parent, nodesize=0, md5='', archive=archive) @@ -383,7 +379,6 @@ class Noder: withpath: bool = False, withdepth: bool = False, withstorage: bool = False, - recalcparent: bool = False, raw: bool = False) -> None: """ print a node @@ -392,7 +387,6 @@ class Noder: @withpath: print the node path @withdepth: print the node depth info @withstorage: print the node storage it belongs to - @recalcparent: get relpath from tree instead of relpath field @raw: print raw size rather than human readable """ if node.type == nodes.TYPE_TOP: @@ -403,14 +397,13 @@ class Noder: # node of type file node.__class__ = NodeFile name = node.name + storage = self._get_storage(node) if withpath: - if recalcparent: - name = os.sep.join([self._get_parents(node.parent), name]) - else: - name = node.relpath + name = os.sep.join([ + storage.name, + self._get_parents(node.parent), + name]) name = name.lstrip(os.sep) - if withstorage: - storage = self._get_storage(node) attr_str = '' if node.md5: attr_str = f', md5:{node.md5}' @@ -424,17 +417,16 @@ class Noder: # node of type directory node.__class__ = NodeDir name = node.name + storage = self._get_storage(node) if withpath: - if recalcparent: - name = os.sep.join([self._get_parents(node.parent), name]) - else: - name = node.relpath + name = os.sep.join([ + storage.name, + self._get_parents(node.parent), + name]) name = name.lstrip(os.sep) depth = 0 if withdepth: depth = len(node.children) - if withstorage: - storage = self._get_storage(node) attr: List[Tuple[str, str]] = [] if node.nodesize: attr.append(('totsize', size_to_str(node.nodesize, raw=raw))) @@ -569,7 +561,6 @@ class Noder: script: bool = False, only_dir: bool = False, startnode: Optional[NodeAny] = None, - parentfromtree: bool = False, fmt: str = 'native', raw: bool = False) -> List[NodeAny]: """ @@ -579,7 +570,6 @@ class Noder: @script: output script @directory: only search for directories @startpath: node to start with - @parentfromtree: get path from parent instead of stored relpath @fmt: output format @raw: raw size output returns the found nodes @@ -598,16 +588,10 @@ class Noder: paths = {} for item in found: item.name = fix_badchars(item.name) - if hasattr(item, 'relpath'): - item.relpath = fix_badchars(item.relpath) storage = self._get_storage(item) - if parentfromtree: - parent = self._get_parents(item) - key = f'{storage}/{parent}/{item.relpath}' - paths[parent] = item - else: - key = f'{storage}/{item.path}' - paths[key] = item + parents = self._get_parents(item) + key = f'{storage}/{parents}/{item.name}' + paths[parents] = item # handle fzf mode if fmt.startswith('fzf'): @@ -626,7 +610,6 @@ class Noder: self._print_node_native(item, withpath=True, withdepth=True, withstorage=True, - recalcparent=parentfromtree, raw=raw) elif fmt.startswith('csv'): if fmt == 'csv-with-header': diff --git a/catcli/nodes.py b/catcli/nodes.py index b622410..6743958 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -99,7 +99,6 @@ class NodeFile(NodeAny): def __init__(self, # type: ignore[no-untyped-def] name: str, - relpath: str, nodesize: int, md5: str, maccess: float, @@ -109,7 +108,6 @@ class NodeFile(NodeAny): super().__init__() # type: ignore[no-untyped-call] self.name = name self.type = TYPE_FILE - self.relpath = relpath self.nodesize = nodesize self.md5 = md5 self.maccess = maccess @@ -126,7 +124,6 @@ class NodeDir(NodeAny): def __init__(self, # type: ignore[no-untyped-def] name: str, - relpath: str, nodesize: int, maccess: float, parent=None, @@ -135,7 +132,6 @@ class NodeDir(NodeAny): super().__init__() # type: ignore[no-untyped-call] self.name = name self.type = TYPE_DIR - self.relpath = relpath self.nodesize = nodesize self.maccess = maccess self.parent = parent @@ -151,7 +147,6 @@ class NodeArchived(NodeAny): def __init__(self, # type: ignore[no-untyped-def] name: str, - relpath: str, nodesize: int, md5: str, archive: str, @@ -161,7 +156,6 @@ class NodeArchived(NodeAny): super().__init__() # type: ignore[no-untyped-call] self.name = name self.type = TYPE_ARCHIVED - self.relpath = relpath self.nodesize = nodesize self.md5 = md5 self.archive = archive diff --git a/catcli/utils.py b/catcli/utils.py index 4111df8..4dca31b 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -16,13 +16,12 @@ from catcli import nodes from catcli.exceptions import CatcliException -SEPARATOR = '/' WILD = '*' def path_to_top(path: str) -> str: """path pivot under top""" - pre = f'{SEPARATOR}{nodes.NAME_TOP}' + pre = f'{os.path.sep}{nodes.NAME_TOP}' if not path.startswith(pre): # prepend with top node path path = pre + path @@ -32,16 +31,16 @@ def path_to_top(path: str) -> str: def path_to_search_all(path: str) -> str: """path to search for all subs""" if not path: - path = SEPARATOR - if not path.startswith(SEPARATOR): - path = SEPARATOR + path - pre = f'{SEPARATOR}{nodes.NAME_TOP}' + path = os.path.sep + if not path.startswith(os.path.sep): + path = os.path.sep + path + pre = f'{os.path.sep}{nodes.NAME_TOP}' if not path.startswith(pre): # prepend with top node path path = pre + path - if not path.endswith(SEPARATOR): + if not path.endswith(os.path.sep): # ensure ends with a separator - path += SEPARATOR + path += os.path.sep if not path.endswith(WILD): # add wild card path += WILD diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index ff4c561..4a13008 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -7,7 +7,6 @@ "maccess": 1666206037.0786593, "md5": "0c6407a84d412c514007313fb3bca4de", "name": "FUNDING.yml", - "relpath": "/FUNDING.yml", "size": 17, "type": "file" }, @@ -17,7 +16,6 @@ "maccess": 1666206037.078865, "md5": "57699a7a6a03e20e864f220e19f8e197", "name": "pypi-release.yml", - "relpath": "workflows/pypi-release.yml", "size": 691, "type": "file" }, @@ -25,14 +23,12 @@ "maccess": 1678375244.4870229, "md5": "7144a119ef43adb634654522c12ec250", "name": "testing.yml", - "relpath": "workflows/testing.yml", "size": 802, "type": "file" } ], "maccess": 1678375244.4865956, "name": "workflows", - "relpath": "/workflows", "size": 1493, "type": "dir" } diff --git a/tests/helpers.py b/tests/helpers.py index a6da349..ca66377 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -149,21 +149,18 @@ FAKECATALOG = """ { "md5": null, "name": "7544G", - "relpath": "tmpj5602ih7.catcli/7544G", "size": 100, "type": "file" }, { "md5": null, "name": "KF2ZC", - "relpath": "tmpj5602ih7.catcli/KF2ZC", "size": 100, "type": "file" }, { "md5": null, "name": "Z9OII", - "relpath": "tmpj5602ih7.catcli/Z9OII", "size": 100, "type": "file" }, @@ -172,14 +169,12 @@ FAKECATALOG = """ { "md5": null, "name": "M592O9", - "relpath": "tmpj5602ih7.catcli/VNN/M592O9", "size": 100, "type": "file" } ], "md5": null, "name": "VNN", - "relpath": "VNN", "size": 100, "type": "dir" }, @@ -188,21 +183,18 @@ FAKECATALOG = """ { "md5": null, "name": "X37H", - "relpath": "tmpj5602ih7.catcli/P4C/X37H", "size": 100, "type": "file" }, { "md5": null, "name": "I566", - "relpath": "tmpj5602ih7.catcli/P4C/I566", "size": 100, "type": "file" } ], "md5": null, "name": "P4C", - "relpath": "P4C", "size": 200, "type": "dir" } From 1c74290fa1e22e27004eab63a49ec95cf39b9fbb Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 22:45:50 +0100 Subject: [PATCH 02/33] update --- catcli/noder.py | 4 ++-- catcli/walker.py | 19 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index 9a6c6bf..0fd769c 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -198,7 +198,7 @@ class Noder: return top def new_file_node(self, name: str, path: str, - parent: NodeAny, storagepath: str) -> Optional[NodeFile]: + parent: NodeAny) -> Optional[NodeFile]: """create a new node representing a file""" if not os.path.exists(path): Logger.err(f'File \"{path}\" does not exist') @@ -230,7 +230,7 @@ class Noder: return node def new_dir_node(self, name: str, path: str, - parent: NodeAny, storagepath: str) -> NodeDir: + parent: NodeAny) -> NodeDir: """create a new node representing a directory""" path = os.path.abspath(path) maccess = os.path.getmtime(path) diff --git a/catcli/walker.py b/catcli/walker.py index 5942a5b..c5762f6 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -47,8 +47,9 @@ class Walker: """ self._debug(f'indexing starting at {path}') if not parent: - parent = self.noder.new_dir_node(name, path, - parent, storagepath) + parent = self.noder.new_dir_node(name, + path, + parent) if os.path.islink(path): rel = os.readlink(path) @@ -65,8 +66,9 @@ class Walker: continue self._progress(file) self._debug(f'index file {sub}') - node = self.noder.new_file_node(os.path.basename(file), sub, - parent, storagepath) + node = self.noder.new_file_node(os.path.basename(file), + sub, + parent) if node: cnt += 1 for adir in dirs: @@ -76,7 +78,7 @@ class Walker: self._debug(f'index directory {sub}') if not os.path.exists(sub): continue - dummy = self.noder.new_dir_node(base, sub, parent, storagepath) + dummy = self.noder.new_dir_node(base, sub, parent) if not dummy: continue cnt += 1 @@ -118,8 +120,9 @@ class Walker: if node: node.flag() continue - node = self.noder.new_file_node(os.path.basename(file), sub, - parent, storagepath) + node = self.noder.new_file_node(os.path.basename(file), + sub, + parent) if node: node.flag() cnt += 1 @@ -131,7 +134,7 @@ class Walker: reindex, dummy = self._need_reindex(parent, sub, treepath) if reindex: dummy = self.noder.new_dir_node(base, sub, - parent, storagepath) + parent) cnt += 1 if dummy: dummy.flag() From 35d1d0d9c4d9134419ff0895648054b2c54a6752 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 22:51:05 +0100 Subject: [PATCH 03/33] depth to nbchildren --- catcli/nodeprinter.py | 6 +++--- catcli/noder.py | 35 ++++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/catcli/nodeprinter.py b/catcli/nodeprinter.py index 79d6e39..e20b5d5 100644 --- a/catcli/nodeprinter.py +++ b/catcli/nodeprinter.py @@ -49,12 +49,12 @@ class NodePrinter: @classmethod def print_dir_native(cls: Type[CLASSTYPE], pre: str, name: str, - depth: int = 0, + nbchildren: int = 0, attr: Optional[List[Tuple[str, str]]] = None) -> None: """print a directory node""" end = [] - if depth > 0: - end.append(f'{cls.NBFILES}:{depth}') + if nbchildren > 0: + end.append(f'{cls.NBFILES}:{nbchildren}') if attr: end.append(' '.join([f'{x}:{y}' for x, y in attr])) end_string = '' diff --git a/catcli/noder.py b/catcli/noder.py index 0fd769c..15f53e8 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -257,8 +257,10 @@ class Noder: self.attrs_to_string(attrs), parent=parent) - def new_archive_node(self, name: str, path: str, - parent: str, archive: str) -> NodeArchived: + def new_archive_node(self, + name: str, + parent: str, + archive: str) -> NodeArchived: """create a new node for archive data""" return NodeArchived(name=name, parent=parent, nodesize=0, md5='', @@ -377,7 +379,7 @@ class Noder: def _print_node_native(self, node: NodeAny, pre: str = '', withpath: bool = False, - withdepth: bool = False, + withnbchildren: bool = False, withstorage: bool = False, raw: bool = False) -> None: """ @@ -385,7 +387,7 @@ class Noder: @node: the node to print @pre: string to print before node @withpath: print the node path - @withdepth: print the node depth info + @withnbchildren: print the node nb children @withstorage: print the node storage it belongs to @raw: print raw size rather than human readable """ @@ -424,15 +426,18 @@ class Noder: self._get_parents(node.parent), name]) name = name.lstrip(os.sep) - depth = 0 - if withdepth: - depth = len(node.children) + nbchildren = 0 + if withnbchildren: + nbchildren = len(node.children) attr: List[Tuple[str, str]] = [] if node.nodesize: attr.append(('totsize', size_to_str(node.nodesize, raw=raw))) if withstorage: attr.append(('storage', Logger.get_bold_text(storage.name))) - NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) + NodePrinter.print_dir_native(pre, + name, + nbchildren=nbchildren, + attr=attr) elif node.type == nodes.TYPE_STORAGE: # node of type storage node.__class__ = NodeStorage @@ -489,7 +494,7 @@ class Noder: rend = anytree.RenderTree(node, childiter=self._sort_tree) for pre, _, thenode in rend: self._print_node_native(thenode, pre=pre, - withdepth=True, raw=raw) + withnbchildren=True, raw=raw) elif fmt == 'csv': # csv output self._to_csv(node, raw=raw) @@ -608,7 +613,7 @@ class Noder: if fmt == 'native': for _, item in paths.items(): self._print_node_native(item, withpath=True, - withdepth=True, + withnbchildren=True, withstorage=True, raw=raw) elif fmt.startswith('csv'): @@ -692,7 +697,7 @@ class Noder: if fmt == 'native': self._print_node_native(found[0].parent, withpath=False, - withdepth=True, + withnbchildren=True, raw=raw) elif fmt.startswith('csv'): self._node_to_csv(found[0].parent, raw=raw) @@ -706,7 +711,7 @@ class Noder: if fmt == 'native': self._print_node_native(item, withpath=False, pre='- ', - withdepth=True, + withnbchildren=True, raw=raw) elif fmt.startswith('csv'): self._node_to_csv(item, raw=raw) @@ -726,15 +731,15 @@ class Noder: """add an entry to the tree""" entries = name.rstrip(os.sep).split(os.sep) if len(entries) == 1: - self.new_archive_node(name, name, top, top.name) + self.new_archive_node(name, top, top.name) return sub = os.sep.join(entries[:-1]) nodename = entries[-1] try: parent = resolv.get(top, sub) - parent = self.new_archive_node(nodename, name, parent, top.name) + parent = self.new_archive_node(nodename, parent, top.name) except anytree.resolver.ChildResolverError: - self.new_archive_node(nodename, name, top, top.name) + self.new_archive_node(nodename, top, top.name) def list_to_tree(self, parent: NodeAny, names: List[str]) -> None: """convert list of files to a tree""" From 9dfc4da8bfb73deaed5b3de81afde94ac6b46da5 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 23:08:39 +0100 Subject: [PATCH 04/33] check for TODO/FIXME --- tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests.sh b/tests.sh index 366d84e..03b8574 100755 --- a/tests.sh +++ b/tests.sh @@ -61,6 +61,11 @@ pylint -sn setup.py echo "[+] mypy" mypy --strict catcli/ +set +e +grep -R 'TODO' catcli/ && echo "TODO found" && exit 1 +grep -R 'FIXME' catcli/ && echo "FIXME found" && exit 1 +set -e + # unittest echo "[+] unittests" coverage run -p -m pytest tests From 9050c6bcf6c57a83fd0e83743871da6c59a4f8c6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 23:08:48 +0100 Subject: [PATCH 05/33] typecheck --- catcli/catcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index cc88a68..f7277b7 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -12,7 +12,7 @@ import sys import os import datetime from typing import Dict, Any, List -from docopt import docopt +from docopt import docopt # type: ignore # local imports from catcli.version import __version__ as VERSION From f918ea5ae472b8086e6aa926b69e30c963a7baba Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 23:09:04 +0100 Subject: [PATCH 06/33] handle file listing and glob --- catcli/noder.py | 36 ++++++++++++++++++++++++++++++++++-- catcli/utils.py | 6 +++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index 15f53e8..cab21cc 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -376,6 +376,26 @@ class Noder: if len(line) > 0: Logger.stdout_nocolor(line) + def node_has_subs(self, node: Any) -> bool: + """ + node may have children + we explicitely handle all case + for clarity + """ + if not node: + return False + if node.type == nodes.TYPE_TOP: + return True + if node.type == nodes.TYPE_FILE: + return False + if node.type == nodes.TYPE_DIR: + return True + if node.type == nodes.TYPE_STORAGE: + return True + if node.type == nodes.TYPE_ARCHIVED: + return True + return False + def _print_node_native(self, node: NodeAny, pre: str = '', withpath: bool = False, @@ -678,8 +698,20 @@ class Noder: resolv = anytree.resolver.Resolver('name') found = [] try: - # resolve the path in the tree - found = resolv.glob(top, path) + if '*' in path or '?' in path: + # we need to handle glob + found = resolv.glob(top, path) + else: + # we have a canonical path + found = resolv.get(top, path) + if found and self.node_has_subs(found): + # let's find its children as well + print(path) + modpath = os.path.join(path, '*') + found = resolv.glob(top, modpath) + else: + found = [found] + if len(found) < 1: # nothing found self._debug('nothing found') diff --git a/catcli/utils.py b/catcli/utils.py index 4dca31b..60d07d1 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -41,9 +41,9 @@ def path_to_search_all(path: str) -> str: if not path.endswith(os.path.sep): # ensure ends with a separator path += os.path.sep - if not path.endswith(WILD): - # add wild card - path += WILD + # if not path.endswith(WILD): + # # add wild card + # path += WILD return path From e610273dc3be352edbdba25f280f6927d4619ec6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 23:15:01 +0100 Subject: [PATCH 07/33] cleaning --- catcli/noder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catcli/noder.py b/catcli/noder.py index cab21cc..5d13e17 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -706,7 +706,6 @@ class Noder: found = resolv.get(top, path) if found and self.node_has_subs(found): # let's find its children as well - print(path) modpath = os.path.join(path, '*') found = resolv.glob(top, modpath) else: From 3ccaf81abdf6600709b70b0eed763f1ca5c3c2f1 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 3 Jan 2024 23:21:14 +0100 Subject: [PATCH 08/33] fix find --- catcli/noder.py | 5 +++-- tests-ng/find.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index 5d13e17..7c80e25 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -615,8 +615,9 @@ class Noder: item.name = fix_badchars(item.name) storage = self._get_storage(item) parents = self._get_parents(item) - key = f'{storage}/{parents}/{item.name}' - paths[parents] = item + parent_key = f'{storage.name}/{parents}' + key = f'{parent_key}/{item.name}' + paths[parent_key] = item # handle fzf mode if fmt.startswith('fzf'): diff --git a/tests-ng/find.sh b/tests-ng/find.sh index e8201a9..887e57b 100755 --- a/tests-ng/find.sh +++ b/tests-ng/find.sh @@ -40,6 +40,7 @@ ${bin} -B index -c -f --catalog="${catalog}" github2 .github #cat "${catalog}" echo "" +${bin} -B ls -r --catalog="${catalog}" ${bin} -B find --catalog="${catalog}" testing.yml cnt=$(${bin} -B find --catalog="${catalog}" testing.yml | wc -l) [ "${cnt}" != "2" ] && echo "should return 2!" && exit 1 From ddefc662dbf7527a4ed963306a9bf99ef4284137 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 00:00:52 +0100 Subject: [PATCH 09/33] coverage --- .github/codecov.yml | 7 +++++++ .github/workflows/testing.yml | 10 +++++----- .gitignore | 2 ++ tests.sh | 6 ++++-- 4 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..4e5d31b --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,7 @@ +coverage: + status: + project: + default: + target: 90% + threshold: 1% + patch: off \ No newline at end of file diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b444d47..8d425ad 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,9 +21,9 @@ jobs: - name: Run tests run: | ./tests.sh - - name: Coveralls - run: | - pip install coveralls - coveralls --service=github + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + files: coverage.xml env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 8227ca7..99e2cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.pyc .coverage .coverage* +coverages/ +coverage.xml dist/ build/ *.egg-info/ diff --git a/tests.sh b/tests.sh index 03b8574..0896e6d 100755 --- a/tests.sh +++ b/tests.sh @@ -68,7 +68,8 @@ set -e # unittest echo "[+] unittests" -coverage run -p -m pytest tests +mkdir -p coverages/ +coverage run -p --data-file coverages/coverage -m pytest tests # tests-ng echo "[+] tests-ng" @@ -92,7 +93,8 @@ done # merge coverage echo "[+] coverage merge" -coverage combine +coverage combine coverages/* +coverage xml echo "ALL TESTS DONE OK" exit 0 From 07d323f0e6c17dc84c86e86edf0562b2d6c6968b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:15:37 +0100 Subject: [PATCH 10/33] fix tests --- tests-ng/assets/generate.sh | 23 ++++++++++++++++++ tests-ng/assets/github.catalog.csv.txt | 11 +++++---- tests-ng/assets/github.catalog.json | 29 ++++++++++++++--------- tests-ng/assets/github.catalog.native.txt | 7 +++--- tests-ng/compare.sh | 8 ++++--- tests-ng/find.sh | 3 ++- tests-ng/update.sh | 3 ++- 7 files changed, 60 insertions(+), 24 deletions(-) create mode 100755 tests-ng/assets/generate.sh diff --git a/tests-ng/assets/generate.sh b/tests-ng/assets/generate.sh new file mode 100755 index 0000000..4923566 --- /dev/null +++ b/tests-ng/assets/generate.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# run me from the root of the package + +source tests-ng/helper + + +rm -f tests-ng/assets/github.catalog.json +python3 -m catcli.catcli index -c github .github --catalog=tests-ng/assets/github.catalog.json + +# edit catalog +sed -i 's/"free": .*,/"free": 0,/g' "tests-ng/assets/github.catalog.json" +sed -i 's/"total": .*,/"total": 0,/g' "tests-ng/assets/github.catalog.json" + +# native +python3 -m catcli.catcli ls -r -s -B --catalog=tests-ng/assets/github.catalog.json | \ + sed -e 's/free:.*%/free:0.0%/g' \ + -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ + -e 's#du:[^|]* |#du:0/0 |#g' > tests-ng/assets/github.catalog.native.txt + +# csv +python3 -m catcli.catcli ls -r -s -B --catalog=tests-ng/assets/github.catalog.json --format=csv | \ + sed -e 's/"3","[^"]*","[^"]*",""/"3","0","0",""/g' | \ + sed 's/20..-..-.. ..:..:..//g' > tests-ng/assets/github.catalog.csv.txt \ No newline at end of file diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 9d7e3b4..9333720 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,5 +1,6 @@ -"github","storage","","1510","2023-03-09 16:20:59","","","2","0","0","" -"workflows","dir","github/workflows","1493","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" -"pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-09 16:20:59","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","802","2023-03-09 16:20:59","2023-03-09 16:20:44","7144a119ef43adb634654522c12ec250","","","","" -"FUNDING.yml","file","github/FUNDING.yml","17","2023-03-09 16:20:59","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" +"github","storage","","1642","","","","3","0","0","" +"workflows","dir","github/workflows","1521","","","","2","","","" +"pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" +"testing.yml","file","github/workflows/testing.yml","830","","","93c53fc77c983b4686c8971aa25b6b18","","","","" +"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" +"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index 4a13008..c2d31ea 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -10,6 +10,13 @@ "size": 17, "type": "file" }, + { + "maccess": 1704320710.7056112, + "md5": "4203204f75b43cd4bf032402beb3359d", + "name": "codecov.yml", + "size": 104, + "type": "file" + }, { "children": [ { @@ -20,32 +27,32 @@ "type": "file" }, { - "maccess": 1678375244.4870229, - "md5": "7144a119ef43adb634654522c12ec250", + "maccess": 1704320754.2561862, + "md5": "93c53fc77c983b4686c8971aa25b6b18", "name": "testing.yml", - "size": 802, + "size": 830, "type": "file" } ], - "maccess": 1678375244.4865956, + "maccess": 1704320727.2641916, "name": "workflows", - "size": 1493, + "size": 1521, "type": "dir" } ], "free": 0, "name": "github", - "size": 1510, + "size": 1642, "total": 0, - "ts": 1678375259, + "ts": 1704402832, "type": "storage" }, { "attr": { - "access": 1678375259, - "access_version": "0.8.7", - "created": 1678375259, - "created_version": "0.8.7" + "access": 1704402832, + "access_version": "0.9.6", + "created": 1704402832, + "created_version": "0.9.6" }, "name": "meta", "type": "meta" diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 14a0d42..16ee224 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,8 @@ top └── storage: github - nbfiles:2 | totsize:1510 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1493] + nbfiles:3 | totsize:1642 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1521] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:802, md5:7144a119ef43adb634654522c12ec250] + │ └── testing.yml [size:830, md5:93c53fc77c983b4686c8971aa25b6b18] + ├── codecov.yml [size:104, md5:4203204f75b43cd4bf032402beb3359d] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index c7a68b3..4cafd42 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -10,7 +10,8 @@ cd "${prev}" # coverage bin="python3 -m catcli.catcli" if command -v coverage 2>/dev/null; then - bin="coverage run -p --source=catcli -m catcli.catcli" + mkdir -p coverages/ + bin="coverage run -p --data-file coverages/coverage --source=catcli -m catcli.catcli" fi echo "current dir: $(pwd)" @@ -114,13 +115,14 @@ csv="${tmpd}/csv.txt" ${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" # modify created csv mod="${tmpd}/csv.mod.txt" -cat "${csv}" | sed -e 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' | \ +cat "${csv}" | \ + sed -e 's/"3","[^"]*","[^"]*",""/"3","0","0",""/g' | \ sed 's/20..-..-.. ..:..:..//g' > "${mod}" # modify original ori="${tmpd}/ori.mod.txt" cat "tests-ng/assets/github.catalog.csv.txt" | \ sed 's/....-..-.. ..:..:..//g' | \ - sed 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' > "${ori}" + sed 's/"3","[^"]*","[^"]*",""/"3","0","0",""/g' > "${ori}" if command -v delta >/dev/null; then delta -s "${ori}" "${mod}" fi diff --git a/tests-ng/find.sh b/tests-ng/find.sh index 887e57b..34a2d1c 100755 --- a/tests-ng/find.sh +++ b/tests-ng/find.sh @@ -10,7 +10,8 @@ cd "${prev}" # coverage bin="python3 -m catcli.catcli" if command -v coverage 2>/dev/null; then - bin="coverage run -p --source=catcli -m catcli.catcli" + mkdir -p coverages/ + bin="coverage run -p --data-file coverages/coverage --source=catcli -m catcli.catcli" fi echo "current dir: $(pwd)" diff --git a/tests-ng/update.sh b/tests-ng/update.sh index 8a93892..d07eec6 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -10,7 +10,8 @@ cd "${prev}" # coverage bin="python3 -m catcli.catcli" if command -v coverage 2>/dev/null; then - bin="coverage run -p --source=catcli -m catcli.catcli" + mkdir -p coverages/ + bin="coverage run -p --data-file coverages/coverage --source=catcli -m catcli.catcli" fi echo "current dir: $(pwd)" From c61ce59b357fb60102885d815acafb413d0632ab Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:17:35 +0100 Subject: [PATCH 11/33] fix actions --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 8d425ad..ad54fb6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,7 +21,7 @@ jobs: - name: Run tests run: | ./tests.sh - - name: Upload coverage reports to Codecov + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: files: coverage.xml From 982f69d4107fbf32fe95522f95d832b946046227 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:23:35 +0100 Subject: [PATCH 12/33] add python 3.11 --- .github/workflows/testing.yml | 6 +++--- catcli/catcli.py | 2 +- tests-ng/assets/github.catalog.csv.txt | 6 +++--- tests-ng/assets/github.catalog.json | 16 ++++++++-------- tests-ng/assets/github.catalog.native.txt | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ad54fb6..069c5e3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,11 +1,11 @@ name: tests -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/catcli/catcli.py b/catcli/catcli.py index f7277b7..fbf7ca4 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -12,7 +12,7 @@ import sys import os import datetime from typing import Dict, Any, List -from docopt import docopt # type: ignore +from docopt import docopt # type: ignore[import-untyped] # local imports from catcli.version import __version__ as VERSION diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 9333720..4256f67 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,6 +1,6 @@ -"github","storage","","1642","","","","3","0","0","" -"workflows","dir","github/workflows","1521","","","","2","","","" +"github","storage","","1669","","","","3","0","0","" +"workflows","dir","github/workflows","1548","","","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","830","","","93c53fc77c983b4686c8971aa25b6b18","","","","" +"testing.yml","file","github/workflows/testing.yml","857","","","19421bdffa97bff7f391e4927a88fe45","","","","" "codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" "FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index c2d31ea..69d993c 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -27,31 +27,31 @@ "type": "file" }, { - "maccess": 1704320754.2561862, - "md5": "93c53fc77c983b4686c8971aa25b6b18", + "maccess": 1704403201.272202, + "md5": "19421bdffa97bff7f391e4927a88fe45", "name": "testing.yml", - "size": 830, + "size": 857, "type": "file" } ], "maccess": 1704320727.2641916, "name": "workflows", - "size": 1521, + "size": 1548, "type": "dir" } ], "free": 0, "name": "github", - "size": 1642, + "size": 1669, "total": 0, - "ts": 1704402832, + "ts": 1704403315, "type": "storage" }, { "attr": { - "access": 1704402832, + "access": 1704403315, "access_version": "0.9.6", - "created": 1704402832, + "created": 1704403315, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 16ee224..4996f81 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,8 +1,8 @@ top └── storage: github - nbfiles:3 | totsize:1642 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1521] + nbfiles:3 | totsize:1669 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1548] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:830, md5:93c53fc77c983b4686c8971aa25b6b18] + │ └── testing.yml [size:857, md5:19421bdffa97bff7f391e4927a88fe45] ├── codecov.yml [size:104, md5:4203204f75b43cd4bf032402beb3359d] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] From 9d6bba0127da5d5d960693403f3528472c8f85e9 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:27:28 +0100 Subject: [PATCH 13/33] remove python 3.6 --- .github/workflows/testing.yml | 2 +- setup.py | 2 +- tests-ng/assets/github.catalog.csv.txt | 6 +++--- tests-ng/assets/github.catalog.json | 16 ++++++++-------- tests-ng/assets/github.catalog.native.txt | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 069c5e3..5f8577a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 3eb38cb..ce52374 100644 --- a/setup.py +++ b/setup.py @@ -37,11 +37,11 @@ setup( python_requires=REQUIRES_PYTHON, classifiers=[ 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', ], diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 4256f67..0806fac 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,6 +1,6 @@ -"github","storage","","1669","","","","3","0","0","" -"workflows","dir","github/workflows","1548","","","","2","","","" +"github","storage","","1662","","","","3","0","0","" +"workflows","dir","github/workflows","1541","","","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","857","","","19421bdffa97bff7f391e4927a88fe45","","","","" +"testing.yml","file","github/workflows/testing.yml","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" "codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" "FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index 69d993c..f3e1103 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -27,31 +27,31 @@ "type": "file" }, { - "maccess": 1704403201.272202, - "md5": "19421bdffa97bff7f391e4927a88fe45", + "maccess": 1704403569.24789, + "md5": "691df1a4d2f254b5cd04c152e7c6ccaf", "name": "testing.yml", - "size": 857, + "size": 850, "type": "file" } ], "maccess": 1704320727.2641916, "name": "workflows", - "size": 1548, + "size": 1541, "type": "dir" } ], "free": 0, "name": "github", - "size": 1669, + "size": 1662, "total": 0, - "ts": 1704403315, + "ts": 1704403613, "type": "storage" }, { "attr": { - "access": 1704403315, + "access": 1704403613, "access_version": "0.9.6", - "created": 1704403315, + "created": 1704403613, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 4996f81..9a220a4 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,8 +1,8 @@ top └── storage: github - nbfiles:3 | totsize:1669 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1548] + nbfiles:3 | totsize:1662 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1541] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:857, md5:19421bdffa97bff7f391e4927a88fe45] + │ └── testing.yml [size:850, md5:691df1a4d2f254b5cd04c152e7c6ccaf] ├── codecov.yml [size:104, md5:4203204f75b43cd4bf032402beb3359d] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] From 5301126d9027ce1c2f8c4ab44d12b04db09f10bf Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:39:49 +0100 Subject: [PATCH 14/33] fix mypy --- catcli/catcli.py | 2 +- tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index fbf7ca4..cc88a68 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -12,7 +12,7 @@ import sys import os import datetime from typing import Dict, Any, List -from docopt import docopt # type: ignore[import-untyped] +from docopt import docopt # local imports from catcli.version import __version__ as VERSION diff --git a/tests.sh b/tests.sh index 0896e6d..e7a4132 100755 --- a/tests.sh +++ b/tests.sh @@ -59,7 +59,7 @@ pylint -sn setup.py # mypy echo "[+] mypy" -mypy --strict catcli/ +mypy --strict --disable-error-code=import-untyped catcli/ set +e grep -R 'TODO' catcli/ && echo "TODO found" && exit 1 From 5c06e36cc6e2a8d122ef6535d08631fef475f86b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 4 Jan 2024 22:44:21 +0100 Subject: [PATCH 15/33] mypy version --- tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests.sh b/tests.sh index e7a4132..f5e5b48 100755 --- a/tests.sh +++ b/tests.sh @@ -59,6 +59,7 @@ pylint -sn setup.py # mypy echo "[+] mypy" +mypy --version mypy --strict --disable-error-code=import-untyped catcli/ set +e From b0876f3382b4f65fa60d3768ce4245f9e3324b30 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 5 Jan 2024 20:18:22 +0100 Subject: [PATCH 16/33] add mypy.ini --- .mypy.ini | 3 +++ tests.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .mypy.ini diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..424e71f --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,3 @@ +[mypy] +strict = true +disable_error_code = import-untyped \ No newline at end of file diff --git a/tests.sh b/tests.sh index f5e5b48..c74939c 100755 --- a/tests.sh +++ b/tests.sh @@ -60,7 +60,7 @@ pylint -sn setup.py # mypy echo "[+] mypy" mypy --version -mypy --strict --disable-error-code=import-untyped catcli/ +mypy --config-file=.mypy.ini catcli/ set +e grep -R 'TODO' catcli/ && echo "TODO found" && exit 1 From 9326911824e8e4f34f87a6dabd82765754ca0a1e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 5 Jan 2024 20:25:59 +0100 Subject: [PATCH 17/33] coverage badge --- README.md | 2 +- tests-requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 182e0ac..a51d013 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Tests Status](https://github.com/deadc0de6/catcli/workflows/tests/badge.svg?branch=master)](https://github.com/deadc0de6/catcli/actions) [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](http://www.gnu.org/licenses/gpl-3.0) -[![Coveralls](https://img.shields.io/coveralls/github/deadc0de6/catcli)](https://coveralls.io/github/deadc0de6/catcli?branch=master) +[![Coverage](https://codecov.io/gh/deadc0de6/catcli/graph/badge.svg?token=t5dF7UL7K1)](https://codecov.io/gh/deadc0de6/catcli) [![PyPI version](https://badge.fury.io/py/catcli.svg)](https://badge.fury.io/py/catcli) [![AUR](https://img.shields.io/aur/version/catcli-git.svg)](https://aur.archlinux.org/packages/catcli-git) diff --git a/tests-requirements.txt b/tests-requirements.txt index 5174deb..b8a8cd1 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -2,7 +2,6 @@ pycodestyle; python_version >= '3.0' pyflakes; python_version >= '3.0' nose2; python_version >= '3.0' coverage; python_version >= '3.0' -coveralls; python_version >= '3.0' pylint; python_version > '3.0' mypy; python_version > '3.0' pytest; python_version > '3.0' From b7d6f21cc22798689e0a33eee0fa4ee4ae89423d Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 6 Jan 2024 09:50:58 +0100 Subject: [PATCH 18/33] ls sorting --- catcli/noder.py | 11 ++++++++--- requirements.txt | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index 7c80e25..61457f0 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -10,6 +10,7 @@ import shutil import time from typing import List, Union, Tuple, Any, Optional, Dict, cast import anytree # type: ignore +from natsort import os_sort_keygen # local imports from catcli import nodes @@ -723,7 +724,9 @@ class Noder: return found # sort found nodes - found = sorted(found, key=self._sort, reverse=self.sortsize) + #found = os_sorted(found) + found = sorted(found, key=os_sort_keygen(self._sort)) + #found = sorted(found, key=cmp_to_key(self._sort), reverse=self.sortsize) # print the parent if fmt == 'native': @@ -798,8 +801,10 @@ class Noder: @staticmethod def _sort_fs(node: NodeAny) -> Tuple[str, str]: - """sorting nodes dir first and alpha""" - return (node.type, node.name.lstrip('.').lower()) + """sort by name""" + # to sort by types then name + # return (node.type, node.name) + return node.name @staticmethod def _sort_size(node: NodeAny) -> float: diff --git a/requirements.txt b/requirements.txt index 571d8a3..d9cfeb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ types-docopt; python_version >= '3.0' anytree; python_version >= '3.0' pyfzf; python_version >= '3.0' fusepy; python_version >= '3.0' +natsort; python_version >= '3.0' From 691396c96a5d1cd643011e33f6c543dec8a8ffc0 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 6 Jan 2024 14:52:07 +0100 Subject: [PATCH 19/33] add tree command --- catcli/catcli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/catcli/catcli.py b/catcli/catcli.py index cc88a68..0fb2726 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -40,6 +40,7 @@ USAGE = f""" Usage: {NAME} ls [--catalog=] [--format=] [-aBCrVSs] [] + {NAME} tree [--catalog=] [-aBCVSs] [] {NAME} find [--catalog=] [--format=] [-aBCbdVs] [--path=] [] {NAME} index [--catalog=] [--meta=...] @@ -347,6 +348,12 @@ def main() -> bool: Logger.err(f'no such catalog: {catalog_path}') return False cmd_ls(args, noder, top) + elif args['tree']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False + args['--recursive'] = True + cmd_ls(args, noder, top) elif args['mount']: if not catalog.exists(): Logger.err(f'no such catalog: {catalog_path}') From 0dcbfa94bd3bb8320cad287febbdc97b30d59298 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 7 Jan 2024 23:26:44 +0100 Subject: [PATCH 20/33] refactoring --- catcli/catalog.py | 15 +-- catcli/catcli.py | 20 +-- catcli/nodeprinter.py | 73 ---------- catcli/noder.py | 285 ++++++++++----------------------------- catcli/nodes.py | 107 ++++++++++++++- catcli/printer_csv.py | 84 ++++++++++++ catcli/printer_native.py | 151 +++++++++++++++++++++ catcli/utils.py | 25 +++- tests/test_rm.py | 2 +- tests/test_update.py | 2 +- 10 files changed, 451 insertions(+), 313 deletions(-) delete mode 100644 catcli/nodeprinter.py create mode 100644 catcli/printer_csv.py create mode 100644 catcli/printer_native.py diff --git a/catcli/catalog.py b/catcli/catalog.py index 67afe28..d075399 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -96,14 +96,13 @@ class Catalog: def _restore_json(self, string: str) -> Optional[NodeTop]: """restore the tree from json""" imp = JsonImporter(dictimporter=_DictImporter(debug=self.debug)) - self._debug('import from string...') root = imp.import_(string) self._debug(f'Catalog imported from json \"{self.path}\"') self._debug(f'root imported: {root}') if root.type != nodes.TYPE_TOP: return None top = NodeTop(root.name, children=root.children) - self._debug(f'top imported: {top}') + self._debug(f'top imported: {top.name}') return top @@ -126,7 +125,7 @@ class _DictImporter(): assert "parent" not in data attrs = dict(data) # replace attr - attrs = back_attriter(attrs, debug=self.debug) + attrs = back_attriter(attrs) children: Union[str, Any] = attrs.pop("children", []) node = self.nodecls(parent=parent, **attrs) for child in children: @@ -134,16 +133,14 @@ class _DictImporter(): return node -def back_attriter(adict: Dict[str, str], - debug: bool = False) -> Dict[str, str]: +def back_attriter(adict: Dict[str, str]) -> Dict[str, str]: """replace attribute on json restore""" attrs = {} for k, val in adict.items(): + newk = k if k == 'size': - if debug: - Logger.debug(f'changing {k}={val}') - k = 'nodesize' - attrs[k] = val + newk = 'nodesize' + attrs[newk] = val return attrs diff --git a/catcli/catcli.py b/catcli/catcli.py index 0fb2726..593bc71 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -18,6 +18,7 @@ from docopt import docopt from catcli.version import __version__ as VERSION from catcli.nodes import NodeTop, NodeAny from catcli.logger import Logger +from catcli.printer_csv import CsvPrinter from catcli.colors import Colors from catcli.catalog import Catalog from catcli.walker import Walker @@ -116,7 +117,7 @@ def cmd_index(args: Dict[str, Any], except KeyboardInterrupt: Logger.err('aborted') return - node = noder.get_storage_node(top, name) + node = top.get_storage_node() node.parent = None start = datetime.datetime.now() @@ -125,7 +126,7 @@ def cmd_index(args: Dict[str, Any], root = noder.new_storage_node(name, path, top, attr) _, cnt = walker.index(path, root, name) if subsize: - noder.rec_size(root, store=True) + root.nodesize = root.get_rec_size() stop = datetime.datetime.now() diff = stop - start Logger.info(f'Indexed {cnt} file(s) in {diff}') @@ -147,16 +148,17 @@ def cmd_update(args: Dict[str, Any], if not os.path.exists(path): Logger.err(f'\"{path}\" does not exist') return - root = noder.get_storage_node(top, name, newpath=path) - if not root: + storage = noder.find_storage_node_by_name(top, name) + if not storage: Logger.err(f'storage named \"{name}\" does not exist') return + noder.update_storage_path(top, name, path) start = datetime.datetime.now() walker = Walker(noder, usehash=usehash, debug=debug, logpath=logpath) - cnt = walker.reindex(path, root, top) + cnt = walker.reindex(path, storage, top) if subsize: - noder.rec_size(root, store=True) + storage.nodesize = storage.get_rec_size() stop = datetime.datetime.now() diff = stop - start Logger.info(f'updated {cnt} file(s) in {diff}') @@ -189,7 +191,7 @@ def cmd_rm(args: Dict[str, Any], top: NodeTop) -> NodeTop: """rm action""" name = args[''] - node = noder.get_storage_node(top, name) + node = noder.find_storage_node_by_name(top, name) if node: node.parent = None if catalog.save(top): @@ -278,7 +280,7 @@ def print_supported_formats() -> None: """print all supported formats to stdout""" print('"native" : native format') print('"csv" : CSV format') - print(f' {Noder.CSV_HEADER}') + print(f' {CsvPrinter.CSV_HEADER}') print('"fzf-native" : fzf to native (only valid for find)') print('"fzf-csv" : fzf to csv (only valid for find)') @@ -303,7 +305,7 @@ def main() -> bool: return False if args['--verbose']: - print(args) + print(f'args: {args}') # print banner if not args['--no-banner']: diff --git a/catcli/nodeprinter.py b/catcli/nodeprinter.py deleted file mode 100644 index e20b5d5..0000000 --- a/catcli/nodeprinter.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -author: deadc0de6 (https://github.com/deadc0de6) -Copyright (c) 2022, deadc0de6 - -Class for printing nodes -""" - -import sys -from typing import TypeVar, Type, Optional, Tuple, List, \ - Dict, Any - -from catcli.colors import Colors -from catcli.utils import fix_badchars - - -CLASSTYPE = TypeVar('CLASSTYPE', bound='NodePrinter') - - -class NodePrinter: - """a node printer class""" - - STORAGE = 'storage' - ARCHIVE = 'archive' - NBFILES = 'nbfiles' - - @classmethod - def print_storage_native(cls: Type[CLASSTYPE], pre: str, - name: str, args: str, - attr: Dict[str, Any]) -> None: - """print a storage node""" - end = '' - if attr: - end = f' {Colors.GRAY}({attr}){Colors.RESET}' - out = f'{pre}{Colors.UND}{cls.STORAGE}{Colors.RESET}:' - out += ' ' + Colors.PURPLE + fix_badchars(name) + \ - Colors.RESET + end + '\n' - out += f' {Colors.GRAY}{args}{Colors.RESET}' - sys.stdout.write(f'{out}\n') - - @classmethod - def print_file_native(cls: Type[CLASSTYPE], pre: str, - name: str, attr: str) -> None: - """print a file node""" - nobad = fix_badchars(name) - out = f'{pre}{nobad}' - out += f' {Colors.GRAY}[{attr}]{Colors.RESET}' - sys.stdout.write(f'{out}\n') - - @classmethod - def print_dir_native(cls: Type[CLASSTYPE], pre: str, - name: str, - nbchildren: int = 0, - attr: Optional[List[Tuple[str, str]]] = None) -> None: - """print a directory node""" - end = [] - if nbchildren > 0: - end.append(f'{cls.NBFILES}:{nbchildren}') - if attr: - end.append(' '.join([f'{x}:{y}' for x, y in attr])) - end_string = '' - if end: - end_string = f' [{", ".join(end)}]' - out = pre + Colors.BLUE + fix_badchars(name) + Colors.RESET - out += f'{Colors.GRAY}{end_string}{Colors.RESET}' - sys.stdout.write(f'{out}\n') - - @classmethod - def print_archive_native(cls: Type[CLASSTYPE], pre: str, - name: str, archive: str) -> None: - """archive to stdout""" - out = pre + Colors.YELLOW + fix_badchars(name) + Colors.RESET - out += f' {Colors.GRAY}[{cls.ARCHIVE}:{archive}]{Colors.RESET}' - sys.stdout.write(f'{out}\n') diff --git a/catcli/noder.py b/catcli/noder.py index 61457f0..3e4f8f7 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -10,16 +10,17 @@ import shutil import time from typing import List, Union, Tuple, Any, Optional, Dict, cast import anytree # type: ignore -from natsort import os_sort_keygen +from natsort import os_sort_keygen # type: ignore # local imports from catcli import nodes from catcli.nodes import NodeAny, NodeStorage, \ NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta, \ typcast_node -from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars +from catcli.utils import md5sum, fix_badchars, has_attr from catcli.logger import Logger -from catcli.nodeprinter import NodePrinter +from catcli.printer_native import NativePrinter +from catcli.printer_csv import CsvPrinter from catcli.decomp import Decomp from catcli.version import __version__ as VERSION from catcli.exceptions import CatcliException @@ -35,9 +36,7 @@ class Noder: * "file" node representing a file """ - CSV_HEADER = ('name,type,path,size,indexed_at,' - 'maccess,md5,nbfiles,free_space,' - 'total_space,meta') + PRE = ' ' def __init__(self, debug: bool = False, sortsize: bool = False, @@ -53,31 +52,33 @@ class Noder: self.arc = arc if self.arc: self.decomp = Decomp() + self.csv_printer = CsvPrinter() + self.native_printer = NativePrinter() @staticmethod def get_storage_names(top: NodeTop) -> List[str]: """return a list of all storage names""" return [x.name for x in list(top.children)] - def get_storage_node(self, top: NodeTop, - name: str, - newpath: str = '') -> NodeStorage: - """ - return the storage node if any - if newpath is submitted, it will update the media info - """ - found = None + def find_storage_node_by_name(self, top: NodeTop, + name: str) -> Optional[NodeStorage]: + """find a storage node by name""" for node in top.children: if node.type != nodes.TYPE_STORAGE: continue if node.name == name: - found = node - break - if found and newpath and os.path.exists(newpath): - found.free = shutil.disk_usage(newpath).free - found.total = shutil.disk_usage(newpath).total - found.ts = int(time.time()) - return cast(NodeStorage, found) + return cast(NodeStorage, node) + return None + + def update_storage_path(self, top: NodeTop, + name: str, + newpath: str) -> None: + """find and update storage path on update""" + storage = self.find_storage_node_by_name(top, name) + if storage and newpath and os.path.exists(newpath): + storage.free = shutil.disk_usage(newpath).free + storage.total = shutil.disk_usage(newpath).total + storage.ts = int(time.time()) @staticmethod def get_node(top: NodeTop, @@ -115,7 +116,7 @@ class Noder: return node, False # force re-indexing if no maccess maccess = os.path.getmtime(path) - if not self._has_attr(node, 'maccess') or \ + if not has_attr(node, 'maccess') or \ not node.maccess: self._debug('\tchange: no maccess found') return node, True @@ -134,39 +135,6 @@ class Noder: self._debug(f'\tchange: no change for \"{path}\"') return node, False - def rec_size(self, node: Union[NodeDir, NodeStorage], - store: bool = True) -> int: - """ - recursively traverse tree and return size - @store: store the size in the node - """ - if node.type == nodes.TYPE_FILE: - node.__class__ = NodeFile - msg = f'size of {node.type} \"{node.name}\": {node.nodesize}' - self._debug(msg) - return node.nodesize - msg = f'getting node size recursively for \"{node.name}\"' - self._debug(msg) - fullsize: int = 0 - for i in node.children: - if node.type == nodes.TYPE_DIR: - sub_size = self.rec_size(i, store=store) - if store: - i.nodesize = sub_size - fullsize += sub_size - continue - if node.type == nodes.TYPE_STORAGE: - sub_size = self.rec_size(i, store=store) - if store: - i.nodesize = sub_size - fullsize += sub_size - continue - self._debug(f'skipping {node.name}') - if store: - node.nodesize = fullsize - self._debug(f'size of {node.type} \"{node.name}\": {fullsize}') - return fullsize - ############################################################### # public helpers ############################################################### @@ -315,9 +283,9 @@ class Noder: ############################################################### # printing ############################################################### - def _node_to_csv(self, node: NodeAny, - sep: str = ',', - raw: bool = False) -> None: + def _print_node_csv(self, node: NodeAny, + sep: str = ',', + raw: bool = False) -> None: """ print a node to csv @node: the node to consider @@ -329,53 +297,14 @@ class Noder: if node.type == nodes.TYPE_TOP: return - out = [] if node.type == nodes.TYPE_STORAGE: - # handle storage - out.append(node.name) # name - out.append(node.type) # type - out.append('') # fake full path - size = self.rec_size(node, store=False) - out.append(size_to_str(size, raw=raw)) # size - out.append(epoch_to_str(node.ts)) # indexed_at - out.append('') # fake maccess - out.append('') # fake md5 - out.append(str(len(node.children))) # nbfiles - # fake free_space - out.append(size_to_str(node.free, raw=raw)) - # fake total_space - out.append(size_to_str(node.total, raw=raw)) - out.append(node.attr) # meta + self.csv_printer.print_storage(node, + sep=sep, + raw=raw) else: - # handle other nodes - out.append(node.name.replace('"', '""')) # name - out.append(node.type) # type - parents = self._get_parents(node) - storage = self._get_storage(node) - fullpath = os.path.join(storage.name, parents) - out.append(fullpath.replace('"', '""')) # full path - - out.append(size_to_str(node.nodesize, raw=raw)) # size - out.append(epoch_to_str(storage.ts)) # indexed_at - if self._has_attr(node, 'maccess'): - out.append(epoch_to_str(node.maccess)) # maccess - else: - out.append('') # fake maccess - if self._has_attr(node, 'md5'): - out.append(node.md5) # md5 - else: - out.append('') # fake md5 - if node.type == nodes.TYPE_DIR: - out.append(str(len(node.children))) # nbfiles - else: - out.append('') # fake nbfiles - out.append('') # fake free_space - out.append('') # fake total_space - out.append('') # fake meta - - line = sep.join(['"' + o + '"' for o in out]) - if len(line) > 0: - Logger.stdout_nocolor(line) + self.csv_printer.print_node(node, + sep=sep, + raw=raw) def node_has_subs(self, node: Any) -> bool: """ @@ -415,88 +344,35 @@ class Noder: if node.type == nodes.TYPE_TOP: # top node node.__class__ = NodeTop - Logger.stdout_nocolor(f'{pre}{node.name}') + self.native_printer.print_top(pre, node.name) elif node.type == nodes.TYPE_FILE: # node of type file node.__class__ = NodeFile - name = node.name - storage = self._get_storage(node) - if withpath: - name = os.sep.join([ - storage.name, - self._get_parents(node.parent), - name]) - name = name.lstrip(os.sep) - attr_str = '' - if node.md5: - attr_str = f', md5:{node.md5}' - size = size_to_str(node.nodesize, raw=raw) - compl = f'size:{size}{attr_str}' - if withstorage: - content = Logger.get_bold_text(storage.name) - compl += f', storage:{content}' - NodePrinter.print_file_native(pre, name, compl) + self.native_printer.print_file(pre, node, + withpath=withpath, + withstorage=withstorage, + raw=raw) elif node.type == nodes.TYPE_DIR: # node of type directory node.__class__ = NodeDir - name = node.name - storage = self._get_storage(node) - if withpath: - name = os.sep.join([ - storage.name, - self._get_parents(node.parent), - name]) - name = name.lstrip(os.sep) - nbchildren = 0 - if withnbchildren: - nbchildren = len(node.children) - attr: List[Tuple[str, str]] = [] - if node.nodesize: - attr.append(('totsize', size_to_str(node.nodesize, raw=raw))) - if withstorage: - attr.append(('storage', Logger.get_bold_text(storage.name))) - NodePrinter.print_dir_native(pre, - name, - nbchildren=nbchildren, - attr=attr) + self.native_printer.print_dir(pre, + node, + withpath=withpath, + withstorage=withstorage, + withnbchildren=withnbchildren, + raw=raw) elif node.type == nodes.TYPE_STORAGE: # node of type storage node.__class__ = NodeStorage - sztotal = size_to_str(node.total, raw=raw) - szused = size_to_str(node.total - node.free, raw=raw) - nbchildren = len(node.children) - pcent = 0 - if node.total > 0: - pcent = node.free * 100 / node.total - freepercent = f'{pcent:.1f}%' - # get the date - timestamp = '' - if self._has_attr(node, 'ts'): - timestamp = 'date:' - timestamp += epoch_to_str(node.ts) - disksize = '' - # the children size - recsize = self.rec_size(node, store=False) - sizestr = size_to_str(recsize, raw=raw) - disksize = 'totsize:' + f'{sizestr}' - # format the output - name = node.name - args = [ - 'nbfiles:' + f'{nbchildren}', - disksize, - f'free:{freepercent}', - 'du:' + f'{szused}/{sztotal}', - timestamp] - argsstring = ' | '.join(args) - NodePrinter.print_storage_native(pre, - name, - argsstring, - node.attr) + + self.native_printer.print_storage(pre, + node, + raw=raw) elif node.type == nodes.TYPE_ARCHIVED: # archive node node.__class__ = NodeArchived if self.arc: - NodePrinter.print_archive_native(pre, node.name, node.archive) + self.native_printer.print_archive(pre, node.name, node.archive) else: Logger.err(f'bad node encountered: {node}') @@ -518,18 +394,18 @@ class Noder: withnbchildren=True, raw=raw) elif fmt == 'csv': # csv output - self._to_csv(node, raw=raw) + self._print_nodes_csv(node, raw=raw) elif fmt == 'csv-with-header': # csv output - Logger.stdout_nocolor(self.CSV_HEADER) - self._to_csv(node, raw=raw) + self.csv_printer.print_header() + self._print_nodes_csv(node, raw=raw) - def _to_csv(self, node: NodeAny, - raw: bool = False) -> None: + def _print_nodes_csv(self, node: NodeAny, + raw: bool = False) -> None: """print the tree to csv""" rend = anytree.RenderTree(node, childiter=self._sort_tree) for _, _, item in rend: - self._node_to_csv(item, raw=raw) + self._print_node_csv(item, raw=raw) @staticmethod def _fzf_prompt(strings: Any) -> Any: @@ -555,8 +431,8 @@ class Noder: for _, _, rend in rendered: if not rend: continue - parents = self._get_parents(rend) - storage = self._get_storage(rend) + parents = rend.get_parent_hierarchy() + storage = rend.get_storage_node() fullpath = os.path.join(storage.name, parents) the_nodes[fullpath] = rend # prompt with fzf @@ -613,9 +489,10 @@ class Noder: # compile found nodes paths = {} for item in found: + typcast_node(item) item.name = fix_badchars(item.name) - storage = self._get_storage(item) - parents = self._get_parents(item) + storage = item.get_storage_node() + parents = item.get_parent_hierarchy() parent_key = f'{storage.name}/{parents}' key = f'{parent_key}/{item.name}' paths[parent_key] = item @@ -640,9 +517,9 @@ class Noder: raw=raw) elif fmt.startswith('csv'): if fmt == 'csv-with-header': - Logger.stdout_nocolor(self.CSV_HEADER) + self.csv_printer.print_header() for _, item in paths.items(): - self._node_to_csv(item, raw=raw) + self._print_node_csv(item, raw=raw) # execute script if any if script: @@ -695,13 +572,14 @@ class Noder: @fmt: output format @raw: print raw size """ - self._debug(f'walking path: \"{path}\" from {top}') + self._debug(f'walking path: \"{path}\" from {top.name}') resolv = anytree.resolver.Resolver('name') found = [] try: if '*' in path or '?' in path: # we need to handle glob + self._debug(f'glob with top {top.name} and path {path}') found = resolv.glob(top, path) else: # we have a canonical path @@ -724,9 +602,7 @@ class Noder: return found # sort found nodes - #found = os_sorted(found) found = sorted(found, key=os_sort_keygen(self._sort)) - #found = sorted(found, key=cmp_to_key(self._sort), reverse=self.sortsize) # print the parent if fmt == 'native': @@ -735,21 +611,21 @@ class Noder: withnbchildren=True, raw=raw) elif fmt.startswith('csv'): - self._node_to_csv(found[0].parent, raw=raw) + self._print_node_csv(found[0].parent, raw=raw) elif fmt.startswith('fzf'): pass # print all found nodes if fmt == 'csv-with-header': - Logger.stdout_nocolor(self.CSV_HEADER) + self.csv_printer.print_header() for item in found: if fmt == 'native': self._print_node_native(item, withpath=False, - pre='- ', + pre=Noder.PRE, withnbchildren=True, raw=raw) elif fmt.startswith('csv'): - self._node_to_csv(item, raw=raw) + self._print_node_csv(item, raw=raw) elif fmt.startswith('fzf'): self._to_fzf(item, fmt) @@ -800,11 +676,10 @@ class Noder: return self._sort_fs(lst) @staticmethod - def _sort_fs(node: NodeAny) -> Tuple[str, str]: + def _sort_fs(node: NodeAny) -> str: """sort by name""" # to sort by types then name - # return (node.type, node.name) - return node.name + return str(node.name) @staticmethod def _sort_size(node: NodeAny) -> float: @@ -816,28 +691,6 @@ class Noder: except AttributeError: return 0 - def _get_storage(self, node: NodeAny) -> NodeStorage: - """recursively traverse up to find storage""" - if node.type == nodes.TYPE_STORAGE: - return node - return cast(NodeStorage, node.ancestors[1]) - - @staticmethod - def _has_attr(node: NodeAny, attr: str) -> bool: - """return True if node has attr as attribute""" - return attr in node.__dict__.keys() - - def _get_parents(self, node: NodeAny) -> str: - """get all parents recursively""" - if node.type == nodes.TYPE_STORAGE: - return '' - if node.type == nodes.TYPE_TOP: - return '' - parent = self._get_parents(node.parent) - if parent: - return os.sep.join([parent, node.name]) - return str(node.name) - @staticmethod def _get_hash(path: str) -> str: """return md5 hash of node""" diff --git a/catcli/nodes.py b/catcli/nodes.py index 6743958..545ba3f 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -6,9 +6,12 @@ Class that represents a node in the catalog tree """ # pylint: disable=W0622 -from typing import Dict, Any +import os +from typing import Dict, Any, cast from anytree import NodeMixin # type: ignore +from catcli.exceptions import CatcliException + TYPE_TOP = 'top' TYPE_FILE = 'file' @@ -35,6 +38,8 @@ def typcast_node(node: Any) -> None: node.__class__ = NodeStorage elif node.type == TYPE_META: node.__class__ = NodeMeta + else: + raise CatcliException(f"bad node: {node}") class NodeAny(NodeMixin): # type: ignore @@ -60,6 +65,18 @@ class NodeAny(NodeMixin): # type: ignore def __str__(self) -> str: return self._to_str() + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + raise NotImplementedError + + def get_storage_node(self) -> NodeMixin: + """recursively traverse up to find storage""" + return None + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + raise NotImplementedError + def flagged(self) -> bool: """is flagged""" if not hasattr(self, '_flagged'): @@ -90,6 +107,14 @@ class NodeTop(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + return '' + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + return 0 + def __str__(self) -> str: return self._to_str() @@ -115,6 +140,22 @@ class NodeFile(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + typcast_node(self.parent) + path = self.parent.get_parent_hierarchy() + if path: + return os.sep.join([path, self.name]) + return str(self.name) + + def get_storage_node(self) -> NodeAny: + """recursively traverse up to find storage""" + return cast(NodeStorage, self.ancestors[1]) + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + return self.nodesize + def __str__(self) -> str: return self._to_str() @@ -138,6 +179,26 @@ class NodeDir(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + typcast_node(self.parent) + path = self.parent.get_parent_hierarchy() + if path: + return os.sep.join([path, self.name]) + return str(self.name) + + def get_storage_node(self) -> NodeAny: + """recursively traverse up to find storage""" + return cast(NodeStorage, self.ancestors[1]) + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + totsize: int = 0 + for node in self.children: + typcast_node(node) + totsize += node.get_rec_size() + return totsize + def __str__(self) -> str: return self._to_str() @@ -163,6 +224,22 @@ class NodeArchived(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + typcast_node(self.parent) + path = self.parent.get_parent_hierarchy() + if path: + return os.sep.join([path, self.name]) + return str(self.name) + + def get_storage_node(self) -> NodeAny: + """recursively traverse up to find storage""" + return cast(NodeStorage, self.ancestors[1]) + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + return self.nodesize + def __str__(self) -> str: return self._to_str() @@ -192,6 +269,22 @@ class NodeStorage(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + return '' + + def get_storage_node(self) -> NodeAny: + """recursively traverse up to find storage""" + return self + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + totsize: int = 0 + for node in self.children: + typcast_node(node) + totsize += node.get_rec_size() + return totsize + def __str__(self) -> str: return self._to_str() @@ -213,5 +306,17 @@ class NodeMeta(NodeAny): if children: self.children = children + def get_parent_hierarchy(self) -> str: + """get all parents recursively""" + typcast_node(self.parent) + path = self.parent.get_parent_hierarchy() + if path: + return os.sep.join([path, self.name]) + return str(self.name) + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + return 0 + def __str__(self) -> str: return self._to_str() diff --git a/catcli/printer_csv.py b/catcli/printer_csv.py new file mode 100644 index 0000000..3b7a453 --- /dev/null +++ b/catcli/printer_csv.py @@ -0,0 +1,84 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2024, deadc0de6 + +Class for printing nodes in csv format +""" + +import sys +import os +from typing import List + +from catcli.nodes import NodeAny, NodeStorage, TYPE_DIR +from catcli.utils import size_to_str, epoch_to_str, \ + has_attr + + +class CsvPrinter: + """a node printer class""" + + DEFSEP = ',' + CSV_HEADER = ('name,type,path,size,indexed_at,' + 'maccess,md5,nbfiles,free_space,' + 'total_space,meta') + + def _print_entries(self, entries: List[str], sep: str = DEFSEP) -> None: + line = sep.join(['"' + o + '"' for o in entries]) + if len(line) > 0: + sys.stdout.write(f'{line}\n') + + def print_header(self) -> None: + """print csv header""" + sys.stdout.write(f'{self.CSV_HEADER}\n') + + def print_storage(self, node: NodeStorage, + sep: str = DEFSEP, + raw: bool = False) -> None: + """print a storage node""" + out = [] + out.append(node.name) # name + out.append(node.type) # type + out.append('') # fake full path + size = node.get_rec_size() + out.append(size_to_str(size, raw=raw)) # size + out.append(epoch_to_str(node.ts)) # indexed_at + out.append('') # fake maccess + out.append('') # fake md5 + out.append(str(len(node.children))) # nbfiles + # fake free_space + out.append(size_to_str(node.free, raw=raw)) + # fake total_space + out.append(size_to_str(node.total, raw=raw)) + out.append(node.attr) # meta + self._print_entries(out, sep=sep) + + def print_node(self, node: NodeAny, + sep: str = DEFSEP, + raw: bool = False) -> None: + """print other nodes""" + out = [] + out.append(node.name.replace('"', '""')) # name + out.append(node.type) # type + parents = node.get_parent_hierarchy() + storage = node.get_storage_node() + fullpath = os.path.join(storage.name, parents) + out.append(fullpath.replace('"', '""')) # full path + + out.append(size_to_str(node.nodesize, raw=raw)) # size + out.append(epoch_to_str(storage.ts)) # indexed_at + if has_attr(node, 'maccess'): + out.append(epoch_to_str(node.maccess)) # maccess + else: + out.append('') # fake maccess + if has_attr(node, 'md5'): + out.append(node.md5) # md5 + else: + out.append('') # fake md5 + if node.type == TYPE_DIR: + out.append(str(len(node.children))) # nbfiles + else: + out.append('') # fake nbfiles + out.append('') # fake free_space + out.append('') # fake total_space + out.append('') # fake meta + self._print_entries(out, sep=sep) diff --git a/catcli/printer_native.py b/catcli/printer_native.py new file mode 100644 index 0000000..5091eca --- /dev/null +++ b/catcli/printer_native.py @@ -0,0 +1,151 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2022, deadc0de6 + +Class for printing nodes in native format +""" + +import sys +import os + +from catcli.nodes import NodeFile, NodeDir, NodeStorage +from catcli.colors import Colors +from catcli.logger import Logger +from catcli.utils import fix_badchars, size_to_str, \ + has_attr, epoch_to_ls_str + + +TS_LJUST = 13 +SIZE_LJUST = 6 +NAME_LJUST = 20 + +class NativePrinter: + """a node printer class""" + + STORAGE = 'storage' + ARCHIVE = 'archive' + NBFILES = 'nbfiles' + + def print_top(self, pre: str, name: str) -> None: + """print top node""" + sys.stdout.write(f'{pre}{name}\n') + + def print_storage(self, pre: str, + node: NodeStorage, + raw: bool = False) -> None: + """print a storage node""" + # construct name + name = node.name + name = fix_badchars(name) + # construct attrs + attrs = [] + # nb files + attrs.append(f'nbfiles:{len(node.children)}') + # the children size + recsize = node.get_rec_size() + sizestr = size_to_str(recsize, raw=raw) + attrs.append(f'totsize:{sizestr}') + # free + pcent = 0.0 + if node.total > 0: + pcent = node.free * 100 / node.total + attrs.append(f'free:{pcent:.1f}%') + # du + sztotal = size_to_str(node.total, raw=raw) + szused = size_to_str(node.total - node.free, raw=raw) + attrs.append(f'du:{szused}/{sztotal}') + # timestamp + if has_attr(node, 'ts'): + attrs.append(f'date:{epoch_to_ls_str(node.ts)}') + + # print + out = f'{pre}{Colors.UND}{self.STORAGE}{Colors.RESET}: ' + out += f'{Colors.PURPLE}{name}{Colors.RESET}' + if attrs: + out += f'\n{" "*len(name)}{Colors.GRAY}{"|".join(attrs)}{Colors.RESET}' + sys.stdout.write(f'{out}\n') + + def print_file(self, pre: str, + node: NodeFile, + withpath: bool = False, + withstorage: bool = False, + raw: bool = False) -> None: + """print a file node""" + # construct name + name = node.name + storage = node.get_storage_node() + if withpath: + name = os.sep.join([ + storage.name, + node.parent.get_parent_hierarchy(), + name]) + name = fix_badchars(name) + # construct attributes + attrs = [] + if node.md5: + attrs.append(f'md5:{node.md5}') + if withstorage: + content = Logger.get_bold_text(storage.name) + attrs.append(f', storage:{content}') + # print + out = [] + out .append(f'{pre}') + line = name.ljust(NAME_LJUST, ' ') + out.append(f'{line}') + size = 0 + if node.nodesize: + size = node.nodesize + line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ') + out.append(f'{Colors.BLUE}{line}{Colors.RESET}') + line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ') + out.append(f'{Colors.PURPLE}{line}{Colors.RESET}') + if attrs: + out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') + sys.stdout.write(f'{" ".join(out)}\n') + + def print_dir(self, pre: str, + node: NodeDir, + withpath: bool = False, + withstorage: bool = False, + withnbchildren: bool = False, + raw: bool = False) -> None: + """print a directory node""" + # construct name + name = node.name + storage = node.get_storage_node() + if withpath: + name = os.sep.join([ + storage.name, + node.parent.get_parent_hierarchy(), + name]) + name = fix_badchars(name) + # construct attrs + attrs = [] + if withnbchildren: + nbchildren = len(node.children) + attrs.append(f'{self.NBFILES}:{nbchildren}') + if withstorage: + attrs.append(f'storage:{Logger.get_bold_text(storage.name)}') + # print + out = [] + out.append(f'{pre}') + line = name.ljust(NAME_LJUST, ' ') + out.append(f'{Colors.BLUE}{line}{Colors.RESET}') + size = 0 + if node.nodesize: + size = node.nodesize + line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ') + out.append(f'{Colors.GRAY}{line}{Colors.RESET}') + line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ') + out.append(f'{Colors.GRAY}{line}{Colors.RESET}') + if attrs: + out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') + sys.stdout.write(f'{" ".join(out)}\n') + + def print_archive(self, pre: str, + name: str, archive: str) -> None: + """print an archive""" + name = fix_badchars(name) + out = f'{pre}{Colors.YELLOW}{name}{Colors.RESET} ' + out += f'{Colors.GRAY}[{self.ARCHIVE}:{archive}]{Colors.RESET}' + sys.stdout.write(f'{out}\n') diff --git a/catcli/utils.py b/catcli/utils.py index 60d07d1..078fec0 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -17,6 +17,8 @@ from catcli.exceptions import CatcliException WILD = '*' +TS_FORMAT_6 = '%b %d %H:%M' +TS_FORMAT_MORE = '%b %d %Y' def path_to_top(path: str) -> str: @@ -38,9 +40,9 @@ def path_to_search_all(path: str) -> str: if not path.startswith(pre): # prepend with top node path path = pre + path - if not path.endswith(os.path.sep): - # ensure ends with a separator - path += os.path.sep + # if not path.endswith(os.path.sep): + # # ensure ends with a separator + # path += os.path.sep # if not path.endswith(WILD): # # add wild card # path += WILD @@ -95,6 +97,18 @@ def epoch_to_str(epoch: float) -> str: return timestamp.strftime(fmt) +def epoch_to_ls_str(epoch: float) -> str: + """convert epoch to string""" + if not epoch: + return '' + timestamp = datetime.datetime.fromtimestamp(epoch) + delta = datetime.date.today() - datetime.timedelta(days=6*365/12) + fmt = TS_FORMAT_MORE + if timestamp.date() < delta: + fmt = TS_FORMAT_6 + return timestamp.strftime(fmt) + + def ask(question: str) -> bool: """ask the user what to do""" resp = input(f'{question} [y|N] ? ') @@ -117,3 +131,8 @@ def edit(string: str) -> str: def fix_badchars(string: str) -> str: """fix none utf-8 chars in string""" return string.encode('utf-8', 'ignore').decode('utf-8') + + +def has_attr(node: nodes.NodeAny, attr: str) -> bool: + """return True if node has attr as attribute""" + return attr in node.__dict__.keys() diff --git a/tests/test_rm.py b/tests/test_rm.py index e9e1b6f..7087ebf 100644 --- a/tests/test_rm.py +++ b/tests/test_rm.py @@ -46,7 +46,7 @@ class TestRm(unittest.TestCase): top = cmd_rm(args, noder, catalog, top) # ensure there no children anymore - self.assertTrue(len(top.children) == 0) + self.assertEqual(len(top.children), 0) def main(): diff --git a/tests/test_update.py b/tests/test_update.py index 6f75610..7a5b61f 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -125,7 +125,7 @@ class TestUpdate(unittest.TestCase): # explore the top node to find all nodes self.assertEqual(len(top.children), 1) storage = top.children[0] - self.assertTrue(len(storage.children) == 8) + self.assertEqual(len(storage.children), 8) # ensure d1f1 md5 sum has changed in catalog nods = noder.find_name(top, os.path.basename(d1f1)) From e58da8b6bfdfae536a2aa224877dc17d4e9eb086 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 8 Jan 2024 22:17:47 +0100 Subject: [PATCH 21/33] wildcard and refactoring --- catcli/catcli.py | 17 +++--- catcli/colors.py | 2 + catcli/noder.py | 55 ++++++++----------- catcli/printer_native.py | 65 ++++++++++++----------- catcli/utils.py | 14 +++++ tests-ng/assets/generate.sh | 3 +- tests-ng/assets/github.catalog.csv.txt | 6 +-- tests-ng/assets/github.catalog.json | 8 +-- tests-ng/assets/github.catalog.native.txt | 13 +++-- tests-ng/compare.sh | 1 + tests-ng/find.sh | 8 ++- tests-ng/helper | 8 +++ tests-ng/ls.sh | 59 ++++++++++++++++++++ tests-ng/update.sh | 16 +++--- tests/test_update.py | 12 ++--- 15 files changed, 183 insertions(+), 104 deletions(-) create mode 100755 tests-ng/ls.sh diff --git a/catcli/catcli.py b/catcli/catcli.py index 593bc71..245f0ce 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -46,7 +46,8 @@ Usage: [-aBCbdVs] [--path=] [] {NAME} index [--catalog=] [--meta=...] [-aBCcfnV] - {NAME} update [--catalog=] [-aBCcfnV] [--lpath=] + {NAME} update [--catalog=] [-aBCcfnV] + [--lpath=] {NAME} mount [--catalog=] [-V] {NAME} rm [--catalog=] [-BCfV] {NAME} rename [--catalog=] [-BCfV] @@ -176,8 +177,8 @@ def cmd_ls(args: Dict[str, Any], raise BadFormatException('fzf is not supported in ls, use find') found = noder.list(top, path, - rec=args['--recursive'], fmt=fmt, + rec=args['--recursive'], raw=args['--raw-size']) if not found: path = args[''] @@ -213,12 +214,12 @@ def cmd_find(args: Dict[str, Any], search_for = args[''] if args['--verbose']: Logger.debug(f'search for \"{search_for}\" under \"{top.name}\"') - found = noder.find_name(top, search_for, - script=script, - startnode=startpath, - only_dir=directory, - fmt=fmt, - raw=raw) + found = noder.find(top, search_for, + script=script, + startnode=startpath, + only_dir=directory, + fmt=fmt, + raw=raw) return found diff --git a/catcli/colors.py b/catcli/colors.py index 07010c5..6c5da98 100644 --- a/catcli/colors.py +++ b/catcli/colors.py @@ -20,7 +20,9 @@ class Colors: PURPLE = '\033[1;35m' BLUE = '\033[94m' GRAY = '\033[0;37m' + CYAN = '\033[36m' MAGENTA = '\033[95m' + WHITE = '\033[97m' RESET = '\033[0m' EMPH = '\033[33m' BOLD = '\033[1m' diff --git a/catcli/noder.py b/catcli/noder.py index 3e4f8f7..7f5f0f8 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -9,6 +9,7 @@ import os import shutil import time from typing import List, Union, Tuple, Any, Optional, Dict, cast +import fnmatch import anytree # type: ignore from natsort import os_sort_keygen # type: ignore @@ -17,7 +18,8 @@ from catcli import nodes from catcli.nodes import NodeAny, NodeStorage, \ NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta, \ typcast_node -from catcli.utils import md5sum, fix_badchars, has_attr +from catcli.utils import md5sum, fix_badchars, has_attr, \ + get_node_fullpath from catcli.logger import Logger from catcli.printer_native import NativePrinter from catcli.printer_csv import CsvPrinter @@ -35,8 +37,7 @@ class Noder: * "dir" node representing a directory * "file" node representing a file """ - - PRE = ' ' + # pylint: disable=R0904 def __init__(self, debug: bool = False, sortsize: bool = False, @@ -292,11 +293,11 @@ class Noder: @sep: CSV separator character @raw: print raw size rather than human readable """ + typcast_node(node) if not node: return if node.type == nodes.TYPE_TOP: return - if node.type == nodes.TYPE_STORAGE: self.csv_printer.print_storage(node, sep=sep, @@ -341,20 +342,18 @@ class Noder: @withstorage: print the node storage it belongs to @raw: print raw size rather than human readable """ + typcast_node(node) if node.type == nodes.TYPE_TOP: # top node - node.__class__ = NodeTop self.native_printer.print_top(pre, node.name) elif node.type == nodes.TYPE_FILE: # node of type file - node.__class__ = NodeFile self.native_printer.print_file(pre, node, withpath=withpath, withstorage=withstorage, raw=raw) elif node.type == nodes.TYPE_DIR: # node of type directory - node.__class__ = NodeDir self.native_printer.print_dir(pre, node, withpath=withpath, @@ -363,14 +362,11 @@ class Noder: raw=raw) elif node.type == nodes.TYPE_STORAGE: # node of type storage - node.__class__ = NodeStorage - self.native_printer.print_storage(pre, node, raw=raw) elif node.type == nodes.TYPE_ARCHIVED: # archive node - node.__class__ = NodeArchived if self.arc: self.native_printer.print_archive(pre, node.name, node.archive) else: @@ -458,13 +454,13 @@ class Noder: ############################################################### # searching ############################################################### - def find_name(self, top: NodeTop, - key: str, - script: bool = False, - only_dir: bool = False, - startnode: Optional[NodeAny] = None, - fmt: str = 'native', - raw: bool = False) -> List[NodeAny]: + def find(self, top: NodeTop, + key: str, + script: bool = False, + only_dir: bool = False, + startnode: Optional[NodeAny] = None, + fmt: str = 'native', + raw: bool = False) -> List[NodeAny]: """ find files based on their names @top: top node @@ -511,7 +507,8 @@ class Noder: else: if fmt == 'native': for _, item in paths.items(): - self._print_node_native(item, withpath=True, + self._print_node_native(item, + withpath=True, withnbchildren=True, withstorage=True, raw=raw) @@ -533,6 +530,7 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" def find_name(node: NodeAny) -> bool: + path = get_node_fullpath(node) if node.type == nodes.TYPE_STORAGE: # ignore storage nodes return False @@ -549,7 +547,9 @@ class Noder: # filter if not term: return True - if term.lower() in node.name.lower(): + if term in path: + return True + if fnmatch.fnmatch(path, term): return True # ignore @@ -572,17 +572,18 @@ class Noder: @fmt: output format @raw: print raw size """ - self._debug(f'walking path: \"{path}\" from {top.name}') + self._debug(f'walking path: \"{path}\" from \"{top.name}\"') resolv = anytree.resolver.Resolver('name') found = [] try: if '*' in path or '?' in path: # we need to handle glob - self._debug(f'glob with top {top.name} and path {path}') + self._debug('glob ls...') found = resolv.glob(top, path) else: # we have a canonical path + self._debug('get ls...') found = resolv.get(top, path) if found and self.node_has_subs(found): # let's find its children as well @@ -604,24 +605,12 @@ class Noder: # sort found nodes found = sorted(found, key=os_sort_keygen(self._sort)) - # print the parent - if fmt == 'native': - self._print_node_native(found[0].parent, - withpath=False, - withnbchildren=True, - raw=raw) - elif fmt.startswith('csv'): - self._print_node_csv(found[0].parent, raw=raw) - elif fmt.startswith('fzf'): - pass - # print all found nodes if fmt == 'csv-with-header': self.csv_printer.print_header() for item in found: if fmt == 'native': self._print_node_native(item, withpath=False, - pre=Noder.PRE, withnbchildren=True, raw=raw) elif fmt.startswith('csv'): diff --git a/catcli/printer_native.py b/catcli/printer_native.py index 5091eca..b9f86ce 100644 --- a/catcli/printer_native.py +++ b/catcli/printer_native.py @@ -6,18 +6,23 @@ Class for printing nodes in native format """ import sys -import os from catcli.nodes import NodeFile, NodeDir, NodeStorage from catcli.colors import Colors from catcli.logger import Logger from catcli.utils import fix_badchars, size_to_str, \ - has_attr, epoch_to_ls_str + has_attr, epoch_to_ls_str, get_node_fullpath -TS_LJUST = 13 -SIZE_LJUST = 6 -NAME_LJUST = 20 +COLOR_STORAGE = Colors.YELLOW +COLOR_FILE = Colors.WHITE +COLOR_DIRECTORY = Colors.BLUE +COLOR_ARCHIVE = Colors.PURPLE +COLOR_TS = Colors.CYAN +COLOR_SIZE = Colors.GREEN + +FULLPATH_IN_NAME = True + class NativePrinter: """a node printer class""" @@ -60,9 +65,9 @@ class NativePrinter: # print out = f'{pre}{Colors.UND}{self.STORAGE}{Colors.RESET}: ' - out += f'{Colors.PURPLE}{name}{Colors.RESET}' + out += f'{COLOR_STORAGE}{name}{Colors.RESET}' if attrs: - out += f'\n{" "*len(name)}{Colors.GRAY}{"|".join(attrs)}{Colors.RESET}' + out += f' [{Colors.WHITE}{"|".join(attrs)}{Colors.RESET}]' sys.stdout.write(f'{out}\n') def print_file(self, pre: str, @@ -75,32 +80,30 @@ class NativePrinter: name = node.name storage = node.get_storage_node() if withpath: - name = os.sep.join([ - storage.name, - node.parent.get_parent_hierarchy(), - name]) - name = fix_badchars(name) + name = get_node_fullpath(node) # construct attributes attrs = [] if node.md5: attrs.append(f'md5:{node.md5}') if withstorage: content = Logger.get_bold_text(storage.name) - attrs.append(f', storage:{content}') + attrs.append(f'storage:{content}') # print out = [] - out .append(f'{pre}') - line = name.ljust(NAME_LJUST, ' ') - out.append(f'{line}') + out.append(f'{pre}') + out.append(f'{COLOR_FILE}{name}{Colors.RESET}') size = 0 if node.nodesize: size = node.nodesize - line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ') - out.append(f'{Colors.BLUE}{line}{Colors.RESET}') - line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ') - out.append(f'{Colors.PURPLE}{line}{Colors.RESET}') + line = size_to_str(size, raw=raw) + out.append(f'{COLOR_SIZE}{line}{Colors.RESET}') + if has_attr(node, 'maccess'): + line = epoch_to_ls_str(node.maccess) + out.append(f'{COLOR_TS}{line}{Colors.RESET}') if attrs: out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') + + out = [x for x in out if x] sys.stdout.write(f'{" ".join(out)}\n') def print_dir(self, pre: str, @@ -114,11 +117,7 @@ class NativePrinter: name = node.name storage = node.get_storage_node() if withpath: - name = os.sep.join([ - storage.name, - node.parent.get_parent_hierarchy(), - name]) - name = fix_badchars(name) + name = get_node_fullpath(node) # construct attrs attrs = [] if withnbchildren: @@ -129,23 +128,25 @@ class NativePrinter: # print out = [] out.append(f'{pre}') - line = name.ljust(NAME_LJUST, ' ') - out.append(f'{Colors.BLUE}{line}{Colors.RESET}') + out.append(f'{COLOR_DIRECTORY}{name}{Colors.RESET}') size = 0 if node.nodesize: size = node.nodesize - line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ') - out.append(f'{Colors.GRAY}{line}{Colors.RESET}') - line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ') - out.append(f'{Colors.GRAY}{line}{Colors.RESET}') + line = size_to_str(size, raw=raw) + out.append(f'{COLOR_SIZE}{line}{Colors.RESET}') + if has_attr(node, 'maccess'): + line = epoch_to_ls_str(node.maccess) + out.append(f'{COLOR_TS}{line}{Colors.RESET}') if attrs: out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') + + out = [x for x in out if x] sys.stdout.write(f'{" ".join(out)}\n') def print_archive(self, pre: str, name: str, archive: str) -> None: """print an archive""" name = fix_badchars(name) - out = f'{pre}{Colors.YELLOW}{name}{Colors.RESET} ' + out = f'{pre}{COLOR_ARCHIVE}{name}{Colors.RESET} ' out += f'{Colors.GRAY}[{self.ARCHIVE}:{archive}]{Colors.RESET}' sys.stdout.write(f'{out}\n') diff --git a/catcli/utils.py b/catcli/utils.py index 078fec0..e509f25 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -136,3 +136,17 @@ def fix_badchars(string: str) -> str: def has_attr(node: nodes.NodeAny, attr: str) -> bool: """return True if node has attr as attribute""" return attr in node.__dict__.keys() + + +def get_node_fullpath(node: nodes.NodeAny) -> str: + """get node full path""" + nodes.typcast_node(node) + path = node.name + parents = node.get_parent_hierarchy() + if parents: + path = os.sep.join([parents, path]) + storage = node.get_storage_node() + if storage: + path = os.sep.join([storage.name, path]) + path = fix_badchars(path) + return str(path) diff --git a/tests-ng/assets/generate.sh b/tests-ng/assets/generate.sh index 4923566..a13c1c7 100755 --- a/tests-ng/assets/generate.sh +++ b/tests-ng/assets/generate.sh @@ -8,8 +8,7 @@ rm -f tests-ng/assets/github.catalog.json python3 -m catcli.catcli index -c github .github --catalog=tests-ng/assets/github.catalog.json # edit catalog -sed -i 's/"free": .*,/"free": 0,/g' "tests-ng/assets/github.catalog.json" -sed -i 's/"total": .*,/"total": 0,/g' "tests-ng/assets/github.catalog.json" +clean_catalog "tests-ng/assets/github.catalog.json" # native python3 -m catcli.catcli ls -r -s -B --catalog=tests-ng/assets/github.catalog.json | \ diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 0806fac..1cb0158 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,6 +1,6 @@ "github","storage","","1662","","","","3","0","0","" -"workflows","dir","github/workflows","1541","","","","2","","","" +"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" +"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" +"workflows","dir","github/workflows","0","","","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" "testing.yml","file","github/workflows/testing.yml","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" -"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" -"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index f3e1103..e7159ab 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -36,7 +36,7 @@ ], "maccess": 1704320727.2641916, "name": "workflows", - "size": 1541, + "size": 0, "type": "dir" } ], @@ -44,14 +44,14 @@ "name": "github", "size": 1662, "total": 0, - "ts": 1704403613, + "ts": 1704747820, "type": "storage" }, { "attr": { - "access": 1704403613, + "access": 1704747820, "access_version": "0.9.6", - "created": 1704403613, + "created": 1704747820, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 9a220a4..0bc80a6 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,8 +1,7 @@ top -└── storage: github - nbfiles:3 | totsize:1662 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1541] - │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:850, md5:691df1a4d2f254b5cd04c152e7c6ccaf] - ├── codecov.yml [size:104, md5:4203204f75b43cd4bf032402beb3359d] - └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] +└── storage: github [nbfiles:3|totsize:1662|free:0.0%|du:0/0|date:Jan 08 2024] + ├── FUNDING.yml 17 Oct 19 21:00 [md5:0c6407a84d412c514007313fb3bca4de] + ├── codecov.yml 104 Jan 03 2024 [md5:4203204f75b43cd4bf032402beb3359d] + └── workflows 0 Jan 03 2024 [nbfiles:2] + ├── pypi-release.yml 691 Oct 19 21:00 [md5:57699a7a6a03e20e864f220e19f8e197] + └── testing.yml 850 Jan 04 2024 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 4cafd42..6e66abd 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -36,6 +36,7 @@ catalog="${tmpd}/catalog" # index ${bin} -B index -c --catalog="${catalog}" github .github +clean_catalog "${catalog}" ls -laR .github cat "${catalog}" diff --git a/tests-ng/find.sh b/tests-ng/find.sh index 34a2d1c..ad690d8 100755 --- a/tests-ng/find.sh +++ b/tests-ng/find.sh @@ -37,14 +37,20 @@ catalog="${tmpd}/catalog" # index ${bin} -B index -c -f --catalog="${catalog}" github1 .github ${bin} -B index -c -f --catalog="${catalog}" github2 .github +clean_catalog "${catalog}" #cat "${catalog}" echo "" ${bin} -B ls -r --catalog="${catalog}" + ${bin} -B find --catalog="${catalog}" testing.yml cnt=$(${bin} -B find --catalog="${catalog}" testing.yml | wc -l) -[ "${cnt}" != "2" ] && echo "should return 2!" && exit 1 +[ "${cnt}" != "2" ] && echo "should return 2 (not ${cnt})" && exit 1 + +${bin} -B find --catalog="${catalog}" '*.yml' +cnt=$(${bin} -B find --catalog="${catalog}" '*.yml' | wc -l) +[ "${cnt}" != "8" ] && echo "should return 8 (not ${cnt})" && exit 1 # the end echo "" diff --git a/tests-ng/helper b/tests-ng/helper index d7373e8..f7c09bb 100644 --- a/tests-ng/helper +++ b/tests-ng/helper @@ -21,6 +21,14 @@ clear_on_exit() fi } +# clear catalog stuff for testing +# $1: catalog path +clean_catalog() +{ + sed -i 's/"free": .*,/"free": 0,/g' "${1}" + sed -i 's/"total": .*,/"total": 0,/g' "${1}" +} + # clear files on_exit() { diff --git a/tests-ng/ls.sh b/tests-ng/ls.sh new file mode 100755 index 0000000..061466d --- /dev/null +++ b/tests-ng/ls.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2024, deadc0de6 + +set -e +cur=$(cd "$(dirname "${0}")" && pwd) +prev="${cur}/.." +cd "${prev}" + +# coverage +bin="python3 -m catcli.catcli" +if command -v coverage 2>/dev/null; then + mkdir -p coverages/ + bin="coverage run -p --data-file coverages/coverage --source=catcli -m catcli.catcli" +fi + +echo "current dir: $(pwd)" +echo "pythonpath: ${PYTHONPATH}" +echo "bin: ${bin}" +${bin} --version + +# get the helpers +# shellcheck source=tests-ng/helper +source "${cur}"/helper +echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" + +########################################################## +# the test +########################################################## + +# create temp dirs +tmpd=$(mktemp -d) +clear_on_exit "${tmpd}" + +catalog="${tmpd}/catalog" + +# index +${bin} -B index -c -f --catalog="${catalog}" github1 .github +${bin} -B index -c -f --catalog="${catalog}" github2 .github +clean_catalog "${catalog}" + +#cat "${catalog}" +echo "" + +${bin} -B ls -r --catalog="${catalog}" + +${bin} -B ls --catalog="${catalog}" 'github1/*.yml' +cnt=$(${bin} -B ls --catalog="${catalog}" 'github1/*.yml' | wc -l) +[ "${cnt}" != "2" ] && echo "should return 2 (not ${cnt})" && exit 1 + +${bin} -B ls --catalog="${catalog}" 'github*/*.yml' +cnt=$(${bin} -B ls --catalog="${catalog}" 'github*/*.yml' | wc -l) +[ "${cnt}" != "4" ] && echo "should return 4 (not ${cnt})" && exit 1 + +# the end +echo "" +echo "test \"$(basename "$0")\" success" +cd "${cur}" +exit 0 diff --git a/tests-ng/update.sh b/tests-ng/update.sh index d07eec6..ca6fd49 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -41,12 +41,12 @@ echo "abc" > "${tmpd}/dir/a" # index ${bin} -B index --catalog="${catalog}" dir "${tmpd}/dir" -${bin} -B ls --catalog="${catalog}" dir +${bin} -B ls --catalog="${catalog}" # get attributes -freeb=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') -dub=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') -dateb=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') +freeb=$(${bin} -B ls --catalog="${catalog}" | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') +dub=$(${bin} -B ls --catalog="${catalog}" | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') +dateb=$(${bin} -B ls --catalog="${catalog}" | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') echo "before: free:${freeb} | du:${dub} | date:${dateb}" # change content @@ -61,12 +61,12 @@ sleep 1 # update ${bin} -B update -f --catalog="${catalog}" dir "${tmpu}/dir" -${bin} -B ls --catalog="${catalog}" dir +${bin} -B ls --catalog="${catalog}" # get new attributes -freea=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') -dua=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') -datea=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') +freea=$(${bin} -B ls --catalog="${catalog}" | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') +dua=$(${bin} -B ls --catalog="${catalog}" | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') +datea=$(${bin} -B ls --catalog="${catalog}" | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') echo "after: free:${freea} | du:${dua} | date:${datea}" # test they are all different diff --git a/tests/test_update.py b/tests/test_update.py index 7a5b61f..7365377 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -71,11 +71,11 @@ class TestUpdate(unittest.TestCase): self.assertTrue(os.stat(catalogpath).st_size != 0) # ensure md5 sum are in - nods = noder.find_name(top, os.path.basename(file4)) - self.assertTrue(len(nods) == 1) + nods = noder.find(top, os.path.basename(file4)) + self.assertEqual(len(nods), 1) nod = nods[0] self.assertTrue(nod) - self.assertTrue(nod.md5 == f4_md5) + self.assertEqual(nod.md5, f4_md5) # print catalog noder.print_tree(top) @@ -128,7 +128,7 @@ class TestUpdate(unittest.TestCase): self.assertEqual(len(storage.children), 8) # ensure d1f1 md5 sum has changed in catalog - nods = noder.find_name(top, os.path.basename(d1f1)) + nods = noder.find(top, os.path.basename(d1f1)) self.assertTrue(len(nods) == 1) nod = nods[0] self.assertTrue(nod) @@ -136,7 +136,7 @@ class TestUpdate(unittest.TestCase): self.assertTrue(nod.md5 == d1f1_md5_new) # ensure f4 md5 sum has changed in catalog - nods = noder.find_name(top, os.path.basename(file4)) + nods = noder.find(top, os.path.basename(file4)) self.assertTrue(len(nods) == 1) nod = nods[0] self.assertTrue(nod) @@ -144,7 +144,7 @@ class TestUpdate(unittest.TestCase): self.assertTrue(nod.md5 == f4_md5_new) # ensure d2f2 md5 sum has changed in catalog - nods = noder.find_name(top, os.path.basename(d2f2)) + nods = noder.find(top, os.path.basename(d2f2)) self.assertTrue(len(nods) == 1) nod = nods[0] self.assertTrue(nod) From bed81ffee48765204e5c70abc58b2f1e4aa3d32b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 8 Jan 2024 22:39:36 +0100 Subject: [PATCH 22/33] fix ls --- README.md | 3 --- catcli/catcli.py | 16 ++++++---------- catcli/noder.py | 3 ++- catcli/nodes.py | 8 ++++---- catcli/walker.py | 4 +++- tests/test_index.py | 2 +- tests/test_update.py | 3 +-- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a51d013..9e7c844 100644 --- a/README.md +++ b/README.md @@ -148,9 +148,6 @@ directory under `catcli.catalog`. The `--meta` switch allows to add any additional information to store along in the catalog like for example `the blue disk in my office`. -Catcli will calculate and store the total size of each node (directories, storages, etc) -unless the `-n --no-subsize` switch is used. - Using the `-a --archive` switch allows to also index archive files as explained [below](#index-archive-files). diff --git a/catcli/catcli.py b/catcli/catcli.py index 245f0ce..e9e62d3 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -45,8 +45,8 @@ Usage: {NAME} find [--catalog=] [--format=] [-aBCbdVs] [--path=] [] {NAME} index [--catalog=] [--meta=...] - [-aBCcfnV] - {NAME} update [--catalog=] [-aBCcfnV] + [-aBCcfV] + {NAME} update [--catalog=] [-aBCcfV] [--lpath=] {NAME} mount [--catalog=] [-V] {NAME} rm [--catalog=] [-BCfV] @@ -70,7 +70,6 @@ Options: -F --format= see \"print_supported_formats\" [default: native]. -f --force Do not ask when updating the catalog [default: False]. -l --lpath= Path where changes are logged [default: ] - -n --no-subsize Do not store size of directories [default: False]. -p --path= Start path. -r --recursive Recursive [default: False]. -s --raw-size Print raw size [default: False]. @@ -106,7 +105,6 @@ def cmd_index(args: Dict[str, Any], name = args[''] usehash = args['--hash'] debug = args['--verbose'] - subsize = not args['--no-subsize'] if not os.path.exists(path): Logger.err(f'\"{path}\" does not exist') return @@ -119,15 +117,15 @@ def cmd_index(args: Dict[str, Any], Logger.err('aborted') return node = top.get_storage_node() - node.parent = None + if node: + node.parent = None start = datetime.datetime.now() walker = Walker(noder, usehash=usehash, debug=debug) attr = args['--meta'] root = noder.new_storage_node(name, path, top, attr) _, cnt = walker.index(path, root, name) - if subsize: - root.nodesize = root.get_rec_size() + root.nodesize = root.get_rec_size() stop = datetime.datetime.now() diff = stop - start Logger.info(f'Indexed {cnt} file(s) in {diff}') @@ -145,7 +143,6 @@ def cmd_update(args: Dict[str, Any], usehash = args['--hash'] logpath = args['--lpath'] debug = args['--verbose'] - subsize = not args['--no-subsize'] if not os.path.exists(path): Logger.err(f'\"{path}\" does not exist') return @@ -158,8 +155,7 @@ def cmd_update(args: Dict[str, Any], walker = Walker(noder, usehash=usehash, debug=debug, logpath=logpath) cnt = walker.reindex(path, storage, top) - if subsize: - storage.nodesize = storage.get_rec_size() + storage.nodesize = storage.get_rec_size() stop = datetime.datetime.now() diff = stop - start Logger.info(f'updated {cnt} file(s) in {diff}') diff --git a/catcli/noder.py b/catcli/noder.py index 7f5f0f8..d20d5d0 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -610,7 +610,8 @@ class Noder: self.csv_printer.print_header() for item in found: if fmt == 'native': - self._print_node_native(item, withpath=False, + self._print_node_native(item, + withpath=True, withnbchildren=True, raw=raw) elif fmt.startswith('csv'): diff --git a/catcli/nodes.py b/catcli/nodes.py index 545ba3f..fe49ec9 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -146,7 +146,7 @@ class NodeFile(NodeAny): path = self.parent.get_parent_hierarchy() if path: return os.sep.join([path, self.name]) - return str(self.name) + return '' def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" @@ -185,7 +185,7 @@ class NodeDir(NodeAny): path = self.parent.get_parent_hierarchy() if path: return os.sep.join([path, self.name]) - return str(self.name) + return '' def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" @@ -230,7 +230,7 @@ class NodeArchived(NodeAny): path = self.parent.get_parent_hierarchy() if path: return os.sep.join([path, self.name]) - return str(self.name) + return '' def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" @@ -312,7 +312,7 @@ class NodeMeta(NodeAny): path = self.parent.get_parent_hierarchy() if path: return os.sep.join([path, self.name]) - return str(self.name) + return '' def get_rec_size(self) -> int: """recursively traverse tree and return size""" diff --git a/catcli/walker.py b/catcli/walker.py index c5762f6..d57f5cd 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -35,7 +35,8 @@ class Walker: self.debug = debug self.lpath = logpath - def index(self, path: str, + def index(self, + path: str, parent: NodeAny, name: str, storagepath: str = '') -> Tuple[str, int]: @@ -47,6 +48,7 @@ class Walker: """ self._debug(f'indexing starting at {path}') if not parent: + # create the parent parent = self.noder.new_dir_node(name, path, parent) diff --git a/tests/test_index.py b/tests/test_index.py index 088f65a..46734de 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -50,7 +50,7 @@ class TestIndexing(unittest.TestCase): tmpdirname = 'tmpdir' args = {'': dirpath, '': tmpdirname, '--hash': True, '--meta': ['some meta'], - '--no-subsize': False, '--verbose': True} + '--verbose': True} # index the directory cmd_index(args, noder, catalog, top) diff --git a/tests/test_update.py b/tests/test_update.py index 7365377..d70c564 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -62,8 +62,7 @@ class TestUpdate(unittest.TestCase): tmpdirname = 'tmpdir' args = {'': dirpath, '': tmpdirname, '--hash': True, '--meta': ['some meta'], - '--no-subsize': False, '--verbose': True, - '--lpath': None} + '--verbose': True, '--lpath': None} # index the directory unix_tree(dirpath) From 8111327b5338ba3e74c1b1c2c02e5d3667302cab Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 8 Jan 2024 22:47:13 +0100 Subject: [PATCH 23/33] fix find --- catcli/noder.py | 9 ++++----- tests-ng/assets/github.catalog.csv.txt | 10 +++++----- tests-ng/assets/github.catalog.json | 6 +++--- tests-ng/find.sh | 2 ++ 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index d20d5d0..104a95a 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -487,11 +487,8 @@ class Noder: for item in found: typcast_node(item) item.name = fix_badchars(item.name) - storage = item.get_storage_node() - parents = item.get_parent_hierarchy() - parent_key = f'{storage.name}/{parents}' - key = f'{parent_key}/{item.name}' - paths[parent_key] = item + key = get_node_fullpath(item) + paths[key] = item # handle fzf mode if fmt.startswith('fzf'): @@ -549,6 +546,8 @@ class Noder: return True if term in path: return True + if self.debug: + Logger.debug(f'match \"{path}\" with \"{term}\"') if fnmatch.fnmatch(path, term): return True diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 1cb0158..4e415e9 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,6 +1,6 @@ "github","storage","","1662","","","","3","0","0","" -"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" -"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" -"workflows","dir","github/workflows","0","","","","2","","","" -"pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" +"FUNDING.yml","file","github/","17","","","0c6407a84d412c514007313fb3bca4de","","","","" +"codecov.yml","file","github/","104","","","4203204f75b43cd4bf032402beb3359d","","","","" +"workflows","dir","github/","0","","","","2","","","" +"pypi-release.yml","file","github/","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" +"testing.yml","file","github/","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index e7159ab..d4fcf58 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -44,14 +44,14 @@ "name": "github", "size": 1662, "total": 0, - "ts": 1704747820, + "ts": 1704750073, "type": "storage" }, { "attr": { - "access": 1704747820, + "access": 1704750073, "access_version": "0.9.6", - "created": 1704747820, + "created": 1704750073, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/find.sh b/tests-ng/find.sh index ad690d8..c356b36 100755 --- a/tests-ng/find.sh +++ b/tests-ng/find.sh @@ -44,10 +44,12 @@ echo "" ${bin} -B ls -r --catalog="${catalog}" +echo "finding \"testing.yml\"" ${bin} -B find --catalog="${catalog}" testing.yml cnt=$(${bin} -B find --catalog="${catalog}" testing.yml | wc -l) [ "${cnt}" != "2" ] && echo "should return 2 (not ${cnt})" && exit 1 +echo "finding \"*.yml\"" ${bin} -B find --catalog="${catalog}" '*.yml' cnt=$(${bin} -B find --catalog="${catalog}" '*.yml' | wc -l) [ "${cnt}" != "8" ] && echo "should return 8 (not ${cnt})" && exit 1 From 4aaa073603fd81c2a8bbdff958d1d8755fd7b105 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Mon, 8 Jan 2024 22:55:33 +0100 Subject: [PATCH 24/33] mypy --- .mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mypy.ini b/.mypy.ini index 424e71f..20e0ae3 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,3 +1,3 @@ [mypy] strict = true -disable_error_code = import-untyped \ No newline at end of file +disable_error_code = import-untyped,import-not-found \ No newline at end of file From ff02f9bb97c83fbacd81b95411fe6d05da37a665 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 10 Jan 2024 20:11:02 +0100 Subject: [PATCH 25/33] mypy fix --- .mypy.ini | 3 ++- catcli/catalog.py | 6 +++--- catcli/fuser.py | 2 +- catcli/noder.py | 6 +++--- catcli/nodes.py | 2 +- catcli/printer_native.py | 8 ++++---- catcli/utils.py | 14 -------------- tests-ng/assets/github.catalog.json | 6 +++--- tests-ng/assets/github.catalog.native.txt | 12 ++++++------ 9 files changed, 23 insertions(+), 36 deletions(-) diff --git a/.mypy.ini b/.mypy.ini index 20e0ae3..1e6851f 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,3 +1,4 @@ [mypy] strict = true -disable_error_code = import-untyped,import-not-found \ No newline at end of file +disable_error_code = import-untyped,import-not-found +ignore_missing_imports = True \ No newline at end of file diff --git a/catcli/catalog.py b/catcli/catalog.py index d075399..2744430 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -7,9 +7,9 @@ Class that represents the catcli catalog import os from typing import Optional, List, Dict, Tuple, Union, Any -from anytree.exporter import JsonExporter, DictExporter # type: ignore -from anytree.importer import JsonImporter # type: ignore -from anytree import AnyNode # type: ignore +from anytree.exporter import JsonExporter, DictExporter +from anytree.importer import JsonImporter +from anytree import AnyNode # local imports from catcli import nodes diff --git a/catcli/fuser.py b/catcli/fuser.py index f015e36..afbcfd2 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -10,7 +10,7 @@ from time import time from stat import S_IFDIR, S_IFREG from typing import List, Dict, Any, Optional try: - import fuse # type: ignore + import fuse except ModuleNotFoundError: pass diff --git a/catcli/noder.py b/catcli/noder.py index 104a95a..fbc0acc 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -10,8 +10,8 @@ import shutil import time from typing import List, Union, Tuple, Any, Optional, Dict, cast import fnmatch -import anytree # type: ignore -from natsort import os_sort_keygen # type: ignore +import anytree +from natsort import os_sort_keygen # local imports from catcli import nodes @@ -407,7 +407,7 @@ class Noder: def _fzf_prompt(strings: Any) -> Any: """prompt with fzf""" try: - from pyfzf.pyfzf import FzfPrompt # type: ignore # pylint: disable=C0415 # noqa + from pyfzf.pyfzf import FzfPrompt # pylint: disable=C0415 # noqa fzf = FzfPrompt() selected = fzf.prompt(strings) return selected diff --git a/catcli/nodes.py b/catcli/nodes.py index fe49ec9..e1524d9 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -8,7 +8,7 @@ Class that represents a node in the catalog tree import os from typing import Dict, Any, cast -from anytree import NodeMixin # type: ignore +from anytree import NodeMixin from catcli.exceptions import CatcliException diff --git a/catcli/printer_native.py b/catcli/printer_native.py index b9f86ce..38e4fdf 100644 --- a/catcli/printer_native.py +++ b/catcli/printer_native.py @@ -11,7 +11,7 @@ from catcli.nodes import NodeFile, NodeDir, NodeStorage from catcli.colors import Colors from catcli.logger import Logger from catcli.utils import fix_badchars, size_to_str, \ - has_attr, epoch_to_ls_str, get_node_fullpath + has_attr, epoch_to_str, get_node_fullpath COLOR_STORAGE = Colors.YELLOW @@ -61,7 +61,7 @@ class NativePrinter: attrs.append(f'du:{szused}/{sztotal}') # timestamp if has_attr(node, 'ts'): - attrs.append(f'date:{epoch_to_ls_str(node.ts)}') + attrs.append(f'date:{epoch_to_str(node.ts)}') # print out = f'{pre}{Colors.UND}{self.STORAGE}{Colors.RESET}: ' @@ -98,7 +98,7 @@ class NativePrinter: line = size_to_str(size, raw=raw) out.append(f'{COLOR_SIZE}{line}{Colors.RESET}') if has_attr(node, 'maccess'): - line = epoch_to_ls_str(node.maccess) + line = epoch_to_str(node.maccess) out.append(f'{COLOR_TS}{line}{Colors.RESET}') if attrs: out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') @@ -135,7 +135,7 @@ class NativePrinter: line = size_to_str(size, raw=raw) out.append(f'{COLOR_SIZE}{line}{Colors.RESET}') if has_attr(node, 'maccess'): - line = epoch_to_ls_str(node.maccess) + line = epoch_to_str(node.maccess) out.append(f'{COLOR_TS}{line}{Colors.RESET}') if attrs: out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}') diff --git a/catcli/utils.py b/catcli/utils.py index e509f25..8ae5e30 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -17,8 +17,6 @@ from catcli.exceptions import CatcliException WILD = '*' -TS_FORMAT_6 = '%b %d %H:%M' -TS_FORMAT_MORE = '%b %d %Y' def path_to_top(path: str) -> str: @@ -97,18 +95,6 @@ def epoch_to_str(epoch: float) -> str: return timestamp.strftime(fmt) -def epoch_to_ls_str(epoch: float) -> str: - """convert epoch to string""" - if not epoch: - return '' - timestamp = datetime.datetime.fromtimestamp(epoch) - delta = datetime.date.today() - datetime.timedelta(days=6*365/12) - fmt = TS_FORMAT_MORE - if timestamp.date() < delta: - fmt = TS_FORMAT_6 - return timestamp.strftime(fmt) - - def ask(question: str) -> bool: """ask the user what to do""" resp = input(f'{question} [y|N] ? ') diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index d4fcf58..13e594f 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -44,14 +44,14 @@ "name": "github", "size": 1662, "total": 0, - "ts": 1704750073, + "ts": 1704913782, "type": "storage" }, { "attr": { - "access": 1704750073, + "access": 1704913782, "access_version": "0.9.6", - "created": 1704750073, + "created": 1704913782, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 0bc80a6..57d26b9 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top -└── storage: github [nbfiles:3|totsize:1662|free:0.0%|du:0/0|date:Jan 08 2024] - ├── FUNDING.yml 17 Oct 19 21:00 [md5:0c6407a84d412c514007313fb3bca4de] - ├── codecov.yml 104 Jan 03 2024 [md5:4203204f75b43cd4bf032402beb3359d] - └── workflows 0 Jan 03 2024 [nbfiles:2] - ├── pypi-release.yml 691 Oct 19 21:00 [md5:57699a7a6a03e20e864f220e19f8e197] - └── testing.yml 850 Jan 04 2024 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] +└── storage: github [nbfiles:3|totsize:1662|free:0.0%|du:0/0|date:2023-03-09 16:20:59] + ├── FUNDING.yml 17 2022-10-19 21:00:37 [md5:0c6407a84d412c514007313fb3bca4de] + ├── codecov.yml 104 2024-01-03 23:25:10 [md5:4203204f75b43cd4bf032402beb3359d] + └── workflows 0 2024-01-03 23:25:27 [nbfiles:2] + ├── pypi-release.yml 691 2022-10-19 21:00:37 [md5:57699a7a6a03e20e864f220e19f8e197] + └── testing.yml 850 2024-01-04 22:26:09 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] From 340ab62d77671696d836b7532e43dfc55fa99010 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 10 Jan 2024 22:18:43 +0100 Subject: [PATCH 26/33] du --- README.md | 8 +++ catcli/catcli.py | 62 ++++++++++++++---- catcli/noder.py | 91 +++++++++++++++++++-------- catcli/nodes.py | 132 +++++++++++++++++++-------------------- catcli/printer_csv.py | 5 +- catcli/printer_native.py | 22 +++++-- catcli/utils.py | 14 ----- 7 files changed, 209 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 9e7c844..2916b4b 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ catcli ls -r catcli ls log # find files/directories named '*log*' catcli find log +# show directories sizes +catcli du log ``` see [usage](#usage) for specific info @@ -76,6 +78,7 @@ See the [examples](#examples) for an overview of the available features. * [Find files](#find-files) * [Mount catalog](#mount-catalog) * [Display entire hierarchy](#display-entire-hierarchy) + * [Disk usage](#disk-usage) * [Catalog graph](#catalog-graph) * [Edit storage](#edit-storage) * [Update catalog](#update-catalog) @@ -212,6 +215,11 @@ Resulting files can be sorted by size using the `-S --sortsize` switch. See the [examples](#examples) for more. +## Disk usage + +You can get the disk usage with the `du` command. +Resulting files can be sorted by size using the `-S --sortsize` switch. + ## Catalog graph The catalog can be exported in a dot file that can be used to diff --git a/catcli/catcli.py b/catcli/catcli.py index e9e62d3..805de23 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -40,19 +40,21 @@ USAGE = f""" {BANNER} Usage: - {NAME} ls [--catalog=] [--format=] [-aBCrVSs] [] - {NAME} tree [--catalog=] [-aBCVSs] [] - {NAME} find [--catalog=] [--format=] - [-aBCbdVs] [--path=] [] - {NAME} index [--catalog=] [--meta=...] - [-aBCcfV] - {NAME} update [--catalog=] [-aBCcfV] - [--lpath=] - {NAME} mount [--catalog=] [-V] - {NAME} rm [--catalog=] [-BCfV] - {NAME} rename [--catalog=] [-BCfV] - {NAME} edit [--catalog=] [-BCfV] - {NAME} graph [--catalog=] [-BCV] [] + {NAME} ls [--catalog=] [--format=] [-aBCrVSs] [] + {NAME} tree [--catalog=] [-aBCVSs] [] + {NAME} find [--catalog=] [--format=] + [-aBCbdVs] [--path=] [] + {NAME} index [--catalog=] [--meta=...] + [-aBCcfV] + {NAME} update [--catalog=] [-aBCcfV] + [--lpath=] + {NAME} mount [--catalog=] [-V] + {NAME} du [--catalog=] [-BCVSs] [] + {NAME} rm [--catalog=] [-BCfV] + {NAME} rename [--catalog=] [-BCfV] + {NAME} edit [--catalog=] [-BCfV] + {NAME} graph [--catalog=] [-BCV] [] + {NAME} fixsizes [--catalog=] {NAME} print_supported_formats {NAME} help {NAME} --help @@ -163,6 +165,19 @@ def cmd_update(args: Dict[str, Any], catalog.save(top) +def cmd_du(args: Dict[str, Any], + noder: Noder, + top: NodeTop) -> List[NodeAny]: + """du action""" + path = path_to_search_all(args['']) + found = noder.du(top, + path, + raw=args['--raw-size']) + if not found: + path = args[''] + Logger.err(f'\"{path}\": nothing found') + return found + def cmd_ls(args: Dict[str, Any], noder: Noder, top: NodeTop) -> List[NodeAny]: @@ -230,6 +245,17 @@ def cmd_graph(args: Dict[str, Any], Logger.info(f'create graph with \"{cmd}\" (you need graphviz)') +def cmd_fixsizes(top: NodeTop, + noder: Noder, + catalog: Catalog) -> None: + """ + fix each node size by re-calculating + recursively their size + """ + noder.fixsizes(top) + Logger.info('sizes fixed') + + def cmd_rename(args: Dict[str, Any], catalog: Catalog, top: NodeTop) -> None: @@ -379,6 +405,16 @@ def main() -> bool: Logger.err(f'no such catalog: {catalog_path}') return False cmd_edit(args, noder, catalog, top) + elif args['du']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False + cmd_du(args, noder, top) + elif args['fixsizes']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False + cmd_fixsizes(top, noder, catalog) except CatcliException as exc: Logger.stderr_nocolor('ERROR ' + str(exc)) return False diff --git a/catcli/noder.py b/catcli/noder.py index fbc0acc..f735a3f 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -18,8 +18,7 @@ from catcli import nodes from catcli.nodes import NodeAny, NodeStorage, \ NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta, \ typcast_node -from catcli.utils import md5sum, fix_badchars, has_attr, \ - get_node_fullpath +from catcli.utils import md5sum, fix_badchars, has_attr from catcli.logger import Logger from catcli.printer_native import NativePrinter from catcli.printer_csv import CsvPrinter @@ -307,25 +306,16 @@ class Noder: sep=sep, raw=raw) - def node_has_subs(self, node: Any) -> bool: + def _print_node_du(self, node: NodeAny, + raw: bool = False) -> None: """ - node may have children - we explicitely handle all case - for clarity + print node du style """ - if not node: - return False - if node.type == nodes.TYPE_TOP: - return True - if node.type == nodes.TYPE_FILE: - return False - if node.type == nodes.TYPE_DIR: - return True - if node.type == nodes.TYPE_STORAGE: - return True - if node.type == nodes.TYPE_ARCHIVED: - return True - return False + typcast_node(node) + thenodes = self._get_entire_tree(node, + dironly=True) + for thenode in thenodes: + self.native_printer.print_du(thenode, raw=raw) def _print_node_native(self, node: NodeAny, pre: str = '', @@ -427,7 +417,7 @@ class Noder: for _, _, rend in rendered: if not rend: continue - parents = rend.get_parent_hierarchy() + parents = rend.get_fullpath() storage = rend.get_storage_node() fullpath = os.path.join(storage.name, parents) the_nodes[fullpath] = rend @@ -487,7 +477,7 @@ class Noder: for item in found: typcast_node(item) item.name = fix_badchars(item.name) - key = get_node_fullpath(item) + key = item.get_fullpath() paths[key] = item # handle fzf mode @@ -527,7 +517,7 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" def find_name(node: NodeAny) -> bool: - path = get_node_fullpath(node) + path = node.get_fullpath() if node.type == nodes.TYPE_STORAGE: # ignore storage nodes return False @@ -555,6 +545,16 @@ class Noder: return False return find_name + ############################################################### + # fixsizes + ############################################################### + def fixsizes(self, top: NodeTop) -> None: + typcast_node(top) + rend = anytree.RenderTree(top) + for _, _, thenode in rend: + typcast_node(thenode) + thenode.nodesize = thenode.get_rec_size() + ############################################################### # ls ############################################################### @@ -571,8 +571,7 @@ class Noder: @fmt: output format @raw: print raw size """ - self._debug(f'walking path: \"{path}\" from \"{top.name}\"') - + self._debug(f'ls walking path: \"{path}\" from \"{top.name}\"') resolv = anytree.resolver.Resolver('name') found = [] try: @@ -584,7 +583,8 @@ class Noder: # we have a canonical path self._debug('get ls...') found = resolv.get(top, path) - if found and self.node_has_subs(found): + typcast_node(found) + if found and found.may_have_children(): # let's find its children as well modpath = os.path.join(path, '*') found = resolv.glob(top, modpath) @@ -622,6 +622,30 @@ class Noder: pass return found + ############################################################### + # du + ############################################################### + def du(self, top: NodeTop, + path: str, + raw: bool = False) -> List[NodeAny]: + self._debug(f'du walking path: \"{path}\" from \"{top.name}\"') + resolv = anytree.resolver.Resolver('name') + found = [] + try: + # we have a canonical path + self._debug('get du...') + found = resolv.get(top, path) + if not found: + # nothing found + self._debug('nothing found') + return [] + + self._debug(f'du found: {found}') + self._print_node_du(found, raw=raw) + except anytree.resolver.ChildResolverError: + pass + return found + ############################################################### # tree creation ############################################################### @@ -653,6 +677,23 @@ class Noder: ############################################################### # diverse ############################################################### + def _get_entire_tree(self, start: NodeAny, + dironly: bool = False) -> List[NodeAny]: + """ + get entire tree and sort it + """ + typcast_node(start) + rend = anytree.RenderTree(start) + thenodes = [] + if dironly: + for _, _, thenode in rend: + typcast_node(thenode) + if thenode.type == nodes.TYPE_DIR: + thenodes.append(thenode) + else: + [thenodes.append(x) for _, _, x in rend] + return sorted(thenodes, key=os_sort_keygen(self._sort)) + def _sort_tree(self, items: List[NodeAny]) -> List[NodeAny]: """sorting a list of items""" diff --git a/catcli/nodes.py b/catcli/nodes.py index e1524d9..987e478 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -54,6 +54,10 @@ class NodeAny(NodeMixin): # type: ignore if children: self.children = children + def may_have_children(self) -> bool: + """can node contains sub""" + raise NotImplementedError + def _to_str(self) -> str: ret = str(self.__class__) + ": " + str(self.__dict__) if self.children: @@ -65,18 +69,27 @@ class NodeAny(NodeMixin): # type: ignore def __str__(self) -> str: return self._to_str() - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - raise NotImplementedError + def get_fullpath(self) -> str: + """return full path to this node""" + path = self.name + if self.parent: + typcast_node(self.parent) + ppath = self.parent.get_fullpath() + path = os.path.join(ppath, path) + return path + + def get_rec_size(self) -> int: + """recursively traverse tree and return size""" + totsize: int = self.nodesize + for node in self.children: + typcast_node(node) + totsize += node.get_rec_size() + return totsize def get_storage_node(self) -> NodeMixin: """recursively traverse up to find storage""" return None - def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - raise NotImplementedError - def flagged(self) -> bool: """is flagged""" if not hasattr(self, '_flagged'): @@ -107,13 +120,22 @@ class NodeTop(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" + def get_fullpath(self) -> str: + """return full path to this node""" return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return True + def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - return 0 + """ + recursively traverse tree and return size + also ensure to update the size on the way + """ + size = super().get_rec_size() + self.nodesize = size + return size def __str__(self) -> str: return self._to_str() @@ -140,22 +162,14 @@ class NodeFile(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - typcast_node(self.parent) - path = self.parent.get_parent_hierarchy() - if path: - return os.sep.join([path, self.name]) - return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return False def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" return cast(NodeStorage, self.ancestors[1]) - def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - return self.nodesize - def __str__(self) -> str: return self._to_str() @@ -179,26 +193,23 @@ class NodeDir(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - typcast_node(self.parent) - path = self.parent.get_parent_hierarchy() - if path: - return os.sep.join([path, self.name]) - return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return True + + def get_rec_size(self) -> int: + """ + recursively traverse tree and return size + also ensure to update the size on the way + """ + size = super().get_rec_size() + self.nodesize = size + return size def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" return cast(NodeStorage, self.ancestors[1]) - def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - totsize: int = 0 - for node in self.children: - typcast_node(node) - totsize += node.get_rec_size() - return totsize - def __str__(self) -> str: return self._to_str() @@ -224,22 +235,14 @@ class NodeArchived(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - typcast_node(self.parent) - path = self.parent.get_parent_hierarchy() - if path: - return os.sep.join([path, self.name]) - return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return False def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" return cast(NodeStorage, self.ancestors[1]) - def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - return self.nodesize - def __str__(self) -> str: return self._to_str() @@ -269,22 +272,23 @@ class NodeStorage(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return True + + def get_rec_size(self) -> int: + """ + recursively traverse tree and return size + also ensure to update the size on the way + """ + size = super().get_rec_size() + self.nodesize = size + return size def get_storage_node(self) -> NodeAny: """recursively traverse up to find storage""" return self - def get_rec_size(self) -> int: - """recursively traverse tree and return size""" - totsize: int = 0 - for node in self.children: - typcast_node(node) - totsize += node.get_rec_size() - return totsize - def __str__(self) -> str: return self._to_str() @@ -306,13 +310,9 @@ class NodeMeta(NodeAny): if children: self.children = children - def get_parent_hierarchy(self) -> str: - """get all parents recursively""" - typcast_node(self.parent) - path = self.parent.get_parent_hierarchy() - if path: - return os.sep.join([path, self.name]) - return '' + def may_have_children(self) -> bool: + """can node contains sub""" + return False def get_rec_size(self) -> int: """recursively traverse tree and return size""" diff --git a/catcli/printer_csv.py b/catcli/printer_csv.py index 3b7a453..2f62a3c 100644 --- a/catcli/printer_csv.py +++ b/catcli/printer_csv.py @@ -59,12 +59,11 @@ class CsvPrinter: out = [] out.append(node.name.replace('"', '""')) # name out.append(node.type) # type - parents = node.get_parent_hierarchy() - storage = node.get_storage_node() - fullpath = os.path.join(storage.name, parents) + fullpath = node.get_fullpath() out.append(fullpath.replace('"', '""')) # full path out.append(size_to_str(node.nodesize, raw=raw)) # size + storage = node.get_storage_node() out.append(epoch_to_str(storage.ts)) # indexed_at if has_attr(node, 'maccess'): out.append(epoch_to_str(node.maccess)) # maccess diff --git a/catcli/printer_native.py b/catcli/printer_native.py index 38e4fdf..dc3d6bb 100644 --- a/catcli/printer_native.py +++ b/catcli/printer_native.py @@ -7,11 +7,12 @@ Class for printing nodes in native format import sys -from catcli.nodes import NodeFile, NodeDir, NodeStorage +from catcli.nodes import NodeFile, NodeDir, \ + NodeStorage, NodeAny, typcast_node from catcli.colors import Colors from catcli.logger import Logger from catcli.utils import fix_badchars, size_to_str, \ - has_attr, epoch_to_str, get_node_fullpath + has_attr, epoch_to_str COLOR_STORAGE = Colors.YELLOW @@ -31,6 +32,19 @@ class NativePrinter: ARCHIVE = 'archive' NBFILES = 'nbfiles' + def print_du(self, node: NodeAny, + raw: bool = False) -> None: + """print du style""" + typcast_node(node) + name = node.get_fullpath() + size = node.nodesize + + line = size_to_str(size, raw=raw).ljust(10, ' ') + out = f'{COLOR_SIZE}{line}{Colors.RESET}' + out += ' ' + out += f'{COLOR_FILE}{name}{Colors.RESET}' + sys.stdout.write(f'{out}\n') + def print_top(self, pre: str, name: str) -> None: """print top node""" sys.stdout.write(f'{pre}{name}\n') @@ -80,7 +94,7 @@ class NativePrinter: name = node.name storage = node.get_storage_node() if withpath: - name = get_node_fullpath(node) + name = node.get_fullpath() # construct attributes attrs = [] if node.md5: @@ -117,7 +131,7 @@ class NativePrinter: name = node.name storage = node.get_storage_node() if withpath: - name = get_node_fullpath(node) + name = node.get_fullpath() # construct attrs attrs = [] if withnbchildren: diff --git a/catcli/utils.py b/catcli/utils.py index 8ae5e30..7fef081 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -122,17 +122,3 @@ def fix_badchars(string: str) -> str: def has_attr(node: nodes.NodeAny, attr: str) -> bool: """return True if node has attr as attribute""" return attr in node.__dict__.keys() - - -def get_node_fullpath(node: nodes.NodeAny) -> str: - """get node full path""" - nodes.typcast_node(node) - path = node.name - parents = node.get_parent_hierarchy() - if parents: - path = os.sep.join([parents, path]) - storage = node.get_storage_node() - if storage: - path = os.sep.join([storage.name, path]) - path = fix_badchars(path) - return str(path) From ea0cb2f9dae295143d8236ad16185359574e2b9e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 10 Jan 2024 22:28:17 +0100 Subject: [PATCH 27/33] linting --- catcli/catcli.py | 8 +++++--- catcli/noder.py | 22 +++++++++++++--------- catcli/nodes.py | 6 +++++- catcli/printer_csv.py | 1 - tests-ng/assets/github.catalog.csv.txt | 12 ++++++------ tests-ng/assets/github.catalog.json | 10 ++++++---- tests-ng/assets/github.catalog.native.txt | 4 ++-- 7 files changed, 37 insertions(+), 26 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 805de23..459d1d7 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -170,14 +170,15 @@ def cmd_du(args: Dict[str, Any], top: NodeTop) -> List[NodeAny]: """du action""" path = path_to_search_all(args['']) - found = noder.du(top, - path, - raw=args['--raw-size']) + found = noder.diskusage(top, + path, + raw=args['--raw-size']) if not found: path = args[''] Logger.err(f'\"{path}\": nothing found') return found + def cmd_ls(args: Dict[str, Any], noder: Noder, top: NodeTop) -> List[NodeAny]: @@ -253,6 +254,7 @@ def cmd_fixsizes(top: NodeTop, recursively their size """ noder.fixsizes(top) + catalog.save(top) Logger.info('sizes fixed') diff --git a/catcli/noder.py b/catcli/noder.py index f735a3f..6009262 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -517,6 +517,7 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" def find_name(node: NodeAny) -> bool: + typcast_node(node) path = node.get_fullpath() if node.type == nodes.TYPE_STORAGE: # ignore storage nodes @@ -549,6 +550,7 @@ class Noder: # fixsizes ############################################################### def fixsizes(self, top: NodeTop) -> None: + """fix node sizes""" typcast_node(top) rend = anytree.RenderTree(top) for _, _, thenode in rend: @@ -582,14 +584,15 @@ class Noder: else: # we have a canonical path self._debug('get ls...') - found = resolv.get(top, path) - typcast_node(found) - if found and found.may_have_children(): + foundone = resolv.get(top, path) + cast(NodeAny, foundone) + typcast_node(foundone) + if foundone and foundone.may_have_children(): # let's find its children as well modpath = os.path.join(path, '*') found = resolv.glob(top, modpath) else: - found = [found] + found = [foundone] if len(found) < 1: # nothing found @@ -625,12 +628,13 @@ class Noder: ############################################################### # du ############################################################### - def du(self, top: NodeTop, - path: str, - raw: bool = False) -> List[NodeAny]: + def diskusage(self, top: NodeTop, + path: str, + raw: bool = False) -> List[NodeAny]: + """disk usage""" self._debug(f'du walking path: \"{path}\" from \"{top.name}\"') resolv = anytree.resolver.Resolver('name') - found = [] + found: NodeAny try: # we have a canonical path self._debug('get du...') @@ -691,7 +695,7 @@ class Noder: if thenode.type == nodes.TYPE_DIR: thenodes.append(thenode) else: - [thenodes.append(x) for _, _, x in rend] + thenodes = [x for _, _, x in rend] return sorted(thenodes, key=os_sort_keygen(self._sort)) def _sort_tree(self, diff --git a/catcli/nodes.py b/catcli/nodes.py index 987e478..43371ff 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -46,10 +46,14 @@ class NodeAny(NodeMixin): # type: ignore """generic node""" def __init__(self, # type: ignore[no-untyped-def] + name=None, + size=None, parent=None, children=None): """build generic node""" super().__init__() + self.name = name + self.nodesize = size self.parent = parent if children: self.children = children @@ -76,7 +80,7 @@ class NodeAny(NodeMixin): # type: ignore typcast_node(self.parent) ppath = self.parent.get_fullpath() path = os.path.join(ppath, path) - return path + return str(path) def get_rec_size(self) -> int: """recursively traverse tree and return size""" diff --git a/catcli/printer_csv.py b/catcli/printer_csv.py index 2f62a3c..eed7dc5 100644 --- a/catcli/printer_csv.py +++ b/catcli/printer_csv.py @@ -6,7 +6,6 @@ Class for printing nodes in csv format """ import sys -import os from typing import List from catcli.nodes import NodeAny, NodeStorage, TYPE_DIR diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 4e415e9..235abf3 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,6 +1,6 @@ -"github","storage","","1662","","","","3","0","0","" -"FUNDING.yml","file","github/","17","","","0c6407a84d412c514007313fb3bca4de","","","","" -"codecov.yml","file","github/","104","","","4203204f75b43cd4bf032402beb3359d","","","","" -"workflows","dir","github/","0","","","","2","","","" -"pypi-release.yml","file","github/","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" +"github","storage","","4865","","","","3","0","0","" +"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","","" +"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","","" +"workflows","dir","github/workflows","3082","","","","2","","","" +"pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","","" +"testing.yml","file","github/workflows/testing.yml","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index 13e594f..4b02252 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -36,7 +36,7 @@ ], "maccess": 1704320727.2641916, "name": "workflows", - "size": 0, + "size": 1541, "type": "dir" } ], @@ -44,20 +44,22 @@ "name": "github", "size": 1662, "total": 0, - "ts": 1704913782, + "ts": 1704922075, "type": "storage" }, { "attr": { - "access": 1704913782, + "access": 1704922075, "access_version": "0.9.6", - "created": 1704913782, + "created": 1704922075, "created_version": "0.9.6" }, "name": "meta", + "size": null, "type": "meta" } ], "name": "top", + "size": null, "type": "top" } \ No newline at end of file diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 57d26b9..3382e11 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top -└── storage: github [nbfiles:3|totsize:1662|free:0.0%|du:0/0|date:2023-03-09 16:20:59] +└── storage: github [nbfiles:3|totsize:4865|free:0.0%|du:0/0|date:2023-03-09 16:20:59] ├── FUNDING.yml 17 2022-10-19 21:00:37 [md5:0c6407a84d412c514007313fb3bca4de] ├── codecov.yml 104 2024-01-03 23:25:10 [md5:4203204f75b43cd4bf032402beb3359d] - └── workflows 0 2024-01-03 23:25:27 [nbfiles:2] + └── workflows 3082 2024-01-03 23:25:27 [nbfiles:2] ├── pypi-release.yml 691 2022-10-19 21:00:37 [md5:57699a7a6a03e20e864f220e19f8e197] └── testing.yml 850 2024-01-04 22:26:09 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] From 963a8b0518531f3badd3a20da8df29867232da0b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 10 Jan 2024 22:31:03 +0100 Subject: [PATCH 28/33] update ensures dir size are correct --- catcli/catcli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 459d1d7..8aed08f 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -255,7 +255,6 @@ def cmd_fixsizes(top: NodeTop, """ noder.fixsizes(top) catalog.save(top) - Logger.info('sizes fixed') def cmd_rename(args: Dict[str, Any], @@ -365,6 +364,7 @@ def main() -> bool: Logger.err(f'no such catalog: {catalog_path}') return False cmd_update(args, noder, catalog, top) + cmd_fixsizes(top, noder, catalog) elif args['find']: if not catalog.exists(): Logger.err(f'no such catalog: {catalog_path}') From f3c4a86b1dafa213911e617ffc7da245c4c19595 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 10 Jan 2024 22:46:27 +0100 Subject: [PATCH 29/33] tests --- catcli/nodes.py | 2 +- tests-ng/assets/generate.sh | 2 +- tests-ng/assets/github.catalog.json | 6 +++--- tests-ng/assets/github.catalog.native.txt | 10 +++++----- tests-ng/compare.sh | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/catcli/nodes.py b/catcli/nodes.py index 43371ff..3f19553 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -47,7 +47,7 @@ class NodeAny(NodeMixin): # type: ignore def __init__(self, # type: ignore[no-untyped-def] name=None, - size=None, + size=0, parent=None, children=None): """build generic node""" diff --git a/tests-ng/assets/generate.sh b/tests-ng/assets/generate.sh index a13c1c7..26f3acb 100755 --- a/tests-ng/assets/generate.sh +++ b/tests-ng/assets/generate.sh @@ -13,7 +13,7 @@ clean_catalog "tests-ng/assets/github.catalog.json" # native python3 -m catcli.catcli ls -r -s -B --catalog=tests-ng/assets/github.catalog.json | \ sed -e 's/free:.*%/free:0.0%/g' \ - -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ + -e 's/....-..-.. ..:..:../2023-03-09 16:20:59/g' \ -e 's#du:[^|]* |#du:0/0 |#g' > tests-ng/assets/github.catalog.native.txt # csv diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index 4b02252..fbc0269 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -44,14 +44,14 @@ "name": "github", "size": 1662, "total": 0, - "ts": 1704922075, + "ts": 1704923096, "type": "storage" }, { "attr": { - "access": 1704922075, + "access": 1704923096, "access_version": "0.9.6", - "created": 1704922075, + "created": 1704923096, "created_version": "0.9.6" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 3382e11..e1981cc 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top └── storage: github [nbfiles:3|totsize:4865|free:0.0%|du:0/0|date:2023-03-09 16:20:59] - ├── FUNDING.yml 17 2022-10-19 21:00:37 [md5:0c6407a84d412c514007313fb3bca4de] - ├── codecov.yml 104 2024-01-03 23:25:10 [md5:4203204f75b43cd4bf032402beb3359d] - └── workflows 3082 2024-01-03 23:25:27 [nbfiles:2] - ├── pypi-release.yml 691 2022-10-19 21:00:37 [md5:57699a7a6a03e20e864f220e19f8e197] - └── testing.yml 850 2024-01-04 22:26:09 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] + ├── FUNDING.yml 17 2023-03-09 16:20:59 [md5:0c6407a84d412c514007313fb3bca4de] + ├── codecov.yml 104 2023-03-09 16:20:59 [md5:4203204f75b43cd4bf032402beb3359d] + └── workflows 3082 2023-03-09 16:20:59 [nbfiles:2] + ├── pypi-release.yml 691 2023-03-09 16:20:59 [md5:57699a7a6a03e20e864f220e19f8e197] + └── testing.yml 850 2023-03-09 16:20:59 [md5:691df1a4d2f254b5cd04c152e7c6ccaf] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 6e66abd..d239590 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -102,7 +102,7 @@ native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" mod="${tmpd}/native.mod.txt" cat "${native}" | sed -e 's/free:.*%/free:0.0%/g' \ - -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ + -e 's/....-..-.. ..:..:../2023-03-09 16:20:59/g' \ -e 's#du:[^|]* |#du:0/0 |#g' > "${mod}" if command -v delta >/dev/null; then delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" From 638085ce982ce74d1b6b406a2292016cbf34524f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 14 Jan 2024 22:27:18 +0100 Subject: [PATCH 30/33] repl --- catcli/catcli.py | 92 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 2 ++ 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 8aed08f..c7d8a9a 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -12,7 +12,9 @@ import sys import os import datetime from typing import Dict, Any, List +import typing from docopt import docopt +import cmd2 # local imports from catcli.version import __version__ as VERSION @@ -54,6 +56,7 @@ Usage: {NAME} rename [--catalog=] [-BCfV] {NAME} edit [--catalog=] [-BCfV] {NAME} graph [--catalog=] [-BCV] [] + {NAME} [--catalog=] {NAME} fixsizes [--catalog=] {NAME} print_supported_formats {NAME} help @@ -294,6 +297,73 @@ def cmd_edit(args: Dict[str, Any], Logger.err(f'Storage named \"{storage}\" does not exist') +@typing.no_type_check +class CatcliRepl(cmd2.Cmd): + """catcli repl""" + + prompt = 'catcli> ' + intro = '' + + def __init__(self): + super().__init__() + # remove built-ins + del cmd2.Cmd.do_alias + del cmd2.Cmd.do_edit + del cmd2.Cmd.do_macro + del cmd2.Cmd.do_run_pyscript + del cmd2.Cmd.do_run_script + del cmd2.Cmd.do_set + del cmd2.Cmd.do_shell + del cmd2.Cmd.do_shortcuts + self.hidden_commands.append('EOF') + + def cmdloop(self, intro=None): + return cmd2.Cmd.cmdloop(self, intro) + + @cmd2.with_argument_list + def do_ls(self, arglist: List[str]): + """ls """ + arglist.insert(0, '--no-banner') + arglist.insert(0, 'ls') + args, noder, _, _, top = init(arglist) + cmd_ls(args, noder, top) + return False + + @cmd2.with_argument_list + def do_tree(self, arglist): + """tree """ + arglist.insert(0, '--no-banner') + arglist.insert(0, 'tree') + args, noder, _, _, top = init(arglist) + cmd_ls(args, noder, top) + + @cmd2.with_argument_list + def do_find(self, arglist): + """find """ + arglist.insert(0, '--no-banner') + arglist.insert(0, 'find') + args, noder, _, _, top = init(arglist) + cmd_find(args, noder, top) + + @cmd2.with_argument_list + def do_du(self, arglist): + """du """ + arglist.insert(0, '--no-banner') + arglist.insert(0, 'du') + args, noder, _, _, top = init(arglist) + cmd_du(args, noder, top) + + def do_help(self, _): + """help""" + print(USAGE) + return False + + # pylint: disable=C0103 + def do_EOF(self, _): + """exit repl""" + return True + + def banner() -> None: """print banner""" Logger.stderr_nocolor(BANNER) @@ -309,24 +379,23 @@ def print_supported_formats() -> None: print('"fzf-csv" : fzf to csv (only valid for find)') -def main() -> bool: - """entry point""" - args = docopt(USAGE, version=VERSION) +def init(args): + """parse catcli arguments""" + args = docopt(USAGE, argv=args, version=VERSION) if args['help'] or args['--help']: print(USAGE) - return True + sys.exit(0) if args['print_supported_formats']: print_supported_formats() - return True + sys.exit(0) - # check format fmt = args['--format'] if fmt not in FORMATS: Logger.err(f'bad format: {fmt}') print_supported_formats() - return False + sys.exit(0) if args['--verbose']: print(f'args: {args}') @@ -355,6 +424,13 @@ def main() -> bool: meta = noder.update_metanode(top) catalog.set_metanode(meta) + return args, noder, catalog, catalog_path, top + + +def main() -> bool: + """entry point""" + args, noder, catalog, catalog_path, top = init(sys.argv[1:]) + # parse command try: if args['index']: @@ -417,6 +493,8 @@ def main() -> bool: Logger.err(f'no such catalog: {catalog_path}') return False cmd_fixsizes(top, noder, catalog) + else: + CatcliRepl().cmdloop() except CatcliException as exc: Logger.stderr_nocolor('ERROR ' + str(exc)) return False diff --git a/requirements.txt b/requirements.txt index d9cfeb9..f4f334e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ anytree; python_version >= '3.0' pyfzf; python_version >= '3.0' fusepy; python_version >= '3.0' natsort; python_version >= '3.0' +cmd2; python_version >= '3.0' +gnureadline; python_version >= '3.0' From 61649300888d73d2171fec2de3b8dcf6843557f6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 16 Jan 2024 23:06:07 +0100 Subject: [PATCH 31/33] linting --- catcli/catcli.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index c7d8a9a..5191611 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -11,8 +11,8 @@ Catcli command line interface import sys import os import datetime -from typing import Dict, Any, List -import typing +from typing import Dict, Any, List, \ + Tuple from docopt import docopt import cmd2 @@ -297,14 +297,13 @@ def cmd_edit(args: Dict[str, Any], Logger.err(f'Storage named \"{storage}\" does not exist') -@typing.no_type_check -class CatcliRepl(cmd2.Cmd): +class CatcliRepl(cmd2.Cmd): # type: ignore """catcli repl""" prompt = 'catcli> ' intro = '' - def __init__(self): + def __init__(self) -> None: super().__init__() # remove built-ins del cmd2.Cmd.do_alias @@ -317,11 +316,11 @@ class CatcliRepl(cmd2.Cmd): del cmd2.Cmd.do_shortcuts self.hidden_commands.append('EOF') - def cmdloop(self, intro=None): + def cmdloop(self, intro: Any = None) -> Any: return cmd2.Cmd.cmdloop(self, intro) - @cmd2.with_argument_list - def do_ls(self, arglist: List[str]): + @cmd2.with_argument_list # type: ignore + def do_ls(self, arglist: List[str]) -> bool: """ls """ arglist.insert(0, '--no-banner') arglist.insert(0, 'ls') @@ -329,37 +328,40 @@ class CatcliRepl(cmd2.Cmd): cmd_ls(args, noder, top) return False - @cmd2.with_argument_list - def do_tree(self, arglist): + @cmd2.with_argument_list # type: ignore + def do_tree(self, arglist: List[str]) -> bool: """tree """ arglist.insert(0, '--no-banner') arglist.insert(0, 'tree') args, noder, _, _, top = init(arglist) cmd_ls(args, noder, top) + return False - @cmd2.with_argument_list - def do_find(self, arglist): + @cmd2.with_argument_list # type: ignore + def do_find(self, arglist: List[str]) -> bool: """find """ arglist.insert(0, '--no-banner') arglist.insert(0, 'find') args, noder, _, _, top = init(arglist) cmd_find(args, noder, top) + return False - @cmd2.with_argument_list - def do_du(self, arglist): + @cmd2.with_argument_list # type: ignore + def do_du(self, arglist: List[str]) -> bool: """du """ arglist.insert(0, '--no-banner') arglist.insert(0, 'du') args, noder, _, _, top = init(arglist) cmd_du(args, noder, top) + return False - def do_help(self, _): + def do_help(self, _: Any) -> bool: """help""" print(USAGE) return False # pylint: disable=C0103 - def do_EOF(self, _): + def do_EOF(self, _: Any) -> bool: """exit repl""" return True @@ -379,9 +381,13 @@ def print_supported_formats() -> None: print('"fzf-csv" : fzf to csv (only valid for find)') -def init(args): +def init(argv: List[str]) -> Tuple[Dict[str, Any], + Noder, + Catalog, + str, + NodeTop]: """parse catcli arguments""" - args = docopt(USAGE, argv=args, version=VERSION) + args = docopt(USAGE, argv=argv, version=VERSION) if args['help'] or args['--help']: print(USAGE) @@ -435,7 +441,7 @@ def main() -> bool: try: if args['index']: cmd_index(args, noder, catalog, top) - if args['update']: + elif args['update']: if not catalog.exists(): Logger.err(f'no such catalog: {catalog_path}') return False From e63f5b8d79b6d4a4899678d556ff5a7ef56c8370 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 17 Jan 2024 12:09:25 +0100 Subject: [PATCH 32/33] pytype checks --- tests-requirements.txt | 1 + tests.sh | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tests-requirements.txt b/tests-requirements.txt index b8a8cd1..1803f06 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -5,3 +5,4 @@ coverage; python_version >= '3.0' pylint; python_version > '3.0' mypy; python_version > '3.0' pytest; python_version > '3.0' +pytype; python_version > '3.0' diff --git a/tests.sh b/tests.sh index c74939c..3412e78 100755 --- a/tests.sh +++ b/tests.sh @@ -62,6 +62,11 @@ echo "[+] mypy" mypy --version mypy --config-file=.mypy.ini catcli/ +# pytype +echo "[+] pytype" +pytype --version +pytype catcli/ + set +e grep -R 'TODO' catcli/ && echo "TODO found" && exit 1 grep -R 'FIXME' catcli/ && echo "FIXME found" && exit 1 From 6b9e00f11b6578e9c16ce2d3b7832e50f8b594bd Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 17 Jan 2024 12:10:03 +0100 Subject: [PATCH 33/33] fix linting --- .gitignore | 2 ++ .mypy.ini | 3 ++- catcli/noder.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 99e2cbc..ce60cf2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ build/ .mypy_cache .pytest_cache __pycache__ +.pyre +.pytype diff --git a/.mypy.ini b/.mypy.ini index 1e6851f..7d2c387 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,4 +1,5 @@ [mypy] strict = true disable_error_code = import-untyped,import-not-found -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True +warn_unused_ignores = False diff --git a/catcli/noder.py b/catcli/noder.py index 6009262..57ef430 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -86,6 +86,7 @@ class Noder: quiet: bool = False) -> Optional[NodeAny]: """get the node by internal tree path""" resolv = anytree.resolver.Resolver('name') + bpath = '' try: bpath = os.path.basename(path) the_node = resolv.get(top, bpath)