From 04b8d6e770d409754c5379fbf57a804820d93855 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sat, 4 Mar 2023 22:36:26 +0100 Subject: [PATCH 01/36] fuse memory filesystem --- catcli/catcli.py | 13 +++++ catcli/fuser.py | 134 +++++++++++++++++++++++++++++++++++++++++++++++ catcli/utils.py | 3 ++ requirements.txt | 1 + tests.sh | 2 + 5 files changed, 153 insertions(+) create mode 100644 catcli/fuser.py diff --git a/catcli/catcli.py b/catcli/catcli.py index 0655c62..526d701 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -21,6 +21,7 @@ from .catalog import Catalog from .walker import Walker from .noder import Noder from .utils import ask, edit +from .fuser import Fuser from .exceptions import BadFormatException, CatcliException NAME = 'catcli' @@ -43,6 +44,7 @@ Usage: {NAME} find [--catalog=] [--format=] [-aBCbdVsP] [--path=] [] {NAME} index [--catalog=] [--meta=...] [-aBCcfnV] {NAME} update [--catalog=] [-aBCcfnV] [--lpath=] + {NAME} mount [--catalog=] [-V] {NAME} rm [--catalog=] [-BCfV] {NAME} rename [--catalog=] [-BCfV] {NAME} edit [--catalog=] [-BCfV] @@ -76,6 +78,12 @@ Options: """ # nopep8 +def cmd_mount(args, top, noder): + """mount action""" + mountpoint = args[''] + Fuser(mountpoint, top, noder) + + def cmd_index(args, noder, catalog, top): """index action""" path = args[''] @@ -320,6 +328,11 @@ def main(): Logger.err(f'no such catalog: {catalog_path}') return False cmd_ls(args, noder, top) + elif args['mount']: + if not catalog.exists(): + Logger.err(f'no such catalog: {catalog_path}') + return False + cmd_mount(args, top, noder) elif args['rm']: if not catalog.exists(): Logger.err(f'no such catalog: {catalog_path}') diff --git a/catcli/fuser.py b/catcli/fuser.py new file mode 100644 index 0000000..1c62760 --- /dev/null +++ b/catcli/fuser.py @@ -0,0 +1,134 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2023, deadc0de6 + +fuse for catcli +""" + +import os +import logging +from time import time +from stat import S_IFDIR, S_IFREG +import fuse +from .noder import Noder + + +# build custom logger to log in /tmp +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +fh = logging.FileHandler('/tmp/fuse-catcli.log') +fh.setLevel(logging.DEBUG) +logger.addHandler(fh) + +# globals +WILD = '*' +SEPARATOR = '/' + + +class Fuser: + """fuser filesystem""" + + def __init__(self, mountpoint, top, noder): + """fuse filesystem""" + filesystem = CatcliFilesystem(top, noder) + fuse.FUSE(filesystem, + mountpoint, + foreground=True, + allow_other=True, + nothreads=True, + debug=True) + + +class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): + """in-memory filesystem for catcli catalog""" + + def __init__(self, top, noder): + """init fuse filesystem""" + self.top = top + self.noder = noder + + def _get_entry(self, path): + pre = f'{SEPARATOR}{self.noder.NAME_TOP}' + if not path.startswith(pre): + path = pre + path + found = self.noder.list(self.top, path, + rec=False, + fmt='native', + raw=True) + if found: + return found[0] + return [] + + def _get_entries(self, path): + pre = f'{SEPARATOR}{self.noder.NAME_TOP}' + if not path.startswith(pre): + path = pre + path + if not path.endswith(SEPARATOR): + path += SEPARATOR + if not path.endswith(WILD): + path += WILD + found = self.noder.list(self.top, path, + rec=False, + fmt='native', + raw=True) + return found + + def _getattr(self, path): + entry = self._get_entry(path) + if not entry: + return None + + curt = time() + mode = S_IFREG + if entry.type == Noder.TYPE_ARC: + mode = S_IFREG + elif entry.type == Noder.TYPE_DIR: + mode = S_IFDIR + elif entry.type == Noder.TYPE_FILE: + mode = S_IFREG + elif entry.type == Noder.TYPE_STORAGE: + mode = S_IFDIR + elif entry.type == Noder.TYPE_META: + mode = S_IFREG + elif entry.type == Noder.TYPE_TOP: + mode = S_IFREG + return { + 'st_mode': (mode), + 'st_nlink': 1, + 'st_size': 0, + 'st_ctime': curt, + 'st_mtime': curt, + 'st_atime': curt, + 'st_uid': os.getuid(), + 'st_gid': os.getgid(), + } + + def getattr(self, path, _fh=None): + """return attr of file pointed by path""" + logger.info('getattr path: %s', path) + + if path == '/': + # mountpoint + curt = time() + meta = { + 'st_mode': (S_IFDIR), + 'st_nlink': 1, + 'st_size': 0, + 'st_ctime': curt, + 'st_mtime': curt, + 'st_atime': curt, + 'st_uid': os.getuid(), + 'st_gid': os.getgid(), + } + return meta + meta = self._getattr(path) + return meta + + def readdir(self, path, _fh): + """read directory content""" + logger.info('readdir path: %s', path) + content = ['.', '..'] + entries = self._get_entries(path) + for entry in entries: + content.append(entry.name) + return content diff --git a/catcli/utils.py b/catcli/utils.py index 6bc197d..1180455 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -15,6 +15,9 @@ import datetime from catcli.exceptions import CatcliException +SEPARATOR = '/' + + def md5sum(path): """ calculate md5 sum of a file diff --git a/requirements.txt b/requirements.txt index 936d33b..1143736 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ docopt; python_version >= '3.0' anytree; python_version >= '3.0' pyfzf; python_version >= '3.0' +fusepy; python_version >= '3.0' diff --git a/tests.sh b/tests.sh index 040de1a..ee403db 100755 --- a/tests.sh +++ b/tests.sh @@ -21,6 +21,7 @@ pyflakes tests/ # R0915: Too many statements # R0911: Too many return statements # R0903: Too few public methods +# R0801: Similar lines in 2 files pylint --version pylint \ --disable=R0914 \ @@ -29,6 +30,7 @@ pylint \ --disable=R0915 \ --disable=R0911 \ --disable=R0903 \ + --disable=R0801 \ catcli/ pylint \ --disable=W0212 \ From 8659bfb3486560a61b4e0ff90333874f4f6f8387 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 5 Mar 2023 16:00:19 +0100 Subject: [PATCH 02/36] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 860e52b..01fb98a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist/ build/ *.egg-info/ *.catalog +.vscode/ From 047c9bf4ab023104dc98fc8cd1cd603775b53363 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 5 Mar 2023 16:00:36 +0100 Subject: [PATCH 03/36] type hinting --- requirements.txt | 1 + tests-requirements.txt | 1 + tests.sh | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/requirements.txt b/requirements.txt index 1143736..571d8a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ docopt; python_version >= '3.0' +types-docopt; python_version >= '3.0' anytree; python_version >= '3.0' pyfzf; python_version >= '3.0' fusepy; python_version >= '3.0' diff --git a/tests-requirements.txt b/tests-requirements.txt index 8e826f4..fa48b04 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -4,3 +4,4 @@ nose2; python_version >= '3.0' coverage; python_version >= '3.0' coveralls; python_version >= '3.0' pylint; python_version > '3.0' +mypy; python_version > '3.0' diff --git a/tests.sh b/tests.sh index ee403db..4e81638 100755 --- a/tests.sh +++ b/tests.sh @@ -22,6 +22,7 @@ pyflakes tests/ # R0911: Too many return statements # R0903: Too few public methods # R0801: Similar lines in 2 files +# R0902: Too many instance attributes pylint --version pylint \ --disable=R0914 \ @@ -31,6 +32,7 @@ pylint \ --disable=R0911 \ --disable=R0903 \ --disable=R0801 \ + --disable=R0902 \ catcli/ pylint \ --disable=W0212 \ @@ -39,6 +41,10 @@ pylint \ --disable=R0801 \ tests/ +mypy \ + --strict \ + catcli/ + nosebin="nose2" PYTHONPATH=catcli ${nosebin} --with-coverage --coverage=catcli From 4a9e565e742f48ecc9ef8aa50029fff18e07ae97 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 5 Mar 2023 16:00:47 +0100 Subject: [PATCH 04/36] typing --- catcli/__init__.py | 2 +- catcli/catalog.py | 31 ++++--- catcli/catcli.py | 67 ++++++++++----- catcli/colors.py | 7 +- catcli/decomp.py | 17 ++-- catcli/fuser.py | 38 +++++---- catcli/logger.py | 30 +++++-- catcli/nodeprinter.py | 27 ++++-- catcli/noder.py | 193 +++++++++++++++++++++++++----------------- catcli/utils.py | 19 +++-- catcli/walker.py | 37 +++++--- 11 files changed, 297 insertions(+), 171 deletions(-) diff --git a/catcli/__init__.py b/catcli/__init__.py index 3205988..48ed909 100644 --- a/catcli/__init__.py +++ b/catcli/__init__.py @@ -7,7 +7,7 @@ Copyright (c) 2017, deadc0de6 import sys -def main(): +def main() -> None: """entry point""" import catcli.catcli if catcli.catcli.main(): diff --git a/catcli/catalog.py b/catcli/catalog.py index b5df73d..005d081 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -7,8 +7,9 @@ Class that represents the catcli catalog import os import pickle -from anytree.exporter import JsonExporter -from anytree.importer import JsonImporter +import anytree # type: ignore +from anytree.exporter import JsonExporter # type: ignore +from anytree.importer import JsonImporter # type: ignore # local imports from catcli.utils import ask @@ -18,7 +19,10 @@ from catcli.logger import Logger class Catalog: """the catalog""" - def __init__(self, path, usepickle=False, debug=False, force=False): + def __init__(self, path: str, + usepickle: bool = False, + debug: bool = False, + force: bool = False) -> None: """ @path: catalog path @usepickle: use pickle @@ -31,12 +35,13 @@ class Catalog: self.metanode = None self.pickle = usepickle - def set_metanode(self, metanode): + def set_metanode(self, metanode: anytree.AnyNode) -> None: """remove the metanode until tree is re-written""" self.metanode = metanode - self.metanode.parent = None + if self.metanode: + self.metanode.parent = None - def exists(self): + def exists(self) -> bool: """does catalog exist""" if not self.path: return False @@ -44,7 +49,7 @@ class Catalog: return True return False - def restore(self): + def restore(self) -> anytree.AnyNode: """restore the catalog""" if not self.path: return None @@ -56,7 +61,7 @@ class Catalog: content = file.read() return self._restore_json(content) - def save(self, node): + def save(self, node: anytree.AnyNode) -> bool: """save the catalog""" if not self.path: Logger.err('Path not defined') @@ -77,19 +82,19 @@ class Catalog: return self._save_pickle(node) return self._save_json(node) - def _debug(self, text): + def _debug(self, text: str) -> None: if not self.debug: return Logger.debug(text) - def _save_pickle(self, node): + def _save_pickle(self, node: anytree.AnyNode) -> bool: """pickle the catalog""" with open(self.path, 'wb') as file: pickle.dump(node, file) self._debug(f'Catalog saved to pickle \"{self.path}\"') return True - def _restore_pickle(self): + def _restore_pickle(self) -> anytree.AnyNode: """restore the pickled tree""" with open(self.path, 'rb') as file: root = pickle.load(file) @@ -97,7 +102,7 @@ class Catalog: self._debug(msg) return root - def _save_json(self, node): + def _save_json(self, node: anytree.AnyNode) -> bool: """export the catalog in json""" exp = JsonExporter(indent=2, sort_keys=True) with open(self.path, 'w', encoding='UTF-8') as file: @@ -105,7 +110,7 @@ class Catalog: self._debug(f'Catalog saved to json \"{self.path}\"') return True - def _restore_json(self, string): + def _restore_json(self, string: str) -> anytree.AnyNode: """restore the tree from json""" imp = JsonImporter() root = imp.import_(string) diff --git a/catcli/catcli.py b/catcli/catcli.py index 526d701..49f5e0a 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -11,6 +11,8 @@ Catcli command line interface import sys import os import datetime +from typing import Dict, Any, List +import anytree # type: ignore from docopt import docopt # local imports @@ -78,13 +80,20 @@ Options: """ # nopep8 -def cmd_mount(args, top, noder): +def cmd_mount(args: Dict[str, Any], + top: anytree.AnyNode, + noder: Noder) -> None: """mount action""" mountpoint = args[''] - Fuser(mountpoint, top, noder) + debug = args['--verbose'] + Fuser(mountpoint, top, noder, + debug=debug) -def cmd_index(args, noder, catalog, top): +def cmd_index(args: Dict[str, Any], + noder: Noder, + catalog: Catalog, + top: anytree.AnyNode) -> None: """index action""" path = args[''] name = args[''] @@ -119,7 +128,10 @@ def cmd_index(args, noder, catalog, top): catalog.save(top) -def cmd_update(args, noder, catalog, top): +def cmd_update(args: Dict[str, Any], + noder: Noder, + catalog: Catalog, + top: anytree.AnyNode) -> None: """update action""" path = args[''] name = args[''] @@ -147,7 +159,9 @@ def cmd_update(args, noder, catalog, top): catalog.save(top) -def cmd_ls(args, noder, top): +def cmd_ls(args: Dict[str, Any], + noder: Noder, + top: anytree.AnyNode) -> List[anytree.AnyNode]: """ls action""" path = args[''] if not path: @@ -168,7 +182,8 @@ def cmd_ls(args, noder, top): fmt = args['--format'] if fmt.startswith('fzf'): raise BadFormatException('fzf is not supported in ls, use find') - found = noder.list(top, path, + found = noder.list(top, + path, rec=args['--recursive'], fmt=fmt, raw=args['--raw-size']) @@ -178,7 +193,10 @@ def cmd_ls(args, noder, top): return found -def cmd_rm(args, noder, catalog, top): +def cmd_rm(args: Dict[str, Any], + noder: Noder, + catalog: Catalog, + top: anytree.AnyNode) -> anytree.AnyNode: """rm action""" name = args[''] node = noder.get_storage_node(top, name) @@ -191,7 +209,9 @@ def cmd_rm(args, noder, catalog, top): return top -def cmd_find(args, noder, top): +def cmd_find(args: Dict[str, Any], + noder: Noder, + top: anytree.AnyNode) -> List[anytree.AnyNode]: """find action""" fromtree = args['--parent'] directory = args['--directory'] @@ -200,12 +220,18 @@ def cmd_find(args, noder, top): raw = args['--raw-size'] script = args['--script'] search_for = args[''] - return noder.find_name(top, search_for, script=script, - startpath=startpath, only_dir=directory, - parentfromtree=fromtree, fmt=fmt, raw=raw) + found = noder.find_name(top, search_for, + script=script, + startnode=startpath, + only_dir=directory, + parentfromtree=fromtree, + fmt=fmt, raw=raw) + return found -def cmd_graph(args, noder, top): +def cmd_graph(args: Dict[str, Any], + noder: Noder, + top: anytree.AnyNode) -> None: """graph action""" path = args[''] if not path: @@ -214,7 +240,9 @@ def cmd_graph(args, noder, top): Logger.info(f'create graph with \"{cmd}\" (you need graphviz)') -def cmd_rename(args, catalog, top): +def cmd_rename(args: Dict[str, Any], + catalog: Catalog, + top: anytree.AnyNode) -> None: """rename action""" storage = args[''] new = args[''] @@ -227,10 +255,12 @@ def cmd_rename(args, catalog, top): Logger.info(msg) else: Logger.err(f'Storage named \"{storage}\" does not exist') - return top -def cmd_edit(args, noder, catalog, top): +def cmd_edit(args: Dict[str, Any], + noder: Noder, + catalog: Catalog, + top: anytree.AnyNode) -> None: """edit action""" storage = args[''] storages = list(x.name for x in top.children) @@ -245,16 +275,15 @@ def cmd_edit(args, noder, catalog, top): Logger.info(f'Storage \"{storage}\" edited') else: Logger.err(f'Storage named \"{storage}\" does not exist') - return top -def banner(): +def banner() -> None: """print banner""" Logger.stderr_nocolor(BANNER) Logger.stderr_nocolor("") -def print_supported_formats(): +def print_supported_formats() -> None: """print all supported formats to stdout""" print('"native" : native format') print('"csv" : CSV format') @@ -263,7 +292,7 @@ def print_supported_formats(): print('"fzf-csv" : fzf to csv (only for find)') -def main(): +def main() -> bool: """entry point""" args = docopt(USAGE, version=VERSION) diff --git a/catcli/colors.py b/catcli/colors.py index e53cd6c..07010c5 100644 --- a/catcli/colors.py +++ b/catcli/colors.py @@ -5,6 +5,11 @@ Copyright (c) 2022, deadc0de6 shell colors """ +from typing import TypeVar, Type + + +CLASSTYPE = TypeVar('CLASSTYPE', bound='Colors') + class Colors: """shell colors""" @@ -22,7 +27,7 @@ class Colors: UND = '\033[4m' @classmethod - def no_color(cls): + def no_color(cls: Type[CLASSTYPE]) -> None: """disable colors""" Colors.RED = '' Colors.GREEN = '' diff --git a/catcli/decomp.py b/catcli/decomp.py index 13f4ff4..079a428 100644 --- a/catcli/decomp.py +++ b/catcli/decomp.py @@ -8,12 +8,13 @@ Catcli generic compressed data lister import os import tarfile import zipfile +from typing import List class Decomp: """decompressor""" - def __init__(self): + def __init__(self) -> None: self.ext = { 'tar': self._tar, 'tgz': self._tar, @@ -28,29 +29,29 @@ class Decomp: 'tar.bz2': self._tar, 'zip': self._zip} - def get_formats(self): + def get_formats(self) -> List[str]: """return list of supported extensions""" return list(self.ext.keys()) - def get_names(self, path): + def get_names(self, path: str) -> List[str]: """get tree of compressed archive""" ext = os.path.splitext(path)[1][1:].lower() if ext in list(self.ext): return self.ext[ext](path) - return None + return [] @staticmethod - def _tar(path): + def _tar(path: str) -> List[str]: """return list of file names in tar""" if not tarfile.is_tarfile(path): - return None + return [] with tarfile.open(path, "r") as tar: return tar.getnames() @staticmethod - def _zip(path): + def _zip(path: str) -> List[str]: """return list of file names in zip""" if not zipfile.is_zipfile(path): - return None + return [] with zipfile.ZipFile(path) as file: return file.namelist() diff --git a/catcli/fuser.py b/catcli/fuser.py index 1c62760..0b3d011 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -9,7 +9,9 @@ import os import logging from time import time from stat import S_IFDIR, S_IFREG -import fuse +from typing import List, Dict, Any +import anytree # type: ignore +import fuse # type: ignore from .noder import Noder @@ -26,28 +28,33 @@ SEPARATOR = '/' class Fuser: - """fuser filesystem""" + """fuse filesystem mounter""" - def __init__(self, mountpoint, top, noder): + def __init__(self, mountpoint: str, + top: anytree.AnyNode, + noder: Noder, + debug: bool = False): """fuse filesystem""" filesystem = CatcliFilesystem(top, noder) fuse.FUSE(filesystem, mountpoint, - foreground=True, + foreground=debug, allow_other=True, nothreads=True, - debug=True) + debug=debug) -class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): +class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore """in-memory filesystem for catcli catalog""" - def __init__(self, top, noder): + def __init__(self, top: anytree.AnyNode, + noder: Noder): """init fuse filesystem""" self.top = top self.noder = noder - def _get_entry(self, path): + def _get_entry(self, path: str) -> anytree.AnyNode: + """return the node pointed by path""" pre = f'{SEPARATOR}{self.noder.NAME_TOP}' if not path.startswith(pre): path = pre + path @@ -57,9 +64,10 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): raw=True) if found: return found[0] - return [] + return None - def _get_entries(self, path): + def _get_entries(self, path: str) -> List[anytree.AnyNode]: + """return nodes pointed by path""" pre = f'{SEPARATOR}{self.noder.NAME_TOP}' if not path.startswith(pre): path = pre + path @@ -73,13 +81,13 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): raw=True) return found - def _getattr(self, path): + def _getattr(self, path: str) -> Dict[str, Any]: entry = self._get_entry(path) if not entry: - return None + return {} curt = time() - mode = S_IFREG + mode: Any = S_IFREG if entry.type == Noder.TYPE_ARC: mode = S_IFREG elif entry.type == Noder.TYPE_DIR: @@ -103,7 +111,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): 'st_gid': os.getgid(), } - def getattr(self, path, _fh=None): + def getattr(self, path: str, _fh: Any = None) -> Dict[str, Any]: """return attr of file pointed by path""" logger.info('getattr path: %s', path) @@ -124,7 +132,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): meta = self._getattr(path) return meta - def readdir(self, path, _fh): + def readdir(self, path: str, _fh: Any) -> List[str]: """read directory content""" logger.info('readdir path: %s', path) content = ['.', '..'] diff --git a/catcli/logger.py b/catcli/logger.py index 586b033..423a044 100644 --- a/catcli/logger.py +++ b/catcli/logger.py @@ -6,61 +6,75 @@ Logging helper """ import sys +from typing import TypeVar, Type # local imports from catcli.colors import Colors from catcli.utils import fix_badchars +CLASSTYPE = TypeVar('CLASSTYPE', bound='Logger') + + class Logger: """log to stdout/stderr""" @classmethod - def stdout_nocolor(cls, string): + def stdout_nocolor(cls: Type[CLASSTYPE], + string: str) -> None: """to stdout no color""" string = fix_badchars(string) sys.stdout.write(f'{string}\n') @classmethod - def stderr_nocolor(cls, string): + def stderr_nocolor(cls: Type[CLASSTYPE], + string: str) -> None: """to stderr no color""" string = fix_badchars(string) sys.stderr.write(f'{string}\n') @classmethod - def debug(cls, string): + def debug(cls: Type[CLASSTYPE], + string: str) -> None: """to stderr no color""" cls.stderr_nocolor(f'[DBG] {string}\n') @classmethod - def info(cls, string): + def info(cls: Type[CLASSTYPE], + string: str) -> None: """to stdout in color""" string = fix_badchars(string) out = f'{Colors.MAGENTA}{string}{Colors.RESET}' sys.stdout.write(f'{out}\n') @classmethod - def err(cls, string): + def err(cls: Type[CLASSTYPE], + string: str) -> None: """to stderr in RED""" string = fix_badchars(string) out = f'{Colors.RED}{string}{Colors.RESET}' sys.stderr.write(f'{out}\n') @classmethod - def progr(cls, string): + def progr(cls: Type[CLASSTYPE], + string: str) -> None: """print progress""" string = fix_badchars(string) sys.stderr.write(f'{string}\r') sys.stderr.flush() @classmethod - def get_bold_text(cls, string): + def get_bold_text(cls: Type[CLASSTYPE], + string: str) -> str: """make it bold""" string = fix_badchars(string) return f'{Colors.BOLD}{string}{Colors.RESET}' @classmethod - def log_to_file(cls, path, string, append=True): + def log_to_file(cls: Type[CLASSTYPE], + path: str, + string: str, + append: bool = True) -> None: """log to file""" string = fix_badchars(string) mode = 'w' diff --git a/catcli/nodeprinter.py b/catcli/nodeprinter.py index b43613b..26505e6 100644 --- a/catcli/nodeprinter.py +++ b/catcli/nodeprinter.py @@ -6,11 +6,15 @@ Class for printing nodes """ import sys +from typing import TypeVar, Type, Optional, Tuple, List from catcli.colors import Colors from catcli.utils import fix_badchars +CLASSTYPE = TypeVar('CLASSTYPE', bound='NodePrinter') + + class NodePrinter: """a node printer class""" @@ -19,7 +23,9 @@ class NodePrinter: NBFILES = 'nbfiles' @classmethod - def print_storage_native(cls, pre, name, args, attr): + def print_storage_native(cls: Type[CLASSTYPE], pre: str, + name: str, args: str, + attr: str) -> None: """print a storage node""" end = '' if attr: @@ -31,7 +37,8 @@ class NodePrinter: sys.stdout.write(f'{out}\n') @classmethod - def print_file_native(cls, pre, name, attr): + def print_file_native(cls: Type[CLASSTYPE], pre: str, + name: str, attr: str) -> None: """print a file node""" nobad = fix_badchars(name) out = f'{pre}{nobad}' @@ -39,22 +46,26 @@ class NodePrinter: sys.stdout.write(f'{out}\n') @classmethod - def print_dir_native(cls, pre, name, depth='', attr=None): + def print_dir_native(cls: Type[CLASSTYPE], pre: str, + name: str, + depth: int = 0, + attr: Optional[List[Tuple[str, str]]] = None) -> None: """print a directory node""" end = [] - if depth != '': + if depth > 0: end.append(f'{cls.NBFILES}:{depth}') if attr: end.append(' '.join([f'{x}:{y}' for x, y in attr])) + end_string = '' if end: - endstring = ', '.join(end) - end = f' [{endstring}]' + end_string = f' [{", ".join(end)}]' out = pre + Colors.BLUE + fix_badchars(name) + Colors.RESET - out += f'{Colors.GRAY}{end}{Colors.RESET}' + out += f'{Colors.GRAY}{end_string}{Colors.RESET}' sys.stdout.write(f'{out}\n') @classmethod - def print_archive_native(cls, pre, name, archive): + def print_archive_native(cls: Type[CLASSTYPE], pre: str, + name: str, archive: str) -> None: """archive to stdout""" out = pre + Colors.YELLOW + fix_badchars(name) + Colors.RESET out += f' {Colors.GRAY}[{cls.ARCHIVE}:{archive}]{Colors.RESET}' diff --git a/catcli/noder.py b/catcli/noder.py index 7ad39bd..8624e8b 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -2,14 +2,15 @@ author: deadc0de6 (https://github.com/deadc0de6) Copyright (c) 2017, deadc0de6 -Class that represents a node in the catalog tree +Class that process nodes in the catalog tree """ import os import shutil import time -import anytree -from pyfzf.pyfzf import FzfPrompt +from typing import List, Union, Tuple, Any, Optional, Dict +import anytree # type: ignore +from pyfzf.pyfzf import FzfPrompt # type: ignore # local imports from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars @@ -44,7 +45,9 @@ class Noder: 'maccess,md5,nbfiles,free_space,' 'total_space,meta') - def __init__(self, debug=False, sortsize=False, arc=False): + def __init__(self, debug: bool = False, + sortsize: bool = False, + arc: bool = False) -> None: """ @debug: debug mode @sortsize: sort nodes by size @@ -58,11 +61,12 @@ class Noder: self.decomp = Decomp() @staticmethod - def get_storage_names(top): + def get_storage_names(top: anytree.AnyNode) -> List[str]: """return a list of all storage names""" return [x.name for x in list(top.children)] - def get_storage_node(self, top, name, path=None): + def get_storage_node(self, top: anytree.AnyNode, + name: str, path: str = '') -> anytree.AnyNode: """ return the storage node if any if path is submitted, it will update the media info @@ -81,7 +85,8 @@ class Noder: return found @staticmethod - def get_node(top, path, quiet=False): + def get_node(top: str, path: str, + quiet: bool = False) -> anytree.AnyNode: """get the node by internal tree path""" resolv = anytree.resolver.Resolver('name') try: @@ -92,7 +97,10 @@ class Noder: Logger.err(f'No node at path \"{bpath}\"') return None - def get_node_if_changed(self, top, path, treepath): + def get_node_if_changed(self, + top: anytree.AnyNode, + path: str, + treepath: str) -> Tuple[anytree.AnyNode, bool]: """ return the node (if any) and if it has changed @top: top node (storage) @@ -128,17 +136,18 @@ class Noder: self._debug(f'\tchange: no change for \"{path}\"') return node, False - def rec_size(self, node, store=True): + def rec_size(self, node: anytree.AnyNode, + store: bool = True) -> float: """ recursively traverse tree and return size @store: store the size in the node """ if node.type == self.TYPE_FILE: self._debug(f'getting node size for \"{node.name}\"') - return node.size + return float(node.size) msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) - size = 0 + size: float = 0 for i in node.children: if node.type == self.TYPE_DIR: size = self.rec_size(i, store=store) @@ -160,7 +169,7 @@ class Noder: # public helpers ############################################################### @staticmethod - def format_storage_attr(attr): + def format_storage_attr(attr: Union[str, List[str]]) -> str: """format the storage attr for saving""" if not attr: return '' @@ -169,18 +178,19 @@ class Noder: attr = attr.rstrip() return attr - def set_hashing(self, val): + def set_hashing(self, val: bool) -> None: """hash files when indexing""" self.hash = val ############################################################### # node creation ############################################################### - def new_top_node(self): + def new_top_node(self) -> anytree.AnyNode: """create a new top node""" return anytree.AnyNode(name=self.NAME_TOP, type=self.TYPE_TOP) - def new_file_node(self, name, path, parent, storagepath): + def new_file_node(self, name: str, path: str, + parent: str, storagepath: str) -> anytree.AnyNode: """create a new node representing a file""" if not os.path.exists(path): Logger.err(f'File \"{path}\" does not exist') @@ -191,14 +201,16 @@ class Noder: except OSError as exc: Logger.err(f'OSError: {exc}') return None - md5 = None + md5 = '' if self.hash: md5 = self._get_hash(path) relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - node = self._new_generic_node(name, self.TYPE_FILE, relpath, parent, - size=stat.st_size, md5=md5, + node = self._new_generic_node(name, self.TYPE_FILE, + relpath, parent, + size=stat.st_size, + md5=md5, maccess=maccess) if self.arc: ext = os.path.splitext(path)[1][1:] @@ -210,7 +222,8 @@ class Noder: self._debug(f'{path} is NOT an archive') return node - def new_dir_node(self, name, path, parent, storagepath): + def new_dir_node(self, name: str, path: str, + parent: str, storagepath: str) -> anytree.AnyNode: """create a new node representing a directory""" path = os.path.abspath(path) relpath = os.sep.join([storagepath, name]) @@ -218,7 +231,10 @@ class Noder: return self._new_generic_node(name, self.TYPE_DIR, relpath, parent, maccess=maccess) - def new_storage_node(self, name, path, parent, attr=None): + def new_storage_node(self, name: str, + path: str, + parent: str, + attr: Optional[str] = None) -> anytree.AnyNode: """create a new node representing a storage""" path = os.path.abspath(path) free = shutil.disk_usage(path).free @@ -227,15 +243,19 @@ class Noder: return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free, total=total, parent=parent, attr=attr, ts=epoch) - def new_archive_node(self, name, path, parent, archive): + def new_archive_node(self, name: str, path: str, + parent: str, archive: str) -> anytree.AnyNode: """create a new node for archive data""" return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, parent=parent, size=0, md5=None, archive=archive) @staticmethod - def _new_generic_node(name, nodetype, relpath, parent, - size=None, md5=None, maccess=None): + def _new_generic_node(name: str, nodetype: str, + relpath: str, parent: str, + size: float = 0, + md5: str = '', + maccess: float = 0) -> anytree.AnyNode: """generic node creation""" return anytree.AnyNode(name=name, type=nodetype, relpath=relpath, parent=parent, size=size, @@ -244,21 +264,22 @@ class Noder: ############################################################### # node management ############################################################### - def update_metanode(self, top): + def update_metanode(self, top: anytree.AnyNode) -> anytree.AnyNode: """create or update meta node information""" meta = self._get_meta_node(top) epoch = int(time.time()) if not meta: - attr = {} + attr: Dict[str, Any] = {} attr['created'] = epoch attr['created_version'] = VERSION - meta = anytree.AnyNode(name=self.NAME_META, type=self.TYPE_META, + meta = anytree.AnyNode(name=self.NAME_META, + type=self.TYPE_META, attr=attr) meta.attr['access'] = epoch meta.attr['access_version'] = VERSION return meta - def _get_meta_node(self, top): + def _get_meta_node(self, top: anytree.AnyNode) -> anytree.AnyNode: """return the meta node if any""" try: return next(filter(lambda x: x.type == self.TYPE_META, @@ -266,7 +287,7 @@ class Noder: except StopIteration: return None - def clean_not_flagged(self, top): + def clean_not_flagged(self, top: anytree.AnyNode) -> int: """remove any node not flagged and clean flags""" cnt = 0 for node in anytree.PreOrderIter(top): @@ -277,11 +298,11 @@ class Noder: return cnt @staticmethod - def flag(node): + def flag(node: anytree.AnyNode) -> None: """flag a node""" node.flag = True - def _clean(self, node): + def _clean(self, node: anytree.AnyNode) -> bool: """remove node if not flagged""" if not self._has_attr(node, 'flag') or \ not node.flag: @@ -293,7 +314,9 @@ class Noder: ############################################################### # printing ############################################################### - def _node_to_csv(self, node, sep=',', raw=False): + def _node_to_csv(self, node: anytree.AnyNode, + sep: str = ',', + raw: bool = False) -> None: """ print a node to csv @node: the node to consider @@ -353,9 +376,13 @@ class Noder: if len(line) > 0: Logger.stdout_nocolor(line) - def _print_node_native(self, node, pre='', withpath=False, - withdepth=False, withstorage=False, - recalcparent=False, raw=False): + def _print_node_native(self, node: anytree.AnyNode, + pre: str = '', + withpath: bool = False, + withdepth: bool = False, + withstorage: bool = False, + recalcparent: bool = False, + raw: bool = False) -> None: """ print a node @node: the node to print @@ -380,11 +407,11 @@ class Noder: name = name.lstrip(os.sep) if withstorage: storage = self._get_storage(node) - attr = '' + attr_str = '' if node.md5: - attr = f', md5:{node.md5}' + attr_str = f', md5:{node.md5}' size = size_to_str(node.size, raw=raw) - compl = f'size:{size}{attr}' + compl = f'size:{size}{attr_str}' if withstorage: content = Logger.get_bold_text(storage.name) compl += f', storage:{content}' @@ -398,16 +425,16 @@ class Noder: else: name = node.relpath name = name.lstrip(os.sep) - depth = '' + depth = 0 if withdepth: depth = len(node.children) if withstorage: storage = self._get_storage(node) - attr = [] + attr: List[Tuple[str, str]] = [] if node.size: - attr.append(['totsize', size_to_str(node.size, raw=raw)]) + attr.append(('totsize', size_to_str(node.size, raw=raw))) if withstorage: - attr.append(['storage', Logger.get_bold_text(storage.name)]) + attr.append(('storage', Logger.get_bold_text(storage.name))) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) elif node.type == self.TYPE_STORAGE: # node of type storage @@ -423,9 +450,9 @@ class Noder: timestamp += epoch_to_str(node.ts) disksize = '' # the children size - size = self.rec_size(node, store=False) - size = size_to_str(size, raw=raw) - disksize = 'totsize:' + f'{size}' + recsize = self.rec_size(node, store=False) + sizestr = size_to_str(recsize, raw=raw) + disksize = 'totsize:' + f'{sizestr}' # format the output name = node.name args = [ @@ -446,9 +473,9 @@ class Noder: else: Logger.err(f'bad node encountered: {node}') - def print_tree(self, node, - fmt='native', - raw=False): + def print_tree(self, node: anytree.AnyNode, + fmt: str = 'native', + raw: bool = False) -> None: """ print the tree in different format @node: start node @@ -470,20 +497,21 @@ class Noder: Logger.stdout_nocolor(self.CSV_HEADER) self._to_csv(node, raw=raw) - def _to_csv(self, node, raw=False): + def _to_csv(self, node: anytree.AnyNode, + raw: bool = False) -> None: """print the tree to csv""" rend = anytree.RenderTree(node, childiter=self._sort_tree) for _, _, item in rend: self._node_to_csv(item, raw=raw) @staticmethod - def _fzf_prompt(strings): + def _fzf_prompt(strings: Any) -> Any: # prompt with fzf fzf = FzfPrompt() selected = fzf.prompt(strings) return selected - def _to_fzf(self, node, fmt): + def _to_fzf(self, node: anytree.AnyNode, fmt: str) -> None: """ fzf prompt with list and print selected node(s) @node: node to start with @@ -512,7 +540,8 @@ class Noder: self.print_tree(rend, fmt=subfmt) @staticmethod - def to_dot(node, path='tree.dot'): + def to_dot(node: anytree.AnyNode, + path: str = 'tree.dot') -> str: """export to dot for graphing""" anytree.exporter.DotExporter(node).to_dotfile(path) Logger.info(f'dot file created under \"{path}\"') @@ -521,10 +550,14 @@ class Noder: ############################################################### # searching ############################################################### - def find_name(self, top, key, - script=False, only_dir=False, - startpath=None, parentfromtree=False, - fmt='native', raw=False): + def find_name(self, top: anytree.AnyNode, + key: str, + script: bool = False, + only_dir: bool = False, + startnode: anytree.AnyNode = None, + parentfromtree: bool = False, + fmt: str = 'native', + raw: bool = False) -> List[anytree.AnyNode]: """ find files based on their names @top: top node @@ -541,8 +574,8 @@ class Noder: # search for nodes based on path start = top - if startpath: - start = self.get_node(top, startpath) + if startnode: + start = self.get_node(top, startnode) filterfunc = self._callback_find_name(key, only_dir) found = anytree.findall(start, filter_=filterfunc) nbfound = len(found) @@ -591,9 +624,9 @@ class Noder: return list(paths.values()) - def _callback_find_name(self, term, only_dir): + def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" - def find_name(node): + def find_name(node: anytree.AnyNode) -> bool: if node.type == self.TYPE_STORAGE: # ignore storage nodes return False @@ -620,10 +653,11 @@ class Noder: ############################################################### # ls ############################################################### - def list(self, top, path, - rec=False, - fmt='native', - raw=False): + def list(self, top: anytree.AnyNode, + path: str, + rec: bool = False, + fmt: str = 'native', + raw: bool = False) -> List[anytree.AnyNode]: """ list nodes for "ls" @top: top node @@ -684,7 +718,9 @@ class Noder: ############################################################### # tree creation ############################################################### - def _add_entry(self, name, top, resolv): + def _add_entry(self, name: str, + top: anytree.AnyNode, + resolv: Any) -> None: """add an entry to the tree""" entries = name.rstrip(os.sep).split(os.sep) if len(entries) == 1: @@ -698,7 +734,7 @@ class Noder: except anytree.resolver.ChildResolverError: self.new_archive_node(nodename, name, top, top.name) - def list_to_tree(self, parent, names): + def list_to_tree(self, parent: anytree.AnyNode, names: List[str]) -> None: """convert list of files to a tree""" if not names: return @@ -710,43 +746,44 @@ class Noder: ############################################################### # diverse ############################################################### - def _sort_tree(self, items): + def _sort_tree(self, + items: List[anytree.AnyNode]) -> List[anytree.AnyNode]: """sorting a list of items""" return sorted(items, key=self._sort, reverse=self.sortsize) - def _sort(self, lst): + def _sort(self, lst: List[anytree.AnyNode]) -> Any: """sort a list""" if self.sortsize: return self._sort_size(lst) return self._sort_fs(lst) @staticmethod - def _sort_fs(node): + def _sort_fs(node: anytree.AnyNode) -> Tuple[str, str]: """sorting nodes dir first and alpha""" return (node.type, node.name.lstrip('.').lower()) @staticmethod - def _sort_size(node): + def _sort_size(node: anytree.AnyNode) -> float: """sorting nodes by size""" try: if not node.size: return 0 - return node.size + return float(node.size) except AttributeError: return 0 - def _get_storage(self, node): + def _get_storage(self, node: anytree.AnyNode) -> anytree.AnyNode: """recursively traverse up to find storage""" if node.type == self.TYPE_STORAGE: return node return node.ancestors[1] @staticmethod - def _has_attr(node, attr): + def _has_attr(node: anytree.AnyNode, attr: str) -> bool: """return True if node has attr as attribute""" return attr in node.__dict__.keys() - def _get_parents(self, node): + def _get_parents(self, node: anytree.AnyNode) -> str: """get all parents recursively""" if node.type == self.TYPE_STORAGE: return '' @@ -755,25 +792,25 @@ class Noder: parent = self._get_parents(node.parent) if parent: return os.sep.join([parent, node.name]) - return node.name + return str(node.name) @staticmethod - def _get_hash(path): + def _get_hash(path: str) -> str: """return md5 hash of node""" try: return md5sum(path) except CatcliException as exc: Logger.err(str(exc)) - return None + return '' @staticmethod - def _sanitize(node): + def _sanitize(node: anytree.AnyNode) -> anytree.AnyNode: """sanitize node strings""" node.name = fix_badchars(node.name) node.relpath = fix_badchars(node.relpath) return node - def _debug(self, string): + def _debug(self, string: str) -> None: """print debug""" if not self.debug: return diff --git a/catcli/utils.py b/catcli/utils.py index 1180455..38a257e 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -18,7 +18,7 @@ from catcli.exceptions import CatcliException SEPARATOR = '/' -def md5sum(path): +def md5sum(path: str) -> str: """ calculate md5 sum of a file may raise exception @@ -39,10 +39,11 @@ def md5sum(path): pass except OSError as exc: raise CatcliException(f'md5sum error: {exc}') from exc - return None + return '' -def size_to_str(size, raw=True): +def size_to_str(size: float, + raw: bool = True) -> str: """convert size to string, optionally human readable""" div = 1024. suf = ['B', 'K', 'M', 'G', 'T', 'P'] @@ -56,7 +57,7 @@ def size_to_str(size, raw=True): return f'{size:.1f}{sufix}' -def epoch_to_str(epoch): +def epoch_to_str(epoch: int) -> str: """convert epoch to string""" if not epoch: return '' @@ -65,18 +66,18 @@ def epoch_to_str(epoch): return timestamp.strftime(fmt) -def ask(question): +def ask(question: str) -> bool: """ask the user what to do""" resp = input(f'{question} [y|N] ? ') return resp.lower() == 'y' -def edit(string): +def edit(string: str) -> str: """edit the information with the default EDITOR""" - string = string.encode('utf-8') + data = string.encode('utf-8') editor = os.environ.get('EDITOR', 'vim') with tempfile.NamedTemporaryFile(prefix='catcli', suffix='.tmp') as file: - file.write(string) + file.write(data) file.flush() subprocess.call([editor, file.name]) file.seek(0) @@ -84,6 +85,6 @@ def edit(string): return new.decode('utf-8') -def fix_badchars(string): +def fix_badchars(string: str) -> str: """fix none utf-8 chars in string""" return string.encode('utf-8', 'ignore').decode('utf-8') diff --git a/catcli/walker.py b/catcli/walker.py index 4740faf..4d9f79d 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -6,8 +6,11 @@ Catcli filesystem indexer """ import os +from typing import Tuple +import anytree # type: ignore # local imports +from catcli.noder import Noder from catcli.logger import Logger @@ -16,8 +19,10 @@ class Walker: MAXLINELEN = 80 - 15 - def __init__(self, noder, usehash=True, debug=False, - logpath=None): + def __init__(self, noder: Noder, + usehash: bool = True, + debug: bool = False, + logpath: str = ''): """ @noder: the noder to use @hash: calculate hash of nodes @@ -30,7 +35,10 @@ class Walker: self.debug = debug self.lpath = logpath - def index(self, path, parent, name, storagepath=''): + def index(self, path: str, + parent: str, + name: str, + storagepath: str = '') -> Tuple[str, int]: """ index a directory and store in tree @path: path to index @@ -39,7 +47,8 @@ class Walker: """ self._debug(f'indexing starting at {path}') if not parent: - parent = self.noder.new_dir_node(name, path, parent) + parent = self.noder.new_dir_node(name, path, + parent, storagepath) if os.path.islink(path): rel = os.readlink(path) @@ -77,16 +86,19 @@ class Walker: _, cnt2 = self.index(sub, dummy, base, nstoragepath) cnt += cnt2 break - self._progress(None) + self._progress('') return parent, cnt - def reindex(self, path, parent, top): + def reindex(self, path: str, parent: str, top: str) -> int: """reindex a directory and store in tree""" cnt = self._reindex(path, parent, top) cnt += self.noder.clean_not_flagged(parent) return cnt - def _reindex(self, path, parent, top, storagepath=''): + def _reindex(self, path: str, + parent: str, + top: anytree.AnyNode, + storagepath: str = '') -> int: """ reindex a directory and store in tree @path: directory path to re-index @@ -131,7 +143,10 @@ class Walker: break return cnt - def _need_reindex(self, top, path, treepath): + def _need_reindex(self, + top: anytree.AnyNode, + path: str, + treepath: str) -> Tuple[bool, anytree.AnyNode]: """ test if node needs re-indexing @top: top node (storage) @@ -153,13 +168,13 @@ class Walker: cnode.parent = None return True, cnode - def _debug(self, string): + def _debug(self, string: str) -> None: """print to debug""" if not self.debug: return Logger.debug(string) - def _progress(self, string): + def _progress(self, string: str) -> None: """print progress""" if self.debug: return @@ -171,7 +186,7 @@ class Walker: string = string[:self.MAXLINELEN] + '...' Logger.progr(f'indexing: {string:80}') - def _log2file(self, string): + def _log2file(self, string: str) -> None: """log to file""" if not self.lpath: return From 50eb5cf9fddb75d7858717e26a9941171a5ae772 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 5 Mar 2023 23:27:11 +0100 Subject: [PATCH 05/36] add custom node type --- catcli/catalog.py | 21 ++-- catcli/catcli.py | 49 ++++----- catcli/cnode.py | 68 +++++++++++++ catcli/fuser.py | 33 +++--- catcli/noder.py | 234 ++++++++++++++++++++++--------------------- catcli/utils.py | 4 +- catcli/walker.py | 52 +++++----- tests/test_update.py | 2 +- 8 files changed, 275 insertions(+), 188 deletions(-) create mode 100644 catcli/cnode.py diff --git a/catcli/catalog.py b/catcli/catalog.py index 005d081..821327c 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -7,11 +7,12 @@ Class that represents the catcli catalog import os import pickle -import anytree # type: ignore +from typing import Optional, Union, Any, cast from anytree.exporter import JsonExporter # type: ignore from anytree.importer import JsonImporter # type: ignore # local imports +from catcli.cnode import Node from catcli.utils import ask from catcli.logger import Logger @@ -32,10 +33,10 @@ class Catalog: self.path = path self.debug = debug self.force = force - self.metanode = None + self.metanode: Optional[Node] = None self.pickle = usepickle - def set_metanode(self, metanode: anytree.AnyNode) -> None: + def set_metanode(self, metanode: Node) -> None: """remove the metanode until tree is re-written""" self.metanode = metanode if self.metanode: @@ -49,7 +50,7 @@ class Catalog: return True return False - def restore(self) -> anytree.AnyNode: + def restore(self) -> Optional[Node]: """restore the catalog""" if not self.path: return None @@ -61,7 +62,7 @@ class Catalog: content = file.read() return self._restore_json(content) - def save(self, node: anytree.AnyNode) -> bool: + def save(self, node: Node) -> bool: """save the catalog""" if not self.path: Logger.err('Path not defined') @@ -87,14 +88,14 @@ class Catalog: return Logger.debug(text) - def _save_pickle(self, node: anytree.AnyNode) -> bool: + def _save_pickle(self, node: Node) -> bool: """pickle the catalog""" with open(self.path, 'wb') as file: pickle.dump(node, file) self._debug(f'Catalog saved to pickle \"{self.path}\"') return True - def _restore_pickle(self) -> anytree.AnyNode: + def _restore_pickle(self) -> Union[Node, Any]: """restore the pickled tree""" with open(self.path, 'rb') as file: root = pickle.load(file) @@ -102,7 +103,7 @@ class Catalog: self._debug(msg) return root - def _save_json(self, node: anytree.AnyNode) -> bool: + def _save_json(self, node: Node) -> bool: """export the catalog in json""" exp = JsonExporter(indent=2, sort_keys=True) with open(self.path, 'w', encoding='UTF-8') as file: @@ -110,9 +111,9 @@ class Catalog: self._debug(f'Catalog saved to json \"{self.path}\"') return True - def _restore_json(self, string: str) -> anytree.AnyNode: + def _restore_json(self, string: str) -> Node: """restore the tree from json""" imp = JsonImporter() root = imp.import_(string) self._debug(f'Catalog imported from json \"{self.path}\"') - return root + return cast(Node, root) diff --git a/catcli/catcli.py b/catcli/catcli.py index 49f5e0a..efcbfb8 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -12,19 +12,20 @@ import sys import os import datetime from typing import Dict, Any, List -import anytree # type: ignore from docopt import docopt # local imports -from .version import __version__ as VERSION -from .logger import Logger -from .colors import Colors -from .catalog import Catalog -from .walker import Walker -from .noder import Noder -from .utils import ask, edit -from .fuser import Fuser -from .exceptions import BadFormatException, CatcliException +from catcli import cnode +from catcli.version import __version__ as VERSION +from catcli.logger import Logger +from catcli.colors import Colors +from catcli.catalog import Catalog +from catcli.walker import Walker +from catcli.cnode import Node +from catcli.noder import Noder +from catcli.utils import ask, edit +from catcli.fuser import Fuser +from catcli.exceptions import BadFormatException, CatcliException NAME = 'catcli' CUR = os.path.dirname(os.path.abspath(__file__)) @@ -81,7 +82,7 @@ Options: def cmd_mount(args: Dict[str, Any], - top: anytree.AnyNode, + top: Node, noder: Noder) -> None: """mount action""" mountpoint = args[''] @@ -93,7 +94,7 @@ def cmd_mount(args: Dict[str, Any], def cmd_index(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: anytree.AnyNode) -> None: + top: Node) -> None: """index action""" path = args[''] name = args[''] @@ -116,8 +117,8 @@ def cmd_index(args: Dict[str, Any], start = datetime.datetime.now() walker = Walker(noder, usehash=usehash, debug=debug) - attr = noder.format_storage_attr(args['--meta']) - root = noder.new_storage_node(name, path, parent=top, attr=attr) + attr = noder.attrs_to_string(args['--meta']) + root = noder.new_storage_node(name, path, parent=top, attrs=attr) _, cnt = walker.index(path, root, name) if subsize: noder.rec_size(root, store=True) @@ -131,7 +132,7 @@ def cmd_index(args: Dict[str, Any], def cmd_update(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: anytree.AnyNode) -> None: + top: Node) -> None: """update action""" path = args[''] name = args[''] @@ -142,7 +143,7 @@ def cmd_update(args: Dict[str, Any], if not os.path.exists(path): Logger.err(f'\"{path}\" does not exist') return - root = noder.get_storage_node(top, name, path=path) + root = noder.get_storage_node(top, name, newpath=path) if not root: Logger.err(f'storage named \"{name}\" does not exist') return @@ -161,7 +162,7 @@ def cmd_update(args: Dict[str, Any], def cmd_ls(args: Dict[str, Any], noder: Noder, - top: anytree.AnyNode) -> List[anytree.AnyNode]: + top: Node) -> List[Node]: """ls action""" path = args[''] if not path: @@ -169,7 +170,7 @@ def cmd_ls(args: Dict[str, Any], if not path.startswith(SEPARATOR): path = SEPARATOR + path # prepend with top node path - pre = f'{SEPARATOR}{noder.NAME_TOP}' + pre = f'{SEPARATOR}{cnode.NAME_TOP}' if not path.startswith(pre): path = pre + path # ensure ends with a separator @@ -196,7 +197,7 @@ def cmd_ls(args: Dict[str, Any], def cmd_rm(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: anytree.AnyNode) -> anytree.AnyNode: + top: Node) -> Node: """rm action""" name = args[''] node = noder.get_storage_node(top, name) @@ -211,7 +212,7 @@ def cmd_rm(args: Dict[str, Any], def cmd_find(args: Dict[str, Any], noder: Noder, - top: anytree.AnyNode) -> List[anytree.AnyNode]: + top: Node) -> List[Node]: """find action""" fromtree = args['--parent'] directory = args['--directory'] @@ -231,7 +232,7 @@ def cmd_find(args: Dict[str, Any], def cmd_graph(args: Dict[str, Any], noder: Noder, - top: anytree.AnyNode) -> None: + top: Node) -> None: """graph action""" path = args[''] if not path: @@ -242,7 +243,7 @@ def cmd_graph(args: Dict[str, Any], def cmd_rename(args: Dict[str, Any], catalog: Catalog, - top: anytree.AnyNode) -> None: + top: Node) -> None: """rename action""" storage = args[''] new = args[''] @@ -260,7 +261,7 @@ def cmd_rename(args: Dict[str, Any], def cmd_edit(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: anytree.AnyNode) -> None: + top: Node) -> None: """edit action""" storage = args[''] storages = list(x.name for x in top.children) @@ -270,7 +271,7 @@ def cmd_edit(args: Dict[str, Any], if not attr: attr = '' new = edit(attr) - node.attr = noder.format_storage_attr(new) + node.attr = noder.attrs_to_string(new) if catalog.save(top): Logger.info(f'Storage \"{storage}\" edited') else: diff --git a/catcli/cnode.py b/catcli/cnode.py new file mode 100644 index 0000000..599d4fa --- /dev/null +++ b/catcli/cnode.py @@ -0,0 +1,68 @@ +""" +author: deadc0de6 (https://github.com/deadc0de6) +Copyright (c) 2023, deadc0de6 + +Class that represents a node in the catalog tree +""" +# pylint: disable=W0622 + +from anytree import NodeMixin # type: ignore + + +TYPE_TOP = 'top' +TYPE_FILE = 'file' +TYPE_DIR = 'dir' +TYPE_ARC = 'arc' +TYPE_STORAGE = 'storage' +TYPE_META = 'meta' + +NAME_TOP = 'top' +NAME_META = 'meta' + + +class Node(NodeMixin): # type: ignore + """a node in the catalog""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + type: str, + size: float = 0, + relpath: str = '', + md5: str = '', + maccess: float = 0, + free: int = 0, + total: int = 0, + indexed_dt: int = 0, + attr: str = '', + archive: str = '', + parent=None, + children=None): + """build a node""" + super().__init__() + self.name = name + self.type = type + self.size = size + self.relpath = relpath + self.md5 = md5 + self.maccess = maccess + self.free = free + self.total = total + self.indexed_dt = indexed_dt + self.attr = attr + self.archive = archive + self.parent = parent + if children: + self.children = children + self._flagged = False + + def flagged(self) -> bool: + """is flagged""" + return self._flagged + + def flag(self) -> None: + """flag a node""" + self._flagged = True + + def unflag(self) -> None: + """unflag node""" + self._flagged = False diff --git a/catcli/fuser.py b/catcli/fuser.py index 0b3d011..6a4b6cd 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -9,10 +9,13 @@ import os import logging from time import time from stat import S_IFDIR, S_IFREG -from typing import List, Dict, Any -import anytree # type: ignore +from typing import List, Dict, Any, Optional import fuse # type: ignore -from .noder import Noder + +# local imports +from catcli.noder import Noder +from catcli import cnode +from catcli.cnode import Node # build custom logger to log in /tmp @@ -31,7 +34,7 @@ class Fuser: """fuse filesystem mounter""" def __init__(self, mountpoint: str, - top: anytree.AnyNode, + top: Node, noder: Noder, debug: bool = False): """fuse filesystem""" @@ -47,15 +50,15 @@ class Fuser: class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore """in-memory filesystem for catcli catalog""" - def __init__(self, top: anytree.AnyNode, + def __init__(self, top: Node, noder: Noder): """init fuse filesystem""" self.top = top self.noder = noder - def _get_entry(self, path: str) -> anytree.AnyNode: + def _get_entry(self, path: str) -> Optional[Node]: """return the node pointed by path""" - pre = f'{SEPARATOR}{self.noder.NAME_TOP}' + pre = f'{SEPARATOR}{cnode.NAME_TOP}' if not path.startswith(pre): path = pre + path found = self.noder.list(self.top, path, @@ -66,9 +69,9 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore return found[0] return None - def _get_entries(self, path: str) -> List[anytree.AnyNode]: + def _get_entries(self, path: str) -> List[Node]: """return nodes pointed by path""" - pre = f'{SEPARATOR}{self.noder.NAME_TOP}' + pre = f'{SEPARATOR}{cnode.NAME_TOP}' if not path.startswith(pre): path = pre + path if not path.endswith(SEPARATOR): @@ -88,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore curt = time() mode: Any = S_IFREG - if entry.type == Noder.TYPE_ARC: + if entry.type == cnode.TYPE_ARC: mode = S_IFREG - elif entry.type == Noder.TYPE_DIR: + elif entry.type == cnode.TYPE_DIR: mode = S_IFDIR - elif entry.type == Noder.TYPE_FILE: + elif entry.type == cnode.TYPE_FILE: mode = S_IFREG - elif entry.type == Noder.TYPE_STORAGE: + elif entry.type == cnode.TYPE_STORAGE: mode = S_IFDIR - elif entry.type == Noder.TYPE_META: + elif entry.type == cnode.TYPE_META: mode = S_IFREG - elif entry.type == Noder.TYPE_TOP: + elif entry.type == cnode.TYPE_TOP: mode = S_IFREG return { 'st_mode': (mode), diff --git a/catcli/noder.py b/catcli/noder.py index 8624e8b..5acf7f6 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -8,11 +8,13 @@ Class that process nodes in the catalog tree import os import shutil import time -from typing import List, Union, Tuple, Any, Optional, Dict +from typing import List, Union, Tuple, Any, Optional, Dict, cast import anytree # type: ignore from pyfzf.pyfzf import FzfPrompt # type: ignore # local imports +from catcli import cnode +from catcli.cnode import Node from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars from catcli.logger import Logger from catcli.nodeprinter import NodePrinter @@ -31,16 +33,6 @@ class Noder: * "file" node representing a file """ - NAME_TOP = 'top' - NAME_META = 'meta' - - TYPE_TOP = 'top' - TYPE_FILE = 'file' - TYPE_DIR = 'dir' - TYPE_ARC = 'arc' - TYPE_STORAGE = 'storage' - TYPE_META = 'meta' - CSV_HEADER = ('name,type,path,size,indexed_at,' 'maccess,md5,nbfiles,free_space,' 'total_space,meta') @@ -61,46 +53,49 @@ class Noder: self.decomp = Decomp() @staticmethod - def get_storage_names(top: anytree.AnyNode) -> List[str]: + def get_storage_names(top: Node) -> List[str]: """return a list of all storage names""" return [x.name for x in list(top.children)] - def get_storage_node(self, top: anytree.AnyNode, - name: str, path: str = '') -> anytree.AnyNode: + def get_storage_node(self, top: Node, + name: str, + newpath: str = '') -> Node: """ return the storage node if any if path is submitted, it will update the media info """ found = None for node in top.children: - if node.type != self.TYPE_STORAGE: + if node.type != cnode.TYPE_STORAGE: continue if node.name == name: found = node break - if found and path and os.path.exists(path): - found.free = shutil.disk_usage(path).free - found.total = shutil.disk_usage(path).total + if found and newpath and os.path.exists(newpath): + found.free = shutil.disk_usage(newpath).free + found.total = shutil.disk_usage(newpath).total found.ts = int(time.time()) - return found + return cast(Node, found) @staticmethod - def get_node(top: str, path: str, - quiet: bool = False) -> anytree.AnyNode: + def get_node(top: Node, + path: str, + quiet: bool = False) -> Optional[Node]: """get the node by internal tree path""" resolv = anytree.resolver.Resolver('name') try: bpath = os.path.basename(path) - return resolv.get(top, bpath) + the_node = resolv.get(top, bpath) + return cast(Node, the_node) except anytree.resolver.ChildResolverError: if not quiet: Logger.err(f'No node at path \"{bpath}\"') return None def get_node_if_changed(self, - top: anytree.AnyNode, + top: Node, path: str, - treepath: str) -> Tuple[anytree.AnyNode, bool]: + treepath: str) -> Tuple[Optional[Node], bool]: """ return the node (if any) and if it has changed @top: top node (storage) @@ -136,25 +131,25 @@ class Noder: self._debug(f'\tchange: no change for \"{path}\"') return node, False - def rec_size(self, node: anytree.AnyNode, + def rec_size(self, node: Node, store: bool = True) -> float: """ recursively traverse tree and return size @store: store the size in the node """ - if node.type == self.TYPE_FILE: + if node.type == cnode.TYPE_FILE: self._debug(f'getting node size for \"{node.name}\"') return float(node.size) msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) size: float = 0 for i in node.children: - if node.type == self.TYPE_DIR: + if node.type == cnode.TYPE_DIR: size = self.rec_size(i, store=store) if store: i.size = size size += size - if node.type == self.TYPE_STORAGE: + if node.type == cnode.TYPE_STORAGE: size = self.rec_size(i, store=store) if store: i.size = size @@ -169,28 +164,34 @@ class Noder: # public helpers ############################################################### @staticmethod - def format_storage_attr(attr: Union[str, List[str]]) -> str: + def attrs_to_string(attr: Union[List[str], Dict[str, str], str]) -> str: """format the storage attr for saving""" if not attr: return '' if isinstance(attr, list): return ', '.join(attr) + if isinstance(attr, dict): + ret = [] + for key, val in attr.items(): + ret.append(f'{key}={val}') + return ', '.join(ret) attr = attr.rstrip() return attr - def set_hashing(self, val: bool) -> None: + def do_hashing(self, val: bool) -> None: """hash files when indexing""" self.hash = val ############################################################### # node creation ############################################################### - def new_top_node(self) -> anytree.AnyNode: + def new_top_node(self) -> Node: """create a new top node""" - return anytree.AnyNode(name=self.NAME_TOP, type=self.TYPE_TOP) + return Node(cnode.NAME_TOP, + cnode.TYPE_TOP) def new_file_node(self, name: str, path: str, - parent: str, storagepath: str) -> anytree.AnyNode: + parent: Node, storagepath: str) -> Optional[Node]: """create a new node representing a file""" if not os.path.exists(path): Logger.err(f'File \"{path}\" does not exist') @@ -207,7 +208,7 @@ class Noder: relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - node = self._new_generic_node(name, self.TYPE_FILE, + node = self._new_generic_node(name, cnode.TYPE_FILE, relpath, parent, size=stat.st_size, md5=md5, @@ -223,98 +224,107 @@ class Noder: return node def new_dir_node(self, name: str, path: str, - parent: str, storagepath: str) -> anytree.AnyNode: + parent: Node, storagepath: str) -> Node: """create a new node representing a directory""" path = os.path.abspath(path) relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - return self._new_generic_node(name, self.TYPE_DIR, relpath, + return self._new_generic_node(name, cnode.TYPE_DIR, relpath, parent, maccess=maccess) def new_storage_node(self, name: str, path: str, parent: str, - attr: Optional[str] = None) -> anytree.AnyNode: + attrs: str = '') -> Node: """create a new node representing a storage""" path = os.path.abspath(path) free = shutil.disk_usage(path).free total = shutil.disk_usage(path).total epoch = int(time.time()) - return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free, - total=total, parent=parent, attr=attr, ts=epoch) + return Node(name=name, + type=cnode.TYPE_STORAGE, + free=free, + total=total, + parent=parent, + attr=attrs, + indexed_dt=epoch) def new_archive_node(self, name: str, path: str, - parent: str, archive: str) -> anytree.AnyNode: + parent: str, archive: str) -> Node: """create a new node for archive data""" - return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, - parent=parent, size=0, md5=None, - archive=archive) + return Node(name=name, type=cnode.TYPE_ARC, relpath=path, + parent=parent, size=0, md5='', + archive=archive) @staticmethod - def _new_generic_node(name: str, nodetype: str, - relpath: str, parent: str, + def _new_generic_node(name: str, + nodetype: str, + relpath: str, + parent: Node, size: float = 0, md5: str = '', - maccess: float = 0) -> anytree.AnyNode: + maccess: float = 0) -> Node: """generic node creation""" - return anytree.AnyNode(name=name, type=nodetype, relpath=relpath, - parent=parent, size=size, - md5=md5, maccess=maccess) + return Node(name, + nodetype, + size=size, + relpath=relpath, + md5=md5, + maccess=maccess, + parent=parent) ############################################################### # node management ############################################################### - def update_metanode(self, top: anytree.AnyNode) -> anytree.AnyNode: + def update_metanode(self, top: Node) -> Node: """create or update meta node information""" meta = self._get_meta_node(top) epoch = int(time.time()) if not meta: - attr: Dict[str, Any] = {} - attr['created'] = epoch - attr['created_version'] = VERSION - meta = anytree.AnyNode(name=self.NAME_META, - type=self.TYPE_META, - attr=attr) - meta.attr['access'] = epoch - meta.attr['access_version'] = VERSION + attrs: Dict[str, Any] = {} + attrs['created'] = epoch + attrs['created_version'] = VERSION + meta = Node(name=cnode.NAME_META, + type=cnode.TYPE_META, + attr=self.attrs_to_string(attrs)) + if meta.attr: + meta.attr += ', ' + meta.attr += f'access={epoch}' + meta.attr += ', ' + meta.attr += f'access_version={VERSION}' return meta - def _get_meta_node(self, top: anytree.AnyNode) -> anytree.AnyNode: + def _get_meta_node(self, top: Node) -> Optional[Node]: """return the meta node if any""" try: - return next(filter(lambda x: x.type == self.TYPE_META, - top.children)) + found = next(filter(lambda x: x.type == cnode.TYPE_META, + top.children)) + return cast(Node, found) except StopIteration: return None - def clean_not_flagged(self, top: anytree.AnyNode) -> int: + def clean_not_flagged(self, top: Node) -> int: """remove any node not flagged and clean flags""" cnt = 0 for node in anytree.PreOrderIter(top): - if node.type not in [self.TYPE_FILE, self.TYPE_DIR]: + if node.type not in [cnode.TYPE_FILE, cnode.TYPE_DIR]: continue if self._clean(node): cnt += 1 return cnt - @staticmethod - def flag(node: anytree.AnyNode) -> None: - """flag a node""" - node.flag = True - - def _clean(self, node: anytree.AnyNode) -> bool: + def _clean(self, node: Node) -> bool: """remove node if not flagged""" - if not self._has_attr(node, 'flag') or \ - not node.flag: + if not node.flagged(): node.parent = None return True - del node.flag + node.unflag() return False ############################################################### # printing ############################################################### - def _node_to_csv(self, node: anytree.AnyNode, + def _node_to_csv(self, node: Node, sep: str = ',', raw: bool = False) -> None: """ @@ -323,13 +333,13 @@ class Noder: @sep: CSV separator character @raw: print raw size rather than human readable """ - if not node: + if not cnode: return - if node.type == self.TYPE_TOP: + if node.type == node.TYPE_TOP: return out = [] - if node.type == self.TYPE_STORAGE: + if node.type == node.TYPE_STORAGE: # handle storage out.append(node.name) # name out.append(node.type) # type @@ -364,7 +374,7 @@ class Noder: out.append(node.md5) # md5 else: out.append('') # fake md5 - if node.type == self.TYPE_DIR: + if node.type == cnode.TYPE_DIR: out.append(str(len(node.children))) # nbfiles else: out.append('') # fake nbfiles @@ -376,7 +386,7 @@ class Noder: if len(line) > 0: Logger.stdout_nocolor(line) - def _print_node_native(self, node: anytree.AnyNode, + def _print_node_native(self, node: Node, pre: str = '', withpath: bool = False, withdepth: bool = False, @@ -393,10 +403,10 @@ class Noder: @recalcparent: get relpath from tree instead of relpath field @raw: print raw size rather than human readable """ - if node.type == self.TYPE_TOP: + if node.type == cnode.TYPE_TOP: # top node Logger.stdout_nocolor(f'{pre}{node.name}') - elif node.type == self.TYPE_FILE: + elif node.type == cnode.TYPE_FILE: # node of type file name = node.name if withpath: @@ -416,7 +426,7 @@ class Noder: content = Logger.get_bold_text(storage.name) compl += f', storage:{content}' NodePrinter.print_file_native(pre, name, compl) - elif node.type == self.TYPE_DIR: + elif node.type == cnode.TYPE_DIR: # node of type directory name = node.name if withpath: @@ -436,7 +446,7 @@ class Noder: if withstorage: attr.append(('storage', Logger.get_bold_text(storage.name))) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) - elif node.type == self.TYPE_STORAGE: + elif node.type == cnode.TYPE_STORAGE: # node of type storage sztotal = size_to_str(node.total, raw=raw) szused = size_to_str(node.total - node.free, raw=raw) @@ -466,14 +476,14 @@ class Noder: name, argsstring, node.attr) - elif node.type == self.TYPE_ARC: + elif node.type == cnode.TYPE_ARC: # archive node if self.arc: NodePrinter.print_archive_native(pre, node.name, node.archive) else: Logger.err(f'bad node encountered: {node}') - def print_tree(self, node: anytree.AnyNode, + def print_tree(self, node: Node, fmt: str = 'native', raw: bool = False) -> None: """ @@ -497,7 +507,7 @@ class Noder: Logger.stdout_nocolor(self.CSV_HEADER) self._to_csv(node, raw=raw) - def _to_csv(self, node: anytree.AnyNode, + def _to_csv(self, node: Node, raw: bool = False) -> None: """print the tree to csv""" rend = anytree.RenderTree(node, childiter=self._sort_tree) @@ -511,7 +521,7 @@ class Noder: selected = fzf.prompt(strings) return selected - def _to_fzf(self, node: anytree.AnyNode, fmt: str) -> None: + def _to_fzf(self, node: Node, fmt: str) -> None: """ fzf prompt with list and print selected node(s) @node: node to start with @@ -540,7 +550,7 @@ class Noder: self.print_tree(rend, fmt=subfmt) @staticmethod - def to_dot(node: anytree.AnyNode, + def to_dot(node: Node, path: str = 'tree.dot') -> str: """export to dot for graphing""" anytree.exporter.DotExporter(node).to_dotfile(path) @@ -550,14 +560,14 @@ class Noder: ############################################################### # searching ############################################################### - def find_name(self, top: anytree.AnyNode, + def find_name(self, top: Node, key: str, script: bool = False, only_dir: bool = False, - startnode: anytree.AnyNode = None, + startnode: Optional[Node] = None, parentfromtree: bool = False, fmt: str = 'native', - raw: bool = False) -> List[anytree.AnyNode]: + raw: bool = False) -> List[Node]: """ find files based on their names @top: top node @@ -573,7 +583,7 @@ class Noder: self._debug(f'searching for \"{key}\"') # search for nodes based on path - start = top + start: Optional[Node] = top if startnode: start = self.get_node(top, startnode) filterfunc = self._callback_find_name(key, only_dir) @@ -626,17 +636,17 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" - def find_name(node: anytree.AnyNode) -> bool: - if node.type == self.TYPE_STORAGE: + def find_name(node: Node) -> bool: + if node.type == cnode.TYPE_STORAGE: # ignore storage nodes return False - if node.type == self.TYPE_TOP: + if node.type == cnode.TYPE_TOP: # ignore top nodes return False - if node.type == self.TYPE_META: + if node.type == cnode.TYPE_META: # ignore meta nodes return False - if only_dir and node.type != self.TYPE_DIR: + if only_dir and node.type != cnode.TYPE_DIR: # ignore non directory return False @@ -653,11 +663,11 @@ class Noder: ############################################################### # ls ############################################################### - def list(self, top: anytree.AnyNode, + def list(self, top: Node, path: str, rec: bool = False, fmt: str = 'native', - raw: bool = False) -> List[anytree.AnyNode]: + raw: bool = False) -> List[Node]: """ list nodes for "ls" @top: top node @@ -719,7 +729,7 @@ class Noder: # tree creation ############################################################### def _add_entry(self, name: str, - top: anytree.AnyNode, + top: Node, resolv: Any) -> None: """add an entry to the tree""" entries = name.rstrip(os.sep).split(os.sep) @@ -734,7 +744,7 @@ class Noder: except anytree.resolver.ChildResolverError: self.new_archive_node(nodename, name, top, top.name) - def list_to_tree(self, parent: anytree.AnyNode, names: List[str]) -> None: + def list_to_tree(self, parent: Node, names: List[str]) -> None: """convert list of files to a tree""" if not names: return @@ -747,23 +757,23 @@ class Noder: # diverse ############################################################### def _sort_tree(self, - items: List[anytree.AnyNode]) -> List[anytree.AnyNode]: + items: List[Node]) -> List[Node]: """sorting a list of items""" return sorted(items, key=self._sort, reverse=self.sortsize) - def _sort(self, lst: List[anytree.AnyNode]) -> Any: + def _sort(self, lst: Node) -> Any: """sort a list""" if self.sortsize: return self._sort_size(lst) return self._sort_fs(lst) @staticmethod - def _sort_fs(node: anytree.AnyNode) -> Tuple[str, str]: + def _sort_fs(node: Node) -> Tuple[str, str]: """sorting nodes dir first and alpha""" return (node.type, node.name.lstrip('.').lower()) @staticmethod - def _sort_size(node: anytree.AnyNode) -> float: + def _sort_size(node: Node) -> float: """sorting nodes by size""" try: if not node.size: @@ -772,22 +782,22 @@ class Noder: except AttributeError: return 0 - def _get_storage(self, node: anytree.AnyNode) -> anytree.AnyNode: + def _get_storage(self, node: Node) -> Node: """recursively traverse up to find storage""" - if node.type == self.TYPE_STORAGE: + if node.type == cnode.TYPE_STORAGE: return node - return node.ancestors[1] + return cast(Node, node.ancestors[1]) @staticmethod - def _has_attr(node: anytree.AnyNode, attr: str) -> bool: + def _has_attr(node: Node, attr: str) -> bool: """return True if node has attr as attribute""" return attr in node.__dict__.keys() - def _get_parents(self, node: anytree.AnyNode) -> str: + def _get_parents(self, node: Node) -> str: """get all parents recursively""" - if node.type == self.TYPE_STORAGE: + if node.type == cnode.TYPE_STORAGE: return '' - if node.type == self.TYPE_TOP: + if node.type == cnode.TYPE_TOP: return '' parent = self._get_parents(node.parent) if parent: @@ -804,7 +814,7 @@ class Noder: return '' @staticmethod - def _sanitize(node: anytree.AnyNode) -> anytree.AnyNode: + def _sanitize(node: Node) -> Node: """sanitize node strings""" node.name = fix_badchars(node.name) node.relpath = fix_badchars(node.relpath) diff --git a/catcli/utils.py b/catcli/utils.py index 38a257e..dd058e8 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -57,12 +57,12 @@ def size_to_str(size: float, return f'{size:.1f}{sufix}' -def epoch_to_str(epoch: int) -> str: +def epoch_to_str(epoch: float) -> str: """convert epoch to string""" if not epoch: return '' fmt = '%Y-%m-%d %H:%M:%S' - timestamp = datetime.datetime.fromtimestamp(float(epoch)) + timestamp = datetime.datetime.fromtimestamp(epoch) return timestamp.strftime(fmt) diff --git a/catcli/walker.py b/catcli/walker.py index 4d9f79d..e263fce 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -6,10 +6,10 @@ Catcli filesystem indexer """ import os -from typing import Tuple -import anytree # type: ignore +from typing import Tuple, Optional # local imports +from catcli.cnode import Node from catcli.noder import Noder from catcli.logger import Logger @@ -31,12 +31,12 @@ class Walker: """ self.noder = noder self.usehash = usehash - self.noder.set_hashing(self.usehash) + self.noder.do_hashing(self.usehash) self.debug = debug self.lpath = logpath def index(self, path: str, - parent: str, + parent: Node, name: str, storagepath: str = '') -> Tuple[str, int]: """ @@ -89,15 +89,15 @@ class Walker: self._progress('') return parent, cnt - def reindex(self, path: str, parent: str, top: str) -> int: + def reindex(self, path: str, parent: Node, top: Node) -> int: """reindex a directory and store in tree""" cnt = self._reindex(path, parent, top) cnt += self.noder.clean_not_flagged(parent) return cnt def _reindex(self, path: str, - parent: str, - top: anytree.AnyNode, + parent: Node, + top: Node, storagepath: str = '') -> int: """ reindex a directory and store in tree @@ -115,13 +115,15 @@ class Walker: reindex, node = self._need_reindex(parent, sub, treepath) if not reindex: self._debug(f'\tskip file {sub}') - self.noder.flag(node) + if node: + node.flag() continue self._log2file(f'update catalog for \"{sub}\"') node = self.noder.new_file_node(os.path.basename(file), sub, parent, storagepath) - self.noder.flag(node) - cnt += 1 + if node: + node.flag() + cnt += 1 for adir in dirs: self._debug(f'found dir \"{adir}\" under {path}') base = os.path.basename(adir) @@ -133,40 +135,42 @@ class Walker: dummy = self.noder.new_dir_node(base, sub, parent, storagepath) cnt += 1 - self.noder.flag(dummy) + if dummy: + dummy.flag() self._debug(f'reindexing deeper under {sub}') nstoragepath = os.sep.join([storagepath, base]) if not storagepath: nstoragepath = base - cnt2 = self._reindex(sub, dummy, top, nstoragepath) - cnt += cnt2 + if dummy: + cnt2 = self._reindex(sub, dummy, top, nstoragepath) + cnt += cnt2 break return cnt def _need_reindex(self, - top: anytree.AnyNode, + top: Node, path: str, - treepath: str) -> Tuple[bool, anytree.AnyNode]: + treepath: str) -> Tuple[bool, Optional[Node]]: """ test if node needs re-indexing @top: top node (storage) @path: abs path to file @treepath: rel path from indexed directory """ - cnode, changed = self.noder.get_node_if_changed(top, path, treepath) - if not cnode: + node, changed = self.noder.get_node_if_changed(top, path, treepath) + if not node: self._debug(f'\t{path} does not exist') - return True, cnode - if cnode and not changed: + return True, node + if node and not changed: # ignore this node self._debug(f'\t{path} has not changed') - return False, cnode - if cnode and changed: + return False, node + if node and changed: # remove this node and re-add self._debug(f'\t{path} has changed') - self._debug(f'\tremoving node {cnode.name} for {path}') - cnode.parent = None - return True, cnode + self._debug(f'\tremoving node {node.name} for {path}') + node.parent = None + return True, node def _debug(self, string: str) -> None: """print to debug""" diff --git a/tests/test_update.py b/tests/test_update.py index fc478e7..1107b45 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -46,7 +46,7 @@ class TestUpdate(unittest.TestCase): d2f2 = create_rnd_file(dir2, 'dir2file2') noder = Noder(debug=True) - noder.set_hashing(True) + noder.do_hashing(True) top = noder.new_top_node() catalog = Catalog(catalogpath, force=True, debug=False) From 54eb365ee451fbbeb3b20983dc2490c1a6ce017f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 08:54:22 +0100 Subject: [PATCH 06/36] linting --- catcli/noder.py | 2 +- tests.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/catcli/noder.py b/catcli/noder.py index 5acf7f6..b420c35 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -62,7 +62,7 @@ class Noder: newpath: str = '') -> Node: """ return the storage node if any - if path is submitted, it will update the media info + if newpath is submitted, it will update the media info """ found = None for node in top.children: diff --git a/tests.sh b/tests.sh index 4e81638..2c5e08a 100755 --- a/tests.sh +++ b/tests.sh @@ -23,6 +23,7 @@ pyflakes tests/ # R0903: Too few public methods # R0801: Similar lines in 2 files # R0902: Too many instance attributes +# R0201: no-self-used pylint --version pylint \ --disable=R0914 \ @@ -33,6 +34,8 @@ pylint \ --disable=R0903 \ --disable=R0801 \ --disable=R0902 \ + --disable=R0201 \ + --disable=R0022 \ catcli/ pylint \ --disable=W0212 \ From bb0835cd46b40bae9d924272509d0ed4a47c8e4f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 19:48:59 +0100 Subject: [PATCH 07/36] update test --- tests-ng/update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-ng/update.sh b/tests-ng/update.sh index e70041a..15fe9e3 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -63,7 +63,7 @@ echo "after: free:${freea} | du:${dua} | date:${datea}" # test they are all different [ "${freeb}" = "${freea}" ] && echo "WARNING free didn't change!" [ "${dub}" = "${dua}" ] && echo "WARNING du didn't change!" -[ "${dateb}" = "${datea}" ] && echo "date didn't change!" && exit 1 +[ "${dateb}" = "${datea}" ] && echo "WARNING date didn't change!" && exit 1 # pivot back cd ${cwd} From bdf05e9322c8d1a1097d564866ac8811a0a4e819 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 22:37:09 +0100 Subject: [PATCH 08/36] linting --- setup.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index e3e5cd9..bc7e905 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,31 @@ -from setuptools import setup, find_packages -from codecs import open +"""setup.py""" from os import path +from setuptools import setup, find_packages import catcli -readme = 'README.md' +README = 'README.md' here = path.abspath(path.dirname(__file__)) -read_readme = lambda f: open(f, 'r').read() - -VERSION = catcli.__version__ +VERSION = catcli.version.__version__ REQUIRES_PYTHON = '>=3' + +def read_readme(readme_path): + """read readme content""" + with open(readme_path, encoding="utf-8") as file: + return file.read() + + +URL = f'https://github.com/deadc0de6/catcli/archive/v{VERSION}.tar.gz' setup( name='catcli', version=VERSION, description='The command line catalog tool for your offline data', - long_description=read_readme(readme), + long_description=read_readme(README), long_description_content_type='text/markdown', - license_files = ('LICENSE',), + license_files=('LICENSE',), url='https://github.com/deadc0de6/catcli', - download_url = 'https://github.com/deadc0de6/catcli/archive/v'+VERSION+'.tar.gz', + download_url=URL, options={"bdist_wheel": {"python_tag": "py3"}}, # include anything from MANIFEST.in include_package_data=True, From 599852eb7a161c1a9f13d4a782eed40051c085b8 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 22:37:17 +0100 Subject: [PATCH 09/36] ignore coverage --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 01fb98a..2d768e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .coverage +.coverage* dist/ build/ *.egg-info/ From 45c2599a95dd5ec373d4d42fc8c8761e1b5931ce Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 22:37:29 +0100 Subject: [PATCH 10/36] tests --- tests-ng/assets/github.catalog.json | 61 +++++++++++++++++++++ tests-ng/compare.sh | 61 +++++++++++++++++++++ tests-ng/helper | 68 +++++++++++++++++++++++ tests-ng/update.sh | 85 ++++++++++++++++++----------- tests.sh | 46 +++++++++++++--- 5 files changed, 282 insertions(+), 39 deletions(-) create mode 100644 tests-ng/assets/github.catalog.json create mode 100755 tests-ng/compare.sh create mode 100644 tests-ng/helper diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json new file mode 100644 index 0000000..6c5fbe7 --- /dev/null +++ b/tests-ng/assets/github.catalog.json @@ -0,0 +1,61 @@ +{ + "children": [ + { + "attr": "", + "children": [ + { + "maccess": 1666206037.0786593, + "md5": null, + "name": "FUNDING.yml", + "relpath": "/FUNDING.yml", + "size": 17, + "type": "file" + }, + { + "children": [ + { + "maccess": 1666206037.078865, + "md5": null, + "name": "pypi-release.yml", + "relpath": "workflows/pypi-release.yml", + "size": 691, + "type": "file" + }, + { + "maccess": 1677748530.6920426, + "md5": null, + "name": "testing.yml", + "relpath": "workflows/testing.yml", + "size": 595, + "type": "file" + } + ], + "maccess": 1677748530.691944, + "md5": null, + "name": "workflows", + "relpath": "/workflows", + "size": 1190, + "type": "dir" + } + ], + "free": 23459602432, + "name": "github", + "size": 2380, + "total": 245107195904, + "ts": 1678214993, + "type": "storage" + }, + { + "attr": { + "access": 1678214993, + "access_version": "0.8.7", + "created": 1678214993, + "created_version": "0.8.7" + }, + "name": "meta", + "type": "meta" + } + ], + "name": "top", + "type": "top" +} \ No newline at end of file diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh new file mode 100755 index 0000000..2c6289d --- /dev/null +++ b/tests-ng/compare.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2023, deadc0de6 + +# exit on first error +set -e + +# get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +# pivot +prev="${cur}/.." +cd "${prev}" + +# coverage +#export PYTHONPATH=".:${PYTHONPATH}" +bin="python3 -m catcli.catcli" +if hash coverage 2>/dev/null; then + bin="coverage run -p --source=catcli -m catcli.catcli" + #bin="coverage run -p --source=${prev}/catcli -m catcli.catcli" +fi + +echo "current dir: $(pwd)" +echo "pythonpath: ${PYTHONPATH}" +echo "bin: ${bin}" +${bin} --version + +# get the helpers +# shellcheck source=tests-ng/helper +source "${cur}"/helper +echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" + +########################################################## +# the test +########################################################## + +# create temp dirs +tmpd=$(mktemp -d) +clear_on_exit "${tmpd}" + +catalog="${tmpd}/catalog" + +# index +${bin} -B index --catalog="${catalog}" github .github + +# diff +cat "${catalog}" +diff "${cur}/assets/github.catalog.json" "${catalog}" + +# the end +echo "test \"$(basename "$0")\" success" +cd "${cur}" +exit 0 diff --git a/tests-ng/helper b/tests-ng/helper new file mode 100644 index 0000000..d7373e8 --- /dev/null +++ b/tests-ng/helper @@ -0,0 +1,68 @@ +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2023, deadc0de6 +# +# file to be sourced from test scripts +# + +declare -a to_be_cleared + +# add a file/directory to be cleared +# on exit +# +# $1: file path to clear +clear_on_exit() +{ + local len="${#to_be_cleared[*]}" + # shellcheck disable=SC2004 + to_be_cleared[${len}]="$1" + if [ "${len}" = "0" ]; then + # set trap + trap on_exit EXIT + fi +} + +# clear files +on_exit() +{ + for i in "${to_be_cleared[@]}"; do + rm -rf "${i}" + done +} + +# osx tricks +# brew install coreutils gnu-sed +if [[ $OSTYPE == 'darwin'* ]]; then + mktemp() { + gmktemp "$@" + } + stat() { + gstat "$@" + } + sed() { + gsed "$@" + } + wc() { + gwc "$@" + } + date() { + gdate "$@" + } + chmod() { + gchmod "$@" + } + readlink() { + greadlink "$@" + } + realpath() { + grealpath "$@" + } + + export -f mktemp + export -f stat + export -f sed + export -f wc + export -f date + export -f chmod + export -f readlink + export -f realpath +fi \ No newline at end of file diff --git a/tests-ng/update.sh b/tests-ng/update.sh index 15fe9e3..dcab7a6 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -2,62 +2,85 @@ # author: deadc0de6 (https://github.com/deadc0de6) # Copyright (c) 2021, deadc0de6 -cur=$(dirname "$(readlink -f "${0}")") -cwd=`pwd` +# exit on first error +set -e + +# get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! hash ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") # pivot -cd ${cur}/../ -python3 -m catcli.catcli --version +prev="${cur}/.." +cd "${prev}" + +# coverage +#export PYTHONPATH=".:${PYTHONPATH}" +bin="python3 -m catcli.catcli" +if hash coverage 2>/dev/null; then + bin="coverage run -p --source=catcli -m catcli.catcli" + #bin="coverage run -p --source=${prev}/catcli -m catcli.catcli" +fi + +echo "current dir: $(pwd)" +echo "pythonpath: ${PYTHONPATH}" +echo "bin: ${bin}" +${bin} --version + +# get the helpers +# shellcheck source=tests-ng/helper +source "${cur}"/helper +echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" ########################################################## # the test ########################################################## # create temp dirs -tmpd=`mktemp -d` +tmpd=$(mktemp -d) +clear_on_exit "${tmpd}" tmpu="${tmpd}/dir2" -mkdir -p ${tmpu} - -# setup cleaning -clean() { - # clean - rm -rf ${tmpd} ${tmpu} -} -trap clean EXIT +mkdir -p "${tmpu}" catalog="${tmpd}/catalog" -mkdir -p ${tmpd}/dir -echo "abc" > ${tmpd}/dir/a +mkdir -p "${tmpd}/dir" +echo "abc" > "${tmpd}/dir/a" # index -python3 -m catcli.catcli -B index --catalog=${catalog} dir ${tmpd}/dir -python3 -m catcli.catcli -B ls --catalog=${catalog} dir +${bin} -B index --catalog="${catalog}" dir "${tmpd}/dir" +${bin} -B ls --catalog="${catalog}" dir # get attributes -freeb=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g'` -dub=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g'` -dateb=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g'` +freeb=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') +dub=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') +dateb=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') echo "before: free:${freeb} | du:${dub} | date:${dateb}" # change content -echo "abc" >> ${tmpd}/dir/a -echo "abc" > ${tmpd}/dir/b +echo "abc" >> "${tmpd}/dir/a" +echo "abc" > "${tmpd}/dir/b" # move dir -cp -r ${tmpd}/dir ${tmpu}/ +cp -r "${tmpd}/dir" "${tmpu}/" # sleep to force date change sleep 1 # update -python3 -m catcli.catcli -B update -f --catalog=${catalog} dir ${tmpu}/dir -python3 -m catcli.catcli -B ls --catalog=${catalog} dir +${bin} -B update -f --catalog="${catalog}" dir "${tmpu}/dir" +${bin} -B ls --catalog="${catalog}" dir # get new attributes -freea=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g'` -dua=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g'` -datea=`python3 -m catcli.catcli -B ls --catalog=${catalog} dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g'` +freea=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g') +dua=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g') +datea=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g') echo "after: free:${freea} | du:${dua} | date:${datea}" # test they are all different @@ -65,9 +88,7 @@ echo "after: free:${freea} | du:${dua} | date:${datea}" [ "${dub}" = "${dua}" ] && echo "WARNING du didn't change!" [ "${dateb}" = "${datea}" ] && echo "WARNING date didn't change!" && exit 1 -# pivot back -cd ${cwd} - # the end -echo "test \"`basename $0`\" success" +echo "test \"$(basename "$0")\" success" +cd "${cur}" exit 0 diff --git a/tests.sh b/tests.sh index 2c5e08a..6719eec 100755 --- a/tests.sh +++ b/tests.sh @@ -5,16 +5,24 @@ cur=$(dirname "$(readlink -f "${0}")") # stop on first error -set -ev +set -e +#set -v +# pycodestyle +echo "[+] pycodestyle" pycodestyle --version pycodestyle --ignore=W605 catcli/ pycodestyle tests/ +pycodestyle setup.py +# pyflakes +echo "[+] pyflakes" pyflakes --version pyflakes catcli/ pyflakes tests/ +pyflakes setup.py +# pylint # R0914: Too many local variables # R0913: Too many arguments # R0912: Too many branches @@ -24,8 +32,9 @@ pyflakes tests/ # R0801: Similar lines in 2 files # R0902: Too many instance attributes # R0201: no-self-used +echo "[+] pylint" pylint --version -pylint \ +pylint -sn \ --disable=R0914 \ --disable=R0913 \ --disable=R0912 \ @@ -37,23 +46,46 @@ pylint \ --disable=R0201 \ --disable=R0022 \ catcli/ -pylint \ +pylint -sn \ --disable=W0212 \ --disable=R0914 \ --disable=R0915 \ --disable=R0801 \ tests/ +pylint -sn setup.py +# mypy +echo "[+] mypy" mypy \ --strict \ catcli/ -nosebin="nose2" -PYTHONPATH=catcli ${nosebin} --with-coverage --coverage=catcli +# unittest +echo "[+] unittests" +coverage run -p -m pytest tests -for t in ${cur}/tests-ng/*; do - echo "running test \"`basename ${t}`\"" +# tests-ng +echo "[+] tests-ng" +for t in "${cur}"/tests-ng/*.sh; do + echo "running test \"$(basename "${t}")\"" ${t} done +# check shells +echo "[+] shellcheck" +if ! which shellcheck >/dev/null 2>&1; then + echo "Install shellcheck" + exit 1 +fi +shellcheck --version +find . -iname '*.sh' | while read -r script; do + shellcheck -x \ + "${script}" +done + +# merge coverage +echo "[+] coverage merge" +coverage combine + +echo "ALL TESTS DONE OK" exit 0 From 45d3b096f3b1ad4b9188772a9528ae7032292ae6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Tue, 7 Mar 2023 22:37:49 +0100 Subject: [PATCH 11/36] refactor node types --- catcli/catalog.py | 20 ++-- catcli/catcli.py | 24 ++--- catcli/cnode.py | 176 ++++++++++++++++++++++++++++------- catcli/fuser.py | 22 ++--- catcli/nodeprinter.py | 5 +- catcli/noder.py | 212 +++++++++++++++++++----------------------- catcli/walker.py | 14 +-- 7 files changed, 284 insertions(+), 189 deletions(-) diff --git a/catcli/catalog.py b/catcli/catalog.py index 821327c..2cdf8ed 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -12,7 +12,7 @@ from anytree.exporter import JsonExporter # type: ignore from anytree.importer import JsonImporter # type: ignore # local imports -from catcli.cnode import Node +from catcli.cnode import NodeMeta, NodeTop from catcli.utils import ask from catcli.logger import Logger @@ -33,10 +33,10 @@ class Catalog: self.path = path self.debug = debug self.force = force - self.metanode: Optional[Node] = None + self.metanode: Optional[NodeMeta] = None self.pickle = usepickle - def set_metanode(self, metanode: Node) -> None: + def set_metanode(self, metanode: NodeMeta) -> None: """remove the metanode until tree is re-written""" self.metanode = metanode if self.metanode: @@ -50,7 +50,7 @@ class Catalog: return True return False - def restore(self) -> Optional[Node]: + def restore(self) -> Optional[NodeTop]: """restore the catalog""" if not self.path: return None @@ -62,7 +62,7 @@ class Catalog: content = file.read() return self._restore_json(content) - def save(self, node: Node) -> bool: + def save(self, node: NodeTop) -> bool: """save the catalog""" if not self.path: Logger.err('Path not defined') @@ -88,14 +88,14 @@ class Catalog: return Logger.debug(text) - def _save_pickle(self, node: Node) -> bool: + def _save_pickle(self, node: NodeTop) -> bool: """pickle the catalog""" with open(self.path, 'wb') as file: pickle.dump(node, file) self._debug(f'Catalog saved to pickle \"{self.path}\"') return True - def _restore_pickle(self) -> Union[Node, Any]: + def _restore_pickle(self) -> Union[NodeTop, Any]: """restore the pickled tree""" with open(self.path, 'rb') as file: root = pickle.load(file) @@ -103,7 +103,7 @@ class Catalog: self._debug(msg) return root - def _save_json(self, node: Node) -> bool: + def _save_json(self, node: NodeTop) -> bool: """export the catalog in json""" exp = JsonExporter(indent=2, sort_keys=True) with open(self.path, 'w', encoding='UTF-8') as file: @@ -111,9 +111,9 @@ class Catalog: self._debug(f'Catalog saved to json \"{self.path}\"') return True - def _restore_json(self, string: str) -> Node: + def _restore_json(self, string: str) -> NodeTop: """restore the tree from json""" imp = JsonImporter() root = imp.import_(string) self._debug(f'Catalog imported from json \"{self.path}\"') - return cast(Node, root) + return cast(NodeTop, root) diff --git a/catcli/catcli.py b/catcli/catcli.py index efcbfb8..c8ffea9 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -17,11 +17,11 @@ from docopt import docopt # local imports from catcli import cnode from catcli.version import __version__ as VERSION +from catcli.cnode import NodeTop, NodeAny from catcli.logger import Logger from catcli.colors import Colors from catcli.catalog import Catalog from catcli.walker import Walker -from catcli.cnode import Node from catcli.noder import Noder from catcli.utils import ask, edit from catcli.fuser import Fuser @@ -82,7 +82,7 @@ Options: def cmd_mount(args: Dict[str, Any], - top: Node, + top: NodeTop, noder: Noder) -> None: """mount action""" mountpoint = args[''] @@ -94,7 +94,7 @@ def cmd_mount(args: Dict[str, Any], def cmd_index(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: Node) -> None: + top: NodeTop) -> None: """index action""" path = args[''] name = args[''] @@ -117,8 +117,8 @@ def cmd_index(args: Dict[str, Any], start = datetime.datetime.now() walker = Walker(noder, usehash=usehash, debug=debug) - attr = noder.attrs_to_string(args['--meta']) - root = noder.new_storage_node(name, path, parent=top, attrs=attr) + attr = args['--meta'] + root = noder.new_storage_node(name, path, top, attr) _, cnt = walker.index(path, root, name) if subsize: noder.rec_size(root, store=True) @@ -132,7 +132,7 @@ def cmd_index(args: Dict[str, Any], def cmd_update(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: Node) -> None: + top: NodeTop) -> None: """update action""" path = args[''] name = args[''] @@ -162,7 +162,7 @@ def cmd_update(args: Dict[str, Any], def cmd_ls(args: Dict[str, Any], noder: Noder, - top: Node) -> List[Node]: + top: NodeTop) -> List[NodeAny]: """ls action""" path = args[''] if not path: @@ -197,7 +197,7 @@ def cmd_ls(args: Dict[str, Any], def cmd_rm(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: Node) -> Node: + top: NodeTop) -> NodeTop: """rm action""" name = args[''] node = noder.get_storage_node(top, name) @@ -212,7 +212,7 @@ def cmd_rm(args: Dict[str, Any], def cmd_find(args: Dict[str, Any], noder: Noder, - top: Node) -> List[Node]: + top: NodeTop) -> List[NodeAny]: """find action""" fromtree = args['--parent'] directory = args['--directory'] @@ -232,7 +232,7 @@ def cmd_find(args: Dict[str, Any], def cmd_graph(args: Dict[str, Any], noder: Noder, - top: Node) -> None: + top: NodeTop) -> None: """graph action""" path = args[''] if not path: @@ -243,7 +243,7 @@ def cmd_graph(args: Dict[str, Any], def cmd_rename(args: Dict[str, Any], catalog: Catalog, - top: Node) -> None: + top: NodeTop) -> None: """rename action""" storage = args[''] new = args[''] @@ -261,7 +261,7 @@ def cmd_rename(args: Dict[str, Any], def cmd_edit(args: Dict[str, Any], noder: Noder, catalog: Catalog, - top: Node) -> None: + top: NodeTop) -> None: """edit action""" storage = args[''] storages = list(x.name for x in top.children) diff --git a/catcli/cnode.py b/catcli/cnode.py index 599d4fa..3742b87 100644 --- a/catcli/cnode.py +++ b/catcli/cnode.py @@ -6,57 +6,39 @@ Class that represents a node in the catalog tree """ # pylint: disable=W0622 +from typing import Dict, Any from anytree import NodeMixin # type: ignore -TYPE_TOP = 'top' -TYPE_FILE = 'file' -TYPE_DIR = 'dir' -TYPE_ARC = 'arc' -TYPE_STORAGE = 'storage' -TYPE_META = 'meta' +_TYPE_BAD = 'badtype' +_TYPE_TOP = 'top' +_TYPE_FILE = 'file' +_TYPE_DIR = 'dir' +_TYPE_ARC = 'arc' +_TYPE_STORAGE = 'storage' +_TYPE_META = 'meta' NAME_TOP = 'top' NAME_META = 'meta' -class Node(NodeMixin): # type: ignore - """a node in the catalog""" +class NodeAny(NodeMixin): # type: ignore + """generic node""" def __init__(self, # type: ignore[no-untyped-def] - name: str, - type: str, - size: float = 0, - relpath: str = '', - md5: str = '', - maccess: float = 0, - free: int = 0, - total: int = 0, - indexed_dt: int = 0, - attr: str = '', - archive: str = '', parent=None, children=None): - """build a node""" + """build generic node""" super().__init__() - self.name = name - self.type = type - self.size = size - self.relpath = relpath - self.md5 = md5 - self.maccess = maccess - self.free = free - self.total = total - self.indexed_dt = indexed_dt - self.attr = attr - self.archive = archive + self._flagged = False self.parent = parent if children: self.children = children - self._flagged = False def flagged(self) -> bool: """is flagged""" + if not hasattr(self, '_flagged'): + return False return self._flagged def flag(self) -> None: @@ -66,3 +48,133 @@ class Node(NodeMixin): # type: ignore def unflag(self) -> None: """unflag node""" self._flagged = False + del self._flagged + + +class NodeTop(NodeAny): + """a top node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + children=None): + """build a top node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_TOP + self.parent = None + if children: + self.children = children + + +class NodeFile(NodeAny): + """a file node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + relpath: str, + size: int, + md5: str, + maccess: float, + parent=None, + children=None): + """build a file node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_FILE + self.relpath = relpath + self.size = size + self.md5 = md5 + self.maccess = maccess + self.parent = parent + if children: + self.children = children + + +class NodeDir(NodeAny): + """a directory node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + relpath: str, + size: int, + maccess: float, + parent=None, + children=None): + """build a directory node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_DIR + self.relpath = relpath + self.size = size + self.maccess = maccess + self.parent = parent + if children: + self.children = children + + +class NodeArchived(NodeAny): + """an archived node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + relpath: str, + size: int, + md5: str, + archive: str, + parent=None, + children=None): + """build an archived node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_ARC + self.relpath = relpath + self.size = size + self.md5 = md5 + self.archive = archive + self.parent = parent + if children: + self.children = children + + +class NodeStorage(NodeAny): + """a storage node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + free: int, + total: int, + size: int, + indexed_dt: float, + attr: Dict[str, Any], + parent=None, + children=None): + """build a storage node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_STORAGE + self.free = free + self.total = total + self.attr = attr + self.size = size + self.indexed_dt = indexed_dt + self.parent = parent + if children: + self.children = children + + +class NodeMeta(NodeAny): + """a meta node""" + + def __init__(self, # type: ignore[no-untyped-def] + name: str, + attr: str, + parent=None, + children=None): + """build a meta node""" + super().__init__() # type: ignore[no-untyped-call] + self.name = name + self.type = _TYPE_META + self.attr = attr + self.parent = parent + if children: + self.children = children diff --git a/catcli/fuser.py b/catcli/fuser.py index 6a4b6cd..ef89a47 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -14,8 +14,8 @@ import fuse # type: ignore # local imports from catcli.noder import Noder +from catcli.cnode import NodeTop, NodeAny from catcli import cnode -from catcli.cnode import Node # build custom logger to log in /tmp @@ -34,7 +34,7 @@ class Fuser: """fuse filesystem mounter""" def __init__(self, mountpoint: str, - top: Node, + top: NodeTop, noder: Noder, debug: bool = False): """fuse filesystem""" @@ -50,13 +50,13 @@ class Fuser: class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore """in-memory filesystem for catcli catalog""" - def __init__(self, top: Node, + def __init__(self, top: NodeTop, noder: Noder): """init fuse filesystem""" self.top = top self.noder = noder - def _get_entry(self, path: str) -> Optional[Node]: + def _get_entry(self, path: str) -> Optional[NodeAny]: """return the node pointed by path""" pre = f'{SEPARATOR}{cnode.NAME_TOP}' if not path.startswith(pre): @@ -69,7 +69,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore return found[0] return None - def _get_entries(self, path: str) -> List[Node]: + def _get_entries(self, path: str) -> List[NodeAny]: """return nodes pointed by path""" pre = f'{SEPARATOR}{cnode.NAME_TOP}' if not path.startswith(pre): @@ -91,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore curt = time() mode: Any = S_IFREG - if entry.type == cnode.TYPE_ARC: + if isinstance(entry, cnode.NodeArchived): mode = S_IFREG - elif entry.type == cnode.TYPE_DIR: + elif isinstance(entry, cnode.NodeDir): mode = S_IFDIR - elif entry.type == cnode.TYPE_FILE: + elif isinstance(entry, cnode.NodeFile): mode = S_IFREG - elif entry.type == cnode.TYPE_STORAGE: + elif isinstance(entry, cnode.NodeStorage): mode = S_IFDIR - elif entry.type == cnode.TYPE_META: + elif isinstance(entry, cnode.NodeMeta): mode = S_IFREG - elif entry.type == cnode.TYPE_TOP: + elif isinstance(entry, cnode.NodeTop): mode = S_IFREG return { 'st_mode': (mode), diff --git a/catcli/nodeprinter.py b/catcli/nodeprinter.py index 26505e6..79d6e39 100644 --- a/catcli/nodeprinter.py +++ b/catcli/nodeprinter.py @@ -6,7 +6,8 @@ Class for printing nodes """ import sys -from typing import TypeVar, Type, Optional, Tuple, List +from typing import TypeVar, Type, Optional, Tuple, List, \ + Dict, Any from catcli.colors import Colors from catcli.utils import fix_badchars @@ -25,7 +26,7 @@ class NodePrinter: @classmethod def print_storage_native(cls: Type[CLASSTYPE], pre: str, name: str, args: str, - attr: str) -> None: + attr: Dict[str, Any]) -> None: """print a storage node""" end = '' if attr: diff --git a/catcli/noder.py b/catcli/noder.py index b420c35..dc4d855 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -14,7 +14,8 @@ from pyfzf.pyfzf import FzfPrompt # type: ignore # local imports from catcli import cnode -from catcli.cnode import Node +from catcli.cnode import NodeAny, NodeStorage, \ + NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars from catcli.logger import Logger from catcli.nodeprinter import NodePrinter @@ -53,20 +54,20 @@ class Noder: self.decomp = Decomp() @staticmethod - def get_storage_names(top: Node) -> List[str]: + def get_storage_names(top: NodeTop) -> List[str]: """return a list of all storage names""" return [x.name for x in list(top.children)] - def get_storage_node(self, top: Node, + def get_storage_node(self, top: NodeTop, name: str, - newpath: str = '') -> Node: + newpath: str = '') -> NodeStorage: """ return the storage node if any if newpath is submitted, it will update the media info """ found = None for node in top.children: - if node.type != cnode.TYPE_STORAGE: + if not isinstance(node, cnode.NodeStorage): continue if node.name == name: found = node @@ -75,27 +76,27 @@ class Noder: found.free = shutil.disk_usage(newpath).free found.total = shutil.disk_usage(newpath).total found.ts = int(time.time()) - return cast(Node, found) + return cast(NodeStorage, found) @staticmethod - def get_node(top: Node, + def get_node(top: NodeTop, path: str, - quiet: bool = False) -> Optional[Node]: + quiet: bool = False) -> Optional[NodeAny]: """get the node by internal tree path""" resolv = anytree.resolver.Resolver('name') try: bpath = os.path.basename(path) the_node = resolv.get(top, bpath) - return cast(Node, the_node) + return cast(NodeAny, the_node) except anytree.resolver.ChildResolverError: if not quiet: Logger.err(f'No node at path \"{bpath}\"') return None def get_node_if_changed(self, - top: Node, + top: NodeTop, path: str, - treepath: str) -> Tuple[Optional[Node], bool]: + treepath: str) -> Tuple[Optional[NodeAny], bool]: """ return the node (if any) and if it has changed @top: top node (storage) @@ -131,25 +132,25 @@ class Noder: self._debug(f'\tchange: no change for \"{path}\"') return node, False - def rec_size(self, node: Node, - store: bool = True) -> float: + def rec_size(self, node: Union[NodeDir, NodeStorage], + store: bool = True) -> int: """ recursively traverse tree and return size @store: store the size in the node """ - if node.type == cnode.TYPE_FILE: + if isinstance(node, cnode.NodeFile): self._debug(f'getting node size for \"{node.name}\"') - return float(node.size) + return node.size msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) - size: float = 0 + size: int = 0 for i in node.children: - if node.type == cnode.TYPE_DIR: + if isinstance(node, cnode.NodeDir): size = self.rec_size(i, store=store) if store: i.size = size size += size - if node.type == cnode.TYPE_STORAGE: + if isinstance(node, cnode.NodeStorage): size = self.rec_size(i, store=store) if store: i.size = size @@ -185,13 +186,12 @@ class Noder: ############################################################### # node creation ############################################################### - def new_top_node(self) -> Node: + def new_top_node(self) -> NodeTop: """create a new top node""" - return Node(cnode.NAME_TOP, - cnode.TYPE_TOP) + return NodeTop(cnode.NAME_TOP) def new_file_node(self, name: str, path: str, - parent: Node, storagepath: str) -> Optional[Node]: + parent: NodeAny, storagepath: str) -> Optional[NodeFile]: """create a new node representing a file""" if not os.path.exists(path): Logger.err(f'File \"{path}\" does not exist') @@ -208,11 +208,12 @@ class Noder: relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - node = self._new_generic_node(name, cnode.TYPE_FILE, - relpath, parent, - size=stat.st_size, - md5=md5, - maccess=maccess) + node = NodeFile(name, + relpath, + stat.st_size, + md5, + maccess, + parent=parent) if self.arc: ext = os.path.splitext(path)[1][1:] if ext.lower() in self.decomp.get_formats(): @@ -224,59 +225,46 @@ class Noder: return node def new_dir_node(self, name: str, path: str, - parent: Node, storagepath: str) -> Node: + parent: NodeAny, storagepath: str) -> NodeDir: """create a new node representing a directory""" path = os.path.abspath(path) relpath = os.sep.join([storagepath, name]) maccess = os.path.getmtime(path) - return self._new_generic_node(name, cnode.TYPE_DIR, relpath, - parent, maccess=maccess) + return NodeDir(name, + relpath, + 0, + maccess, + parent=parent) def new_storage_node(self, name: str, path: str, parent: str, - attrs: str = '') -> Node: + attrs: Dict[str, Any]) \ + -> NodeStorage: """create a new node representing a storage""" path = os.path.abspath(path) free = shutil.disk_usage(path).free total = shutil.disk_usage(path).total epoch = int(time.time()) - return Node(name=name, - type=cnode.TYPE_STORAGE, - free=free, - total=total, - parent=parent, - attr=attrs, - indexed_dt=epoch) + return NodeStorage(name, + free, + total, + 0, + epoch, + attrs, + parent=parent) def new_archive_node(self, name: str, path: str, - parent: str, archive: str) -> Node: + parent: str, archive: str) -> NodeArchived: """create a new node for archive data""" - return Node(name=name, type=cnode.TYPE_ARC, relpath=path, - parent=parent, size=0, md5='', - archive=archive) - - @staticmethod - def _new_generic_node(name: str, - nodetype: str, - relpath: str, - parent: Node, - size: float = 0, - md5: str = '', - maccess: float = 0) -> Node: - """generic node creation""" - return Node(name, - nodetype, - size=size, - relpath=relpath, - md5=md5, - maccess=maccess, - parent=parent) + return NodeArchived(name=name, relpath=path, + parent=parent, size=0, md5='', + archive=archive) ############################################################### # node management ############################################################### - def update_metanode(self, top: Node) -> Node: + def update_metanode(self, top: NodeTop) -> NodeMeta: """create or update meta node information""" meta = self._get_meta_node(top) epoch = int(time.time()) @@ -284,9 +272,8 @@ class Noder: attrs: Dict[str, Any] = {} attrs['created'] = epoch attrs['created_version'] = VERSION - meta = Node(name=cnode.NAME_META, - type=cnode.TYPE_META, - attr=self.attrs_to_string(attrs)) + meta = NodeMeta(name=cnode.NAME_META, + attr=self.attrs_to_string(attrs)) if meta.attr: meta.attr += ', ' meta.attr += f'access={epoch}' @@ -294,26 +281,26 @@ class Noder: meta.attr += f'access_version={VERSION}' return meta - def _get_meta_node(self, top: Node) -> Optional[Node]: + def _get_meta_node(self, top: NodeTop) -> Optional[NodeMeta]: """return the meta node if any""" try: - found = next(filter(lambda x: x.type == cnode.TYPE_META, + found = next(filter(lambda x: isinstance(x, cnode.NodeMeta), top.children)) - return cast(Node, found) + return cast(NodeMeta, found) except StopIteration: return None - def clean_not_flagged(self, top: Node) -> int: + def clean_not_flagged(self, top: NodeTop) -> int: """remove any node not flagged and clean flags""" cnt = 0 for node in anytree.PreOrderIter(top): - if node.type not in [cnode.TYPE_FILE, cnode.TYPE_DIR]: + if not isinstance(node, (cnode.NodeDir, cnode.NodeFile)): continue if self._clean(node): cnt += 1 return cnt - def _clean(self, node: Node) -> bool: + def _clean(self, node: NodeAny) -> bool: """remove node if not flagged""" if not node.flagged(): node.parent = None @@ -324,7 +311,7 @@ class Noder: ############################################################### # printing ############################################################### - def _node_to_csv(self, node: Node, + def _node_to_csv(self, node: NodeAny, sep: str = ',', raw: bool = False) -> None: """ @@ -333,7 +320,7 @@ class Noder: @sep: CSV separator character @raw: print raw size rather than human readable """ - if not cnode: + if not node: return if node.type == node.TYPE_TOP: return @@ -374,7 +361,7 @@ class Noder: out.append(node.md5) # md5 else: out.append('') # fake md5 - if node.type == cnode.TYPE_DIR: + if isinstance(node, cnode.NodeDir): out.append(str(len(node.children))) # nbfiles else: out.append('') # fake nbfiles @@ -386,7 +373,7 @@ class Noder: if len(line) > 0: Logger.stdout_nocolor(line) - def _print_node_native(self, node: Node, + def _print_node_native(self, node: NodeAny, pre: str = '', withpath: bool = False, withdepth: bool = False, @@ -403,10 +390,10 @@ class Noder: @recalcparent: get relpath from tree instead of relpath field @raw: print raw size rather than human readable """ - if node.type == cnode.TYPE_TOP: + if isinstance(node, cnode.NodeTop): # top node Logger.stdout_nocolor(f'{pre}{node.name}') - elif node.type == cnode.TYPE_FILE: + elif isinstance(node, cnode.NodeFile): # node of type file name = node.name if withpath: @@ -426,7 +413,7 @@ class Noder: content = Logger.get_bold_text(storage.name) compl += f', storage:{content}' NodePrinter.print_file_native(pre, name, compl) - elif node.type == cnode.TYPE_DIR: + elif isinstance(node, cnode.NodeDir): # node of type directory name = node.name if withpath: @@ -446,7 +433,7 @@ class Noder: if withstorage: attr.append(('storage', Logger.get_bold_text(storage.name))) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) - elif node.type == cnode.TYPE_STORAGE: + elif isinstance(node, cnode.NodeStorage): # node of type storage sztotal = size_to_str(node.total, raw=raw) szused = size_to_str(node.total - node.free, raw=raw) @@ -476,14 +463,14 @@ class Noder: name, argsstring, node.attr) - elif node.type == cnode.TYPE_ARC: + elif isinstance(node, cnode.NodeArchived): # archive node if self.arc: NodePrinter.print_archive_native(pre, node.name, node.archive) else: Logger.err(f'bad node encountered: {node}') - def print_tree(self, node: Node, + def print_tree(self, node: NodeAny, fmt: str = 'native', raw: bool = False) -> None: """ @@ -507,7 +494,7 @@ class Noder: Logger.stdout_nocolor(self.CSV_HEADER) self._to_csv(node, raw=raw) - def _to_csv(self, node: Node, + def _to_csv(self, node: NodeAny, raw: bool = False) -> None: """print the tree to csv""" rend = anytree.RenderTree(node, childiter=self._sort_tree) @@ -521,7 +508,7 @@ class Noder: selected = fzf.prompt(strings) return selected - def _to_fzf(self, node: Node, fmt: str) -> None: + def _to_fzf(self, node: NodeAny, fmt: str) -> None: """ fzf prompt with list and print selected node(s) @node: node to start with @@ -550,24 +537,24 @@ class Noder: self.print_tree(rend, fmt=subfmt) @staticmethod - def to_dot(node: Node, + def to_dot(top: NodeTop, path: str = 'tree.dot') -> str: """export to dot for graphing""" - anytree.exporter.DotExporter(node).to_dotfile(path) + anytree.exporter.DotExporter(top).to_dotfile(path) Logger.info(f'dot file created under \"{path}\"') return f'dot {path} -T png -o /tmp/tree.png' ############################################################### # searching ############################################################### - def find_name(self, top: Node, + def find_name(self, top: NodeTop, key: str, script: bool = False, only_dir: bool = False, - startnode: Optional[Node] = None, + startnode: Optional[NodeAny] = None, parentfromtree: bool = False, fmt: str = 'native', - raw: bool = False) -> List[Node]: + raw: bool = False) -> List[NodeAny]: """ find files based on their names @top: top node @@ -583,7 +570,7 @@ class Noder: self._debug(f'searching for \"{key}\"') # search for nodes based on path - start: Optional[Node] = top + start: Optional[NodeAny] = top if startnode: start = self.get_node(top, startnode) filterfunc = self._callback_find_name(key, only_dir) @@ -594,7 +581,9 @@ class Noder: # compile found nodes paths = {} for item in found: - item = self._sanitize(item) + item.name = fix_badchars(item.name) + if hasattr(item, 'relpath'): + item.relpath = fix_badchars(item.relpath) if parentfromtree: paths[self._get_parents(item)] = item else: @@ -636,17 +625,17 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" - def find_name(node: Node) -> bool: - if node.type == cnode.TYPE_STORAGE: + def find_name(node: NodeAny) -> bool: + if isinstance(node, cnode.NodeStorage): # ignore storage nodes return False - if node.type == cnode.TYPE_TOP: + if isinstance(node, cnode.NodeTop): # ignore top nodes return False - if node.type == cnode.TYPE_META: + if isinstance(node, cnode.NodeMeta): # ignore meta nodes return False - if only_dir and node.type != cnode.TYPE_DIR: + if only_dir and isinstance(node, cnode.NodeDir): # ignore non directory return False @@ -663,11 +652,11 @@ class Noder: ############################################################### # ls ############################################################### - def list(self, top: Node, + def list(self, top: NodeTop, path: str, rec: bool = False, fmt: str = 'native', - raw: bool = False) -> List[Node]: + raw: bool = False) -> List[NodeAny]: """ list nodes for "ls" @top: top node @@ -729,7 +718,7 @@ class Noder: # tree creation ############################################################### def _add_entry(self, name: str, - top: Node, + top: NodeTop, resolv: Any) -> None: """add an entry to the tree""" entries = name.rstrip(os.sep).split(os.sep) @@ -744,7 +733,7 @@ class Noder: except anytree.resolver.ChildResolverError: self.new_archive_node(nodename, name, top, top.name) - def list_to_tree(self, parent: Node, names: List[str]) -> None: + def list_to_tree(self, parent: NodeAny, names: List[str]) -> None: """convert list of files to a tree""" if not names: return @@ -757,23 +746,23 @@ class Noder: # diverse ############################################################### def _sort_tree(self, - items: List[Node]) -> List[Node]: + items: List[NodeAny]) -> List[NodeAny]: """sorting a list of items""" return sorted(items, key=self._sort, reverse=self.sortsize) - def _sort(self, lst: Node) -> Any: + def _sort(self, lst: NodeAny) -> Any: """sort a list""" if self.sortsize: return self._sort_size(lst) return self._sort_fs(lst) @staticmethod - def _sort_fs(node: Node) -> Tuple[str, str]: + def _sort_fs(node: NodeAny) -> Tuple[str, str]: """sorting nodes dir first and alpha""" return (node.type, node.name.lstrip('.').lower()) @staticmethod - def _sort_size(node: Node) -> float: + def _sort_size(node: NodeAny) -> float: """sorting nodes by size""" try: if not node.size: @@ -782,22 +771,22 @@ class Noder: except AttributeError: return 0 - def _get_storage(self, node: Node) -> Node: + def _get_storage(self, node: NodeAny) -> NodeStorage: """recursively traverse up to find storage""" - if node.type == cnode.TYPE_STORAGE: + if isinstance(node, cnode.NodeStorage): return node - return cast(Node, node.ancestors[1]) + return cast(NodeStorage, node.ancestors[1]) @staticmethod - def _has_attr(node: Node, attr: str) -> bool: + def _has_attr(node: NodeAny, attr: str) -> bool: """return True if node has attr as attribute""" return attr in node.__dict__.keys() - def _get_parents(self, node: Node) -> str: + def _get_parents(self, node: NodeAny) -> str: """get all parents recursively""" - if node.type == cnode.TYPE_STORAGE: + if isinstance(node, cnode.NodeStorage): return '' - if node.type == cnode.TYPE_TOP: + if isinstance(node, cnode.NodeTop): return '' parent = self._get_parents(node.parent) if parent: @@ -813,13 +802,6 @@ class Noder: Logger.err(str(exc)) return '' - @staticmethod - def _sanitize(node: Node) -> Node: - """sanitize node strings""" - node.name = fix_badchars(node.name) - node.relpath = fix_badchars(node.relpath) - return node - def _debug(self, string: str) -> None: """print debug""" if not self.debug: diff --git a/catcli/walker.py b/catcli/walker.py index e263fce..94fe17c 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -9,9 +9,9 @@ import os from typing import Tuple, Optional # local imports -from catcli.cnode import Node from catcli.noder import Noder from catcli.logger import Logger +from catcli.cnode import NodeAny, NodeTop class Walker: @@ -36,7 +36,7 @@ class Walker: self.lpath = logpath def index(self, path: str, - parent: Node, + parent: NodeAny, name: str, storagepath: str = '') -> Tuple[str, int]: """ @@ -89,15 +89,15 @@ class Walker: self._progress('') return parent, cnt - def reindex(self, path: str, parent: Node, top: Node) -> int: + def reindex(self, path: str, parent: NodeAny, top: NodeTop) -> int: """reindex a directory and store in tree""" cnt = self._reindex(path, parent, top) cnt += self.noder.clean_not_flagged(parent) return cnt def _reindex(self, path: str, - parent: Node, - top: Node, + parent: NodeAny, + top: NodeTop, storagepath: str = '') -> int: """ reindex a directory and store in tree @@ -148,9 +148,9 @@ class Walker: return cnt def _need_reindex(self, - top: Node, + top: NodeTop, path: str, - treepath: str) -> Tuple[bool, Optional[Node]]: + treepath: str) -> Tuple[bool, Optional[NodeTop]]: """ test if node needs re-indexing @top: top node (storage) From a333f60fa7b227c3bf0db5dbae5db8c7a6cce449 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 10:35:59 +0100 Subject: [PATCH 12/36] refactoring --- catcli/catalog.py | 24 +++++++----- catcli/catcli.py | 18 +++++---- catcli/fuser.py | 20 +++++----- catcli/logger.py | 2 +- catcli/noder.py | 70 +++++++++++++++++------------------ catcli/{cnode.py => nodes.py} | 45 +++++++++++++++++----- catcli/walker.py | 2 +- 7 files changed, 109 insertions(+), 72 deletions(-) rename catcli/{cnode.py => nodes.py} (83%) diff --git a/catcli/catalog.py b/catcli/catalog.py index 2cdf8ed..e415a39 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -7,12 +7,12 @@ Class that represents the catcli catalog import os import pickle -from typing import Optional, Union, Any, cast +from typing import Optional from anytree.exporter import JsonExporter # type: ignore from anytree.importer import JsonImporter # type: ignore # local imports -from catcli.cnode import NodeMeta, NodeTop +from catcli.nodes import NodeMeta, NodeTop from catcli.utils import ask from catcli.logger import Logger @@ -88,32 +88,38 @@ class Catalog: return Logger.debug(text) - def _save_pickle(self, node: NodeTop) -> bool: + def _save_pickle(self, top: NodeTop) -> bool: """pickle the catalog""" with open(self.path, 'wb') as file: - pickle.dump(node, file) + pickle.dump(top, file) self._debug(f'Catalog saved to pickle \"{self.path}\"') return True - def _restore_pickle(self) -> Union[NodeTop, Any]: + def _restore_pickle(self) -> NodeTop: """restore the pickled tree""" with open(self.path, 'rb') as file: root = pickle.load(file) msg = f'Catalog imported from pickle \"{self.path}\"' self._debug(msg) - return root + top = NodeTop(root) + return top - def _save_json(self, node: NodeTop) -> bool: + def _save_json(self, top: NodeTop) -> bool: """export the catalog in json""" + Logger.debug(f'saving {top} to json...') exp = JsonExporter(indent=2, sort_keys=True) with open(self.path, 'w', encoding='UTF-8') as file: - exp.write(node, file) + exp.write(top, file) self._debug(f'Catalog saved to json \"{self.path}\"') return True def _restore_json(self, string: str) -> NodeTop: """restore the tree from json""" imp = JsonImporter() + Logger.debug(f'import from string: {string}') root = imp.import_(string) self._debug(f'Catalog imported from json \"{self.path}\"') - return cast(NodeTop, root) + top = NodeTop(root) + Logger.debug(f'top imported: {top}') + return top + # return cast(NodeTop, root) diff --git a/catcli/catcli.py b/catcli/catcli.py index c8ffea9..0323f4f 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -15,9 +15,9 @@ from typing import Dict, Any, List from docopt import docopt # local imports -from catcli import cnode +from catcli import nodes from catcli.version import __version__ as VERSION -from catcli.cnode import NodeTop, NodeAny +from catcli.nodes import NodeTop, NodeAny from catcli.logger import Logger from catcli.colors import Colors from catcli.catalog import Catalog @@ -44,8 +44,10 @@ USAGE = f""" Usage: {NAME} ls [--catalog=] [--format=] [-aBCrVSs] [] - {NAME} find [--catalog=] [--format=] [-aBCbdVsP] [--path=] [] - {NAME} index [--catalog=] [--meta=...] [-aBCcfnV] + {NAME} find [--catalog=] [--format=] + [-aBCbdVsP] [--path=] [] + {NAME} index [--catalog=] [--meta=...] + [-aBCcfnV] {NAME} update [--catalog=] [-aBCcfnV] [--lpath=] {NAME} mount [--catalog=] [-V] {NAME} rm [--catalog=] [-BCfV] @@ -66,14 +68,14 @@ Options: -C --no-color Do not output colors [default: False]. -c --hash Calculate md5 hash [default: False]. -d --directory Only directory [default: False]. - -F --format= Print format, see command \"print_supported_formats\" [default: native]. + -F --format= see \"print_supported_formats\" [default: native]. -f --force Do not ask when updating the catalog [default: False]. -l --lpath= Path where changes are logged [default: ] -n --no-subsize Do not store size of directories [default: False]. -P --parent Ignore stored relpath [default: True]. -p --path= Start path. -r --recursive Recursive [default: False]. - -s --raw-size Print raw size rather than human readable [default: False]. + -s --raw-size Print raw size [default: False]. -S --sortsize Sort by size, largest first [default: False]. -V --verbose Be verbose [default: False]. -v --version Show version. @@ -113,6 +115,8 @@ def cmd_index(args: Dict[str, Any], Logger.err('aborted') return node = noder.get_storage_node(top, name) + Logger.debug(f'top node: {top}') + Logger.debug(f'storage node: {node}') node.parent = None start = datetime.datetime.now() @@ -170,7 +174,7 @@ def cmd_ls(args: Dict[str, Any], if not path.startswith(SEPARATOR): path = SEPARATOR + path # prepend with top node path - pre = f'{SEPARATOR}{cnode.NAME_TOP}' + pre = f'{SEPARATOR}{nodes.NAME_TOP}' if not path.startswith(pre): path = pre + path # ensure ends with a separator diff --git a/catcli/fuser.py b/catcli/fuser.py index ef89a47..4e8701c 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -14,8 +14,8 @@ import fuse # type: ignore # local imports from catcli.noder import Noder -from catcli.cnode import NodeTop, NodeAny -from catcli import cnode +from catcli.nodes import NodeTop, NodeAny +from catcli import nodes # build custom logger to log in /tmp @@ -58,7 +58,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def _get_entry(self, path: str) -> Optional[NodeAny]: """return the node pointed by path""" - pre = f'{SEPARATOR}{cnode.NAME_TOP}' + pre = f'{SEPARATOR}{nodes.NAME_TOP}' if not path.startswith(pre): path = pre + path found = self.noder.list(self.top, path, @@ -71,7 +71,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def _get_entries(self, path: str) -> List[NodeAny]: """return nodes pointed by path""" - pre = f'{SEPARATOR}{cnode.NAME_TOP}' + pre = f'{SEPARATOR}{nodes.NAME_TOP}' if not path.startswith(pre): path = pre + path if not path.endswith(SEPARATOR): @@ -91,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore curt = time() mode: Any = S_IFREG - if isinstance(entry, cnode.NodeArchived): + if isinstance(entry, nodes.NodeArchived): mode = S_IFREG - elif isinstance(entry, cnode.NodeDir): + elif isinstance(entry, nodes.NodeDir): mode = S_IFDIR - elif isinstance(entry, cnode.NodeFile): + elif isinstance(entry, nodes.NodeFile): mode = S_IFREG - elif isinstance(entry, cnode.NodeStorage): + elif isinstance(entry, nodes.NodeStorage): mode = S_IFDIR - elif isinstance(entry, cnode.NodeMeta): + elif isinstance(entry, nodes.NodeMeta): mode = S_IFREG - elif isinstance(entry, cnode.NodeTop): + elif isinstance(entry, nodes.NodeTop): mode = S_IFREG return { 'st_mode': (mode), diff --git a/catcli/logger.py b/catcli/logger.py index 423a044..b56cabe 100644 --- a/catcli/logger.py +++ b/catcli/logger.py @@ -37,7 +37,7 @@ class Logger: def debug(cls: Type[CLASSTYPE], string: str) -> None: """to stderr no color""" - cls.stderr_nocolor(f'[DBG] {string}\n') + cls.stderr_nocolor(f'[DBG] {string}') @classmethod def info(cls: Type[CLASSTYPE], diff --git a/catcli/noder.py b/catcli/noder.py index dc4d855..43677ed 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -13,8 +13,8 @@ import anytree # type: ignore from pyfzf.pyfzf import FzfPrompt # type: ignore # local imports -from catcli import cnode -from catcli.cnode import NodeAny, NodeStorage, \ +from catcli import nodes +from catcli.nodes import NodeAny, NodeStorage, \ NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars from catcli.logger import Logger @@ -67,7 +67,7 @@ class Noder: """ found = None for node in top.children: - if not isinstance(node, cnode.NodeStorage): + if not isinstance(node, nodes.NodeStorage): continue if node.name == name: found = node @@ -138,25 +138,26 @@ class Noder: recursively traverse tree and return size @store: store the size in the node """ - if isinstance(node, cnode.NodeFile): + if isinstance(node, nodes.NodeFile): self._debug(f'getting node size for \"{node.name}\"') return node.size msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) size: int = 0 for i in node.children: - if isinstance(node, cnode.NodeDir): + if isinstance(node, nodes.NodeDir): size = self.rec_size(i, store=store) if store: i.size = size size += size - if isinstance(node, cnode.NodeStorage): + if isinstance(node, nodes.NodeStorage): size = self.rec_size(i, store=store) if store: i.size = size size += size else: continue + self._debug(f'size of {node.name} is {size}') if store: node.size = size return size @@ -188,7 +189,9 @@ class Noder: ############################################################### def new_top_node(self) -> NodeTop: """create a new top node""" - return NodeTop(cnode.NAME_TOP) + top = NodeTop(nodes.NAME_TOP) + self._debug(f'new top node: {top}') + return top def new_file_node(self, name: str, path: str, parent: NodeAny, storagepath: str) -> Optional[NodeFile]: @@ -251,7 +254,7 @@ class Noder: total, 0, epoch, - attrs, + self.attrs_to_string(attrs), parent=parent) def new_archive_node(self, name: str, path: str, @@ -272,19 +275,16 @@ class Noder: attrs: Dict[str, Any] = {} attrs['created'] = epoch attrs['created_version'] = VERSION - meta = NodeMeta(name=cnode.NAME_META, - attr=self.attrs_to_string(attrs)) - if meta.attr: - meta.attr += ', ' - meta.attr += f'access={epoch}' - meta.attr += ', ' - meta.attr += f'access_version={VERSION}' + meta = NodeMeta(name=nodes.NAME_META, + attr=attrs) + meta.attr['access'] = epoch + meta.attr['access_version'] = VERSION return meta def _get_meta_node(self, top: NodeTop) -> Optional[NodeMeta]: """return the meta node if any""" try: - found = next(filter(lambda x: isinstance(x, cnode.NodeMeta), + found = next(filter(lambda x: isinstance(x, nodes.NodeMeta), top.children)) return cast(NodeMeta, found) except StopIteration: @@ -294,7 +294,7 @@ class Noder: """remove any node not flagged and clean flags""" cnt = 0 for node in anytree.PreOrderIter(top): - if not isinstance(node, (cnode.NodeDir, cnode.NodeFile)): + if not isinstance(node, (nodes.NodeDir, nodes.NodeFile)): continue if self._clean(node): cnt += 1 @@ -361,7 +361,7 @@ class Noder: out.append(node.md5) # md5 else: out.append('') # fake md5 - if isinstance(node, cnode.NodeDir): + if isinstance(node, nodes.NodeDir): out.append(str(len(node.children))) # nbfiles else: out.append('') # fake nbfiles @@ -390,10 +390,10 @@ class Noder: @recalcparent: get relpath from tree instead of relpath field @raw: print raw size rather than human readable """ - if isinstance(node, cnode.NodeTop): + if isinstance(node, nodes.NodeTop): # top node Logger.stdout_nocolor(f'{pre}{node.name}') - elif isinstance(node, cnode.NodeFile): + elif isinstance(node, nodes.NodeFile): # node of type file name = node.name if withpath: @@ -413,7 +413,7 @@ class Noder: content = Logger.get_bold_text(storage.name) compl += f', storage:{content}' NodePrinter.print_file_native(pre, name, compl) - elif isinstance(node, cnode.NodeDir): + elif isinstance(node, nodes.NodeDir): # node of type directory name = node.name if withpath: @@ -433,7 +433,7 @@ class Noder: if withstorage: attr.append(('storage', Logger.get_bold_text(storage.name))) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) - elif isinstance(node, cnode.NodeStorage): + elif isinstance(node, nodes.NodeStorage): # node of type storage sztotal = size_to_str(node.total, raw=raw) szused = size_to_str(node.total - node.free, raw=raw) @@ -463,7 +463,7 @@ class Noder: name, argsstring, node.attr) - elif isinstance(node, cnode.NodeArchived): + elif isinstance(node, nodes.NodeArchived): # archive node if self.arc: NodePrinter.print_archive_native(pre, node.name, node.archive) @@ -515,7 +515,7 @@ class Noder: @fmt: output format for selected nodes """ rendered = anytree.RenderTree(node, childiter=self._sort_tree) - nodes = {} + the_nodes = {} # construct node names list for _, _, rend in rendered: if not rend: @@ -523,17 +523,17 @@ class Noder: parents = self._get_parents(rend) storage = self._get_storage(rend) fullpath = os.path.join(storage.name, parents) - nodes[fullpath] = rend + the_nodes[fullpath] = rend # prompt with fzf - paths = self._fzf_prompt(nodes.keys()) + paths = self._fzf_prompt(the_nodes.keys()) # print the resulting tree subfmt = fmt.replace('fzf-', '') for path in paths: if not path: continue - if path not in nodes: + if path not in the_nodes: continue - rend = nodes[path] + rend = the_nodes[path] self.print_tree(rend, fmt=subfmt) @staticmethod @@ -626,16 +626,16 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" def find_name(node: NodeAny) -> bool: - if isinstance(node, cnode.NodeStorage): + if isinstance(node, nodes.NodeStorage): # ignore storage nodes return False - if isinstance(node, cnode.NodeTop): + if isinstance(node, nodes.NodeTop): # ignore top nodes return False - if isinstance(node, cnode.NodeMeta): + if isinstance(node, nodes.NodeMeta): # ignore meta nodes return False - if only_dir and isinstance(node, cnode.NodeDir): + if only_dir and isinstance(node, nodes.NodeDir): # ignore non directory return False @@ -773,7 +773,7 @@ class Noder: def _get_storage(self, node: NodeAny) -> NodeStorage: """recursively traverse up to find storage""" - if isinstance(node, cnode.NodeStorage): + if isinstance(node, nodes.NodeStorage): return node return cast(NodeStorage, node.ancestors[1]) @@ -784,9 +784,9 @@ class Noder: def _get_parents(self, node: NodeAny) -> str: """get all parents recursively""" - if isinstance(node, cnode.NodeStorage): + if isinstance(node, nodes.NodeStorage): return '' - if isinstance(node, cnode.NodeTop): + if isinstance(node, nodes.NodeTop): return '' parent = self._get_parents(node.parent) if parent: diff --git a/catcli/cnode.py b/catcli/nodes.py similarity index 83% rename from catcli/cnode.py rename to catcli/nodes.py index 3742b87..76308ec 100644 --- a/catcli/cnode.py +++ b/catcli/nodes.py @@ -10,7 +10,6 @@ from typing import Dict, Any from anytree import NodeMixin # type: ignore -_TYPE_BAD = 'badtype' _TYPE_TOP = 'top' _TYPE_FILE = 'file' _TYPE_DIR = 'dir' @@ -30,11 +29,21 @@ class NodeAny(NodeMixin): # type: ignore children=None): """build generic node""" super().__init__() - self._flagged = False self.parent = parent if children: self.children = children + def _to_str(self) -> str: + ret = str(self.__class__) + ": " + str(self.__dict__) + if self.children: + ret += '\n' + for child in self.children: + ret += ' child => ' + str(child) + return ret + + def __str__(self) -> str: + return self._to_str() + def flagged(self) -> bool: """is flagged""" if not hasattr(self, '_flagged'): @@ -43,12 +52,12 @@ class NodeAny(NodeMixin): # type: ignore def flag(self) -> None: """flag a node""" - self._flagged = True + self._flagged = True # pylint: disable=W0201 def unflag(self) -> None: """unflag node""" - self._flagged = False - del self._flagged + self._flagged = False # pylint: disable=W0201 + delattr(self, '_flagged') class NodeTop(NodeAny): @@ -65,6 +74,9 @@ class NodeTop(NodeAny): if children: self.children = children + def __str__(self) -> str: + return self._to_str() + class NodeFile(NodeAny): """a file node""" @@ -89,6 +101,9 @@ class NodeFile(NodeAny): if children: self.children = children + def __str__(self) -> str: + return self._to_str() + class NodeDir(NodeAny): """a directory node""" @@ -111,6 +126,9 @@ class NodeDir(NodeAny): if children: self.children = children + def __str__(self) -> str: + return self._to_str() + class NodeArchived(NodeAny): """an archived node""" @@ -135,6 +153,9 @@ class NodeArchived(NodeAny): if children: self.children = children + def __str__(self) -> str: + return self._to_str() + class NodeStorage(NodeAny): """a storage node""" @@ -144,8 +165,8 @@ class NodeStorage(NodeAny): free: int, total: int, size: int, - indexed_dt: float, - attr: Dict[str, Any], + ts: float, + attr: str, parent=None, children=None): """build a storage node""" @@ -156,18 +177,21 @@ class NodeStorage(NodeAny): self.total = total self.attr = attr self.size = size - self.indexed_dt = indexed_dt + self.ts = ts self.parent = parent if children: self.children = children + def __str__(self) -> str: + return self._to_str() + class NodeMeta(NodeAny): """a meta node""" def __init__(self, # type: ignore[no-untyped-def] name: str, - attr: str, + attr: Dict[str, Any], parent=None, children=None): """build a meta node""" @@ -178,3 +202,6 @@ class NodeMeta(NodeAny): self.parent = parent if children: self.children = children + + def __str__(self) -> str: + return self._to_str() diff --git a/catcli/walker.py b/catcli/walker.py index 94fe17c..ee93b4b 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -11,7 +11,7 @@ from typing import Tuple, Optional # local imports from catcli.noder import Noder from catcli.logger import Logger -from catcli.cnode import NodeAny, NodeTop +from catcli.nodes import NodeAny, NodeTop class Walker: From d294aa59e104f0238c200507e847340825ef81f7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 20:57:35 +0100 Subject: [PATCH 13/36] fix formats --- catcli/catalog.py | 38 +++++++++----------------------------- catcli/catcli.py | 4 ++-- catcli/fuser.py | 12 ++++++------ catcli/noder.py | 38 +++++++++++++++++++------------------- catcli/nodes.py | 26 +++++++++++++------------- 5 files changed, 49 insertions(+), 69 deletions(-) diff --git a/catcli/catalog.py b/catcli/catalog.py index e415a39..65993d1 100644 --- a/catcli/catalog.py +++ b/catcli/catalog.py @@ -6,12 +6,12 @@ Class that represents the catcli catalog """ import os -import pickle from typing import Optional from anytree.exporter import JsonExporter # type: ignore from anytree.importer import JsonImporter # type: ignore # local imports +from catcli import nodes from catcli.nodes import NodeMeta, NodeTop from catcli.utils import ask from catcli.logger import Logger @@ -21,7 +21,6 @@ class Catalog: """the catalog""" def __init__(self, path: str, - usepickle: bool = False, debug: bool = False, force: bool = False) -> None: """ @@ -34,7 +33,6 @@ class Catalog: self.debug = debug self.force = force self.metanode: Optional[NodeMeta] = None - self.pickle = usepickle def set_metanode(self, metanode: NodeMeta) -> None: """remove the metanode until tree is re-written""" @@ -56,8 +54,6 @@ class Catalog: return None if not os.path.exists(self.path): return None - if self.pickle: - return self._restore_pickle() with open(self.path, 'r', encoding='UTF-8') as file: content = file.read() return self._restore_json(content) @@ -79,8 +75,6 @@ class Catalog: return False if self.metanode: self.metanode.parent = node - if self.pickle: - return self._save_pickle(node) return self._save_json(node) def _debug(self, text: str) -> None: @@ -88,38 +82,24 @@ class Catalog: return Logger.debug(text) - def _save_pickle(self, top: NodeTop) -> bool: - """pickle the catalog""" - with open(self.path, 'wb') as file: - pickle.dump(top, file) - self._debug(f'Catalog saved to pickle \"{self.path}\"') - return True - - def _restore_pickle(self) -> NodeTop: - """restore the pickled tree""" - with open(self.path, 'rb') as file: - root = pickle.load(file) - msg = f'Catalog imported from pickle \"{self.path}\"' - self._debug(msg) - top = NodeTop(root) - return top - def _save_json(self, top: NodeTop) -> bool: """export the catalog in json""" - Logger.debug(f'saving {top} to json...') + self._debug(f'saving {top} to json...') exp = JsonExporter(indent=2, sort_keys=True) with open(self.path, 'w', encoding='UTF-8') as file: exp.write(top, file) self._debug(f'Catalog saved to json \"{self.path}\"') return True - def _restore_json(self, string: str) -> NodeTop: + def _restore_json(self, string: str) -> Optional[NodeTop]: """restore the tree from json""" imp = JsonImporter() - Logger.debug(f'import from string: {string}') + self._debug(f'import from string: {string}') root = imp.import_(string) self._debug(f'Catalog imported from json \"{self.path}\"') - top = NodeTop(root) - Logger.debug(f'top imported: {top}') + self._debug(f'root imported: {root}') + if root.type != nodes.TYPE_TOP: + return None + top = NodeTop(root.name, children=root.children) + self._debug(f'top imported: {top}') return top - # return cast(NodeTop, root) diff --git a/catcli/catcli.py b/catcli/catcli.py index 0323f4f..46b4c6c 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -293,8 +293,8 @@ def print_supported_formats() -> None: print('"native" : native format') print('"csv" : CSV format') print(f' {Noder.CSV_HEADER}') - print('"fzf-native" : fzf to native (only for find)') - print('"fzf-csv" : fzf to csv (only for find)') + print('"fzf-native" : fzf to native (only valid for find)') + print('"fzf-csv" : fzf to csv (only valid for find)') def main() -> bool: diff --git a/catcli/fuser.py b/catcli/fuser.py index 4e8701c..6641bc2 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -91,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore curt = time() mode: Any = S_IFREG - if isinstance(entry, nodes.NodeArchived): + if entry.type == nodes.TYPE_ARCHIVED: mode = S_IFREG - elif isinstance(entry, nodes.NodeDir): + elif entry.type == nodes.TYPE_DIR: mode = S_IFDIR - elif isinstance(entry, nodes.NodeFile): + elif entry.type == nodes.TYPE_FILE: mode = S_IFREG - elif isinstance(entry, nodes.NodeStorage): + elif entry.type == nodes.TYPE_STORAGE: mode = S_IFDIR - elif isinstance(entry, nodes.NodeMeta): + elif entry.type == nodes.TYPE_META: mode = S_IFREG - elif isinstance(entry, nodes.NodeTop): + elif entry.type == nodes.TYPE_TOP: mode = S_IFREG return { 'st_mode': (mode), diff --git a/catcli/noder.py b/catcli/noder.py index 43677ed..996c733 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -67,7 +67,7 @@ class Noder: """ found = None for node in top.children: - if not isinstance(node, nodes.NodeStorage): + if node.type != nodes.TYPE_STORAGE: continue if node.name == name: found = node @@ -138,19 +138,19 @@ class Noder: recursively traverse tree and return size @store: store the size in the node """ - if isinstance(node, nodes.NodeFile): + if node.type == nodes.TYPE_FILE: self._debug(f'getting node size for \"{node.name}\"') return node.size msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) size: int = 0 for i in node.children: - if isinstance(node, nodes.NodeDir): + if node.type == nodes.TYPE_DIR: size = self.rec_size(i, store=store) if store: i.size = size size += size - if isinstance(node, nodes.NodeStorage): + if node.type == nodes.TYPE_STORAGE: size = self.rec_size(i, store=store) if store: i.size = size @@ -284,7 +284,7 @@ class Noder: def _get_meta_node(self, top: NodeTop) -> Optional[NodeMeta]: """return the meta node if any""" try: - found = next(filter(lambda x: isinstance(x, nodes.NodeMeta), + found = next(filter(lambda x: x.type == nodes.TYPE_META, top.children)) return cast(NodeMeta, found) except StopIteration: @@ -294,7 +294,7 @@ class Noder: """remove any node not flagged and clean flags""" cnt = 0 for node in anytree.PreOrderIter(top): - if not isinstance(node, (nodes.NodeDir, nodes.NodeFile)): + if node.type not in [nodes.TYPE_DIR, nodes.TYPE_FILE]: continue if self._clean(node): cnt += 1 @@ -361,7 +361,7 @@ class Noder: out.append(node.md5) # md5 else: out.append('') # fake md5 - if isinstance(node, nodes.NodeDir): + if node.type == nodes.TYPE_DIR: out.append(str(len(node.children))) # nbfiles else: out.append('') # fake nbfiles @@ -390,10 +390,10 @@ class Noder: @recalcparent: get relpath from tree instead of relpath field @raw: print raw size rather than human readable """ - if isinstance(node, nodes.NodeTop): + if node.type == nodes.TYPE_TOP: # top node Logger.stdout_nocolor(f'{pre}{node.name}') - elif isinstance(node, nodes.NodeFile): + elif node.type == nodes.TYPE_FILE: # node of type file name = node.name if withpath: @@ -413,7 +413,7 @@ class Noder: content = Logger.get_bold_text(storage.name) compl += f', storage:{content}' NodePrinter.print_file_native(pre, name, compl) - elif isinstance(node, nodes.NodeDir): + elif node.type == nodes.TYPE_DIR: # node of type directory name = node.name if withpath: @@ -433,7 +433,7 @@ class Noder: if withstorage: attr.append(('storage', Logger.get_bold_text(storage.name))) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) - elif isinstance(node, nodes.NodeStorage): + elif node.type == nodes.TYPE_STORAGE: # node of type storage sztotal = size_to_str(node.total, raw=raw) szused = size_to_str(node.total - node.free, raw=raw) @@ -463,7 +463,7 @@ class Noder: name, argsstring, node.attr) - elif isinstance(node, nodes.NodeArchived): + elif node.type == nodes.TYPE_ARCHIVED: # archive node if self.arc: NodePrinter.print_archive_native(pre, node.name, node.archive) @@ -626,16 +626,16 @@ class Noder: def _callback_find_name(self, term: str, only_dir: bool) -> Any: """callback for finding files""" def find_name(node: NodeAny) -> bool: - if isinstance(node, nodes.NodeStorage): + if node.type == nodes.TYPE_STORAGE: # ignore storage nodes return False - if isinstance(node, nodes.NodeTop): + if node.type == nodes.TYPE_TOP: # ignore top nodes return False - if isinstance(node, nodes.NodeMeta): + if node.type == nodes.TYPE_META: # ignore meta nodes return False - if only_dir and isinstance(node, nodes.NodeDir): + if only_dir and node.type == nodes.TYPE_DIR: # ignore non directory return False @@ -773,7 +773,7 @@ class Noder: def _get_storage(self, node: NodeAny) -> NodeStorage: """recursively traverse up to find storage""" - if isinstance(node, nodes.NodeStorage): + if node.type == nodes.TYPE_STORAGE: return node return cast(NodeStorage, node.ancestors[1]) @@ -784,9 +784,9 @@ class Noder: def _get_parents(self, node: NodeAny) -> str: """get all parents recursively""" - if isinstance(node, nodes.NodeStorage): + if node.type == nodes.TYPE_STORAGE: return '' - if isinstance(node, nodes.NodeTop): + if node.type == nodes.TYPE_TOP: return '' parent = self._get_parents(node.parent) if parent: diff --git a/catcli/nodes.py b/catcli/nodes.py index 76308ec..e8f30ba 100644 --- a/catcli/nodes.py +++ b/catcli/nodes.py @@ -10,12 +10,12 @@ from typing import Dict, Any from anytree import NodeMixin # type: ignore -_TYPE_TOP = 'top' -_TYPE_FILE = 'file' -_TYPE_DIR = 'dir' -_TYPE_ARC = 'arc' -_TYPE_STORAGE = 'storage' -_TYPE_META = 'meta' +TYPE_TOP = 'top' +TYPE_FILE = 'file' +TYPE_DIR = 'dir' +TYPE_ARCHIVED = 'arc' +TYPE_STORAGE = 'storage' +TYPE_META = 'meta' NAME_TOP = 'top' NAME_META = 'meta' @@ -69,7 +69,7 @@ class NodeTop(NodeAny): """build a top node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_TOP + self.type = TYPE_TOP self.parent = None if children: self.children = children @@ -92,7 +92,7 @@ class NodeFile(NodeAny): """build a file node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_FILE + self.type = TYPE_FILE self.relpath = relpath self.size = size self.md5 = md5 @@ -118,7 +118,7 @@ class NodeDir(NodeAny): """build a directory node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_DIR + self.type = TYPE_DIR self.relpath = relpath self.size = size self.maccess = maccess @@ -144,7 +144,7 @@ class NodeArchived(NodeAny): """build an archived node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_ARC + self.type = TYPE_ARCHIVED self.relpath = relpath self.size = size self.md5 = md5 @@ -172,12 +172,12 @@ class NodeStorage(NodeAny): """build a storage node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_STORAGE + self.type = TYPE_STORAGE self.free = free self.total = total self.attr = attr self.size = size - self.ts = ts + self.ts = ts # pylint: disable=C0103 self.parent = parent if children: self.children = children @@ -197,7 +197,7 @@ class NodeMeta(NodeAny): """build a meta node""" super().__init__() # type: ignore[no-untyped-call] self.name = name - self.type = _TYPE_META + self.type = TYPE_META self.attr = attr self.parent = parent if children: From e39088f6cb4cf92c431c45944cce61aa78b79518 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 21:14:36 +0100 Subject: [PATCH 14/36] fix tests --- .github/workflows/testing.yml | 1 + catcli/noder.py | 6 +++--- tests-ng/assets/github.catalog.csv.txt | 5 +++++ tests-ng/assets/github.catalog.json | 14 +++++++------- tests-ng/assets/github.catalog.native.txt | 7 +++++++ tests-ng/compare.sh | 15 +++++++++++---- 6 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 tests-ng/assets/github.catalog.csv.txt create mode 100644 tests-ng/assets/github.catalog.native.txt diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c7fca14..4c9e3fb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,6 +17,7 @@ jobs: python -m pip install --upgrade pip pip install -r tests-requirements.txt pip install -r requirements.txt + sudo apt-get install shellcheck - name: Run tests run: | ./tests.sh diff --git a/catcli/noder.py b/catcli/noder.py index 996c733..c6390fe 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -322,11 +322,11 @@ class Noder: """ if not node: return - if node.type == node.TYPE_TOP: + if node.type == nodes.TYPE_TOP: return out = [] - if node.type == node.TYPE_STORAGE: + if node.type == nodes.TYPE_STORAGE: # handle storage out.append(node.name) # name out.append(node.type) # type @@ -357,7 +357,7 @@ class Noder: out.append(epoch_to_str(node.maccess)) # maccess else: out.append('') # fake maccess - if node.md5: + if self._has_attr(node, 'md5'): out.append(node.md5) # md5 else: out.append('') # fake md5 diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt new file mode 100644 index 0000000..27c7f31 --- /dev/null +++ b/tests-ng/assets/github.catalog.csv.txt @@ -0,0 +1,5 @@ +"github","storage","","2380","2023-03-08 20:58:18","","","2","23343173632","245107195904","" +"workflows","dir","github/workflows","1190","2023-03-08 20:58:18","2023-03-02 10:15:30","","2","","","" +"pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-08 20:58:18","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" +"testing.yml","file","github/workflows/testing.yml","595","2023-03-08 20:58:18","2023-03-02 10:15:30","f4f4bc5acb9deae488b55f0c9c507120","","","","" +"FUNDING.yml","file","github/FUNDING.yml","17","2023-03-08 20:58:18","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index 6c5fbe7..339358b 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -5,7 +5,7 @@ "children": [ { "maccess": 1666206037.0786593, - "md5": null, + "md5": "0c6407a84d412c514007313fb3bca4de", "name": "FUNDING.yml", "relpath": "/FUNDING.yml", "size": 17, @@ -15,7 +15,7 @@ "children": [ { "maccess": 1666206037.078865, - "md5": null, + "md5": "57699a7a6a03e20e864f220e19f8e197", "name": "pypi-release.yml", "relpath": "workflows/pypi-release.yml", "size": 691, @@ -23,7 +23,7 @@ }, { "maccess": 1677748530.6920426, - "md5": null, + "md5": "f4f4bc5acb9deae488b55f0c9c507120", "name": "testing.yml", "relpath": "workflows/testing.yml", "size": 595, @@ -38,18 +38,18 @@ "type": "dir" } ], - "free": 23459602432, + "free": 23343173632, "name": "github", "size": 2380, "total": 245107195904, - "ts": 1678214993, + "ts": 1678305498, "type": "storage" }, { "attr": { - "access": 1678214993, + "access": 1678305498, "access_version": "0.8.7", - "created": 1678214993, + "created": 1678305498, "created_version": "0.8.7" }, "name": "meta", diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt new file mode 100644 index 0000000..40f187e --- /dev/null +++ b/tests-ng/assets/github.catalog.native.txt @@ -0,0 +1,7 @@ +top +└── storage: github + nbfiles:2 | totsize:2380 | free:9.5% | du:221764022272/245107195904 | date:2023-03-08 20:58:18 + ├── workflows [nbfiles:2, totsize:1190] + │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] + │ └── testing.yml [size:595, md5:f4f4bc5acb9deae488b55f0c9c507120] + └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 2c6289d..7b92d87 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -49,11 +49,18 @@ clear_on_exit "${tmpd}" catalog="${tmpd}/catalog" # index -${bin} -B index --catalog="${catalog}" github .github - -# diff +${bin} -B index -c --catalog="${catalog}" github .github cat "${catalog}" -diff "${cur}/assets/github.catalog.json" "${catalog}" + +# make sure we still get the same output in native format +native="${tmpd}/native.txt" +${bin} -B ls -r -s --format=native --catalog="${catalog}" > "${native}" +diff -I '^.*| totsize.*$' "${cur}/assets/github.catalog.native.txt" "${native}" + +# make sure we still get the same output in csv +csv="${tmpd}/csv.txt" +${bin} -B ls -r -s --format=csv --catalog="${catalog}" > "${csv}" +diff -I '^.*| totsize.*$' "${cur}/assets/github.catalog.csv.txt" "${csv}" # the end echo "test \"$(basename "$0")\" success" From e4cda7948c038a07ceaa995898374d6dffa8dc6e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 21:38:46 +0100 Subject: [PATCH 15/36] fix fuse --- catcli/catcli.py | 22 ++-------------------- catcli/fuser.py | 42 ++++++++++++++++++++---------------------- catcli/utils.py | 30 ++++++++++++++++++++++++++++++ tests.sh | 17 ++++++++++------- 4 files changed, 62 insertions(+), 49 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 46b4c6c..87c65a9 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -15,7 +15,6 @@ from typing import Dict, Any, List from docopt import docopt # local imports -from catcli import nodes from catcli.version import __version__ as VERSION from catcli.nodes import NodeTop, NodeAny from catcli.logger import Logger @@ -23,7 +22,7 @@ from catcli.colors import Colors from catcli.catalog import Catalog from catcli.walker import Walker from catcli.noder import Noder -from catcli.utils import ask, edit +from catcli.utils import ask, edit, path_to_search_all from catcli.fuser import Fuser from catcli.exceptions import BadFormatException, CatcliException @@ -31,8 +30,6 @@ NAME = 'catcli' CUR = os.path.dirname(os.path.abspath(__file__)) CATALOGPATH = f'{NAME}.catalog' GRAPHPATH = f'/tmp/{NAME}.dot' -SEPARATOR = '/' -WILD = '*' FORMATS = ['native', 'csv', 'csv-with-header', 'fzf-native', 'fzf-csv'] BANNER = f""" +-+-+-+-+-+-+ @@ -168,22 +165,7 @@ def cmd_ls(args: Dict[str, Any], noder: Noder, top: NodeTop) -> List[NodeAny]: """ls action""" - path = args[''] - if not path: - path = SEPARATOR - if not path.startswith(SEPARATOR): - path = SEPARATOR + path - # prepend with top node path - pre = f'{SEPARATOR}{nodes.NAME_TOP}' - if not path.startswith(pre): - path = pre + path - # ensure ends with a separator - if not path.endswith(SEPARATOR): - path += SEPARATOR - # add wild card - if not path.endswith(WILD): - path += WILD - + path = path_to_search_all(args['']) fmt = args['--format'] if fmt.startswith('fzf'): raise BadFormatException('fzf is not supported in ls, use find') diff --git a/catcli/fuser.py b/catcli/fuser.py index 6641bc2..177c075 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -15,6 +15,7 @@ import fuse # type: ignore # local imports from catcli.noder import Noder from catcli.nodes import NodeTop, NodeAny +from catcli.utils import path_to_search_all, path_to_top from catcli import nodes @@ -25,10 +26,6 @@ fh = logging.FileHandler('/tmp/fuse-catcli.log') fh.setLevel(logging.DEBUG) logger.addHandler(fh) -# globals -WILD = '*' -SEPARATOR = '/' - class Fuser: """fuse filesystem mounter""" @@ -58,9 +55,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def _get_entry(self, path: str) -> Optional[NodeAny]: """return the node pointed by path""" - pre = f'{SEPARATOR}{nodes.NAME_TOP}' - if not path.startswith(pre): - path = pre + path + path = path_to_top(path) found = self.noder.list(self.top, path, rec=False, fmt='native', @@ -71,13 +66,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def _get_entries(self, path: str) -> List[NodeAny]: """return nodes pointed by path""" - pre = f'{SEPARATOR}{nodes.NAME_TOP}' - if not path.startswith(pre): - path = pre + path - if not path.endswith(SEPARATOR): - path += SEPARATOR - if not path.endswith(WILD): - path += WILD + path = path_to_search_all(path) found = self.noder.list(self.top, path, rec=False, fmt='native', @@ -89,27 +78,36 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore if not entry: return {} - curt = time() + maccess = time() mode: Any = S_IFREG + size: int = 0 if entry.type == nodes.TYPE_ARCHIVED: mode = S_IFREG + size = entry.size elif entry.type == nodes.TYPE_DIR: mode = S_IFDIR + size = entry.size + maccess = entry.maccess elif entry.type == nodes.TYPE_FILE: mode = S_IFREG + size = entry.size + maccess = entry.maccess elif entry.type == nodes.TYPE_STORAGE: mode = S_IFDIR + size = entry.size + maccess = entry.ts elif entry.type == nodes.TYPE_META: mode = S_IFREG elif entry.type == nodes.TYPE_TOP: mode = S_IFREG + mode = mode | 0o777 return { - 'st_mode': (mode), - 'st_nlink': 1, - 'st_size': 0, - 'st_ctime': curt, - 'st_mtime': curt, - 'st_atime': curt, + 'st_mode': (mode), # file type + 'st_nlink': 1, # count hard link + 'st_size': size, + 'st_ctime': maccess, # attr last modified + 'st_mtime': maccess, # content last modified + 'st_atime': maccess, # access time 'st_uid': os.getuid(), 'st_gid': os.getgid(), } @@ -122,7 +120,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore # mountpoint curt = time() meta = { - 'st_mode': (S_IFDIR), + 'st_mode': (S_IFDIR | 0o777), 'st_nlink': 1, 'st_size': 0, 'st_ctime': curt, diff --git a/catcli/utils.py b/catcli/utils.py index dd058e8..4111df8 100644 --- a/catcli/utils.py +++ b/catcli/utils.py @@ -12,10 +12,40 @@ import subprocess import datetime # local imports +from catcli import nodes from catcli.exceptions import CatcliException SEPARATOR = '/' +WILD = '*' + + +def path_to_top(path: str) -> str: + """path pivot under top""" + pre = f'{SEPARATOR}{nodes.NAME_TOP}' + if not path.startswith(pre): + # prepend with top node path + path = pre + path + return path + + +def path_to_search_all(path: str) -> str: + """path to search for all subs""" + if not path: + path = SEPARATOR + if not path.startswith(SEPARATOR): + path = SEPARATOR + path + pre = f'{SEPARATOR}{nodes.NAME_TOP}' + if not path.startswith(pre): + # prepend with top node path + path = pre + path + if not path.endswith(SEPARATOR): + # ensure ends with a separator + path += SEPARATOR + if not path.endswith(WILD): + # add wild card + path += WILD + return path def md5sum(path: str) -> str: diff --git a/tests.sh b/tests.sh index 6719eec..0d8d145 100755 --- a/tests.sh +++ b/tests.sh @@ -11,7 +11,7 @@ set -e # pycodestyle echo "[+] pycodestyle" pycodestyle --version -pycodestyle --ignore=W605 catcli/ +pycodestyle catcli/ pycodestyle tests/ pycodestyle setup.py @@ -29,7 +29,6 @@ pyflakes setup.py # R0915: Too many statements # R0911: Too many return statements # R0903: Too few public methods -# R0801: Similar lines in 2 files # R0902: Too many instance attributes # R0201: no-self-used echo "[+] pylint" @@ -41,24 +40,28 @@ pylint -sn \ --disable=R0915 \ --disable=R0911 \ --disable=R0903 \ - --disable=R0801 \ --disable=R0902 \ --disable=R0201 \ --disable=R0022 \ catcli/ + + + +# R0801: Similar lines in 2 files +# W0212: Access to a protected member +# R0914: Too many local variables +# R0915: Too many statements pylint -sn \ + --disable=R0801 \ --disable=W0212 \ --disable=R0914 \ --disable=R0915 \ - --disable=R0801 \ tests/ pylint -sn setup.py # mypy echo "[+] mypy" -mypy \ - --strict \ - catcli/ +mypy --strict catcli/ # unittest echo "[+] unittests" From 63a052fc00de97714dc2047e28ac513b9e196310 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:10:52 +0100 Subject: [PATCH 16/36] update ignored files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2d768e5..8227ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ build/ *.egg-info/ *.catalog .vscode/ +.mypy_cache +.pytest_cache +__pycache__ From 3c6cce6b4c4e01e0725ac76954d2b780ecc8d6c6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:11:07 +0100 Subject: [PATCH 17/36] update doc --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 87206d8..077523b 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Features: * Index any directories in a catalog * Ability to search for files by name in the catalog * Ability to navigate through indexed data à la `ls` + * Support for fuse to mount the indexed data as a virtual filesystem * Handle archive files (zip, tar, ...) and index their content * Save catalog to json for easy versioning with git * Command line interface FTW @@ -73,6 +74,7 @@ See the [examples](#examples) for an overview of the available features. * [Index archive files](#index-archive-files) * [Walk indexed files with ls](#walk-indexed-files-with-ls) * [Find files](#find-files) + * [Mount catalog](#mount-catalog) * [Display entire hierarchy](#display-entire-hierarchy) * [Catalog graph](#catalog-graph) * [Edit storage](#edit-storage) @@ -185,6 +187,27 @@ searching: See the [examples](#examples) for more. +## Mount catalog + +The catalog can be mounted with [fuse](https://www.kernel.org/doc/html/next/filesystems/fuse.html) +and navigate like any filesystem. + +```bash +$ mkdir /tmp/mnt +$ catcli index -c github .github +$ catcli mount /tmp/mnt +$ ls -laR /tmp/mnt +drwxrwxrwx - user 8 Mar 22:08 github + +mnt/github: +.rwxrwxrwx 17 user 19 Oct 2022 FUNDING.yml +drwxrwxrwx - user 2 Mar 10:15 workflows + +mnt/github/workflows: +.rwxrwxrwx 691 user 19 Oct 2022 pypi-release.yml +.rwxrwxrwx 635 user 8 Mar 21:08 testing.yml +``` + ## Display entire hierarchy The entire catalog can be shown using the `ls -r` command. From a69a7515444920c7cf13429f609162a2c63a0d4e Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:11:16 +0100 Subject: [PATCH 18/36] add pylint as deps --- tests-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests-requirements.txt b/tests-requirements.txt index fa48b04..5174deb 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -5,3 +5,4 @@ coverage; python_version >= '3.0' coveralls; python_version >= '3.0' pylint; python_version > '3.0' mypy; python_version > '3.0' +pytest; python_version > '3.0' From c318a629419fc7f9a38a11054a906ae7fe56c9e4 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:11:29 +0100 Subject: [PATCH 19/36] logging --- catcli/catcli.py | 2 -- catcli/fuser.py | 12 ------------ catcli/logger.py | 13 ------------- catcli/walker.py | 9 --------- 4 files changed, 36 deletions(-) diff --git a/catcli/catcli.py b/catcli/catcli.py index 87c65a9..65da0d6 100755 --- a/catcli/catcli.py +++ b/catcli/catcli.py @@ -112,8 +112,6 @@ def cmd_index(args: Dict[str, Any], Logger.err('aborted') return node = noder.get_storage_node(top, name) - Logger.debug(f'top node: {top}') - Logger.debug(f'storage node: {node}') node.parent = None start = datetime.datetime.now() diff --git a/catcli/fuser.py b/catcli/fuser.py index 177c075..6f220a5 100644 --- a/catcli/fuser.py +++ b/catcli/fuser.py @@ -6,7 +6,6 @@ fuse for catcli """ import os -import logging from time import time from stat import S_IFDIR, S_IFREG from typing import List, Dict, Any, Optional @@ -19,14 +18,6 @@ from catcli.utils import path_to_search_all, path_to_top from catcli import nodes -# build custom logger to log in /tmp -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -fh = logging.FileHandler('/tmp/fuse-catcli.log') -fh.setLevel(logging.DEBUG) -logger.addHandler(fh) - - class Fuser: """fuse filesystem mounter""" @@ -114,8 +105,6 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def getattr(self, path: str, _fh: Any = None) -> Dict[str, Any]: """return attr of file pointed by path""" - logger.info('getattr path: %s', path) - if path == '/': # mountpoint curt = time() @@ -135,7 +124,6 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore def readdir(self, path: str, _fh: Any) -> List[str]: """read directory content""" - logger.info('readdir path: %s', path) content = ['.', '..'] entries = self._get_entries(path) for entry in entries: diff --git a/catcli/logger.py b/catcli/logger.py index b56cabe..645b6ba 100644 --- a/catcli/logger.py +++ b/catcli/logger.py @@ -69,16 +69,3 @@ class Logger: """make it bold""" string = fix_badchars(string) return f'{Colors.BOLD}{string}{Colors.RESET}' - - @classmethod - def log_to_file(cls: Type[CLASSTYPE], - path: str, - string: str, - append: bool = True) -> None: - """log to file""" - string = fix_badchars(string) - mode = 'w' - if append: - mode = 'a' - with open(path, mode, encoding='UTF-8') as file: - file.write(string) diff --git a/catcli/walker.py b/catcli/walker.py index ee93b4b..5942a5b 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -118,7 +118,6 @@ class Walker: if node: node.flag() continue - self._log2file(f'update catalog for \"{sub}\"') node = self.noder.new_file_node(os.path.basename(file), sub, parent, storagepath) if node: @@ -131,7 +130,6 @@ class Walker: treepath = os.path.join(storagepath, adir) reindex, dummy = self._need_reindex(parent, sub, treepath) if reindex: - self._log2file(f'update catalog for \"{sub}\"') dummy = self.noder.new_dir_node(base, sub, parent, storagepath) cnt += 1 @@ -189,10 +187,3 @@ class Walker: if len(string) > self.MAXLINELEN: string = string[:self.MAXLINELEN] + '...' Logger.progr(f'indexing: {string:80}') - - def _log2file(self, string: str) -> None: - """log to file""" - if not self.lpath: - return - line = f'{string}\n' - Logger.log_to_file(self.lpath, line, append=True) From 8ed43d5262a6eada280ceff945702714c8dc7047 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:57:08 +0100 Subject: [PATCH 20/36] remove compare --- tests-ng/assets/github.catalog.csv.txt | 5 -- tests-ng/assets/github.catalog.json | 61 -------------------- tests-ng/assets/github.catalog.native.txt | 7 --- tests-ng/compare.sh | 68 ----------------------- 4 files changed, 141 deletions(-) delete mode 100644 tests-ng/assets/github.catalog.csv.txt delete mode 100644 tests-ng/assets/github.catalog.json delete mode 100644 tests-ng/assets/github.catalog.native.txt delete mode 100755 tests-ng/compare.sh diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt deleted file mode 100644 index 27c7f31..0000000 --- a/tests-ng/assets/github.catalog.csv.txt +++ /dev/null @@ -1,5 +0,0 @@ -"github","storage","","2380","2023-03-08 20:58:18","","","2","23343173632","245107195904","" -"workflows","dir","github/workflows","1190","2023-03-08 20:58:18","2023-03-02 10:15:30","","2","","","" -"pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-08 20:58:18","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","595","2023-03-08 20:58:18","2023-03-02 10:15:30","f4f4bc5acb9deae488b55f0c9c507120","","","","" -"FUNDING.yml","file","github/FUNDING.yml","17","2023-03-08 20:58:18","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json deleted file mode 100644 index 339358b..0000000 --- a/tests-ng/assets/github.catalog.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "children": [ - { - "attr": "", - "children": [ - { - "maccess": 1666206037.0786593, - "md5": "0c6407a84d412c514007313fb3bca4de", - "name": "FUNDING.yml", - "relpath": "/FUNDING.yml", - "size": 17, - "type": "file" - }, - { - "children": [ - { - "maccess": 1666206037.078865, - "md5": "57699a7a6a03e20e864f220e19f8e197", - "name": "pypi-release.yml", - "relpath": "workflows/pypi-release.yml", - "size": 691, - "type": "file" - }, - { - "maccess": 1677748530.6920426, - "md5": "f4f4bc5acb9deae488b55f0c9c507120", - "name": "testing.yml", - "relpath": "workflows/testing.yml", - "size": 595, - "type": "file" - } - ], - "maccess": 1677748530.691944, - "md5": null, - "name": "workflows", - "relpath": "/workflows", - "size": 1190, - "type": "dir" - } - ], - "free": 23343173632, - "name": "github", - "size": 2380, - "total": 245107195904, - "ts": 1678305498, - "type": "storage" - }, - { - "attr": { - "access": 1678305498, - "access_version": "0.8.7", - "created": 1678305498, - "created_version": "0.8.7" - }, - "name": "meta", - "type": "meta" - } - ], - "name": "top", - "type": "top" -} \ No newline at end of file diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt deleted file mode 100644 index 40f187e..0000000 --- a/tests-ng/assets/github.catalog.native.txt +++ /dev/null @@ -1,7 +0,0 @@ -top -└── storage: github - nbfiles:2 | totsize:2380 | free:9.5% | du:221764022272/245107195904 | date:2023-03-08 20:58:18 - ├── workflows [nbfiles:2, totsize:1190] - │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:595, md5:f4f4bc5acb9deae488b55f0c9c507120] - └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh deleted file mode 100755 index 7b92d87..0000000 --- a/tests-ng/compare.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# author: deadc0de6 (https://github.com/deadc0de6) -# Copyright (c) 2023, deadc0de6 - -# exit on first error -set -e - -# get current path -rl="readlink -f" -if ! ${rl} "${0}" >/dev/null 2>&1; then - rl="realpath" - - if ! hash ${rl}; then - echo "\"${rl}\" not found !" && exit 1 - fi -fi -cur=$(dirname "$(${rl} "${0}")") - -# pivot -prev="${cur}/.." -cd "${prev}" - -# coverage -#export PYTHONPATH=".:${PYTHONPATH}" -bin="python3 -m catcli.catcli" -if hash coverage 2>/dev/null; then - bin="coverage run -p --source=catcli -m catcli.catcli" - #bin="coverage run -p --source=${prev}/catcli -m catcli.catcli" -fi - -echo "current dir: $(pwd)" -echo "pythonpath: ${PYTHONPATH}" -echo "bin: ${bin}" -${bin} --version - -# get the helpers -# shellcheck source=tests-ng/helper -source "${cur}"/helper -echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" - -########################################################## -# the test -########################################################## - -# create temp dirs -tmpd=$(mktemp -d) -clear_on_exit "${tmpd}" - -catalog="${tmpd}/catalog" - -# index -${bin} -B index -c --catalog="${catalog}" github .github -cat "${catalog}" - -# make sure we still get the same output in native format -native="${tmpd}/native.txt" -${bin} -B ls -r -s --format=native --catalog="${catalog}" > "${native}" -diff -I '^.*| totsize.*$' "${cur}/assets/github.catalog.native.txt" "${native}" - -# make sure we still get the same output in csv -csv="${tmpd}/csv.txt" -${bin} -B ls -r -s --format=csv --catalog="${catalog}" > "${csv}" -diff -I '^.*| totsize.*$' "${cur}/assets/github.catalog.csv.txt" "${csv}" - -# the end -echo "test \"$(basename "$0")\" success" -cd "${cur}" -exit 0 From 933f6efd2dc043d3cb79aefc9959c5830eff5500 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:57:15 +0100 Subject: [PATCH 21/36] percentage --- catcli/noder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catcli/noder.py b/catcli/noder.py index c6390fe..24e7b28 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -438,7 +438,9 @@ class Noder: sztotal = size_to_str(node.total, raw=raw) szused = size_to_str(node.total - node.free, raw=raw) nbchildren = len(node.children) - pcent = node.free * 100 / node.total + pcent = 0 + if node.total > 0: + pcent = node.free * 100 / node.total freepercent = f'{pcent:.1f}%' # get the date timestamp = '' From 0740626ef89d610f03234c5a3acc9fba5818eadf Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Wed, 8 Mar 2023 22:59:17 +0100 Subject: [PATCH 22/36] linting --- tests-ng/update.sh | 4 ++-- tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests-ng/update.sh b/tests-ng/update.sh index dcab7a6..26964bf 100755 --- a/tests-ng/update.sh +++ b/tests-ng/update.sh @@ -10,7 +10,7 @@ rl="readlink -f" if ! ${rl} "${0}" >/dev/null 2>&1; then rl="realpath" - if ! hash ${rl}; then + if ! command -v ${rl}; then echo "\"${rl}\" not found !" && exit 1 fi fi @@ -23,7 +23,7 @@ cd "${prev}" # coverage #export PYTHONPATH=".:${PYTHONPATH}" bin="python3 -m catcli.catcli" -if hash coverage 2>/dev/null; then +if command -v coverage 2>/dev/null; then bin="coverage run -p --source=catcli -m catcli.catcli" #bin="coverage run -p --source=${prev}/catcli -m catcli.catcli" fi diff --git a/tests.sh b/tests.sh index 0d8d145..5cc11a0 100755 --- a/tests.sh +++ b/tests.sh @@ -76,7 +76,7 @@ done # check shells echo "[+] shellcheck" -if ! which shellcheck >/dev/null 2>&1; then +if ! command -v shellcheck >/dev/null 2>&1; then echo "Install shellcheck" exit 1 fi From b838166a3729b6b7dca83727ae5ffa153186d1ad Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 9 Mar 2023 21:40:56 +0100 Subject: [PATCH 23/36] deps --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4c9e3fb..cc6f831 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,7 +17,7 @@ jobs: python -m pip install --upgrade pip pip install -r tests-requirements.txt pip install -r requirements.txt - sudo apt-get install shellcheck + sudo apt-get -y install shellcheck jq - name: Run tests run: | ./tests.sh From 0da1adcca07810ebb3e957e4c5170e0fa7f45f83 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 9 Mar 2023 21:41:12 +0100 Subject: [PATCH 24/36] test assets --- tests-ng/assets/github.catalog.csv.txt | 5 ++ tests-ng/assets/github.catalog.json | 60 +++++++++++++++++++++++ tests-ng/assets/github.catalog.native.txt | 7 +++ 3 files changed, 72 insertions(+) create mode 100644 tests-ng/assets/github.catalog.csv.txt create mode 100644 tests-ng/assets/github.catalog.json create mode 100644 tests-ng/assets/github.catalog.native.txt diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt new file mode 100644 index 0000000..07479cc --- /dev/null +++ b/tests-ng/assets/github.catalog.csv.txt @@ -0,0 +1,5 @@ +"github","storage","","2380","2023-03-09 16:20:59","","","2","0","0","" +"workflows","dir","github/workflows","1190","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" +"pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-09 16:20:59","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" +"testing.yml","file","github/workflows/testing.yml","595","2023-03-09 16:20:59","2023-03-09 16:20:44","f4f4bc5acb9deae488b55f0c9c507120","","","","" +"FUNDING.yml","file","github/FUNDING.yml","17","2023-03-09 16:20:59","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json new file mode 100644 index 0000000..a22a8be --- /dev/null +++ b/tests-ng/assets/github.catalog.json @@ -0,0 +1,60 @@ +{ + "children": [ + { + "attr": "", + "children": [ + { + "maccess": 1666206037.0786593, + "md5": "0c6407a84d412c514007313fb3bca4de", + "name": "FUNDING.yml", + "relpath": "/FUNDING.yml", + "size": 17, + "type": "file" + }, + { + "children": [ + { + "maccess": 1666206037.078865, + "md5": "57699a7a6a03e20e864f220e19f8e197", + "name": "pypi-release.yml", + "relpath": "workflows/pypi-release.yml", + "size": 691, + "type": "file" + }, + { + "maccess": 1678375244.4870229, + "md5": "f4f4bc5acb9deae488b55f0c9c507120", + "name": "testing.yml", + "relpath": "workflows/testing.yml", + "size": 595, + "type": "file" + } + ], + "maccess": 1678375244.4865956, + "name": "workflows", + "relpath": "/workflows", + "size": 1190, + "type": "dir" + } + ], + "free": 0, + "name": "github", + "size": 2380, + "total": 0, + "ts": 1678375259, + "type": "storage" + }, + { + "attr": { + "access": 1678375259, + "access_version": "0.8.7", + "created": 1678375259, + "created_version": "0.8.7" + }, + "name": "meta", + "type": "meta" + } + ], + "name": "top", + "type": "top" +} \ No newline at end of file diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt new file mode 100644 index 0000000..8784024 --- /dev/null +++ b/tests-ng/assets/github.catalog.native.txt @@ -0,0 +1,7 @@ +top +└── storage: github + nbfiles:2 | totsize:2380 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1190] + │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] + │ └── testing.yml [size:595, md5:f4f4bc5acb9deae488b55f0c9c507120] + └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] From 3ac55e72944f1a614c7b3bae560342e16300eebc Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 9 Mar 2023 21:41:20 +0100 Subject: [PATCH 25/36] new test --- tests-ng/compare.sh | 76 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 tests-ng/compare.sh diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh new file mode 100755 index 0000000..1fde1dd --- /dev/null +++ b/tests-ng/compare.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# author: deadc0de6 (https://github.com/deadc0de6) +# Copyright (c) 2023, deadc0de6 + +# exit on first error +set -e + +# get current path +rl="readlink -f" +if ! ${rl} "${0}" >/dev/null 2>&1; then + rl="realpath" + + if ! command -v ${rl}; then + echo "\"${rl}\" not found !" && exit 1 + fi +fi +cur=$(dirname "$(${rl} "${0}")") + +# pivot +prev="${cur}/.." +cd "${prev}" + +# coverage +#export PYTHONPATH=".:${PYTHONPATH}" +bin="python3 -m catcli.catcli" +if command -v coverage 2>/dev/null; then + bin="coverage run -p --source=catcli -m catcli.catcli" + #bin="coverage run -p --source=${prev}/catcli -m catcli.catcli" +fi + +echo "current dir: $(pwd)" +echo "pythonpath: ${PYTHONPATH}" +echo "bin: ${bin}" +${bin} --version + +# get the helpers +# shellcheck source=tests-ng/helper +source "${cur}"/helper +echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)" + +########################################################## +# the test +########################################################## + +# create temp dirs +tmpd=$(mktemp -d) +clear_on_exit "${tmpd}" + +catalog="${tmpd}/catalog" + +# index +${bin} -B index -c --catalog="${catalog}" github .github + +cat "${catalog}" +echo "" + +# compare keys +src="tests-ng/assets/github.catalog.json" +src_keys="${tmpd}/src-keys" +dst_keys="${tmpd}/dst-keys" +cat "${src}" | jq '.. | keys?' | jq '.[]' > "${src_keys}" +cat "${catalog}" | jq '.. | keys?' | jq '.[]' > "${dst_keys}" +diff "${src_keys}" "${dst_keys}" + +# native +native="${tmpd}/native.txt" +${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" + +# csv +csv="${tmpd}/csv.txt" +${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" + +# the end +echo "test \"$(basename "$0")\" success" +cd "${cur}" +exit 0 From b76dff46d64485e2846ee8cb34ff8673a73f96f3 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Thu, 9 Mar 2023 21:41:27 +0100 Subject: [PATCH 26/36] shellcheck linting --- tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests.sh b/tests.sh index 5cc11a0..31bffe3 100755 --- a/tests.sh +++ b/tests.sh @@ -83,6 +83,7 @@ fi shellcheck --version find . -iname '*.sh' | while read -r script; do shellcheck -x \ + --exclude SC2002 \ "${script}" done From 5e27cc40bae02d0104e7f36e00851c868d413ad5 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 10 Mar 2023 22:25:05 +0100 Subject: [PATCH 27/36] more tests --- tests-ng/compare.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 1fde1dd..aafa77f 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -62,6 +62,27 @@ cat "${src}" | jq '.. | keys?' | jq '.[]' > "${src_keys}" cat "${catalog}" | jq '.. | keys?' | jq '.[]' > "${dst_keys}" diff "${src_keys}" "${dst_keys}" +# compare children 1 +src_keys="${tmpd}/src-child1" +dst_keys="${tmpd}/dst-child1" +cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[].name' > "${dst_keys}" +diff "${src_keys}" "${dst_keys}" + +# compare children 2 +src_keys="${tmpd}/src-child2" +dst_keys="${tmpd}/dst-child2" +cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${dst_keys}" +diff "${src_keys}" "${dst_keys}" + +# compare children 3 +src_keys="${tmpd}/src-child3" +dst_keys="${tmpd}/dst-child3" +cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${dst_keys}" +diff "${src_keys}" "${dst_keys}" + # native native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" From 9deed263d3bd53045fa3e03bacc75874ca0db419 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Fri, 10 Mar 2023 23:01:36 +0100 Subject: [PATCH 28/36] compare --- tests-ng/assets/github.catalog.csv.txt | 6 +++--- tests-ng/assets/github.catalog.native.txt | 6 +++--- tests-ng/compare.sh | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 07479cc..1108085 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,5 +1,5 @@ -"github","storage","","2380","2023-03-09 16:20:59","","","2","0","0","" -"workflows","dir","github/workflows","1190","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" +"github","storage","","2564","2023-03-09 16:20:59","","","2","0","0","" +"workflows","dir","github/workflows","1282","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-09 16:20:59","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","595","2023-03-09 16:20:59","2023-03-09 16:20:44","f4f4bc5acb9deae488b55f0c9c507120","","","","" +"testing.yml","file","github/workflows/testing.yml","641","2023-03-09 16:20:59","2023-03-09 16:20:44","941c01fa956b71133d6b5339824589b7","","","","" "FUNDING.yml","file","github/FUNDING.yml","17","2023-03-09 16:20:59","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 8784024..d681b9b 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top └── storage: github - nbfiles:2 | totsize:2380 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1190] + nbfiles:2 | totsize:2564 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1282] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:595, md5:f4f4bc5acb9deae488b55f0c9c507120] + │ └── testing.yml [size:641, md5:941c01fa956b71133d6b5339824589b7] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index aafa77f..3a92cdc 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -86,10 +86,24 @@ diff "${src_keys}" "${dst_keys}" # native native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" +mod="${tmpd}/native.mod.txt" +cat "${native}" | sed -e 's/free:.*%/free:0.0%/g' \ + -e 's/date:.*$/date:2023-03-09 16:20:59/g' \ + -e 's#du:[^|]* |#du:0/0 |#g' > "${mod}" +#delta "tests-ng/assets/github.catalog.native.txt" "${mod}" +diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" # csv csv="${tmpd}/csv.txt" ${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" +mod="${tmpd}/csv.mod.txt" +cat "${csv}" | sed -e 's/"2",".*",".*",""$/"2","0","0",""/g' | \ + sed 's/20..-..-.. ..:..:..//g' > "${mod}" +ori="${tmpd}/ori.mod.txt" +cat "tests-ng/assets/github.catalog.csv.txt" | \ + sed 's/20..-..-.. ..:..:..//g' > "${ori}" +#delta "${ori}" "${mod}" +diff "${ori}" "${mod}" # the end echo "test \"$(basename "$0")\" success" From 8fdaba5aea003f77af88aa3fa2c24997f38a1e34 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 11:54:52 +0100 Subject: [PATCH 29/36] fix tests --- tests-ng/compare.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 3a92cdc..d6831f2 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -88,22 +88,25 @@ native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" mod="${tmpd}/native.mod.txt" cat "${native}" | sed -e 's/free:.*%/free:0.0%/g' \ - -e 's/date:.*$/date:2023-03-09 16:20:59/g' \ + -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ -e 's#du:[^|]* |#du:0/0 |#g' > "${mod}" -#delta "tests-ng/assets/github.catalog.native.txt" "${mod}" +#delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" # csv csv="${tmpd}/csv.txt" ${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" +# modify created csv mod="${tmpd}/csv.mod.txt" -cat "${csv}" | sed -e 's/"2",".*",".*",""$/"2","0","0",""/g' | \ +cat "${csv}" | sed -e 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' | \ sed 's/20..-..-.. ..:..:..//g' > "${mod}" +# modify original ori="${tmpd}/ori.mod.txt" cat "tests-ng/assets/github.catalog.csv.txt" | \ - sed 's/20..-..-.. ..:..:..//g' > "${ori}" -#delta "${ori}" "${mod}" -diff "${ori}" "${mod}" + sed 's/....-..-.. ..:..:..//g' | \ + sed 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' > "${ori}" +delta -s "${ori}" "${mod}" +#diff "${ori}" "${mod}" # the end echo "test \"$(basename "$0")\" success" From 30c1af0d155fedbc9fa3e2a1ea15367ffa2f2056 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 12:05:28 +0100 Subject: [PATCH 30/36] add coveralls --- .github/workflows/testing.yml | 6 ++++++ tests-ng/assets/github.catalog.csv.txt | 6 +++--- tests-ng/assets/github.catalog.json | 4 ++-- tests-ng/assets/github.catalog.native.txt | 6 +++--- tests-ng/compare.sh | 12 ++++++++---- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cc6f831..b444d47 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,3 +21,9 @@ jobs: - name: Run tests run: | ./tests.sh + - name: Coveralls + run: | + pip install coveralls + coveralls --service=github + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 1108085..40e6a28 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,5 +1,5 @@ -"github","storage","","2564","2023-03-09 16:20:59","","","2","0","0","" -"workflows","dir","github/workflows","1282","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" +"github","storage","","3208","2023-03-09 16:20:59","","","2","0","0","" +"workflows","dir","github/workflows","1604","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-09 16:20:59","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" -"testing.yml","file","github/workflows/testing.yml","641","2023-03-09 16:20:59","2023-03-09 16:20:44","941c01fa956b71133d6b5339824589b7","","","","" +"testing.yml","file","github/workflows/testing.yml","802","2023-03-09 16:20:59","2023-03-09 16:20:44","7144a119ef43adb634654522c12ec250","","","","" "FUNDING.yml","file","github/FUNDING.yml","17","2023-03-09 16:20:59","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index a22a8be..cf67e71 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -23,10 +23,10 @@ }, { "maccess": 1678375244.4870229, - "md5": "f4f4bc5acb9deae488b55f0c9c507120", + "md5": "7144a119ef43adb634654522c12ec250", "name": "testing.yml", "relpath": "workflows/testing.yml", - "size": 595, + "size": 802, "type": "file" } ], diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index d681b9b..9ca05fd 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top └── storage: github - nbfiles:2 | totsize:2564 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1282] + nbfiles:2 | totsize:3208 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1604] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] - │ └── testing.yml [size:641, md5:941c01fa956b71133d6b5339824589b7] + │ └── testing.yml [size:802, md5:7144a119ef43adb634654522c12ec250] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de] diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index d6831f2..03ff812 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -51,7 +51,7 @@ catalog="${tmpd}/catalog" # index ${bin} -B index -c --catalog="${catalog}" github .github -cat "${catalog}" +#cat "${catalog}" echo "" # compare keys @@ -90,7 +90,9 @@ mod="${tmpd}/native.mod.txt" cat "${native}" | sed -e 's/free:.*%/free:0.0%/g' \ -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ -e 's#du:[^|]* |#du:0/0 |#g' > "${mod}" -#delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" +if command -v delta; then + delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" +fi diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" # csv @@ -105,8 +107,10 @@ ori="${tmpd}/ori.mod.txt" cat "tests-ng/assets/github.catalog.csv.txt" | \ sed 's/....-..-.. ..:..:..//g' | \ sed 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' > "${ori}" -delta -s "${ori}" "${mod}" -#diff "${ori}" "${mod}" +if command -v delta; then + delta -s "${ori}" "${mod}" +fi +diff "${ori}" "${mod}" # the end echo "test \"$(basename "$0")\" success" From ac79557d5de949a2f802addcb32a8ecf60f3dd1b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 12:08:39 +0100 Subject: [PATCH 31/36] verbosity --- tests-ng/compare.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 03ff812..9373eeb 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -55,6 +55,7 @@ ${bin} -B index -c --catalog="${catalog}" github .github echo "" # compare keys +echo "compare keys" src="tests-ng/assets/github.catalog.json" src_keys="${tmpd}/src-keys" dst_keys="${tmpd}/dst-keys" @@ -63,6 +64,7 @@ cat "${catalog}" | jq '.. | keys?' | jq '.[]' > "${dst_keys}" diff "${src_keys}" "${dst_keys}" # compare children 1 +echo "compare children 1" src_keys="${tmpd}/src-child1" dst_keys="${tmpd}/dst-child1" cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' > "${src_keys}" @@ -70,6 +72,7 @@ cat "${catalog}" | jq '. | select(.type=="top") | .children | .[].name' > "${dst diff "${src_keys}" "${dst_keys}" # compare children 2 +echo "compare children 2" src_keys="${tmpd}/src-child2" dst_keys="${tmpd}/dst-child2" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${src_keys}" @@ -77,6 +80,7 @@ cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name diff "${src_keys}" "${dst_keys}" # compare children 3 +echo "compare children 3" src_keys="${tmpd}/src-child3" dst_keys="${tmpd}/dst-child3" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${src_keys}" @@ -84,6 +88,7 @@ cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name diff "${src_keys}" "${dst_keys}" # native +echo "compare native output" native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" mod="${tmpd}/native.mod.txt" @@ -96,6 +101,7 @@ fi diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" # csv +echo "compare csv output" csv="${tmpd}/csv.txt" ${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" # modify created csv From 611ddd64a88e51e03ec75caa5b0e5e6770f5f14b Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 12:15:19 +0100 Subject: [PATCH 32/36] tests --- README.md | 2 +- tests-ng/compare.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 077523b..4e37aa1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/deadc0de6/catcli.svg?branch=master)](https://travis-ci.org/deadc0de6/catcli) [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](http://www.gnu.org/licenses/gpl-3.0) -[![Coverage Status](https://coveralls.io/repos/github/deadc0de6/catcli/badge.svg?branch=master)](https://coveralls.io/github/deadc0de6/catcli?branch=master) +[![Coveralls](https://img.shields.io/coveralls/github/deadc0de6/catcli)](https://coveralls.io/github/deadc0de6/catcli?branch=master) [![PyPI version](https://badge.fury.io/py/catcli.svg)](https://badge.fury.io/py/catcli) [![AUR](https://img.shields.io/aur/version/catcli-git.svg)](https://aur.archlinux.org/packages/catcli-git) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 9373eeb..090358b 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -61,7 +61,12 @@ src_keys="${tmpd}/src-keys" dst_keys="${tmpd}/dst-keys" cat "${src}" | jq '.. | keys?' | jq '.[]' > "${src_keys}" cat "${catalog}" | jq '.. | keys?' | jq '.[]' > "${dst_keys}" +echo "src:" +cat "${src_keys}" +echo "dst:" +cat "${dst_keys}" diff "${src_keys}" "${dst_keys}" +echo "ok!" # compare children 1 echo "compare children 1" @@ -69,7 +74,12 @@ src_keys="${tmpd}/src-child1" dst_keys="${tmpd}/dst-child1" cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' > "${src_keys}" cat "${catalog}" | jq '. | select(.type=="top") | .children | .[].name' > "${dst_keys}" +echo "src:" +cat "${src_keys}" +echo "dst:" +cat "${dst_keys}" diff "${src_keys}" "${dst_keys}" +echo "ok!" # compare children 2 echo "compare children 2" @@ -77,7 +87,12 @@ src_keys="${tmpd}/src-child2" dst_keys="${tmpd}/dst-child2" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${src_keys}" cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${dst_keys}" +echo "src:" +cat "${src_keys}" +echo "dst:" +cat "${dst_keys}" diff "${src_keys}" "${dst_keys}" +echo "ok!" # compare children 3 echo "compare children 3" @@ -85,7 +100,12 @@ src_keys="${tmpd}/src-child3" dst_keys="${tmpd}/dst-child3" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${src_keys}" cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${dst_keys}" +echo "src:" +cat "${src_keys}" +echo "dst:" +cat "${dst_keys}" diff "${src_keys}" "${dst_keys}" +echo "ok!" # native echo "compare native output" @@ -99,6 +119,7 @@ if command -v delta; then delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" fi diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" +echo "ok!" # csv echo "compare csv output" @@ -117,6 +138,7 @@ if command -v delta; then delta -s "${ori}" "${mod}" fi diff "${ori}" "${mod}" +echo "ok!" # the end echo "test \"$(basename "$0")\" success" From e50166810296fd4167228d1094ca850e3f898bc2 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 12:19:21 +0100 Subject: [PATCH 33/36] sort --- tests-ng/compare.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index 090358b..ba33d66 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -59,8 +59,8 @@ echo "compare keys" src="tests-ng/assets/github.catalog.json" src_keys="${tmpd}/src-keys" dst_keys="${tmpd}/dst-keys" -cat "${src}" | jq '.. | keys?' | jq '.[]' > "${src_keys}" -cat "${catalog}" | jq '.. | keys?' | jq '.[]' > "${dst_keys}" +cat "${src}" | jq '.. | keys?' | jq '.[]' | sort > "${src_keys}" +cat "${catalog}" | jq '.. | keys?' | jq '.[]' | sort > "${dst_keys}" echo "src:" cat "${src_keys}" echo "dst:" @@ -72,8 +72,8 @@ echo "ok!" echo "compare children 1" src_keys="${tmpd}/src-child1" dst_keys="${tmpd}/dst-child1" -cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' > "${src_keys}" -cat "${catalog}" | jq '. | select(.type=="top") | .children | .[].name' > "${dst_keys}" +cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' | sort > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[].name' | sort > "${dst_keys}" echo "src:" cat "${src_keys}" echo "dst:" @@ -85,8 +85,8 @@ echo "ok!" echo "compare children 2" src_keys="${tmpd}/src-child2" dst_keys="${tmpd}/dst-child2" -cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${src_keys}" -cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' > "${dst_keys}" +cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' | sort > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' | sort > "${dst_keys}" echo "src:" cat "${src_keys}" echo "dst:" @@ -98,8 +98,8 @@ echo "ok!" echo "compare children 3" src_keys="${tmpd}/src-child3" dst_keys="${tmpd}/dst-child3" -cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${src_keys}" -cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' > "${dst_keys}" +cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' | sort > "${src_keys}" +cat "${catalog}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' | sort > "${dst_keys}" echo "src:" cat "${src_keys}" echo "dst:" From 44a13b526fe81bea945f5f28a56f63f5adbe3be6 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 20:31:36 +0100 Subject: [PATCH 34/36] update readme --- README.md | 2 +- tests-ng/compare.sh | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4e37aa1..0129cf1 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ Each line contains the following fields: * **indexed_at**: when this entry was indexed * **maccess**: the entry modification date/time * **md5**: the entry checksum (if any) -* **nbfiles**: the number of children (empty for not storage or directory nodes) +* **nbfiles**: the number of children (empty for nodes that are not storage or directory) * **free_space**: free space (empty for not storage nodes) * **total_space**: total space (empty for not storage nodes) * **meta**: meta information (empty for not storage nodes) diff --git a/tests-ng/compare.sh b/tests-ng/compare.sh index ba33d66..32e383c 100755 --- a/tests-ng/compare.sh +++ b/tests-ng/compare.sh @@ -50,12 +50,14 @@ catalog="${tmpd}/catalog" # index ${bin} -B index -c --catalog="${catalog}" github .github +ls -laR .github +cat "${catalog}" #cat "${catalog}" echo "" # compare keys -echo "compare keys" +echo "[+] compare keys" src="tests-ng/assets/github.catalog.json" src_keys="${tmpd}/src-keys" dst_keys="${tmpd}/dst-keys" @@ -69,7 +71,7 @@ diff "${src_keys}" "${dst_keys}" echo "ok!" # compare children 1 -echo "compare children 1" +echo "[+] compare children 1" src_keys="${tmpd}/src-child1" dst_keys="${tmpd}/dst-child1" cat "${src}" | jq '. | select(.type=="top") | .children | .[].name' | sort > "${src_keys}" @@ -82,7 +84,7 @@ diff "${src_keys}" "${dst_keys}" echo "ok!" # compare children 2 -echo "compare children 2" +echo "[+] compare children 2" src_keys="${tmpd}/src-child2" dst_keys="${tmpd}/dst-child2" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[].name' | sort > "${src_keys}" @@ -95,7 +97,7 @@ diff "${src_keys}" "${dst_keys}" echo "ok!" # compare children 3 -echo "compare children 3" +echo "[+] compare children 3" src_keys="${tmpd}/src-child3" dst_keys="${tmpd}/dst-child3" cat "${src}" | jq '. | select(.type=="top") | .children | .[] | select(.name=="github") | .children | .[] | select(.name=="workflows") | .children | .[].name' | sort > "${src_keys}" @@ -108,21 +110,21 @@ diff "${src_keys}" "${dst_keys}" echo "ok!" # native -echo "compare native output" +echo "[+] compare native output" native="${tmpd}/native.txt" ${bin} -B ls -s -r --format=native --catalog="${catalog}" > "${native}" mod="${tmpd}/native.mod.txt" cat "${native}" | sed -e 's/free:.*%/free:0.0%/g' \ -e 's/date:....-..-.. ..:..:../date:2023-03-09 16:20:59/g' \ -e 's#du:[^|]* |#du:0/0 |#g' > "${mod}" -if command -v delta; then +if command -v delta >/dev/null; then delta -s "tests-ng/assets/github.catalog.native.txt" "${mod}" fi diff --color=always "tests-ng/assets/github.catalog.native.txt" "${mod}" echo "ok!" # csv -echo "compare csv output" +echo "[+] compare csv output" csv="${tmpd}/csv.txt" ${bin} -B ls -s -r --format=csv --catalog="${catalog}" > "${csv}" # modify created csv @@ -134,7 +136,7 @@ ori="${tmpd}/ori.mod.txt" cat "tests-ng/assets/github.catalog.csv.txt" | \ sed 's/....-..-.. ..:..:..//g' | \ sed 's/"2","[^"]*","[^"]*",""/"2","0","0",""/g' > "${ori}" -if command -v delta; then +if command -v delta >/dev/null; then delta -s "${ori}" "${mod}" fi diff "${ori}" "${mod}" From 49160f2b1d4433deeaf19852722da09d54b64c6f Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 20:57:35 +0100 Subject: [PATCH 35/36] fix recursive size bug --- catcli/noder.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/catcli/noder.py b/catcli/noder.py index 24e7b28..187d3d9 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -139,27 +139,28 @@ class Noder: @store: store the size in the node """ if node.type == nodes.TYPE_FILE: - self._debug(f'getting node size for \"{node.name}\"') + self._debug(f'size of {node.type} \"{node.name}\": {node.size}') return node.size msg = f'getting node size recursively for \"{node.name}\"' self._debug(msg) size: int = 0 for i in node.children: if node.type == nodes.TYPE_DIR: - size = self.rec_size(i, store=store) + sub_size = self.rec_size(i, store=store) if store: - i.size = size - size += size + i.size = sub_size + size += sub_size + continue if node.type == nodes.TYPE_STORAGE: - size = self.rec_size(i, store=store) + sub_size = self.rec_size(i, store=store) if store: - i.size = size - size += size - else: + i.size = sub_size + size += sub_size continue - self._debug(f'size of {node.name} is {size}') + self._debug(f'skipping {node.name}') if store: node.size = size + self._debug(f'size of {node.type} \"{node.name}\": {size}') return size ############################################################### From ee2cf80d9d23326b13eaeae80a7e30ec142b00d7 Mon Sep 17 00:00:00 2001 From: deadc0de6 Date: Sun, 12 Mar 2023 20:57:43 +0100 Subject: [PATCH 36/36] fix tests --- tests-ng/assets/github.catalog.csv.txt | 4 ++-- tests-ng/assets/github.catalog.json | 4 ++-- tests-ng/assets/github.catalog.native.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests-ng/assets/github.catalog.csv.txt b/tests-ng/assets/github.catalog.csv.txt index 40e6a28..9d7e3b4 100644 --- a/tests-ng/assets/github.catalog.csv.txt +++ b/tests-ng/assets/github.catalog.csv.txt @@ -1,5 +1,5 @@ -"github","storage","","3208","2023-03-09 16:20:59","","","2","0","0","" -"workflows","dir","github/workflows","1604","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" +"github","storage","","1510","2023-03-09 16:20:59","","","2","0","0","" +"workflows","dir","github/workflows","1493","2023-03-09 16:20:59","2023-03-09 16:20:44","","2","","","" "pypi-release.yml","file","github/workflows/pypi-release.yml","691","2023-03-09 16:20:59","2022-10-19 21:00:37","57699a7a6a03e20e864f220e19f8e197","","","","" "testing.yml","file","github/workflows/testing.yml","802","2023-03-09 16:20:59","2023-03-09 16:20:44","7144a119ef43adb634654522c12ec250","","","","" "FUNDING.yml","file","github/FUNDING.yml","17","2023-03-09 16:20:59","2022-10-19 21:00:37","0c6407a84d412c514007313fb3bca4de","","","","" diff --git a/tests-ng/assets/github.catalog.json b/tests-ng/assets/github.catalog.json index cf67e71..ff4c561 100644 --- a/tests-ng/assets/github.catalog.json +++ b/tests-ng/assets/github.catalog.json @@ -33,13 +33,13 @@ "maccess": 1678375244.4865956, "name": "workflows", "relpath": "/workflows", - "size": 1190, + "size": 1493, "type": "dir" } ], "free": 0, "name": "github", - "size": 2380, + "size": 1510, "total": 0, "ts": 1678375259, "type": "storage" diff --git a/tests-ng/assets/github.catalog.native.txt b/tests-ng/assets/github.catalog.native.txt index 9ca05fd..14a0d42 100644 --- a/tests-ng/assets/github.catalog.native.txt +++ b/tests-ng/assets/github.catalog.native.txt @@ -1,7 +1,7 @@ top └── storage: github - nbfiles:2 | totsize:3208 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 - ├── workflows [nbfiles:2, totsize:1604] + nbfiles:2 | totsize:1510 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59 + ├── workflows [nbfiles:2, totsize:1493] │ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197] │ └── testing.yml [size:802, md5:7144a119ef43adb634654522c12ec250] └── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de]