refactoring

catcli/catalog.py

@@ -7,12 +7,12 @@ Class that represents the catcli catalog
 
 import os
 import pickle
-from typing import Optional, Union, Any, cast
+from typing import Optional
 from anytree.exporter import JsonExporter  # type: ignore
 from anytree.importer import JsonImporter  # type: ignore
 
 # local imports
-from catcli.cnode import NodeMeta, NodeTop
+from catcli.nodes import NodeMeta, NodeTop
 from catcli.utils import ask
 from catcli.logger import Logger
 
@@ -88,32 +88,38 @@ class Catalog:
             return
         Logger.debug(text)
 
-    def _save_pickle(self, node: NodeTop) -> bool:
+    def _save_pickle(self, top: NodeTop) -> bool:
         """pickle the catalog"""
         with open(self.path, 'wb') as file:
-            pickle.dump(node, file)
+            pickle.dump(top, file)
         self._debug(f'Catalog saved to pickle \"{self.path}\"')
         return True
 
-    def _restore_pickle(self) -> Union[NodeTop, Any]:
+    def _restore_pickle(self) -> NodeTop:
         """restore the pickled tree"""
         with open(self.path, 'rb') as file:
             root = pickle.load(file)
         msg = f'Catalog imported from pickle \"{self.path}\"'
         self._debug(msg)
-        return root
+        top = NodeTop(root)
+        return top
 
-    def _save_json(self, node: NodeTop) -> bool:
+    def _save_json(self, top: NodeTop) -> bool:
         """export the catalog in json"""
+        Logger.debug(f'saving {top} to json...')
         exp = JsonExporter(indent=2, sort_keys=True)
         with open(self.path, 'w', encoding='UTF-8') as file:
-            exp.write(node, file)
+            exp.write(top, file)
         self._debug(f'Catalog saved to json \"{self.path}\"')
         return True
 
     def _restore_json(self, string: str) -> NodeTop:
         """restore the tree from json"""
         imp = JsonImporter()
+        Logger.debug(f'import from string: {string}')
         root = imp.import_(string)
         self._debug(f'Catalog imported from json \"{self.path}\"')
-        return cast(NodeTop, root)
+        top = NodeTop(root)
+        Logger.debug(f'top imported: {top}')
+        return top
+        # return cast(NodeTop, root)

catcli/catcli.py

@@ -15,9 +15,9 @@ from typing import Dict, Any, List
 from docopt import docopt
 
 # local imports
-from catcli import cnode
+from catcli import nodes
 from catcli.version import __version__ as VERSION
-from catcli.cnode import NodeTop, NodeAny
+from catcli.nodes import NodeTop, NodeAny
 from catcli.logger import Logger
 from catcli.colors import Colors
 from catcli.catalog import Catalog
@@ -44,8 +44,10 @@ USAGE = f"""
 
 Usage:
     {NAME} ls     [--catalog=<path>] [--format=<fmt>] [-aBCrVSs] [<path>]
-    {NAME} find   [--catalog=<path>] [--format=<fmt>] [-aBCbdVsP] [--path=<path>] [<term>]
-    {NAME} index  [--catalog=<path>] [--meta=<meta>...] [-aBCcfnV] <name> <path>
+    {NAME} find   [--catalog=<path>] [--format=<fmt>]
+                  [-aBCbdVsP] [--path=<path>] [<term>]
+    {NAME} index  [--catalog=<path>] [--meta=<meta>...]
+                  [-aBCcfnV] <name> <path>
     {NAME} update [--catalog=<path>] [-aBCcfnV] [--lpath=<path>] <name> <path>
     {NAME} mount  [--catalog=<path>] [-V] <mountpoint>
     {NAME} rm     [--catalog=<path>] [-BCfV] <storage>
@@ -66,14 +68,14 @@ Options:
     -C --no-color       Do not output colors [default: False].
     -c --hash           Calculate md5 hash [default: False].
     -d --directory      Only directory [default: False].
-    -F --format=<fmt>   Print format, see command \"print_supported_formats\" [default: native].
+    -F --format=<fmt>   see \"print_supported_formats\" [default: native].
     -f --force          Do not ask when updating the catalog [default: False].
     -l --lpath=<path>   Path where changes are logged [default: ]
     -n --no-subsize     Do not store size of directories [default: False].
     -P --parent         Ignore stored relpath [default: True].
     -p --path=<path>    Start path.
     -r --recursive      Recursive [default: False].
-    -s --raw-size       Print raw size rather than human readable [default: False].
+    -s --raw-size       Print raw size [default: False].
     -S --sortsize       Sort by size, largest first [default: False].
     -V --verbose        Be verbose [default: False].
     -v --version        Show version.
@@ -113,6 +115,8 @@ def cmd_index(args: Dict[str, Any],
             Logger.err('aborted')
             return
         node = noder.get_storage_node(top, name)
+        Logger.debug(f'top node: {top}')
+        Logger.debug(f'storage node: {node}')
         node.parent = None
 
     start = datetime.datetime.now()
@@ -170,7 +174,7 @@ def cmd_ls(args: Dict[str, Any],
     if not path.startswith(SEPARATOR):
         path = SEPARATOR + path
     # prepend with top node path
-    pre = f'{SEPARATOR}{cnode.NAME_TOP}'
+    pre = f'{SEPARATOR}{nodes.NAME_TOP}'
     if not path.startswith(pre):
         path = pre + path
     # ensure ends with a separator

catcli/fuser.py

@@ -14,8 +14,8 @@ import fuse  # type: ignore
 
 # local imports
 from catcli.noder import Noder
-from catcli.cnode import NodeTop, NodeAny
-from catcli import cnode
+from catcli.nodes import NodeTop, NodeAny
+from catcli import nodes
 
 
 # build custom logger to log in /tmp
@@ -58,7 +58,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations):  # type: ignore
 
     def _get_entry(self, path: str) -> Optional[NodeAny]:
         """return the node pointed by path"""
-        pre = f'{SEPARATOR}{cnode.NAME_TOP}'
+        pre = f'{SEPARATOR}{nodes.NAME_TOP}'
         if not path.startswith(pre):
             path = pre + path
         found = self.noder.list(self.top, path,
@@ -71,7 +71,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations):  # type: ignore
 
     def _get_entries(self, path: str) -> List[NodeAny]:
         """return nodes pointed by path"""
-        pre = f'{SEPARATOR}{cnode.NAME_TOP}'
+        pre = f'{SEPARATOR}{nodes.NAME_TOP}'
         if not path.startswith(pre):
             path = pre + path
         if not path.endswith(SEPARATOR):
@@ -91,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations):  # type: ignore
 
         curt = time()
         mode: Any = S_IFREG
-        if isinstance(entry, cnode.NodeArchived):
+        if isinstance(entry, nodes.NodeArchived):
             mode = S_IFREG
-        elif isinstance(entry, cnode.NodeDir):
+        elif isinstance(entry, nodes.NodeDir):
             mode = S_IFDIR
-        elif isinstance(entry, cnode.NodeFile):
+        elif isinstance(entry, nodes.NodeFile):
             mode = S_IFREG
-        elif isinstance(entry, cnode.NodeStorage):
+        elif isinstance(entry, nodes.NodeStorage):
             mode = S_IFDIR
-        elif isinstance(entry, cnode.NodeMeta):
+        elif isinstance(entry, nodes.NodeMeta):
             mode = S_IFREG
-        elif isinstance(entry, cnode.NodeTop):
+        elif isinstance(entry, nodes.NodeTop):
             mode = S_IFREG
         return {
             'st_mode': (mode),

catcli/logger.py

@@ -37,7 +37,7 @@ class Logger:
     def debug(cls: Type[CLASSTYPE],
               string: str) -> None:
         """to stderr no color"""
-        cls.stderr_nocolor(f'[DBG] {string}\n')
+        cls.stderr_nocolor(f'[DBG] {string}')
 
     @classmethod
     def info(cls: Type[CLASSTYPE],

catcli/noder.py

@@ -13,8 +13,8 @@ import anytree  # type: ignore
 from pyfzf.pyfzf import FzfPrompt  # type: ignore
 
 # local imports
-from catcli import cnode
-from catcli.cnode import NodeAny, NodeStorage, \
+from catcli import nodes
+from catcli.nodes import NodeAny, NodeStorage, \
     NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta
 from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars
 from catcli.logger import Logger
@@ -67,7 +67,7 @@ class Noder:
         """
         found = None
         for node in top.children:
-            if not isinstance(node, cnode.NodeStorage):
+            if not isinstance(node, nodes.NodeStorage):
                 continue
             if node.name == name:
                 found = node
@@ -138,25 +138,26 @@ class Noder:
         recursively traverse tree and return size
         @store: store the size in the node
         """
-        if isinstance(node, cnode.NodeFile):
+        if isinstance(node, nodes.NodeFile):
             self._debug(f'getting node size for \"{node.name}\"')
             return node.size
         msg = f'getting node size recursively for \"{node.name}\"'
         self._debug(msg)
         size: int = 0
         for i in node.children:
-            if isinstance(node, cnode.NodeDir):
+            if isinstance(node, nodes.NodeDir):
                 size = self.rec_size(i, store=store)
                 if store:
                     i.size = size
                 size += size
-            if isinstance(node, cnode.NodeStorage):
+            if isinstance(node, nodes.NodeStorage):
                 size = self.rec_size(i, store=store)
                 if store:
                     i.size = size
                 size += size
             else:
                 continue
+        self._debug(f'size of {node.name} is {size}')
         if store:
             node.size = size
         return size
@@ -188,7 +189,9 @@ class Noder:
     ###############################################################
     def new_top_node(self) -> NodeTop:
         """create a new top node"""
-        return NodeTop(cnode.NAME_TOP)
+        top = NodeTop(nodes.NAME_TOP)
+        self._debug(f'new top node: {top}')
+        return top
 
     def new_file_node(self, name: str, path: str,
                       parent: NodeAny, storagepath: str) -> Optional[NodeFile]:
@@ -251,7 +254,7 @@ class Noder:
                            total,
                            0,
                            epoch,
-                           attrs,
+                           self.attrs_to_string(attrs),
                            parent=parent)
 
     def new_archive_node(self, name: str, path: str,
@@ -272,19 +275,16 @@ class Noder:
             attrs: Dict[str, Any] = {}
             attrs['created'] = epoch
             attrs['created_version'] = VERSION
-            meta = NodeMeta(name=cnode.NAME_META,
-                            attr=self.attrs_to_string(attrs))
-        if meta.attr:
-            meta.attr += ', '
-        meta.attr += f'access={epoch}'
-        meta.attr += ', '
-        meta.attr += f'access_version={VERSION}'
+            meta = NodeMeta(name=nodes.NAME_META,
+                            attr=attrs)
+        meta.attr['access'] = epoch
+        meta.attr['access_version'] = VERSION
         return meta
 
     def _get_meta_node(self, top: NodeTop) -> Optional[NodeMeta]:
         """return the meta node if any"""
         try:
-            found = next(filter(lambda x: isinstance(x, cnode.NodeMeta),
+            found = next(filter(lambda x: isinstance(x, nodes.NodeMeta),
                          top.children))
             return cast(NodeMeta, found)
         except StopIteration:
@@ -294,7 +294,7 @@ class Noder:
         """remove any node not flagged and clean flags"""
         cnt = 0
         for node in anytree.PreOrderIter(top):
-            if not isinstance(node, (cnode.NodeDir, cnode.NodeFile)):
+            if not isinstance(node, (nodes.NodeDir, nodes.NodeFile)):
                 continue
             if self._clean(node):
                 cnt += 1
@@ -361,7 +361,7 @@ class Noder:
                 out.append(node.md5)  # md5
             else:
                 out.append('')  # fake md5
-            if isinstance(node, cnode.NodeDir):
+            if isinstance(node, nodes.NodeDir):
                 out.append(str(len(node.children)))  # nbfiles
             else:
                 out.append('')  # fake nbfiles
@@ -390,10 +390,10 @@ class Noder:
         @recalcparent: get relpath from tree instead of relpath field
         @raw: print raw size rather than human readable
         """
-        if isinstance(node, cnode.NodeTop):
+        if isinstance(node, nodes.NodeTop):
             # top node
             Logger.stdout_nocolor(f'{pre}{node.name}')
-        elif isinstance(node, cnode.NodeFile):
+        elif isinstance(node, nodes.NodeFile):
             # node of type file
             name = node.name
             if withpath:
@@ -413,7 +413,7 @@ class Noder:
                 content = Logger.get_bold_text(storage.name)
                 compl += f', storage:{content}'
             NodePrinter.print_file_native(pre, name, compl)
-        elif isinstance(node, cnode.NodeDir):
+        elif isinstance(node, nodes.NodeDir):
             # node of type directory
             name = node.name
             if withpath:
@@ -433,7 +433,7 @@ class Noder:
             if withstorage:
                 attr.append(('storage', Logger.get_bold_text(storage.name)))
             NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr)
-        elif isinstance(node, cnode.NodeStorage):
+        elif isinstance(node, nodes.NodeStorage):
             # node of type storage
             sztotal = size_to_str(node.total, raw=raw)
             szused = size_to_str(node.total - node.free, raw=raw)
@@ -463,7 +463,7 @@ class Noder:
                                              name,
                                              argsstring,
                                              node.attr)
-        elif isinstance(node, cnode.NodeArchived):
+        elif isinstance(node, nodes.NodeArchived):
             # archive node
             if self.arc:
                 NodePrinter.print_archive_native(pre, node.name, node.archive)
@@ -515,7 +515,7 @@ class Noder:
         @fmt: output format for selected nodes
         """
         rendered = anytree.RenderTree(node, childiter=self._sort_tree)
-        nodes = {}
+        the_nodes = {}
         # construct node names list
         for _, _, rend in rendered:
             if not rend:
@@ -523,17 +523,17 @@ class Noder:
             parents = self._get_parents(rend)
             storage = self._get_storage(rend)
             fullpath = os.path.join(storage.name, parents)
-            nodes[fullpath] = rend
+            the_nodes[fullpath] = rend
         # prompt with fzf
-        paths = self._fzf_prompt(nodes.keys())
+        paths = self._fzf_prompt(the_nodes.keys())
         # print the resulting tree
         subfmt = fmt.replace('fzf-', '')
         for path in paths:
             if not path:
                 continue
-            if path not in nodes:
+            if path not in the_nodes:
                 continue
-            rend = nodes[path]
+            rend = the_nodes[path]
             self.print_tree(rend, fmt=subfmt)
 
     @staticmethod
@@ -626,16 +626,16 @@ class Noder:
     def _callback_find_name(self, term: str, only_dir: bool) -> Any:
         """callback for finding files"""
         def find_name(node: NodeAny) -> bool:
-            if isinstance(node, cnode.NodeStorage):
+            if isinstance(node, nodes.NodeStorage):
                 # ignore storage nodes
                 return False
-            if isinstance(node, cnode.NodeTop):
+            if isinstance(node, nodes.NodeTop):
                 # ignore top nodes
                 return False
-            if isinstance(node, cnode.NodeMeta):
+            if isinstance(node, nodes.NodeMeta):
                 # ignore meta nodes
                 return False
-            if only_dir and isinstance(node, cnode.NodeDir):
+            if only_dir and isinstance(node, nodes.NodeDir):
                 # ignore non directory
                 return False
 
@@ -773,7 +773,7 @@ class Noder:
 
     def _get_storage(self, node: NodeAny) -> NodeStorage:
         """recursively traverse up to find storage"""
-        if isinstance(node, cnode.NodeStorage):
+        if isinstance(node, nodes.NodeStorage):
             return node
         return cast(NodeStorage, node.ancestors[1])
 
@@ -784,9 +784,9 @@ class Noder:
 
     def _get_parents(self, node: NodeAny) -> str:
         """get all parents recursively"""
-        if isinstance(node, cnode.NodeStorage):
+        if isinstance(node, nodes.NodeStorage):
             return ''
-        if isinstance(node, cnode.NodeTop):
+        if isinstance(node, nodes.NodeTop):
             return ''
         parent = self._get_parents(node.parent)
         if parent:

catcli/nodes.py

@@ -10,7 +10,6 @@ from typing import Dict, Any
 from anytree import NodeMixin  # type: ignore
 
 
-_TYPE_BAD = 'badtype'
 _TYPE_TOP = 'top'
 _TYPE_FILE = 'file'
 _TYPE_DIR = 'dir'
@@ -30,11 +29,21 @@ class NodeAny(NodeMixin):  # type: ignore
                  children=None):
         """build generic node"""
         super().__init__()
-        self._flagged = False
         self.parent = parent
         if children:
             self.children = children
 
+    def _to_str(self) -> str:
+        ret = str(self.__class__) + ": " + str(self.__dict__)
+        if self.children:
+            ret += '\n'
+        for child in self.children:
+            ret += '  child => ' + str(child)
+        return ret
+
+    def __str__(self) -> str:
+        return self._to_str()
+
     def flagged(self) -> bool:
         """is flagged"""
         if not hasattr(self, '_flagged'):
@@ -43,12 +52,12 @@ class NodeAny(NodeMixin):  # type: ignore
 
     def flag(self) -> None:
         """flag a node"""
-        self._flagged = True
+        self._flagged = True  # pylint: disable=W0201
 
     def unflag(self) -> None:
         """unflag node"""
-        self._flagged = False
-        del self._flagged
+        self._flagged = False  # pylint: disable=W0201
+        delattr(self, '_flagged')
 
 
 class NodeTop(NodeAny):
@@ -65,6 +74,9 @@ class NodeTop(NodeAny):
         if children:
             self.children = children
 
+    def __str__(self) -> str:
+        return self._to_str()
+
 
 class NodeFile(NodeAny):
     """a file node"""
@@ -89,6 +101,9 @@ class NodeFile(NodeAny):
         if children:
             self.children = children
 
+    def __str__(self) -> str:
+        return self._to_str()
+
 
 class NodeDir(NodeAny):
     """a directory node"""
@@ -111,6 +126,9 @@ class NodeDir(NodeAny):
         if children:
             self.children = children
 
+    def __str__(self) -> str:
+        return self._to_str()
+
 
 class NodeArchived(NodeAny):
     """an archived node"""
@@ -135,6 +153,9 @@ class NodeArchived(NodeAny):
         if children:
             self.children = children
 
+    def __str__(self) -> str:
+        return self._to_str()
+
 
 class NodeStorage(NodeAny):
     """a storage node"""
@@ -144,8 +165,8 @@ class NodeStorage(NodeAny):
                  free: int,
                  total: int,
                  size: int,
-                 indexed_dt: float,
-                 attr: Dict[str, Any],
+                 ts: float,
+                 attr: str,
                  parent=None,
                  children=None):
         """build a storage node"""
@@ -156,18 +177,21 @@ class NodeStorage(NodeAny):
         self.total = total
         self.attr = attr
         self.size = size
-        self.indexed_dt = indexed_dt
+        self.ts = ts
         self.parent = parent
         if children:
             self.children = children
 
+    def __str__(self) -> str:
+        return self._to_str()
+
 
 class NodeMeta(NodeAny):
     """a meta node"""
 
     def __init__(self,  # type: ignore[no-untyped-def]
                  name: str,
-                 attr: str,
+                 attr: Dict[str, Any],
                  parent=None,
                  children=None):
         """build a meta node"""
@@ -178,3 +202,6 @@ class NodeMeta(NodeAny):
         self.parent = parent
         if children:
             self.children = children
+
+    def __str__(self) -> str:
+        return self._to_str()

catcli/walker.py

@@ -11,7 +11,7 @@ from typing import Tuple, Optional
 # local imports
 from catcli.noder import Noder
 from catcli.logger import Logger
-from catcli.cnode import NodeAny, NodeTop
+from catcli.nodes import NodeAny, NodeTop
 
 
 class Walker: