Archives
/
catcli
mirror of https://github.com/deadc0de6/catcli
2
1
Fork 0
Code Issues Releases Wiki Activity

refactor node types

Browse Source
pull/30/head
deadc0de6 1 year ago
parent 45c2599a95
commit 45d3b096f3
7 changed files with 284 additions and 189 deletions
Show Stats Download Patch File Download Diff File

20
catcli/catalog.py
Unescape Escape View File

@ -12,7 +12,7 @@ from anytree.exporter import JsonExporter # type: ignore
from anytree.importer import JsonImporter # type: ignore from anytree.importer import JsonImporter # type: ignore
# local imports # local imports
from catcli.cnode import Node from catcli.cnode import NodeMeta, NodeTop
from catcli.utils import ask from catcli.utils import ask
from catcli.logger import Logger from catcli.logger import Logger
@ -33,10 +33,10 @@ class Catalog:
self.path = path self.path = path
self.debug = debug self.debug = debug
self.force = force self.force = force
self.metanode: Optional[Node] = None self.metanode: Optional[NodeMeta] = None
self.pickle = usepickle self.pickle = usepickle
def set_metanode(self, metanode: Node) -> None: def set_metanode(self, metanode: NodeMeta) -> None:
"""remove the metanode until tree is re-written""" """remove the metanode until tree is re-written"""
self.metanode = metanode self.metanode = metanode
if self.metanode: if self.metanode:
@ -50,7 +50,7 @@ class Catalog:
return True return True
return False return False
def restore(self) -> Optional[Node]: def restore(self) -> Optional[NodeTop]:
"""restore the catalog""" """restore the catalog"""
if not self.path: if not self.path:
return None return None
@ -62,7 +62,7 @@ class Catalog:
content = file.read() content = file.read()
return self._restore_json(content) return self._restore_json(content)
def save(self, node: Node) -> bool: def save(self, node: NodeTop) -> bool:
"""save the catalog""" """save the catalog"""
if not self.path: if not self.path:
Logger.err('Path not defined') Logger.err('Path not defined')
@ -88,14 +88,14 @@ class Catalog:
return return
Logger.debug(text) Logger.debug(text)
def _save_pickle(self, node: Node) -> bool: def _save_pickle(self, node: NodeTop) -> bool:
"""pickle the catalog""" """pickle the catalog"""
with open(self.path, 'wb') as file: with open(self.path, 'wb') as file:
pickle.dump(node, file) pickle.dump(node, file)
self._debug(f'Catalog saved to pickle \"{self.path}\"') self._debug(f'Catalog saved to pickle \"{self.path}\"')
return True return True
def _restore_pickle(self) -> Union[Node, Any]: def _restore_pickle(self) -> Union[NodeTop, Any]:
"""restore the pickled tree""" """restore the pickled tree"""
with open(self.path, 'rb') as file: with open(self.path, 'rb') as file:
root = pickle.load(file) root = pickle.load(file)
@ -103,7 +103,7 @@ class Catalog:
self._debug(msg) self._debug(msg)
return root return root
def _save_json(self, node: Node) -> bool: def _save_json(self, node: NodeTop) -> bool:
"""export the catalog in json""" """export the catalog in json"""
exp = JsonExporter(indent=2, sort_keys=True) exp = JsonExporter(indent=2, sort_keys=True)
with open(self.path, 'w', encoding='UTF-8') as file: with open(self.path, 'w', encoding='UTF-8') as file:
@ -111,9 +111,9 @@ class Catalog:
self._debug(f'Catalog saved to json \"{self.path}\"') self._debug(f'Catalog saved to json \"{self.path}\"')
return True return True
def _restore_json(self, string: str) -> Node: def _restore_json(self, string: str) -> NodeTop:
"""restore the tree from json""" """restore the tree from json"""
imp = JsonImporter() imp = JsonImporter()
root = imp.import_(string) root = imp.import_(string)
self._debug(f'Catalog imported from json \"{self.path}\"') self._debug(f'Catalog imported from json \"{self.path}\"')
return cast(Node, root) return cast(NodeTop, root)

24
catcli/catcli.py
Unescape Escape View File

@ -17,11 +17,11 @@ from docopt import docopt
# local imports # local imports
from catcli import cnode from catcli import cnode
from catcli.version import __version__ as VERSION from catcli.version import __version__ as VERSION
from catcli.cnode import NodeTop, NodeAny
from catcli.logger import Logger from catcli.logger import Logger
from catcli.colors import Colors from catcli.colors import Colors
from catcli.catalog import Catalog from catcli.catalog import Catalog
from catcli.walker import Walker from catcli.walker import Walker
from catcli.cnode import Node
from catcli.noder import Noder from catcli.noder import Noder
from catcli.utils import ask, edit from catcli.utils import ask, edit
from catcli.fuser import Fuser from catcli.fuser import Fuser
@ -82,7 +82,7 @@ Options:
def cmd_mount(args: Dict[str, Any], def cmd_mount(args: Dict[str, Any],
top: Node, top: NodeTop,
noder: Noder) -> None: noder: Noder) -> None:
"""mount action""" """mount action"""
mountpoint = args['<mountpoint>'] mountpoint = args['<mountpoint>']
@ -94,7 +94,7 @@ def cmd_mount(args: Dict[str, Any],
def cmd_index(args: Dict[str, Any], def cmd_index(args: Dict[str, Any],
noder: Noder, noder: Noder,
catalog: Catalog, catalog: Catalog,
top: Node) -> None: top: NodeTop) -> None:
"""index action""" """index action"""
path = args['<path>'] path = args['<path>']
name = args['<name>'] name = args['<name>']
@ -117,8 +117,8 @@ def cmd_index(args: Dict[str, Any],
start = datetime.datetime.now() start = datetime.datetime.now()
walker = Walker(noder, usehash=usehash, debug=debug) walker = Walker(noder, usehash=usehash, debug=debug)
attr = noder.attrs_to_string(args['--meta']) attr = args['--meta']
root = noder.new_storage_node(name, path, parent=top, attrs=attr) root = noder.new_storage_node(name, path, top, attr)
_, cnt = walker.index(path, root, name) _, cnt = walker.index(path, root, name)
if subsize: if subsize:
noder.rec_size(root, store=True) noder.rec_size(root, store=True)
@ -132,7 +132,7 @@ def cmd_index(args: Dict[str, Any],
def cmd_update(args: Dict[str, Any], def cmd_update(args: Dict[str, Any],
noder: Noder, noder: Noder,
catalog: Catalog, catalog: Catalog,
top: Node) -> None: top: NodeTop) -> None:
"""update action""" """update action"""
path = args['<path>'] path = args['<path>']
name = args['<name>'] name = args['<name>']
@ -162,7 +162,7 @@ def cmd_update(args: Dict[str, Any],
def cmd_ls(args: Dict[str, Any], def cmd_ls(args: Dict[str, Any],
noder: Noder, noder: Noder,
top: Node) -> List[Node]: top: NodeTop) -> List[NodeAny]:
"""ls action""" """ls action"""
path = args['<path>'] path = args['<path>']
if not path: if not path:
@ -197,7 +197,7 @@ def cmd_ls(args: Dict[str, Any],
def cmd_rm(args: Dict[str, Any], def cmd_rm(args: Dict[str, Any],
noder: Noder, noder: Noder,
catalog: Catalog, catalog: Catalog,
top: Node) -> Node: top: NodeTop) -> NodeTop:
"""rm action""" """rm action"""
name = args['<storage>'] name = args['<storage>']
node = noder.get_storage_node(top, name) node = noder.get_storage_node(top, name)
@ -212,7 +212,7 @@ def cmd_rm(args: Dict[str, Any],
def cmd_find(args: Dict[str, Any], def cmd_find(args: Dict[str, Any],
noder: Noder, noder: Noder,
top: Node) -> List[Node]: top: NodeTop) -> List[NodeAny]:
"""find action""" """find action"""
fromtree = args['--parent'] fromtree = args['--parent']
directory = args['--directory'] directory = args['--directory']
@ -232,7 +232,7 @@ def cmd_find(args: Dict[str, Any],
def cmd_graph(args: Dict[str, Any], def cmd_graph(args: Dict[str, Any],
noder: Noder, noder: Noder,
top: Node) -> None: top: NodeTop) -> None:
"""graph action""" """graph action"""
path = args['<path>'] path = args['<path>']
if not path: if not path:
@ -243,7 +243,7 @@ def cmd_graph(args: Dict[str, Any],
def cmd_rename(args: Dict[str, Any], def cmd_rename(args: Dict[str, Any],
catalog: Catalog, catalog: Catalog,
top: Node) -> None: top: NodeTop) -> None:
"""rename action""" """rename action"""
storage = args['<storage>'] storage = args['<storage>']
new = args['<name>'] new = args['<name>']
@ -261,7 +261,7 @@ def cmd_rename(args: Dict[str, Any],
def cmd_edit(args: Dict[str, Any], def cmd_edit(args: Dict[str, Any],
noder: Noder, noder: Noder,
catalog: Catalog, catalog: Catalog,
top: Node) -> None: top: NodeTop) -> None:
"""edit action""" """edit action"""
storage = args['<storage>'] storage = args['<storage>']
storages = list(x.name for x in top.children) storages = list(x.name for x in top.children)

176
catcli/cnode.py
Unescape Escape View File

@ -6,57 +6,39 @@ Class that represents a node in the catalog tree
""" """
# pylint: disable=W0622 # pylint: disable=W0622
from typing import Dict, Any
from anytree import NodeMixin # type: ignore from anytree import NodeMixin # type: ignore
TYPE_TOP = 'top' _TYPE_BAD = 'badtype'
TYPE_FILE = 'file' _TYPE_TOP = 'top'
TYPE_DIR = 'dir' _TYPE_FILE = 'file'
TYPE_ARC = 'arc' _TYPE_DIR = 'dir'
TYPE_STORAGE = 'storage' _TYPE_ARC = 'arc'
TYPE_META = 'meta' _TYPE_STORAGE = 'storage'
_TYPE_META = 'meta'
NAME_TOP = 'top' NAME_TOP = 'top'
NAME_META = 'meta' NAME_META = 'meta'
class Node(NodeMixin): # type: ignore class NodeAny(NodeMixin): # type: ignore
"""a node in the catalog""" """generic node"""
def __init__(self, # type: ignore[no-untyped-def] def __init__(self, # type: ignore[no-untyped-def]
name: str,
type: str,
size: float = 0,
relpath: str = '',
md5: str = '',
maccess: float = 0,
free: int = 0,
total: int = 0,
indexed_dt: int = 0,
attr: str = '',
archive: str = '',
parent=None, parent=None,
children=None): children=None):
"""build a node""" """build generic node"""
super().__init__() super().__init__()
self.name = name self._flagged = False
self.type = type
self.size = size
self.relpath = relpath
self.md5 = md5
self.maccess = maccess
self.free = free
self.total = total
self.indexed_dt = indexed_dt
self.attr = attr
self.archive = archive
self.parent = parent self.parent = parent
if children: if children:
self.children = children self.children = children
self._flagged = False
def flagged(self) -> bool: def flagged(self) -> bool:
"""is flagged""" """is flagged"""
if not hasattr(self, '_flagged'):
return False
return self._flagged return self._flagged
def flag(self) -> None: def flag(self) -> None:
@ -66,3 +48,133 @@ class Node(NodeMixin): # type: ignore
def unflag(self) -> None: def unflag(self) -> None:
"""unflag node""" """unflag node"""
self._flagged = False self._flagged = False
del self._flagged
class NodeTop(NodeAny):
"""a top node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
children=None):
"""build a top node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_TOP
self.parent = None
if children:
self.children = children
class NodeFile(NodeAny):
"""a file node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
relpath: str,
size: int,
md5: str,
maccess: float,
parent=None,
children=None):
"""build a file node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_FILE
self.relpath = relpath
self.size = size
self.md5 = md5
self.maccess = maccess
self.parent = parent
if children:
self.children = children
class NodeDir(NodeAny):
"""a directory node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
relpath: str,
size: int,
maccess: float,
parent=None,
children=None):
"""build a directory node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_DIR
self.relpath = relpath
self.size = size
self.maccess = maccess
self.parent = parent
if children:
self.children = children
class NodeArchived(NodeAny):
"""an archived node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
relpath: str,
size: int,
md5: str,
archive: str,
parent=None,
children=None):
"""build an archived node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_ARC
self.relpath = relpath
self.size = size
self.md5 = md5
self.archive = archive
self.parent = parent
if children:
self.children = children
class NodeStorage(NodeAny):
"""a storage node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
free: int,
total: int,
size: int,
indexed_dt: float,
attr: Dict[str, Any],
parent=None,
children=None):
"""build a storage node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_STORAGE
self.free = free
self.total = total
self.attr = attr
self.size = size
self.indexed_dt = indexed_dt
self.parent = parent
if children:
self.children = children
class NodeMeta(NodeAny):
"""a meta node"""
def __init__(self, # type: ignore[no-untyped-def]
name: str,
attr: str,
parent=None,
children=None):
"""build a meta node"""
super().__init__() # type: ignore[no-untyped-call]
self.name = name
self.type = _TYPE_META
self.attr = attr
self.parent = parent
if children:
self.children = children

22
catcli/fuser.py
Unescape Escape View File

@ -14,8 +14,8 @@ import fuse # type: ignore
# local imports # local imports
from catcli.noder import Noder from catcli.noder import Noder
from catcli.cnode import NodeTop, NodeAny
from catcli import cnode from catcli import cnode
from catcli.cnode import Node
# build custom logger to log in /tmp # build custom logger to log in /tmp
@ -34,7 +34,7 @@ class Fuser:
"""fuse filesystem mounter""" """fuse filesystem mounter"""
def __init__(self, mountpoint: str, def __init__(self, mountpoint: str,
top: Node, top: NodeTop,
noder: Noder, noder: Noder,
debug: bool = False): debug: bool = False):
"""fuse filesystem""" """fuse filesystem"""
@ -50,13 +50,13 @@ class Fuser:
class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore
"""in-memory filesystem for catcli catalog""" """in-memory filesystem for catcli catalog"""
def __init__(self, top: Node, def __init__(self, top: NodeTop,
noder: Noder): noder: Noder):
"""init fuse filesystem""" """init fuse filesystem"""
self.top = top self.top = top
self.noder = noder self.noder = noder
def _get_entry(self, path: str) -> Optional[Node]: def _get_entry(self, path: str) -> Optional[NodeAny]:
"""return the node pointed by path""" """return the node pointed by path"""
pre = f'{SEPARATOR}{cnode.NAME_TOP}' pre = f'{SEPARATOR}{cnode.NAME_TOP}'
if not path.startswith(pre): if not path.startswith(pre):
@ -69,7 +69,7 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore
return found[0] return found[0]
return None return None
def _get_entries(self, path: str) -> List[Node]: def _get_entries(self, path: str) -> List[NodeAny]:
"""return nodes pointed by path""" """return nodes pointed by path"""
pre = f'{SEPARATOR}{cnode.NAME_TOP}' pre = f'{SEPARATOR}{cnode.NAME_TOP}'
if not path.startswith(pre): if not path.startswith(pre):
@ -91,17 +91,17 @@ class CatcliFilesystem(fuse.LoggingMixIn, fuse.Operations): # type: ignore
curt = time() curt = time()
mode: Any = S_IFREG mode: Any = S_IFREG
if entry.type == cnode.TYPE_ARC: if isinstance(entry, cnode.NodeArchived):
mode = S_IFREG mode = S_IFREG
elif entry.type == cnode.TYPE_DIR: elif isinstance(entry, cnode.NodeDir):
mode = S_IFDIR mode = S_IFDIR
elif entry.type == cnode.TYPE_FILE: elif isinstance(entry, cnode.NodeFile):
mode = S_IFREG mode = S_IFREG
elif entry.type == cnode.TYPE_STORAGE: elif isinstance(entry, cnode.NodeStorage):
mode = S_IFDIR mode = S_IFDIR
elif entry.type == cnode.TYPE_META: elif isinstance(entry, cnode.NodeMeta):
mode = S_IFREG mode = S_IFREG
elif entry.type == cnode.TYPE_TOP: elif isinstance(entry, cnode.NodeTop):
mode = S_IFREG mode = S_IFREG
return { return {
'st_mode': (mode), 'st_mode': (mode),

5
catcli/nodeprinter.py
Unescape Escape View File

@ -6,7 +6,8 @@ Class for printing nodes
""" """
import sys import sys
from typing import TypeVar, Type, Optional, Tuple, List from typing import TypeVar, Type, Optional, Tuple, List, \
Dict, Any
from catcli.colors import Colors from catcli.colors import Colors
from catcli.utils import fix_badchars from catcli.utils import fix_badchars
@ -25,7 +26,7 @@ class NodePrinter:
@classmethod @classmethod
def print_storage_native(cls: Type[CLASSTYPE], pre: str, def print_storage_native(cls: Type[CLASSTYPE], pre: str,
name: str, args: str, name: str, args: str,
attr: str) -> None: attr: Dict[str, Any]) -> None:
"""print a storage node""" """print a storage node"""
end = '' end = ''
if attr: if attr:

212
catcli/noder.py
Unescape Escape View File

@ -14,7 +14,8 @@ from pyfzf.pyfzf import FzfPrompt # type: ignore
# local imports # local imports
from catcli import cnode from catcli import cnode
from catcli.cnode import Node from catcli.cnode import NodeAny, NodeStorage, \
NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta
from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars
from catcli.logger import Logger from catcli.logger import Logger
from catcli.nodeprinter import NodePrinter from catcli.nodeprinter import NodePrinter
@ -53,20 +54,20 @@ class Noder:
self.decomp = Decomp() self.decomp = Decomp()
@staticmethod @staticmethod
def get_storage_names(top: Node) -> List[str]: def get_storage_names(top: NodeTop) -> List[str]:
"""return a list of all storage names""" """return a list of all storage names"""
return [x.name for x in list(top.children)] return [x.name for x in list(top.children)]
def get_storage_node(self, top: Node, def get_storage_node(self, top: NodeTop,
name: str, name: str,
newpath: str = '') -> Node: newpath: str = '') -> NodeStorage:
""" """
return the storage node if any return the storage node if any
if newpath is submitted, it will update the media info if newpath is submitted, it will update the media info
""" """
found = None found = None
for node in top.children: for node in top.children:
if node.type != cnode.TYPE_STORAGE: if not isinstance(node, cnode.NodeStorage):
continue continue
if node.name == name: if node.name == name:
found = node found = node
@ -75,27 +76,27 @@ class Noder:
found.free = shutil.disk_usage(newpath).free found.free = shutil.disk_usage(newpath).free
found.total = shutil.disk_usage(newpath).total found.total = shutil.disk_usage(newpath).total
found.ts = int(time.time()) found.ts = int(time.time())
return cast(Node, found) return cast(NodeStorage, found)
@staticmethod @staticmethod
def get_node(top: Node, def get_node(top: NodeTop,
path: str, path: str,
quiet: bool = False) -> Optional[Node]: quiet: bool = False) -> Optional[NodeAny]:
"""get the node by internal tree path""" """get the node by internal tree path"""
resolv = anytree.resolver.Resolver('name') resolv = anytree.resolver.Resolver('name')
try: try:
bpath = os.path.basename(path) bpath = os.path.basename(path)
the_node = resolv.get(top, bpath) the_node = resolv.get(top, bpath)
return cast(Node, the_node) return cast(NodeAny, the_node)
except anytree.resolver.ChildResolverError: except anytree.resolver.ChildResolverError:
if not quiet: if not quiet:
Logger.err(f'No node at path \"{bpath}\"') Logger.err(f'No node at path \"{bpath}\"')
return None return None
def get_node_if_changed(self, def get_node_if_changed(self,
top: Node, top: NodeTop,
path: str, path: str,
treepath: str) -> Tuple[Optional[Node], bool]: treepath: str) -> Tuple[Optional[NodeAny], bool]:
""" """
return the node (if any) and if it has changed return the node (if any) and if it has changed
@top: top node (storage) @top: top node (storage)
@ -131,25 +132,25 @@ class Noder:
self._debug(f'\tchange: no change for \"{path}\"') self._debug(f'\tchange: no change for \"{path}\"')
return node, False return node, False
def rec_size(self, node: Node, def rec_size(self, node: Union[NodeDir, NodeStorage],
store: bool = True) -> float: store: bool = True) -> int:
""" """
recursively traverse tree and return size recursively traverse tree and return size
@store: store the size in the node @store: store the size in the node
""" """
if node.type == cnode.TYPE_FILE: if isinstance(node, cnode.NodeFile):
self._debug(f'getting node size for \"{node.name}\"') self._debug(f'getting node size for \"{node.name}\"')
return float(node.size) return node.size
msg = f'getting node size recursively for \"{node.name}\"' msg = f'getting node size recursively for \"{node.name}\"'
self._debug(msg) self._debug(msg)
size: float = 0 size: int = 0
for i in node.children: for i in node.children:
if node.type == cnode.TYPE_DIR: if isinstance(node, cnode.NodeDir):
size = self.rec_size(i, store=store) size = self.rec_size(i, store=store)
if store: if store:
i.size = size i.size = size
size += size size += size
if node.type == cnode.TYPE_STORAGE: if isinstance(node, cnode.NodeStorage):
size = self.rec_size(i, store=store) size = self.rec_size(i, store=store)
if store: if store:
i.size = size i.size = size
@ -185,13 +186,12 @@ class Noder:
############################################################### ###############################################################
# node creation # node creation
############################################################### ###############################################################
def new_top_node(self) -> Node: def new_top_node(self) -> NodeTop:
"""create a new top node""" """create a new top node"""
return Node(cnode.NAME_TOP, return NodeTop(cnode.NAME_TOP)
cnode.TYPE_TOP)
def new_file_node(self, name: str, path: str, def new_file_node(self, name: str, path: str,
parent: Node, storagepath: str) -> Optional[Node]: parent: NodeAny, storagepath: str) -> Optional[NodeFile]:
"""create a new node representing a file""" """create a new node representing a file"""
if not os.path.exists(path): if not os.path.exists(path):
Logger.err(f'File \"{path}\" does not exist') Logger.err(f'File \"{path}\" does not exist')
@ -208,11 +208,12 @@ class Noder:
relpath = os.sep.join([storagepath, name]) relpath = os.sep.join([storagepath, name])
maccess = os.path.getmtime(path) maccess = os.path.getmtime(path)
node = self._new_generic_node(name, cnode.TYPE_FILE, node = NodeFile(name,
relpath, parent, relpath,
size=stat.st_size, stat.st_size,
md5=md5, md5,
maccess=maccess) maccess,
parent=parent)
if self.arc: if self.arc:
ext = os.path.splitext(path)[1][1:] ext = os.path.splitext(path)[1][1:]
if ext.lower() in self.decomp.get_formats(): if ext.lower() in self.decomp.get_formats():
@ -224,59 +225,46 @@ class Noder:
return node return node
def new_dir_node(self, name: str, path: str, def new_dir_node(self, name: str, path: str,
parent: Node, storagepath: str) -> Node: parent: NodeAny, storagepath: str) -> NodeDir:
"""create a new node representing a directory""" """create a new node representing a directory"""
path = os.path.abspath(path) path = os.path.abspath(path)
relpath = os.sep.join([storagepath, name]) relpath = os.sep.join([storagepath, name])
maccess = os.path.getmtime(path) maccess = os.path.getmtime(path)
return self._new_generic_node(name, cnode.TYPE_DIR, relpath, return NodeDir(name,
parent, maccess=maccess) relpath,
0,
maccess,
parent=parent)
def new_storage_node(self, name: str, def new_storage_node(self, name: str,
path: str, path: str,
parent: str, parent: str,
attrs: str = '') -> Node: attrs: Dict[str, Any]) \
-> NodeStorage:
"""create a new node representing a storage""" """create a new node representing a storage"""
path = os.path.abspath(path) path = os.path.abspath(path)
free = shutil.disk_usage(path).free free = shutil.disk_usage(path).free
total = shutil.disk_usage(path).total total = shutil.disk_usage(path).total
epoch = int(time.time()) epoch = int(time.time())
return Node(name=name, return NodeStorage(name,
type=cnode.TYPE_STORAGE, free,
free=free, total,
total=total, 0,
parent=parent, epoch,
attr=attrs, attrs,
indexed_dt=epoch) parent=parent)
def new_archive_node(self, name: str, path: str, def new_archive_node(self, name: str, path: str,
parent: str, archive: str) -> Node: parent: str, archive: str) -> NodeArchived:
"""create a new node for archive data""" """create a new node for archive data"""
return Node(name=name, type=cnode.TYPE_ARC, relpath=path, return NodeArchived(name=name, relpath=path,
parent=parent, size=0, md5='', parent=parent, size=0, md5='',
archive=archive) archive=archive)
@staticmethod
def _new_generic_node(name: str,
nodetype: str,
relpath: str,
parent: Node,
size: float = 0,
md5: str = '',
maccess: float = 0) -> Node:
"""generic node creation"""
return Node(name,
nodetype,
size=size,
relpath=relpath,
md5=md5,
maccess=maccess,
parent=parent)
############################################################### ###############################################################
# node management # node management
############################################################### ###############################################################
def update_metanode(self, top: Node) -> Node: def update_metanode(self, top: NodeTop) -> NodeMeta:
"""create or update meta node information""" """create or update meta node information"""
meta = self._get_meta_node(top) meta = self._get_meta_node(top)
epoch = int(time.time()) epoch = int(time.time())
@ -284,9 +272,8 @@ class Noder:
attrs: Dict[str, Any] = {} attrs: Dict[str, Any] = {}
attrs['created'] = epoch attrs['created'] = epoch
attrs['created_version'] = VERSION attrs['created_version'] = VERSION
meta = Node(name=cnode.NAME_META, meta = NodeMeta(name=cnode.NAME_META,
type=cnode.TYPE_META, attr=self.attrs_to_string(attrs))
attr=self.attrs_to_string(attrs))
if meta.attr: if meta.attr:
meta.attr += ', ' meta.attr += ', '
meta.attr += f'access={epoch}' meta.attr += f'access={epoch}'
@ -294,26 +281,26 @@ class Noder:
meta.attr += f'access_version={VERSION}' meta.attr += f'access_version={VERSION}'
return meta return meta
def _get_meta_node(self, top: Node) -> Optional[Node]: def _get_meta_node(self, top: NodeTop) -> Optional[NodeMeta]:
"""return the meta node if any""" """return the meta node if any"""
try: try:
found = next(filter(lambda x: x.type == cnode.TYPE_META, found = next(filter(lambda x: isinstance(x, cnode.NodeMeta),
top.children)) top.children))
return cast(Node, found) return cast(NodeMeta, found)
except StopIteration: except StopIteration:
return None return None
def clean_not_flagged(self, top: Node) -> int: def clean_not_flagged(self, top: NodeTop) -> int:
"""remove any node not flagged and clean flags""" """remove any node not flagged and clean flags"""
cnt = 0 cnt = 0
for node in anytree.PreOrderIter(top): for node in anytree.PreOrderIter(top):
if node.type not in [cnode.TYPE_FILE, cnode.TYPE_DIR]: if not isinstance(node, (cnode.NodeDir, cnode.NodeFile)):
continue continue
if self._clean(node): if self._clean(node):
cnt += 1 cnt += 1
return cnt return cnt
def _clean(self, node: Node) -> bool: def _clean(self, node: NodeAny) -> bool:
"""remove node if not flagged""" """remove node if not flagged"""
if not node.flagged(): if not node.flagged():
node.parent = None node.parent = None
@ -324,7 +311,7 @@ class Noder:
############################################################### ###############################################################
# printing # printing
############################################################### ###############################################################
def _node_to_csv(self, node: Node, def _node_to_csv(self, node: NodeAny,
sep: str = ',', sep: str = ',',
raw: bool = False) -> None: raw: bool = False) -> None:
""" """
@ -333,7 +320,7 @@ class Noder:
@sep: CSV separator character @sep: CSV separator character
@raw: print raw size rather than human readable @raw: print raw size rather than human readable
""" """
if not cnode: if not node:
return return
if node.type == node.TYPE_TOP: if node.type == node.TYPE_TOP:
return return
@ -374,7 +361,7 @@ class Noder:
out.append(node.md5) # md5 out.append(node.md5) # md5
else: else:
out.append('') # fake md5 out.append('') # fake md5
if node.type == cnode.TYPE_DIR: if isinstance(node, cnode.NodeDir):
out.append(str(len(node.children))) # nbfiles out.append(str(len(node.children))) # nbfiles
else: else:
out.append('') # fake nbfiles out.append('') # fake nbfiles
@ -386,7 +373,7 @@ class Noder:
if len(line) > 0: if len(line) > 0:
Logger.stdout_nocolor(line) Logger.stdout_nocolor(line)
def _print_node_native(self, node: Node, def _print_node_native(self, node: NodeAny,
pre: str = '', pre: str = '',
withpath: bool = False, withpath: bool = False,
withdepth: bool = False, withdepth: bool = False,
@ -403,10 +390,10 @@ class Noder:
@recalcparent: get relpath from tree instead of relpath field @recalcparent: get relpath from tree instead of relpath field
@raw: print raw size rather than human readable @raw: print raw size rather than human readable
""" """
if node.type == cnode.TYPE_TOP: if isinstance(node, cnode.NodeTop):
# top node # top node
Logger.stdout_nocolor(f'{pre}{node.name}') Logger.stdout_nocolor(f'{pre}{node.name}')
elif node.type == cnode.TYPE_FILE: elif isinstance(node, cnode.NodeFile):
# node of type file # node of type file
name = node.name name = node.name
if withpath: if withpath:
@ -426,7 +413,7 @@ class Noder:
content = Logger.get_bold_text(storage.name) content = Logger.get_bold_text(storage.name)
compl += f', storage:{content}' compl += f', storage:{content}'
NodePrinter.print_file_native(pre, name, compl) NodePrinter.print_file_native(pre, name, compl)
elif node.type == cnode.TYPE_DIR: elif isinstance(node, cnode.NodeDir):
# node of type directory # node of type directory
name = node.name name = node.name
if withpath: if withpath:
@ -446,7 +433,7 @@ class Noder:
if withstorage: if withstorage:
attr.append(('storage', Logger.get_bold_text(storage.name))) attr.append(('storage', Logger.get_bold_text(storage.name)))
NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr) NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr)
elif node.type == cnode.TYPE_STORAGE: elif isinstance(node, cnode.NodeStorage):
# node of type storage # node of type storage
sztotal = size_to_str(node.total, raw=raw) sztotal = size_to_str(node.total, raw=raw)
szused = size_to_str(node.total - node.free, raw=raw) szused = size_to_str(node.total - node.free, raw=raw)
@ -476,14 +463,14 @@ class Noder:
name, name,
argsstring, argsstring,
node.attr) node.attr)
elif node.type == cnode.TYPE_ARC: elif isinstance(node, cnode.NodeArchived):
# archive node # archive node
if self.arc: if self.arc:
NodePrinter.print_archive_native(pre, node.name, node.archive) NodePrinter.print_archive_native(pre, node.name, node.archive)
else: else:
Logger.err(f'bad node encountered: {node}') Logger.err(f'bad node encountered: {node}')
def print_tree(self, node: Node, def print_tree(self, node: NodeAny,
fmt: str = 'native', fmt: str = 'native',
raw: bool = False) -> None: raw: bool = False) -> None:
""" """
@ -507,7 +494,7 @@ class Noder:
Logger.stdout_nocolor(self.CSV_HEADER) Logger.stdout_nocolor(self.CSV_HEADER)
self._to_csv(node, raw=raw) self._to_csv(node, raw=raw)
def _to_csv(self, node: Node, def _to_csv(self, node: NodeAny,
raw: bool = False) -> None: raw: bool = False) -> None:
"""print the tree to csv""" """print the tree to csv"""
rend = anytree.RenderTree(node, childiter=self._sort_tree) rend = anytree.RenderTree(node, childiter=self._sort_tree)
@ -521,7 +508,7 @@ class Noder:
selected = fzf.prompt(strings) selected = fzf.prompt(strings)
return selected return selected
def _to_fzf(self, node: Node, fmt: str) -> None: def _to_fzf(self, node: NodeAny, fmt: str) -> None:
""" """
fzf prompt with list and print selected node(s) fzf prompt with list and print selected node(s)
@node: node to start with @node: node to start with
@ -550,24 +537,24 @@ class Noder:
self.print_tree(rend, fmt=subfmt) self.print_tree(rend, fmt=subfmt)
@staticmethod @staticmethod
def to_dot(node: Node, def to_dot(top: NodeTop,
path: str = 'tree.dot') -> str: path: str = 'tree.dot') -> str:
"""export to dot for graphing""" """export to dot for graphing"""
anytree.exporter.DotExporter(node).to_dotfile(path) anytree.exporter.DotExporter(top).to_dotfile(path)
Logger.info(f'dot file created under \"{path}\"') Logger.info(f'dot file created under \"{path}\"')
return f'dot {path} -T png -o /tmp/tree.png' return f'dot {path} -T png -o /tmp/tree.png'
############################################################### ###############################################################
# searching # searching
############################################################### ###############################################################
def find_name(self, top: Node, def find_name(self, top: NodeTop,
key: str, key: str,
script: bool = False, script: bool = False,
only_dir: bool = False, only_dir: bool = False,
startnode: Optional[Node] = None, startnode: Optional[NodeAny] = None,
parentfromtree: bool = False, parentfromtree: bool = False,
fmt: str = 'native', fmt: str = 'native',
raw: bool = False) -> List[Node]: raw: bool = False) -> List[NodeAny]:
""" """
find files based on their names find files based on their names
@top: top node @top: top node
@ -583,7 +570,7 @@ class Noder:
self._debug(f'searching for \"{key}\"') self._debug(f'searching for \"{key}\"')
# search for nodes based on path # search for nodes based on path
start: Optional[Node] = top start: Optional[NodeAny] = top
if startnode: if startnode:
start = self.get_node(top, startnode) start = self.get_node(top, startnode)
filterfunc = self._callback_find_name(key, only_dir) filterfunc = self._callback_find_name(key, only_dir)
@ -594,7 +581,9 @@ class Noder:
# compile found nodes # compile found nodes
paths = {} paths = {}
for item in found: for item in found:
item = self._sanitize(item) item.name = fix_badchars(item.name)
if hasattr(item, 'relpath'):
item.relpath = fix_badchars(item.relpath)
if parentfromtree: if parentfromtree:
paths[self._get_parents(item)] = item paths[self._get_parents(item)] = item
else: else:
@ -636,17 +625,17 @@ class Noder:
def _callback_find_name(self, term: str, only_dir: bool) -> Any: def _callback_find_name(self, term: str, only_dir: bool) -> Any:
"""callback for finding files""" """callback for finding files"""
def find_name(node: Node) -> bool: def find_name(node: NodeAny) -> bool:
if node.type == cnode.TYPE_STORAGE: if isinstance(node, cnode.NodeStorage):
# ignore storage nodes # ignore storage nodes
return False return False
if node.type == cnode.TYPE_TOP: if isinstance(node, cnode.NodeTop):
# ignore top nodes # ignore top nodes
return False return False
if node.type == cnode.TYPE_META: if isinstance(node, cnode.NodeMeta):
# ignore meta nodes # ignore meta nodes
return False return False
if only_dir and node.type != cnode.TYPE_DIR: if only_dir and isinstance(node, cnode.NodeDir):
# ignore non directory # ignore non directory
return False return False
@ -663,11 +652,11 @@ class Noder:
############################################################### ###############################################################
# ls # ls
############################################################### ###############################################################
def list(self, top: Node, def list(self, top: NodeTop,
path: str, path: str,
rec: bool = False, rec: bool = False,
fmt: str = 'native', fmt: str = 'native',
raw: bool = False) -> List[Node]: raw: bool = False) -> List[NodeAny]:
""" """
list nodes for "ls" list nodes for "ls"
@top: top node @top: top node
@ -729,7 +718,7 @@ class Noder:
# tree creation # tree creation
############################################################### ###############################################################
def _add_entry(self, name: str, def _add_entry(self, name: str,
top: Node, top: NodeTop,
resolv: Any) -> None: resolv: Any) -> None:
"""add an entry to the tree""" """add an entry to the tree"""
entries = name.rstrip(os.sep).split(os.sep) entries = name.rstrip(os.sep).split(os.sep)
@ -744,7 +733,7 @@ class Noder:
except anytree.resolver.ChildResolverError: except anytree.resolver.ChildResolverError:
self.new_archive_node(nodename, name, top, top.name) self.new_archive_node(nodename, name, top, top.name)
def list_to_tree(self, parent: Node, names: List[str]) -> None: def list_to_tree(self, parent: NodeAny, names: List[str]) -> None:
"""convert list of files to a tree""" """convert list of files to a tree"""
if not names: if not names:
return return
@ -757,23 +746,23 @@ class Noder:
# diverse # diverse
############################################################### ###############################################################
def _sort_tree(self, def _sort_tree(self,
items: List[Node]) -> List[Node]: items: List[NodeAny]) -> List[NodeAny]:
"""sorting a list of items""" """sorting a list of items"""
return sorted(items, key=self._sort, reverse=self.sortsize) return sorted(items, key=self._sort, reverse=self.sortsize)
def _sort(self, lst: Node) -> Any: def _sort(self, lst: NodeAny) -> Any:
"""sort a list""" """sort a list"""
if self.sortsize: if self.sortsize:
return self._sort_size(lst) return self._sort_size(lst)
return self._sort_fs(lst) return self._sort_fs(lst)
@staticmethod @staticmethod
def _sort_fs(node: Node) -> Tuple[str, str]: def _sort_fs(node: NodeAny) -> Tuple[str, str]:
"""sorting nodes dir first and alpha""" """sorting nodes dir first and alpha"""
return (node.type, node.name.lstrip('.').lower()) return (node.type, node.name.lstrip('.').lower())
@staticmethod @staticmethod
def _sort_size(node: Node) -> float: def _sort_size(node: NodeAny) -> float:
"""sorting nodes by size""" """sorting nodes by size"""
try: try:
if not node.size: if not node.size:
@ -782,22 +771,22 @@ class Noder:
except AttributeError: except AttributeError:
return 0 return 0
def _get_storage(self, node: Node) -> Node: def _get_storage(self, node: NodeAny) -> NodeStorage:
"""recursively traverse up to find storage""" """recursively traverse up to find storage"""
if node.type == cnode.TYPE_STORAGE: if isinstance(node, cnode.NodeStorage):
return node return node
return cast(Node, node.ancestors[1]) return cast(NodeStorage, node.ancestors[1])
@staticmethod @staticmethod
def _has_attr(node: Node, attr: str) -> bool: def _has_attr(node: NodeAny, attr: str) -> bool:
"""return True if node has attr as attribute""" """return True if node has attr as attribute"""
return attr in node.__dict__.keys() return attr in node.__dict__.keys()
def _get_parents(self, node: Node) -> str: def _get_parents(self, node: NodeAny) -> str:
"""get all parents recursively""" """get all parents recursively"""
if node.type == cnode.TYPE_STORAGE: if isinstance(node, cnode.NodeStorage):
return '' return ''
if node.type == cnode.TYPE_TOP: if isinstance(node, cnode.NodeTop):
return '' return ''
parent = self._get_parents(node.parent) parent = self._get_parents(node.parent)
if parent: if parent:
@ -813,13 +802,6 @@ class Noder:
Logger.err(str(exc)) Logger.err(str(exc))
return '' return ''
@staticmethod
def _sanitize(node: Node) -> Node:
"""sanitize node strings"""
node.name = fix_badchars(node.name)
node.relpath = fix_badchars(node.relpath)
return node
def _debug(self, string: str) -> None: def _debug(self, string: str) -> None:
"""print debug""" """print debug"""
if not self.debug: if not self.debug:

14
catcli/walker.py
Unescape Escape View File

@ -9,9 +9,9 @@ import os
from typing import Tuple, Optional from typing import Tuple, Optional
# local imports # local imports
from catcli.cnode import Node
from catcli.noder import Noder from catcli.noder import Noder
from catcli.logger import Logger from catcli.logger import Logger
from catcli.cnode import NodeAny, NodeTop
class Walker: class Walker:
@ -36,7 +36,7 @@ class Walker:
self.lpath = logpath self.lpath = logpath
def index(self, path: str, def index(self, path: str,
parent: Node, parent: NodeAny,
name: str, name: str,
storagepath: str = '') -> Tuple[str, int]: storagepath: str = '') -> Tuple[str, int]:
""" """
@ -89,15 +89,15 @@ class Walker:
self._progress('') self._progress('')
return parent, cnt return parent, cnt
def reindex(self, path: str, parent: Node, top: Node) -> int: def reindex(self, path: str, parent: NodeAny, top: NodeTop) -> int:
"""reindex a directory and store in tree""" """reindex a directory and store in tree"""
cnt = self._reindex(path, parent, top) cnt = self._reindex(path, parent, top)
cnt += self.noder.clean_not_flagged(parent) cnt += self.noder.clean_not_flagged(parent)
return cnt return cnt
def _reindex(self, path: str, def _reindex(self, path: str,
parent: Node, parent: NodeAny,
top: Node, top: NodeTop,
storagepath: str = '') -> int: storagepath: str = '') -> int:
""" """
reindex a directory and store in tree reindex a directory and store in tree
@ -148,9 +148,9 @@ class Walker:
return cnt return cnt
def _need_reindex(self, def _need_reindex(self,
top: Node, top: NodeTop,
path: str, path: str,
treepath: str) -> Tuple[bool, Optional[Node]]: treepath: str) -> Tuple[bool, Optional[NodeTop]]:
""" """
test if node needs re-indexing test if node needs re-indexing
@top: top node (storage) @top: top node (storage)

Write Preview
Loading…
Cancel
Save