|
|
|
@ -6,45 +6,50 @@ Class that represents a node in the catalog tree
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
import os
|
|
|
|
import os
|
|
|
|
import anytree
|
|
|
|
|
|
|
|
import shutil
|
|
|
|
import shutil
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
|
|
|
|
import anytree
|
|
|
|
|
|
|
|
from pyfzf.pyfzf import FzfPrompt
|
|
|
|
|
|
|
|
|
|
|
|
# local imports
|
|
|
|
# local imports
|
|
|
|
from . import __version__ as VERSION
|
|
|
|
from catcli.utils import size_to_str, epoch_to_str, md5sum, fix_badchars
|
|
|
|
import catcli.utils as utils
|
|
|
|
|
|
|
|
from catcli.logger import Logger
|
|
|
|
from catcli.logger import Logger
|
|
|
|
|
|
|
|
from catcli.nodeprinter import NodePrinter
|
|
|
|
from catcli.decomp import Decomp
|
|
|
|
from catcli.decomp import Decomp
|
|
|
|
|
|
|
|
from catcli.version import __version__ as VERSION
|
|
|
|
|
|
|
|
from catcli.exceptions import CatcliException
|
|
|
|
|
|
|
|
|
|
|
|
'''
|
|
|
|
|
|
|
|
There are 4 types of node:
|
|
|
|
class Noder:
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
handles node in the catalog tree
|
|
|
|
|
|
|
|
There are 4 types of node:
|
|
|
|
* "top" node representing the top node (generic node)
|
|
|
|
* "top" node representing the top node (generic node)
|
|
|
|
* "storage" node representing a storage
|
|
|
|
* "storage" node representing a storage
|
|
|
|
* "dir" node representing a directory
|
|
|
|
* "dir" node representing a directory
|
|
|
|
* "file" node representing a file
|
|
|
|
* "file" node representing a file
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
NAME_TOP = 'top'
|
|
|
|
|
|
|
|
NAME_META = 'meta'
|
|
|
|
|
|
|
|
|
|
|
|
class Noder:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TOPNAME = 'top'
|
|
|
|
|
|
|
|
METANAME = 'meta'
|
|
|
|
|
|
|
|
TYPE_TOP = 'top'
|
|
|
|
TYPE_TOP = 'top'
|
|
|
|
TYPE_FILE = 'file'
|
|
|
|
TYPE_FILE = 'file'
|
|
|
|
TYPE_DIR = 'dir'
|
|
|
|
TYPE_DIR = 'dir'
|
|
|
|
TYPE_ARC = 'arc'
|
|
|
|
TYPE_ARC = 'arc'
|
|
|
|
TYPE_STORAGE = 'storage'
|
|
|
|
TYPE_STORAGE = 'storage'
|
|
|
|
TYPE_META = 'meta'
|
|
|
|
TYPE_META = 'meta'
|
|
|
|
|
|
|
|
|
|
|
|
CSV_HEADER = ('name,type,path,size,indexed_at,'
|
|
|
|
CSV_HEADER = ('name,type,path,size,indexed_at,'
|
|
|
|
'maccess,md5,nbfiles,free_space,'
|
|
|
|
'maccess,md5,nbfiles,free_space,'
|
|
|
|
'total_space,meta')
|
|
|
|
'total_space,meta')
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, debug=False, sortsize=False, arc=False):
|
|
|
|
def __init__(self, debug=False, sortsize=False, arc=False):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
@debug: debug mode
|
|
|
|
@debug: debug mode
|
|
|
|
@sortsize: sort nodes by size
|
|
|
|
@sortsize: sort nodes by size
|
|
|
|
@arch: handle archive
|
|
|
|
@arch: handle archive
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
self.hash = True
|
|
|
|
self.hash = True
|
|
|
|
self.debug = debug
|
|
|
|
self.debug = debug
|
|
|
|
self.sortsize = sortsize
|
|
|
|
self.sortsize = sortsize
|
|
|
|
@ -52,21 +57,22 @@ class Noder:
|
|
|
|
if self.arc:
|
|
|
|
if self.arc:
|
|
|
|
self.decomp = Decomp()
|
|
|
|
self.decomp = Decomp()
|
|
|
|
|
|
|
|
|
|
|
|
def get_storage_names(self, top):
|
|
|
|
@staticmethod
|
|
|
|
'''return a list of all storage names'''
|
|
|
|
def get_storage_names(top):
|
|
|
|
|
|
|
|
"""return a list of all storage names"""
|
|
|
|
return [x.name for x in list(top.children)]
|
|
|
|
return [x.name for x in list(top.children)]
|
|
|
|
|
|
|
|
|
|
|
|
def get_storage_node(self, top, name, path=None):
|
|
|
|
def get_storage_node(self, top, name, path=None):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
return the storage node if any
|
|
|
|
return the storage node if any
|
|
|
|
if path is submitted, it will update the media info
|
|
|
|
if path is submitted, it will update the media info
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
found = None
|
|
|
|
found = None
|
|
|
|
for n in top.children:
|
|
|
|
for node in top.children:
|
|
|
|
if n.type != self.TYPE_STORAGE:
|
|
|
|
if node.type != self.TYPE_STORAGE:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
if n.name == name:
|
|
|
|
if node.name == name:
|
|
|
|
found = n
|
|
|
|
found = node
|
|
|
|
break
|
|
|
|
break
|
|
|
|
if found and path and os.path.exists(path):
|
|
|
|
if found and path and os.path.exists(path):
|
|
|
|
found.free = shutil.disk_usage(path).free
|
|
|
|
found.free = shutil.disk_usage(path).free
|
|
|
|
@ -74,24 +80,25 @@ class Noder:
|
|
|
|
found.ts = int(time.time())
|
|
|
|
found.ts = int(time.time())
|
|
|
|
return found
|
|
|
|
return found
|
|
|
|
|
|
|
|
|
|
|
|
def get_node(self, top, path, quiet=False):
|
|
|
|
@staticmethod
|
|
|
|
'''get the node by internal tree path'''
|
|
|
|
def get_node(top, path, quiet=False):
|
|
|
|
r = anytree.resolver.Resolver('name')
|
|
|
|
"""get the node by internal tree path"""
|
|
|
|
|
|
|
|
resolv = anytree.resolver.Resolver('name')
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
p = os.path.basename(path)
|
|
|
|
bpath = os.path.basename(path)
|
|
|
|
return r.get(top, p)
|
|
|
|
return resolv.get(top, bpath)
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
if not quiet:
|
|
|
|
if not quiet:
|
|
|
|
Logger.err('No node at path \"{}\"'.format(p))
|
|
|
|
Logger.err(f'No node at path \"{bpath}\"')
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def get_node_if_changed(self, top, path, treepath):
|
|
|
|
def get_node_if_changed(self, top, path, treepath):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
return the node (if any) and if it has changed
|
|
|
|
return the node (if any) and if it has changed
|
|
|
|
@top: top node (storage)
|
|
|
|
@top: top node (storage)
|
|
|
|
@path: abs path to file
|
|
|
|
@path: abs path to file
|
|
|
|
@treepath: rel path from indexed directory
|
|
|
|
@treepath: rel path from indexed directory
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
treepath = treepath.lstrip(os.sep)
|
|
|
|
treepath = treepath.lstrip(os.sep)
|
|
|
|
node = self.get_node(top, treepath, quiet=True)
|
|
|
|
node = self.get_node(top, treepath, quiet=True)
|
|
|
|
# node does not exist
|
|
|
|
# node does not exist
|
|
|
|
@ -109,105 +116,80 @@ class Noder:
|
|
|
|
# maccess changed
|
|
|
|
# maccess changed
|
|
|
|
old_maccess = node.maccess
|
|
|
|
old_maccess = node.maccess
|
|
|
|
if float(maccess) != float(old_maccess):
|
|
|
|
if float(maccess) != float(old_maccess):
|
|
|
|
self._debug('\tchange: maccess changed for \"{}\"'.format(path))
|
|
|
|
self._debug(f'\tchange: maccess changed for \"{path}\"')
|
|
|
|
return node, True
|
|
|
|
return node, True
|
|
|
|
# test hash
|
|
|
|
# test hash
|
|
|
|
if self.hash and node.md5:
|
|
|
|
if self.hash and node.md5:
|
|
|
|
md5 = self._get_hash(path)
|
|
|
|
md5 = self._get_hash(path)
|
|
|
|
if md5 != node.md5:
|
|
|
|
if md5 and md5 != node.md5:
|
|
|
|
m = '\tchange: checksum changed for \"{}\"'.format(path)
|
|
|
|
msg = f'\tchange: checksum changed for \"{path}\"'
|
|
|
|
self._debug(m)
|
|
|
|
self._debug(msg)
|
|
|
|
return node, True
|
|
|
|
return node, True
|
|
|
|
self._debug('\tchange: no change for \"{}\"'.format(path))
|
|
|
|
self._debug(f'\tchange: no change for \"{path}\"')
|
|
|
|
return node, False
|
|
|
|
return node, False
|
|
|
|
|
|
|
|
|
|
|
|
def _rec_size(self, node, store=True):
|
|
|
|
def rec_size(self, node, store=True):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
recursively traverse tree and return size
|
|
|
|
recursively traverse tree and return size
|
|
|
|
@store: store the size in the node
|
|
|
|
@store: store the size in the node
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
if node.type == self.TYPE_FILE:
|
|
|
|
if node.type == self.TYPE_FILE:
|
|
|
|
self._debug('getting node size for \"{}\"'.format(node.name))
|
|
|
|
self._debug(f'getting node size for \"{node.name}\"')
|
|
|
|
return node.size
|
|
|
|
return node.size
|
|
|
|
m = 'getting node size recursively for \"{}\"'.format(node.name)
|
|
|
|
msg = f'getting node size recursively for \"{node.name}\"'
|
|
|
|
self._debug(m)
|
|
|
|
self._debug(msg)
|
|
|
|
size = 0
|
|
|
|
size = 0
|
|
|
|
for i in node.children:
|
|
|
|
for i in node.children:
|
|
|
|
if node.type == self.TYPE_DIR:
|
|
|
|
if node.type == self.TYPE_DIR:
|
|
|
|
sz = self._rec_size(i, store=store)
|
|
|
|
size = self.rec_size(i, store=store)
|
|
|
|
if store:
|
|
|
|
if store:
|
|
|
|
i.size = sz
|
|
|
|
i.size = size
|
|
|
|
size += sz
|
|
|
|
size += size
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
sz = self._rec_size(i, store=store)
|
|
|
|
size = self.rec_size(i, store=store)
|
|
|
|
if store:
|
|
|
|
if store:
|
|
|
|
i.size = sz
|
|
|
|
i.size = size
|
|
|
|
size += sz
|
|
|
|
size += size
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
if store:
|
|
|
|
if store:
|
|
|
|
node.size = size
|
|
|
|
node.size = size
|
|
|
|
return size
|
|
|
|
return size
|
|
|
|
|
|
|
|
|
|
|
|
def rec_size(self, node):
|
|
|
|
|
|
|
|
'''recursively traverse tree and store dir size'''
|
|
|
|
|
|
|
|
return self._rec_size(node, store=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# public helpers
|
|
|
|
# public helpers
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def format_storage_attr(self, attr):
|
|
|
|
@staticmethod
|
|
|
|
'''format the storage attr for saving'''
|
|
|
|
def format_storage_attr(attr):
|
|
|
|
|
|
|
|
"""format the storage attr for saving"""
|
|
|
|
if not attr:
|
|
|
|
if not attr:
|
|
|
|
return ''
|
|
|
|
return ''
|
|
|
|
if type(attr) is list:
|
|
|
|
if isinstance(attr, list):
|
|
|
|
return ', '.join(attr)
|
|
|
|
return ', '.join(attr)
|
|
|
|
attr = attr.rstrip()
|
|
|
|
attr = attr.rstrip()
|
|
|
|
return attr
|
|
|
|
return attr
|
|
|
|
|
|
|
|
|
|
|
|
def set_hashing(self, val):
|
|
|
|
def set_hashing(self, val):
|
|
|
|
'''hash files when indexing'''
|
|
|
|
"""hash files when indexing"""
|
|
|
|
self.hash = val
|
|
|
|
self.hash = val
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# node creationg
|
|
|
|
# node creation
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def new_top_node(self):
|
|
|
|
def new_top_node(self):
|
|
|
|
'''create a new top node'''
|
|
|
|
"""create a new top node"""
|
|
|
|
return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP)
|
|
|
|
return anytree.AnyNode(name=self.NAME_TOP, type=self.TYPE_TOP)
|
|
|
|
|
|
|
|
|
|
|
|
def update_metanode(self, top):
|
|
|
|
|
|
|
|
'''create or update meta node information'''
|
|
|
|
|
|
|
|
meta = self._get_meta_node(top)
|
|
|
|
|
|
|
|
epoch = int(time.time())
|
|
|
|
|
|
|
|
if not meta:
|
|
|
|
|
|
|
|
attr = {}
|
|
|
|
|
|
|
|
attr['created'] = epoch
|
|
|
|
|
|
|
|
attr['created_version'] = VERSION
|
|
|
|
|
|
|
|
meta = anytree.AnyNode(name=self.METANAME, type=self.TYPE_META,
|
|
|
|
|
|
|
|
attr=attr)
|
|
|
|
|
|
|
|
meta.attr['access'] = epoch
|
|
|
|
|
|
|
|
meta.attr['access_version'] = VERSION
|
|
|
|
|
|
|
|
return meta
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_meta_node(self, top):
|
|
|
|
|
|
|
|
'''return the meta node if any'''
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
return next(filter(lambda x: x.type == self.TYPE_META,
|
|
|
|
|
|
|
|
top.children))
|
|
|
|
|
|
|
|
except StopIteration:
|
|
|
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def file_node(self, name, path, parent, storagepath):
|
|
|
|
def new_file_node(self, name, path, parent, storagepath):
|
|
|
|
'''create a new node representing a file'''
|
|
|
|
"""create a new node representing a file"""
|
|
|
|
if not os.path.exists(path):
|
|
|
|
if not os.path.exists(path):
|
|
|
|
Logger.err('File \"{}\" does not exist'.format(path))
|
|
|
|
Logger.err(f'File \"{path}\" does not exist')
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
st = os.lstat(path)
|
|
|
|
stat = os.lstat(path)
|
|
|
|
except OSError as e:
|
|
|
|
except OSError as exc:
|
|
|
|
Logger.err('OSError: {}'.format(e))
|
|
|
|
Logger.err(f'OSError: {exc}')
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
md5 = None
|
|
|
|
md5 = None
|
|
|
|
if self.hash:
|
|
|
|
if self.hash:
|
|
|
|
@ -215,42 +197,92 @@ class Noder:
|
|
|
|
relpath = os.sep.join([storagepath, name])
|
|
|
|
relpath = os.sep.join([storagepath, name])
|
|
|
|
|
|
|
|
|
|
|
|
maccess = os.path.getmtime(path)
|
|
|
|
maccess = os.path.getmtime(path)
|
|
|
|
n = self._node(name, self.TYPE_FILE, relpath, parent,
|
|
|
|
node = self._new_generic_node(name, self.TYPE_FILE, relpath, parent,
|
|
|
|
size=st.st_size, md5=md5, maccess=maccess)
|
|
|
|
size=stat.st_size, md5=md5,
|
|
|
|
|
|
|
|
maccess=maccess)
|
|
|
|
if self.arc:
|
|
|
|
if self.arc:
|
|
|
|
ext = os.path.splitext(path)[1][1:]
|
|
|
|
ext = os.path.splitext(path)[1][1:]
|
|
|
|
if ext.lower() in self.decomp.get_formats():
|
|
|
|
if ext.lower() in self.decomp.get_formats():
|
|
|
|
self._debug('{} is an archive'.format(path))
|
|
|
|
self._debug(f'{path} is an archive')
|
|
|
|
names = self.decomp.get_names(path)
|
|
|
|
names = self.decomp.get_names(path)
|
|
|
|
self.list_to_tree(n, names)
|
|
|
|
self.list_to_tree(node, names)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
self._debug('{} is NOT an archive'.format(path))
|
|
|
|
self._debug(f'{path} is NOT an archive')
|
|
|
|
return n
|
|
|
|
return node
|
|
|
|
|
|
|
|
|
|
|
|
def dir_node(self, name, path, parent, storagepath):
|
|
|
|
def new_dir_node(self, name, path, parent, storagepath):
|
|
|
|
'''create a new node representing a directory'''
|
|
|
|
"""create a new node representing a directory"""
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
relpath = os.sep.join([storagepath, name])
|
|
|
|
relpath = os.sep.join([storagepath, name])
|
|
|
|
maccess = os.path.getmtime(path)
|
|
|
|
maccess = os.path.getmtime(path)
|
|
|
|
return self._node(name, self.TYPE_DIR, relpath,
|
|
|
|
return self._new_generic_node(name, self.TYPE_DIR, relpath,
|
|
|
|
parent, maccess=maccess)
|
|
|
|
parent, maccess=maccess)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def new_storage_node(self, name, path, parent, attr=None):
|
|
|
|
|
|
|
|
"""create a new node representing a storage"""
|
|
|
|
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
|
|
|
|
free = shutil.disk_usage(path).free
|
|
|
|
|
|
|
|
total = shutil.disk_usage(path).total
|
|
|
|
|
|
|
|
epoch = int(time.time())
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free,
|
|
|
|
|
|
|
|
total=total, parent=parent, attr=attr, ts=epoch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def new_archive_node(self, name, path, parent, archive):
|
|
|
|
|
|
|
|
"""create a new node for archive data"""
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path,
|
|
|
|
|
|
|
|
parent=parent, size=0, md5=None,
|
|
|
|
|
|
|
|
archive=archive)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def _new_generic_node(name, nodetype, relpath, parent,
|
|
|
|
|
|
|
|
size=None, md5=None, maccess=None):
|
|
|
|
|
|
|
|
"""generic node creation"""
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=nodetype, relpath=relpath,
|
|
|
|
|
|
|
|
parent=parent, size=size,
|
|
|
|
|
|
|
|
md5=md5, maccess=maccess)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
|
|
|
|
# node management
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
|
|
|
|
def update_metanode(self, top):
|
|
|
|
|
|
|
|
"""create or update meta node information"""
|
|
|
|
|
|
|
|
meta = self._get_meta_node(top)
|
|
|
|
|
|
|
|
epoch = int(time.time())
|
|
|
|
|
|
|
|
if not meta:
|
|
|
|
|
|
|
|
attr = {}
|
|
|
|
|
|
|
|
attr['created'] = epoch
|
|
|
|
|
|
|
|
attr['created_version'] = VERSION
|
|
|
|
|
|
|
|
meta = anytree.AnyNode(name=self.NAME_META, type=self.TYPE_META,
|
|
|
|
|
|
|
|
attr=attr)
|
|
|
|
|
|
|
|
meta.attr['access'] = epoch
|
|
|
|
|
|
|
|
meta.attr['access_version'] = VERSION
|
|
|
|
|
|
|
|
return meta
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_meta_node(self, top):
|
|
|
|
|
|
|
|
"""return the meta node if any"""
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
return next(filter(lambda x: x.type == self.TYPE_META,
|
|
|
|
|
|
|
|
top.children))
|
|
|
|
|
|
|
|
except StopIteration:
|
|
|
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def clean_not_flagged(self, top):
|
|
|
|
def clean_not_flagged(self, top):
|
|
|
|
'''remove any node not flagged and clean flags'''
|
|
|
|
"""remove any node not flagged and clean flags"""
|
|
|
|
cnt = 0
|
|
|
|
cnt = 0
|
|
|
|
for node in anytree.PreOrderIter(top):
|
|
|
|
for node in anytree.PreOrderIter(top):
|
|
|
|
if node.type != self.TYPE_FILE and node.type != self.TYPE_DIR:
|
|
|
|
if node.type not in [self.TYPE_FILE, self.TYPE_DIR]:
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
if self._clean(node):
|
|
|
|
if self._clean(node):
|
|
|
|
cnt += 1
|
|
|
|
cnt += 1
|
|
|
|
return cnt
|
|
|
|
return cnt
|
|
|
|
|
|
|
|
|
|
|
|
def flag(self, node):
|
|
|
|
@staticmethod
|
|
|
|
'''flag a node'''
|
|
|
|
def flag(node):
|
|
|
|
|
|
|
|
"""flag a node"""
|
|
|
|
node.flag = True
|
|
|
|
node.flag = True
|
|
|
|
|
|
|
|
|
|
|
|
def _clean(self, node):
|
|
|
|
def _clean(self, node):
|
|
|
|
'''remove node if not flagged'''
|
|
|
|
"""remove node if not flagged"""
|
|
|
|
if not self._has_attr(node, 'flag') or \
|
|
|
|
if not self._has_attr(node, 'flag') or \
|
|
|
|
not node.flag:
|
|
|
|
not node.flag:
|
|
|
|
node.parent = None
|
|
|
|
node.parent = None
|
|
|
|
@ -258,42 +290,20 @@ class Noder:
|
|
|
|
del node.flag
|
|
|
|
del node.flag
|
|
|
|
return False
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
def storage_node(self, name, path, parent, attr=None):
|
|
|
|
|
|
|
|
'''create a new node representing a storage'''
|
|
|
|
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
|
|
|
|
free = shutil.disk_usage(path).free
|
|
|
|
|
|
|
|
total = shutil.disk_usage(path).total
|
|
|
|
|
|
|
|
epoch = int(time.time())
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free,
|
|
|
|
|
|
|
|
total=total, parent=parent, attr=attr, ts=epoch)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def archive_node(self, name, path, parent, archive):
|
|
|
|
|
|
|
|
'''crete a new node for archive data'''
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path,
|
|
|
|
|
|
|
|
parent=parent, size=0, md5=None,
|
|
|
|
|
|
|
|
archive=archive)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _node(self, name, type, relpath, parent,
|
|
|
|
|
|
|
|
size=None, md5=None, maccess=None):
|
|
|
|
|
|
|
|
'''generic node creation'''
|
|
|
|
|
|
|
|
return anytree.AnyNode(name=name, type=type, relpath=relpath,
|
|
|
|
|
|
|
|
parent=parent, size=size,
|
|
|
|
|
|
|
|
md5=md5, maccess=maccess)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# printing
|
|
|
|
# printing
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def _node_to_csv(self, node, sep=',', raw=False):
|
|
|
|
def _node_to_csv(self, node, sep=',', raw=False):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
print a node to csv
|
|
|
|
print a node to csv
|
|
|
|
@node: the node to consider
|
|
|
|
@node: the node to consider
|
|
|
|
@sep: CSV separator character
|
|
|
|
@sep: CSV separator character
|
|
|
|
@raw: print raw size rather than human readable
|
|
|
|
@raw: print raw size rather than human readable
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
if not node:
|
|
|
|
if not node:
|
|
|
|
return ''
|
|
|
|
return
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
return ''
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
out = []
|
|
|
|
out = []
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
@ -301,16 +311,16 @@ class Noder:
|
|
|
|
out.append(node.name) # name
|
|
|
|
out.append(node.name) # name
|
|
|
|
out.append(node.type) # type
|
|
|
|
out.append(node.type) # type
|
|
|
|
out.append('') # fake full path
|
|
|
|
out.append('') # fake full path
|
|
|
|
sz = self._rec_size(node, store=False)
|
|
|
|
size = self.rec_size(node, store=False)
|
|
|
|
out.append(utils.size_to_str(sz, raw=raw)) # size
|
|
|
|
out.append(size_to_str(size, raw=raw)) # size
|
|
|
|
out.append(utils.epoch_to_str(node.ts)) # indexed_at
|
|
|
|
out.append(epoch_to_str(node.ts)) # indexed_at
|
|
|
|
out.append('') # fake maccess
|
|
|
|
out.append('') # fake maccess
|
|
|
|
out.append('') # fake md5
|
|
|
|
out.append('') # fake md5
|
|
|
|
out.append(str(len(node.children))) # nbfiles
|
|
|
|
out.append(str(len(node.children))) # nbfiles
|
|
|
|
# fake free_space
|
|
|
|
# fake free_space
|
|
|
|
out.append(utils.size_to_str(node.free, raw=raw))
|
|
|
|
out.append(size_to_str(node.free, raw=raw))
|
|
|
|
# fake total_space
|
|
|
|
# fake total_space
|
|
|
|
out.append(utils.size_to_str(node.total, raw=raw))
|
|
|
|
out.append(size_to_str(node.total, raw=raw))
|
|
|
|
out.append(node.attr) # meta
|
|
|
|
out.append(node.attr) # meta
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
# handle other nodes
|
|
|
|
# handle other nodes
|
|
|
|
@ -321,10 +331,10 @@ class Noder:
|
|
|
|
fullpath = os.path.join(storage.name, parents)
|
|
|
|
fullpath = os.path.join(storage.name, parents)
|
|
|
|
out.append(fullpath.replace('"', '""')) # full path
|
|
|
|
out.append(fullpath.replace('"', '""')) # full path
|
|
|
|
|
|
|
|
|
|
|
|
out.append(utils.size_to_str(node.size, raw=raw)) # size
|
|
|
|
out.append(size_to_str(node.size, raw=raw)) # size
|
|
|
|
out.append(utils.epoch_to_str(storage.ts)) # indexed_at
|
|
|
|
out.append(epoch_to_str(storage.ts)) # indexed_at
|
|
|
|
if self._has_attr(node, 'maccess'):
|
|
|
|
if self._has_attr(node, 'maccess'):
|
|
|
|
out.append(utils.epoch_to_str(node.maccess)) # maccess
|
|
|
|
out.append(epoch_to_str(node.maccess)) # maccess
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
out.append('') # fake maccess
|
|
|
|
out.append('') # fake maccess
|
|
|
|
if node.md5:
|
|
|
|
if node.md5:
|
|
|
|
@ -341,12 +351,12 @@ class Noder:
|
|
|
|
|
|
|
|
|
|
|
|
line = sep.join(['"' + o + '"' for o in out])
|
|
|
|
line = sep.join(['"' + o + '"' for o in out])
|
|
|
|
if len(line) > 0:
|
|
|
|
if len(line) > 0:
|
|
|
|
Logger.out(line)
|
|
|
|
Logger.stdout_nocolor(line)
|
|
|
|
|
|
|
|
|
|
|
|
def _print_node(self, node, pre='', withpath=False,
|
|
|
|
def _print_node_native(self, node, pre='', withpath=False,
|
|
|
|
withdepth=False, withstorage=False,
|
|
|
|
withdepth=False, withstorage=False,
|
|
|
|
recalcparent=False, raw=False):
|
|
|
|
recalcparent=False, raw=False):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
print a node
|
|
|
|
print a node
|
|
|
|
@node: the node to print
|
|
|
|
@node: the node to print
|
|
|
|
@pre: string to print before node
|
|
|
|
@pre: string to print before node
|
|
|
|
@ -355,10 +365,10 @@ class Noder:
|
|
|
|
@withstorage: print the node storage it belongs to
|
|
|
|
@withstorage: print the node storage it belongs to
|
|
|
|
@recalcparent: get relpath from tree instead of relpath field
|
|
|
|
@recalcparent: get relpath from tree instead of relpath field
|
|
|
|
@raw: print raw size rather than human readable
|
|
|
|
@raw: print raw size rather than human readable
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
# top node
|
|
|
|
# top node
|
|
|
|
Logger.out('{}{}'.format(pre, node.name))
|
|
|
|
Logger.stdout_nocolor(f'{pre}{node.name}')
|
|
|
|
elif node.type == self.TYPE_FILE:
|
|
|
|
elif node.type == self.TYPE_FILE:
|
|
|
|
# node of type file
|
|
|
|
# node of type file
|
|
|
|
name = node.name
|
|
|
|
name = node.name
|
|
|
|
@ -372,12 +382,13 @@ class Noder:
|
|
|
|
storage = self._get_storage(node)
|
|
|
|
storage = self._get_storage(node)
|
|
|
|
attr = ''
|
|
|
|
attr = ''
|
|
|
|
if node.md5:
|
|
|
|
if node.md5:
|
|
|
|
attr = ', md5:{}'.format(node.md5)
|
|
|
|
attr = f', md5:{node.md5}'
|
|
|
|
sz = utils.size_to_str(node.size, raw=raw)
|
|
|
|
size = size_to_str(node.size, raw=raw)
|
|
|
|
compl = 'size:{}{}'.format(sz, attr)
|
|
|
|
compl = f'size:{size}{attr}'
|
|
|
|
if withstorage:
|
|
|
|
if withstorage:
|
|
|
|
compl += ', storage:{}'.format(Logger.bold(storage.name))
|
|
|
|
content = Logger.get_bold_text(storage.name)
|
|
|
|
Logger.file(pre, name, compl)
|
|
|
|
compl += f', storage:{content}'
|
|
|
|
|
|
|
|
NodePrinter.print_file_native(pre, name, compl)
|
|
|
|
elif node.type == self.TYPE_DIR:
|
|
|
|
elif node.type == self.TYPE_DIR:
|
|
|
|
# node of type directory
|
|
|
|
# node of type directory
|
|
|
|
name = node.name
|
|
|
|
name = node.name
|
|
|
|
@ -394,154 +405,243 @@ class Noder:
|
|
|
|
storage = self._get_storage(node)
|
|
|
|
storage = self._get_storage(node)
|
|
|
|
attr = []
|
|
|
|
attr = []
|
|
|
|
if node.size:
|
|
|
|
if node.size:
|
|
|
|
attr.append(['totsize', utils.size_to_str(node.size, raw=raw)])
|
|
|
|
attr.append(['totsize', size_to_str(node.size, raw=raw)])
|
|
|
|
if withstorage:
|
|
|
|
if withstorage:
|
|
|
|
attr.append(['storage', Logger.bold(storage.name)])
|
|
|
|
attr.append(['storage', Logger.get_bold_text(storage.name)])
|
|
|
|
Logger.dir(pre, name, depth=depth, attr=attr)
|
|
|
|
NodePrinter.print_dir_native(pre, name, depth=depth, attr=attr)
|
|
|
|
elif node.type == self.TYPE_STORAGE:
|
|
|
|
elif node.type == self.TYPE_STORAGE:
|
|
|
|
# node of type storage
|
|
|
|
# node of type storage
|
|
|
|
hf = utils.size_to_str(node.free, raw=raw)
|
|
|
|
sztotal = size_to_str(node.total, raw=raw)
|
|
|
|
ht = utils.size_to_str(node.total, raw=raw)
|
|
|
|
szused = size_to_str(node.total - node.free, raw=raw)
|
|
|
|
nbchildren = len(node.children)
|
|
|
|
nbchildren = len(node.children)
|
|
|
|
freepercent = '{:.1f}%'.format(
|
|
|
|
pcent = node.free * 100 / node.total
|
|
|
|
node.free * 100 / node.total
|
|
|
|
freepercent = f'{pcent:.1f}%'
|
|
|
|
)
|
|
|
|
|
|
|
|
# get the date
|
|
|
|
# get the date
|
|
|
|
dt = ''
|
|
|
|
timestamp = ''
|
|
|
|
if self._has_attr(node, 'ts'):
|
|
|
|
if self._has_attr(node, 'ts'):
|
|
|
|
dt = 'date:'
|
|
|
|
timestamp = 'date:'
|
|
|
|
dt += '{}'.format(utils.epoch_to_str(node.ts))
|
|
|
|
timestamp += epoch_to_str(node.ts)
|
|
|
|
ds = ''
|
|
|
|
disksize = ''
|
|
|
|
# the children size
|
|
|
|
# the children size
|
|
|
|
sz = self._rec_size(node, store=False)
|
|
|
|
size = self.rec_size(node, store=False)
|
|
|
|
sz = utils.size_to_str(sz, raw=raw)
|
|
|
|
size = size_to_str(size, raw=raw)
|
|
|
|
ds = 'totsize:' + '{}'.format(sz)
|
|
|
|
disksize = 'totsize:' + f'{size}'
|
|
|
|
# format the output
|
|
|
|
# format the output
|
|
|
|
name = '{}'.format(node.name)
|
|
|
|
name = node.name
|
|
|
|
args = [
|
|
|
|
args = [
|
|
|
|
'nbfiles:' + '{}'.format(nbchildren),
|
|
|
|
'nbfiles:' + f'{nbchildren}',
|
|
|
|
ds,
|
|
|
|
disksize,
|
|
|
|
'free:{}'.format(freepercent),
|
|
|
|
f'free:{freepercent}',
|
|
|
|
'du:' + '{}/{}'.format(hf, ht),
|
|
|
|
'du:' + f'{szused}/{sztotal}',
|
|
|
|
dt]
|
|
|
|
timestamp]
|
|
|
|
Logger.storage(pre,
|
|
|
|
argsstring = ' | '.join(args)
|
|
|
|
name,
|
|
|
|
NodePrinter.print_storage_native(pre,
|
|
|
|
'{}'.format(' | '.join(args)),
|
|
|
|
name,
|
|
|
|
node.attr)
|
|
|
|
argsstring,
|
|
|
|
|
|
|
|
node.attr)
|
|
|
|
elif node.type == self.TYPE_ARC:
|
|
|
|
elif node.type == self.TYPE_ARC:
|
|
|
|
# archive node
|
|
|
|
# archive node
|
|
|
|
if self.arc:
|
|
|
|
if self.arc:
|
|
|
|
Logger.arc(pre, node.name, node.archive)
|
|
|
|
NodePrinter.print_archive_native(pre, node.name, node.archive)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
Logger.err('bad node encountered: {}'.format(node))
|
|
|
|
Logger.err(f'bad node encountered: {node}')
|
|
|
|
|
|
|
|
|
|
|
|
def print_tree(self, node, style=anytree.ContRoundStyle(),
|
|
|
|
def print_tree(self, node,
|
|
|
|
fmt='native', header=False, raw=False):
|
|
|
|
fmt='native',
|
|
|
|
'''
|
|
|
|
raw=False):
|
|
|
|
print the tree similar to unix tool "tree"
|
|
|
|
"""
|
|
|
|
|
|
|
|
print the tree in different format
|
|
|
|
@node: start node
|
|
|
|
@node: start node
|
|
|
|
@style: when fmt=native, defines the tree style
|
|
|
|
@style: when fmt=native, defines the tree style
|
|
|
|
@fmt: output format
|
|
|
|
@fmt: output format
|
|
|
|
@header: when fmt=csv, print the header
|
|
|
|
|
|
|
|
@raw: print the raw size rather than human readable
|
|
|
|
@raw: print the raw size rather than human readable
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
if fmt == 'native':
|
|
|
|
if fmt == 'native':
|
|
|
|
|
|
|
|
# "tree" style
|
|
|
|
rend = anytree.RenderTree(node, childiter=self._sort_tree)
|
|
|
|
rend = anytree.RenderTree(node, childiter=self._sort_tree)
|
|
|
|
for pre, fill, node in rend:
|
|
|
|
for pre, _, thenode in rend:
|
|
|
|
self._print_node(node, pre=pre, withdepth=True, raw=raw)
|
|
|
|
self._print_node_native(thenode, pre=pre,
|
|
|
|
|
|
|
|
withdepth=True, raw=raw)
|
|
|
|
elif fmt == 'csv':
|
|
|
|
elif fmt == 'csv':
|
|
|
|
self._to_csv(node, with_header=header, raw=raw)
|
|
|
|
# csv output
|
|
|
|
|
|
|
|
self._to_csv(node, raw=raw)
|
|
|
|
def _to_csv(self, node, with_header=False, raw=False):
|
|
|
|
elif fmt == 'csv-with-header':
|
|
|
|
'''print the tree to csv'''
|
|
|
|
# csv output
|
|
|
|
|
|
|
|
Logger.stdout_nocolor(self.CSV_HEADER)
|
|
|
|
|
|
|
|
self._to_csv(node, raw=raw)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _to_csv(self, node, raw=False):
|
|
|
|
|
|
|
|
"""print the tree to csv"""
|
|
|
|
rend = anytree.RenderTree(node, childiter=self._sort_tree)
|
|
|
|
rend = anytree.RenderTree(node, childiter=self._sort_tree)
|
|
|
|
if with_header:
|
|
|
|
for _, _, item in rend:
|
|
|
|
Logger.out(self.CSV_HEADER)
|
|
|
|
self._node_to_csv(item, raw=raw)
|
|
|
|
for _, _, node in rend:
|
|
|
|
|
|
|
|
self._node_to_csv(node, raw=raw)
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def _fzf_prompt(strings):
|
|
|
|
|
|
|
|
# prompt with fzf
|
|
|
|
|
|
|
|
fzf = FzfPrompt()
|
|
|
|
|
|
|
|
selected = fzf.prompt(strings)
|
|
|
|
|
|
|
|
return selected
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _to_fzf(self, node, fmt):
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
fzf prompt with list and print selected node(s)
|
|
|
|
|
|
|
|
@node: node to start with
|
|
|
|
|
|
|
|
@fmt: output format for selected nodes
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
rendered = anytree.RenderTree(node, childiter=self._sort_tree)
|
|
|
|
|
|
|
|
nodes = {}
|
|
|
|
|
|
|
|
# construct node names list
|
|
|
|
|
|
|
|
for _, _, rend in rendered:
|
|
|
|
|
|
|
|
if not rend:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
parents = self._get_parents(rend)
|
|
|
|
|
|
|
|
storage = self._get_storage(rend)
|
|
|
|
|
|
|
|
fullpath = os.path.join(storage.name, parents)
|
|
|
|
|
|
|
|
nodes[fullpath] = rend
|
|
|
|
|
|
|
|
# prompt with fzf
|
|
|
|
|
|
|
|
paths = self._fzf_prompt(nodes.keys())
|
|
|
|
|
|
|
|
# print the resulting tree
|
|
|
|
|
|
|
|
subfmt = fmt.replace('fzf-', '')
|
|
|
|
|
|
|
|
for path in paths:
|
|
|
|
|
|
|
|
if not path:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
if path not in nodes:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
rend = nodes[path]
|
|
|
|
|
|
|
|
self.print_tree(rend, fmt=subfmt)
|
|
|
|
|
|
|
|
|
|
|
|
def to_dot(self, node, path='tree.dot'):
|
|
|
|
@staticmethod
|
|
|
|
'''export to dot for graphing'''
|
|
|
|
def to_dot(node, path='tree.dot'):
|
|
|
|
|
|
|
|
"""export to dot for graphing"""
|
|
|
|
anytree.exporter.DotExporter(node).to_dotfile(path)
|
|
|
|
anytree.exporter.DotExporter(node).to_dotfile(path)
|
|
|
|
Logger.info('dot file created under \"{}\"'.format(path))
|
|
|
|
Logger.info(f'dot file created under \"{path}\"')
|
|
|
|
return 'dot {} -T png -o /tmp/tree.png'.format(path)
|
|
|
|
return f'dot {path} -T png -o /tmp/tree.png'
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# searching
|
|
|
|
# searching
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def find_name(self, root, key,
|
|
|
|
def find_name(self, top, key,
|
|
|
|
script=False, directory=False,
|
|
|
|
script=False, only_dir=False,
|
|
|
|
startpath=None, parentfromtree=False,
|
|
|
|
startpath=None, parentfromtree=False,
|
|
|
|
fmt='native', raw=False):
|
|
|
|
fmt='native', raw=False):
|
|
|
|
'''
|
|
|
|
"""
|
|
|
|
find files based on their names
|
|
|
|
find files based on their names
|
|
|
|
|
|
|
|
@top: top node
|
|
|
|
|
|
|
|
@key: term to search for
|
|
|
|
@script: output script
|
|
|
|
@script: output script
|
|
|
|
@directory: only search for directories
|
|
|
|
@directory: only search for directories
|
|
|
|
@startpath: node to start with
|
|
|
|
@startpath: node to start with
|
|
|
|
@parentfromtree: get path from parent instead of stored relpath
|
|
|
|
@parentfromtree: get path from parent instead of stored relpath
|
|
|
|
@fmt: output format
|
|
|
|
@fmt: output format
|
|
|
|
'''
|
|
|
|
@raw: raw size output
|
|
|
|
self._debug('searching for \"{}\"'.format(key))
|
|
|
|
returns the found nodes
|
|
|
|
start = root
|
|
|
|
"""
|
|
|
|
if startpath:
|
|
|
|
self._debug(f'searching for \"{key}\"')
|
|
|
|
start = self.get_node(root, startpath)
|
|
|
|
|
|
|
|
self.term = key
|
|
|
|
|
|
|
|
found = anytree.findall(start, filter_=self._find_name)
|
|
|
|
|
|
|
|
paths = []
|
|
|
|
|
|
|
|
for f in found:
|
|
|
|
|
|
|
|
if f.type == self.TYPE_STORAGE:
|
|
|
|
|
|
|
|
# ignore storage nodes
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
if directory and f.type != self.TYPE_DIR:
|
|
|
|
|
|
|
|
# ignore non directory
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# print the node
|
|
|
|
|
|
|
|
if fmt == 'native':
|
|
|
|
|
|
|
|
self._print_node(f, withpath=True,
|
|
|
|
|
|
|
|
withdepth=True,
|
|
|
|
|
|
|
|
withstorage=True,
|
|
|
|
|
|
|
|
recalcparent=parentfromtree,
|
|
|
|
|
|
|
|
raw=raw)
|
|
|
|
|
|
|
|
elif fmt == 'csv':
|
|
|
|
|
|
|
|
self._node_to_csv(f, raw=raw)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# search for nodes based on path
|
|
|
|
|
|
|
|
start = top
|
|
|
|
|
|
|
|
if startpath:
|
|
|
|
|
|
|
|
start = self.get_node(top, startpath)
|
|
|
|
|
|
|
|
filterfunc = self._callback_find_name(key, only_dir)
|
|
|
|
|
|
|
|
found = anytree.findall(start, filter_=filterfunc)
|
|
|
|
|
|
|
|
nbfound = len(found)
|
|
|
|
|
|
|
|
self._debug(f'found {nbfound} node(s)')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# compile found nodes
|
|
|
|
|
|
|
|
paths = {}
|
|
|
|
|
|
|
|
for item in found:
|
|
|
|
|
|
|
|
item = self._sanitize(item)
|
|
|
|
if parentfromtree:
|
|
|
|
if parentfromtree:
|
|
|
|
paths.append(self._get_parents(f))
|
|
|
|
paths[self._get_parents(item)] = item
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
paths.append(f.relpath)
|
|
|
|
paths[item.relpath] = item
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# handle fzf mode
|
|
|
|
|
|
|
|
if fmt.startswith('fzf'):
|
|
|
|
|
|
|
|
selected = self._fzf_prompt(paths.keys())
|
|
|
|
|
|
|
|
newpaths = {}
|
|
|
|
|
|
|
|
subfmt = fmt.replace('fzf-', '')
|
|
|
|
|
|
|
|
for item in selected:
|
|
|
|
|
|
|
|
if item not in paths:
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
newpaths[item] = paths[item]
|
|
|
|
|
|
|
|
self.print_tree(newpaths[item], fmt=subfmt)
|
|
|
|
|
|
|
|
paths = newpaths
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
if fmt == 'native':
|
|
|
|
|
|
|
|
for _, item in paths.items():
|
|
|
|
|
|
|
|
self._print_node_native(item, withpath=True,
|
|
|
|
|
|
|
|
withdepth=True,
|
|
|
|
|
|
|
|
withstorage=True,
|
|
|
|
|
|
|
|
recalcparent=parentfromtree,
|
|
|
|
|
|
|
|
raw=raw)
|
|
|
|
|
|
|
|
elif fmt.startswith('csv'):
|
|
|
|
|
|
|
|
if fmt == 'csv-with-header':
|
|
|
|
|
|
|
|
Logger.stdout_nocolor(self.CSV_HEADER)
|
|
|
|
|
|
|
|
for _, item in paths.items():
|
|
|
|
|
|
|
|
self._node_to_csv(item, raw=raw)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# execute script if any
|
|
|
|
if script:
|
|
|
|
if script:
|
|
|
|
tmp = ['${source}/' + x for x in paths]
|
|
|
|
tmp = ['${source}/' + x for x in paths]
|
|
|
|
cmd = 'op=file; source=/media/mnt; $op {}'.format(' '.join(tmp))
|
|
|
|
tmpstr = ' '.join(tmp)
|
|
|
|
|
|
|
|
cmd = f'op=file; source=/media/mnt; $op {tmpstr}'
|
|
|
|
Logger.info(cmd)
|
|
|
|
Logger.info(cmd)
|
|
|
|
|
|
|
|
|
|
|
|
return found
|
|
|
|
return list(paths.values())
|
|
|
|
|
|
|
|
|
|
|
|
def _find_name(self, node):
|
|
|
|
def _callback_find_name(self, term, only_dir):
|
|
|
|
'''callback for finding files'''
|
|
|
|
"""callback for finding files"""
|
|
|
|
if self.term.lower() in node.name.lower():
|
|
|
|
def find_name(node):
|
|
|
|
return True
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
return False
|
|
|
|
# ignore storage nodes
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
|
|
|
|
# ignore top nodes
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
if node.type == self.TYPE_META:
|
|
|
|
|
|
|
|
# ignore meta nodes
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
if only_dir and node.type != self.TYPE_DIR:
|
|
|
|
|
|
|
|
# ignore non directory
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# filter
|
|
|
|
|
|
|
|
if not term:
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
if term.lower() in node.name.lower():
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ignore
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
return find_name
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# climbing
|
|
|
|
# ls
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def walk(self, root, path, rec=False, fmt='native', raw=False):
|
|
|
|
def list(self, top, path,
|
|
|
|
'''
|
|
|
|
rec=False,
|
|
|
|
walk the tree for ls based on names
|
|
|
|
fmt='native',
|
|
|
|
@root: start node
|
|
|
|
raw=False):
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
list nodes for "ls"
|
|
|
|
|
|
|
|
@top: top node
|
|
|
|
|
|
|
|
@path: path to search for
|
|
|
|
@rec: recursive walk
|
|
|
|
@rec: recursive walk
|
|
|
|
@fmt: output format
|
|
|
|
@fmt: output format
|
|
|
|
'''
|
|
|
|
@raw: print raw size
|
|
|
|
self._debug('walking path: \"{}\"'.format(path))
|
|
|
|
"""
|
|
|
|
|
|
|
|
self._debug(f'walking path: \"{path}\" from {top}')
|
|
|
|
|
|
|
|
|
|
|
|
r = anytree.resolver.Resolver('name')
|
|
|
|
resolv = anytree.resolver.Resolver('name')
|
|
|
|
found = []
|
|
|
|
found = []
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
found = r.glob(root, path)
|
|
|
|
# resolve the path in the tree
|
|
|
|
|
|
|
|
found = resolv.glob(top, path)
|
|
|
|
if len(found) < 1:
|
|
|
|
if len(found) < 1:
|
|
|
|
# nothing found
|
|
|
|
# nothing found
|
|
|
|
|
|
|
|
self._debug('nothing found')
|
|
|
|
return []
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
|
|
if rec:
|
|
|
|
if rec:
|
|
|
|
@ -554,20 +654,28 @@ class Noder:
|
|
|
|
|
|
|
|
|
|
|
|
# print the parent
|
|
|
|
# print the parent
|
|
|
|
if fmt == 'native':
|
|
|
|
if fmt == 'native':
|
|
|
|
self._print_node(found[0].parent,
|
|
|
|
self._print_node_native(found[0].parent,
|
|
|
|
withpath=False, withdepth=True, raw=raw)
|
|
|
|
withpath=False,
|
|
|
|
elif fmt == 'csv':
|
|
|
|
withdepth=True,
|
|
|
|
|
|
|
|
raw=raw)
|
|
|
|
|
|
|
|
elif fmt.startswith('csv'):
|
|
|
|
self._node_to_csv(found[0].parent, raw=raw)
|
|
|
|
self._node_to_csv(found[0].parent, raw=raw)
|
|
|
|
|
|
|
|
elif fmt.startswith('fzf'):
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
# print all found nodes
|
|
|
|
# print all found nodes
|
|
|
|
for f in found:
|
|
|
|
if fmt == 'csv-with-header':
|
|
|
|
|
|
|
|
Logger.stdout_nocolor(self.CSV_HEADER)
|
|
|
|
|
|
|
|
for item in found:
|
|
|
|
if fmt == 'native':
|
|
|
|
if fmt == 'native':
|
|
|
|
self._print_node(f, withpath=False,
|
|
|
|
self._print_node_native(item, withpath=False,
|
|
|
|
pre='- ',
|
|
|
|
pre='- ',
|
|
|
|
withdepth=True,
|
|
|
|
withdepth=True,
|
|
|
|
raw=raw)
|
|
|
|
raw=raw)
|
|
|
|
elif fmt == 'csv':
|
|
|
|
elif fmt.startswith('csv'):
|
|
|
|
self._node_to_csv(f, raw=raw)
|
|
|
|
self._node_to_csv(item, raw=raw)
|
|
|
|
|
|
|
|
elif fmt.startswith('fzf'):
|
|
|
|
|
|
|
|
self._to_fzf(item, fmt)
|
|
|
|
|
|
|
|
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
@ -577,78 +685,96 @@ class Noder:
|
|
|
|
# tree creation
|
|
|
|
# tree creation
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def _add_entry(self, name, top, resolv):
|
|
|
|
def _add_entry(self, name, top, resolv):
|
|
|
|
'''add an entry to the tree'''
|
|
|
|
"""add an entry to the tree"""
|
|
|
|
entries = name.rstrip(os.sep).split(os.sep)
|
|
|
|
entries = name.rstrip(os.sep).split(os.sep)
|
|
|
|
if len(entries) == 1:
|
|
|
|
if len(entries) == 1:
|
|
|
|
self.archive_node(name, name, top, top.name)
|
|
|
|
self.new_archive_node(name, name, top, top.name)
|
|
|
|
return
|
|
|
|
return
|
|
|
|
sub = os.sep.join(entries[:-1])
|
|
|
|
sub = os.sep.join(entries[:-1])
|
|
|
|
f = entries[-1]
|
|
|
|
nodename = entries[-1]
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
parent = resolv.get(top, sub)
|
|
|
|
parent = resolv.get(top, sub)
|
|
|
|
parent = self.archive_node(f, name, parent, top.name)
|
|
|
|
parent = self.new_archive_node(nodename, name, parent, top.name)
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
except anytree.resolver.ChildResolverError:
|
|
|
|
self.archive_node(f, name, top, top.name)
|
|
|
|
self.new_archive_node(nodename, name, top, top.name)
|
|
|
|
|
|
|
|
|
|
|
|
def list_to_tree(self, parent, names):
|
|
|
|
def list_to_tree(self, parent, names):
|
|
|
|
'''convert list of files to a tree'''
|
|
|
|
"""convert list of files to a tree"""
|
|
|
|
if not names:
|
|
|
|
if not names:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
r = anytree.resolver.Resolver('name')
|
|
|
|
resolv = anytree.resolver.Resolver('name')
|
|
|
|
for name in names:
|
|
|
|
for name in names:
|
|
|
|
name = name.rstrip(os.sep)
|
|
|
|
name = name.rstrip(os.sep)
|
|
|
|
self._add_entry(name, parent, r)
|
|
|
|
self._add_entry(name, parent, resolv)
|
|
|
|
|
|
|
|
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
# diverse
|
|
|
|
# diverse
|
|
|
|
###############################################################
|
|
|
|
###############################################################
|
|
|
|
def _sort_tree(self, items):
|
|
|
|
def _sort_tree(self, items):
|
|
|
|
'''sorting a list of items'''
|
|
|
|
"""sorting a list of items"""
|
|
|
|
return sorted(items, key=self._sort, reverse=self.sortsize)
|
|
|
|
return sorted(items, key=self._sort, reverse=self.sortsize)
|
|
|
|
|
|
|
|
|
|
|
|
def _sort(self, x):
|
|
|
|
def _sort(self, lst):
|
|
|
|
'''sort a list'''
|
|
|
|
"""sort a list"""
|
|
|
|
if self.sortsize:
|
|
|
|
if self.sortsize:
|
|
|
|
return self._sort_size(x)
|
|
|
|
return self._sort_size(lst)
|
|
|
|
return self._sort_fs(x)
|
|
|
|
return self._sort_fs(lst)
|
|
|
|
|
|
|
|
|
|
|
|
def _sort_fs(self, n):
|
|
|
|
@staticmethod
|
|
|
|
'''sorting nodes dir first and alpha'''
|
|
|
|
def _sort_fs(node):
|
|
|
|
return (n.type, n.name.lstrip('\.').lower())
|
|
|
|
"""sorting nodes dir first and alpha"""
|
|
|
|
|
|
|
|
return (node.type, node.name.lstrip('.').lower())
|
|
|
|
|
|
|
|
|
|
|
|
def _sort_size(self, n):
|
|
|
|
@staticmethod
|
|
|
|
'''sorting nodes by size'''
|
|
|
|
def _sort_size(node):
|
|
|
|
|
|
|
|
"""sorting nodes by size"""
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
if not n.size:
|
|
|
|
if not node.size:
|
|
|
|
return 0
|
|
|
|
return 0
|
|
|
|
return n.size
|
|
|
|
return node.size
|
|
|
|
except AttributeError:
|
|
|
|
except AttributeError:
|
|
|
|
return 0
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
def _get_storage(self, node):
|
|
|
|
def _get_storage(self, node):
|
|
|
|
'''recursively traverse up to find storage'''
|
|
|
|
"""recursively traverse up to find storage"""
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
return node
|
|
|
|
return node
|
|
|
|
return node.ancestors[1]
|
|
|
|
return node.ancestors[1]
|
|
|
|
|
|
|
|
|
|
|
|
def _has_attr(self, node, attr):
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def _has_attr(node, attr):
|
|
|
|
|
|
|
|
"""return True if node has attr as attribute"""
|
|
|
|
return attr in node.__dict__.keys()
|
|
|
|
return attr in node.__dict__.keys()
|
|
|
|
|
|
|
|
|
|
|
|
def _get_parents(self, node):
|
|
|
|
def _get_parents(self, node):
|
|
|
|
'''get all parents recursively'''
|
|
|
|
"""get all parents recursively"""
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
if node.type == self.TYPE_STORAGE:
|
|
|
|
return ''
|
|
|
|
return ''
|
|
|
|
|
|
|
|
if node.type == self.TYPE_TOP:
|
|
|
|
|
|
|
|
return ''
|
|
|
|
parent = self._get_parents(node.parent)
|
|
|
|
parent = self._get_parents(node.parent)
|
|
|
|
if parent:
|
|
|
|
if parent:
|
|
|
|
return os.sep.join([parent, node.name])
|
|
|
|
return os.sep.join([parent, node.name])
|
|
|
|
return node.name
|
|
|
|
return node.name
|
|
|
|
|
|
|
|
|
|
|
|
def _get_hash(self, path):
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def _get_hash(path):
|
|
|
|
"""return md5 hash of node"""
|
|
|
|
"""return md5 hash of node"""
|
|
|
|
return utils.md5sum(path)
|
|
|
|
try:
|
|
|
|
|
|
|
|
return md5sum(path)
|
|
|
|
|
|
|
|
except CatcliException as exc:
|
|
|
|
|
|
|
|
Logger.err(str(exc))
|
|
|
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
|
|
|
def _sanitize(node):
|
|
|
|
|
|
|
|
"""sanitize node strings"""
|
|
|
|
|
|
|
|
node.name = fix_badchars(node.name)
|
|
|
|
|
|
|
|
node.relpath = fix_badchars(node.relpath)
|
|
|
|
|
|
|
|
return node
|
|
|
|
|
|
|
|
|
|
|
|
def _debug(self, string):
|
|
|
|
def _debug(self, string):
|
|
|
|
'''print debug'''
|
|
|
|
"""print debug"""
|
|
|
|
if not self.debug:
|
|
|
|
if not self.debug:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
Logger.debug(string)
|
|
|
|
Logger.debug(string)
|
|
|
|
|
deadc0de
1 year ago
committed by
GitHub