Archives
/
catcli
mirror of https://github.com/deadc0de6/catcli
2
1
Fork 0
Code Issues Releases Wiki Activity

linting

Browse Source
pull/29/head
deadc0de6 2 years ago
parent cf3e465525
commit c0afb0feeb
18 changed files with 484 additions and 400 deletions
Show Stats Download Patch File Download Diff File

4
catcli/__init__.py
Unescape Escape View File

@ -3,12 +3,12 @@ author: deadc0de6 (https://github.com/deadc0de6)
Copyright (c) 2017, deadc0de6 Copyright (c) 2017, deadc0de6
""" """
# pylint: disable=C0415
import sys import sys
__version__ = '0.8.7'
def main(): def main():
"""entry point"""
import catcli.catcli import catcli.catcli
if catcli.catcli.main(): if catcli.catcli.main():
sys.exit(0) sys.exit(0)

57
catcli/catalog.py
Unescape Escape View File

@ -11,54 +11,57 @@ from anytree.exporter import JsonExporter
from anytree.importer import JsonImporter from anytree.importer import JsonImporter
# local imports # local imports
import catcli.utils as utils from catcli.utils import ask
from catcli.logger import Logger from catcli.logger import Logger
class Catalog: class Catalog:
"""the catalog"""
def __init__(self, path, pickle=False, debug=False, force=False): def __init__(self, path, usepickle=False, debug=False, force=False):
''' """
@path: catalog path @path: catalog path
@pickle: use pickle @usepickle: use pickle
@debug: debug mode @debug: debug mode
@force: force overwrite if exists @force: force overwrite if exists
''' """
self.path = path self.path = path
self.debug = debug self.debug = debug
self.force = force self.force = force
self.metanode = None self.metanode = None
self.pickle = pickle self.pickle = usepickle
def set_metanode(self, metanode): def set_metanode(self, metanode):
'''remove the metanode until tree is re-written''' """remove the metanode until tree is re-written"""
self.metanode = metanode self.metanode = metanode
self.metanode.parent = None self.metanode.parent = None
def restore(self): def restore(self):
'''restore the catalog''' """restore the catalog"""
if not self.path: if not self.path:
return None return None
if not os.path.exists(self.path): if not os.path.exists(self.path):
return None return None
if self.pickle: if self.pickle:
return self._restore_pickle() return self._restore_pickle()
return self._restore_json(open(self.path, 'r').read()) with open(self.path, 'r', encoding='UTF-8') as file:
content = file.read()
return self._restore_json(content)
def save(self, node): def save(self, node):
'''save the catalog''' """save the catalog"""
if not self.path: if not self.path:
Logger.err('Path not defined') Logger.err('Path not defined')
return False return False
d = os.path.dirname(self.path) directory = os.path.dirname(self.path)
if d and not os.path.exists(d): if directory and not os.path.exists(directory):
os.makedirs(d) os.makedirs(directory)
elif os.path.exists(self.path) and not self.force: elif os.path.exists(self.path) and not self.force:
if not utils.ask(f'Update catalog \"{self.path}\"'): if not ask(f'Update catalog \"{self.path}\"'):
Logger.info('Catalog not saved') Logger.info('Catalog not saved')
return False return False
if d and not os.path.exists(d): if directory and not os.path.exists(directory):
Logger.err(f'Cannot write to \"{d}\"') Logger.err(f'Cannot write to \"{directory}\"')
return False return False
if self.metanode: if self.metanode:
self.metanode.parent = node self.metanode.parent = node
@ -72,28 +75,30 @@ class Catalog:
Logger.debug(text) Logger.debug(text)
def _save_pickle(self, node): def _save_pickle(self, node):
'''pickle the catalog''' """pickle the catalog"""
pickle.dump(node, open(self.path, 'wb')) with open(self.path, 'wb') as file:
pickle.dump(node, file)
self._debug(f'Catalog saved to pickle \"{self.path}\"') self._debug(f'Catalog saved to pickle \"{self.path}\"')
return True return True
def _restore_pickle(self): def _restore_pickle(self):
'''restore the pickled tree''' """restore the pickled tree"""
root = pickle.load(open(self.path, 'rb')) with open(self.path, 'rb') as file:
m = f'Catalog imported from pickle \"{self.path}\"' root = pickle.load(file)
self._debug(m) msg = f'Catalog imported from pickle \"{self.path}\"'
self._debug(msg)
return root return root
def _save_json(self, node): def _save_json(self, node):
'''export the catalog in json''' """export the catalog in json"""
exp = JsonExporter(indent=2, sort_keys=True) exp = JsonExporter(indent=2, sort_keys=True)
with open(self.path, 'w') as f: with open(self.path, 'w', encoding='UTF-8') as file:
exp.write(node, f) exp.write(node, file)
self._debug(f'Catalog saved to json \"{self.path}\"') self._debug(f'Catalog saved to json \"{self.path}\"')
return True return True
def _restore_json(self, string): def _restore_json(self, string):
'''restore the tree from json''' """restore the tree from json"""
imp = JsonImporter() imp = JsonImporter()
root = imp.import_(string) root = imp.import_(string)
self._debug(f'Catalog imported from json \"{self.path}\"') self._debug(f'Catalog imported from json \"{self.path}\"')

31
catcli/catcli.py
Unescape Escape View File

@ -14,7 +14,7 @@ import datetime
from docopt import docopt from docopt import docopt
# local imports # local imports
from . import __version__ as VERSION from .version import __version__ as VERSION
from .logger import Logger from .logger import Logger
from .catalog import Catalog from .catalog import Catalog
from .walker import Walker from .walker import Walker
@ -77,9 +77,10 @@ Options:
def cmd_index(args, noder, catalog, top): def cmd_index(args, noder, catalog, top):
"""index action"""
path = args['<path>'] path = args['<path>']
name = args['<name>'] name = args['<name>']
hash = args['--hash'] usehash = args['--hash']
debug = args['--verbose'] debug = args['--verbose']
subsize = not args['--no-subsize'] subsize = not args['--no-subsize']
if not os.path.exists(path): if not os.path.exists(path):
@ -97,7 +98,7 @@ def cmd_index(args, noder, catalog, top):
node.parent = None node.parent = None
start = datetime.datetime.now() start = datetime.datetime.now()
walker = Walker(noder, hash=hash, debug=debug) walker = Walker(noder, usehash=usehash, debug=debug)
attr = noder.format_storage_attr(args['--meta']) attr = noder.format_storage_attr(args['--meta'])
root = noder.storage_node(name, path, parent=top, attr=attr) root = noder.storage_node(name, path, parent=top, attr=attr)
_, cnt = walker.index(path, root, name) _, cnt = walker.index(path, root, name)
@ -111,9 +112,10 @@ def cmd_index(args, noder, catalog, top):
def cmd_update(args, noder, catalog, top): def cmd_update(args, noder, catalog, top):
"""update action"""
path = args['<path>'] path = args['<path>']
name = args['<name>'] name = args['<name>']
hash = args['--hash'] usehash = args['--hash']
logpath = args['--lpath'] logpath = args['--lpath']
debug = args['--verbose'] debug = args['--verbose']
subsize = not args['--no-subsize'] subsize = not args['--no-subsize']
@ -125,7 +127,7 @@ def cmd_update(args, noder, catalog, top):
Logger.err(f'storage named \"{name}\" does not exist') Logger.err(f'storage named \"{name}\" does not exist')
return return
start = datetime.datetime.now() start = datetime.datetime.now()
walker = Walker(noder, hash=hash, debug=debug, walker = Walker(noder, usehash=usehash, debug=debug,
logpath=logpath) logpath=logpath)
cnt = walker.reindex(path, root, top) cnt = walker.reindex(path, root, top)
if subsize: if subsize:
@ -138,6 +140,7 @@ def cmd_update(args, noder, catalog, top):
def cmd_ls(args, noder, top): def cmd_ls(args, noder, top):
"""ls action"""
path = args['<path>'] path = args['<path>']
if not path: if not path:
path = SEPARATOR path = SEPARATOR
@ -161,6 +164,7 @@ def cmd_ls(args, noder, top):
def cmd_rm(args, noder, catalog, top): def cmd_rm(args, noder, catalog, top):
"""rm action"""
name = args['<storage>'] name = args['<storage>']
node = noder.get_storage_node(top, name) node = noder.get_storage_node(top, name)
if node: if node:
@ -173,6 +177,7 @@ def cmd_rm(args, noder, catalog, top):
def cmd_find(args, noder, top): def cmd_find(args, noder, top):
"""find action"""
fromtree = args['--parent'] fromtree = args['--parent']
directory = args['--directory'] directory = args['--directory']
startpath = args['--path'] startpath = args['--path']
@ -186,6 +191,7 @@ def cmd_find(args, noder, top):
def cmd_tree(args, noder, top): def cmd_tree(args, noder, top):
"""tree action"""
path = args['<path>'] path = args['<path>']
hdr = args['--header'] hdr = args['--header']
raw = args['--raw-size'] raw = args['--raw-size']
@ -201,6 +207,7 @@ def cmd_tree(args, noder, top):
def cmd_graph(args, noder, top): def cmd_graph(args, noder, top):
"""graph action"""
path = args['<path>'] path = args['<path>']
if not path: if not path:
path = GRAPHPATH path = GRAPHPATH
@ -208,7 +215,8 @@ def cmd_graph(args, noder, top):
Logger.info(f'create graph with \"{cmd}\" (you need graphviz)') Logger.info(f'create graph with \"{cmd}\" (you need graphviz)')
def cmd_rename(args, noder, catalog, top): def cmd_rename(args, catalog, top):
"""rename action"""
storage = args['<storage>'] storage = args['<storage>']
new = args['<name>'] new = args['<name>']
storages = list(x.name for x in top.children) storages = list(x.name for x in top.children)
@ -216,14 +224,15 @@ def cmd_rename(args, noder, catalog, top):
node = next(filter(lambda x: x.name == storage, top.children)) node = next(filter(lambda x: x.name == storage, top.children))
node.name = new node.name = new
if catalog.save(top): if catalog.save(top):
m = f'Storage \"{storage}\" renamed to \"{new}\"' msg = f'Storage \"{storage}\" renamed to \"{new}\"'
Logger.info(m) Logger.info(msg)
else: else:
Logger.err(f'Storage named \"{storage}\" does not exist') Logger.err(f'Storage named \"{storage}\" does not exist')
return top return top
def cmd_edit(args, noder, catalog, top): def cmd_edit(args, noder, catalog, top):
"""edit action"""
storage = args['<storage>'] storage = args['<storage>']
storages = list(x.name for x in top.children) storages = list(x.name for x in top.children)
if storage in storages: if storage in storages:
@ -241,11 +250,13 @@ def cmd_edit(args, noder, catalog, top):
def banner(): def banner():
"""print banner"""
Logger.out_err(BANNER) Logger.out_err(BANNER)
Logger.out_err("") Logger.out_err("")
def print_supported_formats(): def print_supported_formats():
"""print all supported formats to stdout"""
print('"native" : native format') print('"native" : native format')
print('"csv" : CSV format') print('"csv" : CSV format')
print(f' {Noder.CSV_HEADER}') print(f' {Noder.CSV_HEADER}')
@ -254,6 +265,7 @@ def print_supported_formats():
def main(): def main():
"""entry point"""
args = docopt(USAGE, version=VERSION) args = docopt(USAGE, version=VERSION)
if args['help'] or args['--help']: if args['help'] or args['--help']:
@ -313,7 +325,7 @@ def main():
elif args['graph']: elif args['graph']:
cmd_graph(args, noder, top) cmd_graph(args, noder, top)
elif args['rename']: elif args['rename']:
cmd_rename(args, noder, catalog, top) cmd_rename(args, catalog, top)
elif args['edit']: elif args['edit']:
cmd_edit(args, noder, catalog, top) cmd_edit(args, noder, catalog, top)
@ -321,7 +333,6 @@ def main():
if __name__ == '__main__': if __name__ == '__main__':
'''entry point'''
if main(): if main():
sys.exit(0) sys.exit(0)
sys.exit(1) sys.exit(1)

19
catcli/decomp.py
Unescape Escape View File

@ -11,6 +11,7 @@ import zipfile
class Decomp: class Decomp:
"""decompressor"""
def __init__(self): def __init__(self):
self.ext = { self.ext = {
@ -28,26 +29,26 @@ class Decomp:
'zip': self._zip} 'zip': self._zip}
def get_formats(self): def get_formats(self):
'''return list of supported extensions''' """return list of supported extensions"""
return list(self.ext.keys()) return list(self.ext.keys())
def get_names(self, path): def get_names(self, path):
'''get tree of compressed archive''' """get tree of compressed archive"""
ext = os.path.splitext(path)[1][1:].lower() ext = os.path.splitext(path)[1][1:].lower()
if ext in list(self.ext.keys()): if ext in list(self.ext):
return self.ext[ext](path) return self.ext[ext](path)
return None return None
def _tar(self, path): def _tar(self, path):
'''return list of file names in tar''' """return list of file names in tar"""
if not tarfile.is_tarfile(path): if not tarfile.is_tarfile(path):
return None return None
tar = tarfile.open(path, "r") with tarfile.open(path, "r") as tar:
return tar.getnames() return tar.getnames()
def _zip(self, path): def _zip(self, path):
'''return list of file names in zip''' """return list of file names in zip"""
if not zipfile.is_zipfile(path): if not zipfile.is_zipfile(path):
return None return None
z = zipfile.ZipFile(path) with zipfile.ZipFile(path) as file:
return z.namelist() return file.namelist()

105
catcli/logger.py
Unescape Escape View File

@ -9,6 +9,7 @@ import sys
class Logger: class Logger:
"""log to stdout/stderr"""
RED = '\033[91m' RED = '\033[91m'
GREEN = '\033[92m' GREEN = '\033[92m'
@ -26,7 +27,9 @@ class Logger:
ARCHIVE = 'archive' ARCHIVE = 'archive'
NBFILES = 'nbfiles' NBFILES = 'nbfiles'
def no_color(): @classmethod
def no_color(cls):
"""disable colors"""
Logger.RED = '' Logger.RED = ''
Logger.GREEN = '' Logger.GREEN = ''
Logger.YELLOW = '' Logger.YELLOW = ''
@ -39,32 +42,37 @@ class Logger:
Logger.BOLD = '' Logger.BOLD = ''
Logger.UND = '' Logger.UND = ''
def fix_badchars(line): @classmethod
def fix_badchars(cls, line):
"""fix none utf-8 chars in line"""
return line.encode('utf-8', 'ignore').decode('utf-8') return line.encode('utf-8', 'ignore').decode('utf-8')
###################################################################### ######################################################################
# node specific output # node specific output
###################################################################### ######################################################################
def storage(pre, name, args, attr): @classmethod
'''print a storage node''' def storage(cls, pre, name, args, attr):
"""print a storage node"""
end = '' end = ''
if attr: if attr:
end = f' {Logger.GRAY}({attr}){Logger.RESET}' end = f' {Logger.GRAY}({attr}){Logger.RESET}'
s = f'{pre}{Logger.UND}{Logger.STORAGE}{Logger.RESET}:' out = f'{pre}{Logger.UND}{Logger.STORAGE}{Logger.RESET}:'
s += ' ' + Logger.PURPLE + Logger.fix_badchars(name) + \ out += ' ' + Logger.PURPLE + Logger.fix_badchars(name) + \
Logger.RESET + end + '\n' Logger.RESET + end + '\n'
s += f' {Logger.GRAY}{args}{Logger.RESET}' out += f' {Logger.GRAY}{args}{Logger.RESET}'
sys.stdout.write(f'{s}\n') sys.stdout.write(f'{out}\n')
def file(pre, name, attr): @classmethod
'''print a file node''' def file(cls, pre, name, attr):
"""print a file node"""
nobad = Logger.fix_badchars(name) nobad = Logger.fix_badchars(name)
s = f'{pre}{nobad}' out = f'{pre}{nobad}'
s += f' {Logger.GRAY}[{attr}]{Logger.RESET}' out += f' {Logger.GRAY}[{attr}]{Logger.RESET}'
sys.stdout.write(f'{s}\n') sys.stdout.write(f'{out}\n')
def dir(pre, name, depth='', attr=None): @classmethod
'''print a directory node''' def dir(cls, pre, name, depth='', attr=None):
"""print a directory node"""
end = [] end = []
if depth != '': if depth != '':
end.append(f'{Logger.NBFILES}:{depth}') end.append(f'{Logger.NBFILES}:{depth}')
@ -73,60 +81,71 @@ class Logger:
if end: if end:
endstring = ', '.join(end) endstring = ', '.join(end)
end = f' [{endstring}]' end = f' [{endstring}]'
s = pre + Logger.BLUE + Logger.fix_badchars(name) + Logger.RESET out = pre + Logger.BLUE + Logger.fix_badchars(name) + Logger.RESET
s += f'{Logger.GRAY}{end}{Logger.RESET}' out += f'{Logger.GRAY}{end}{Logger.RESET}'
sys.stdout.write(f'{s}\n') sys.stdout.write(f'{out}\n')
def arc(pre, name, archive): @classmethod
s = pre + Logger.YELLOW + Logger.fix_badchars(name) + Logger.RESET def arc(cls, pre, name, archive):
s += f' {Logger.GRAY}[{Logger.ARCHIVE}:{archive}]{Logger.RESET}' """archive to stdout"""
sys.stdout.write(f'{s}\n') out = pre + Logger.YELLOW + Logger.fix_badchars(name) + Logger.RESET
out += f' {Logger.GRAY}[{Logger.ARCHIVE}:{archive}]{Logger.RESET}'
sys.stdout.write(f'{out}\n')
###################################################################### ######################################################################
# generic output # generic output
###################################################################### ######################################################################
def out(string): @classmethod
'''to stdout no color''' def out(cls, string):
"""to stdout no color"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
sys.stdout.write(f'{string}\n') sys.stdout.write(f'{string}\n')
def out_err(string): @classmethod
'''to stderr no color''' def out_err(cls, string):
"""to stderr no color"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
sys.stderr.write(f'{string}\n') sys.stderr.write(f'{string}\n')
def debug(string): @classmethod
'''to stderr no color''' def debug(cls, string):
"""to stderr no color"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
sys.stderr.write(f'[DBG] {string}\n') sys.stderr.write(f'[DBG] {string}\n')
def info(string): @classmethod
'''to stdout in color''' def info(cls, string):
"""to stdout in color"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
s = f'{Logger.MAGENTA}{string}{Logger.RESET}' out = f'{Logger.MAGENTA}{string}{Logger.RESET}'
sys.stdout.write(f'{s}\n') sys.stdout.write(f'{out}\n')
def err(string): @classmethod
'''to stderr in RED''' def err(cls, string):
"""to stderr in RED"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
s = f'{Logger.RED}{string}{Logger.RESET}' out = f'{Logger.RED}{string}{Logger.RESET}'
sys.stderr.write(f'{s}\n') sys.stderr.write(f'{out}\n')
def progr(string): @classmethod
'''print progress''' def progr(cls, string):
"""print progress"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
sys.stderr.write(f'{string}\r') sys.stderr.write(f'{string}\r')
sys.stderr.flush() sys.stderr.flush()
def bold(string): @classmethod
'''make it bold''' def bold(cls, string):
"""make it bold"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
return f'{Logger.BOLD}{string}{Logger.RESET}' return f'{Logger.BOLD}{string}{Logger.RESET}'
def flog(path, string, append=True): @classmethod
def flog(cls, path, string, append=True):
"""log and fix bad chars"""
string = Logger.fix_badchars(string) string = Logger.fix_badchars(string)
mode = 'w' mode = 'w'
if append: if append:
mode = 'a' mode = 'a'
with open(path, mode) as f: with open(path, mode, encoding='UTF-8') as file:
f.write(string) file.write(string)

318
catcli/noder.py
Unescape Escape View File

@ -12,21 +12,21 @@ import anytree
from pyfzf.pyfzf import FzfPrompt from pyfzf.pyfzf import FzfPrompt
# local imports # local imports
import catcli.utils as utils from catcli.utils import size_to_str, epoch_to_str, md5sum
from catcli.logger import Logger from catcli.logger import Logger
from catcli.decomp import Decomp from catcli.decomp import Decomp
from . import __version__ as VERSION from catcli.version import __version__ as VERSION
'''
There are 4 types of node: class Noder:
"""
handles node in the catalog tree
There are 4 types of node:
* "top" node representing the top node (generic node) * "top" node representing the top node (generic node)
* "storage" node representing a storage * "storage" node representing a storage
* "dir" node representing a directory * "dir" node representing a directory
* "file" node representing a file * "file" node representing a file
''' """
class Noder:
TOPNAME = 'top' TOPNAME = 'top'
METANAME = 'meta' METANAME = 'meta'
@ -41,11 +41,11 @@ class Noder:
'total_space,meta') 'total_space,meta')
def __init__(self, debug=False, sortsize=False, arc=False): def __init__(self, debug=False, sortsize=False, arc=False):
''' """
@debug: debug mode @debug: debug mode
@sortsize: sort nodes by size @sortsize: sort nodes by size
@arch: handle archive @arch: handle archive
''' """
self.hash = True self.hash = True
self.debug = debug self.debug = debug
self.sortsize = sortsize self.sortsize = sortsize
@ -54,20 +54,20 @@ class Noder:
self.decomp = Decomp() self.decomp = Decomp()
def get_storage_names(self, top): def get_storage_names(self, top):
'''return a list of all storage names''' """return a list of all storage names"""
return [x.name for x in list(top.children)] return [x.name for x in list(top.children)]
def get_storage_node(self, top, name, path=None): def get_storage_node(self, top, name, path=None):
''' """
return the storage node if any return the storage node if any
if path is submitted, it will update the media info if path is submitted, it will update the media info
''' """
found = None found = None
for n in top.children: for node in top.children:
if n.type != self.TYPE_STORAGE: if node.type != self.TYPE_STORAGE:
continue continue
if n.name == name: if node.name == name:
found = n found = node
break break
if found and path and os.path.exists(path): if found and path and os.path.exists(path):
found.free = shutil.disk_usage(path).free found.free = shutil.disk_usage(path).free
@ -76,23 +76,23 @@ class Noder:
return found return found
def get_node(self, top, path, quiet=False): def get_node(self, top, path, quiet=False):
'''get the node by internal tree path''' """get the node by internal tree path"""
r = anytree.resolver.Resolver('name') resolv = anytree.resolver.Resolver('name')
try: try:
p = os.path.basename(path) bpath = os.path.basename(path)
return r.get(top, p) return resolv.get(top, bpath)
except anytree.resolver.ChildResolverError: except anytree.resolver.ChildResolverError:
if not quiet: if not quiet:
Logger.err(f'No node at path \"{p}\"') Logger.err(f'No node at path \"{bpath}\"')
return None return None
def get_node_if_changed(self, top, path, treepath): def get_node_if_changed(self, top, path, treepath):
''' """
return the node (if any) and if it has changed return the node (if any) and if it has changed
@top: top node (storage) @top: top node (storage)
@path: abs path to file @path: abs path to file
@treepath: rel path from indexed directory @treepath: rel path from indexed directory
''' """
treepath = treepath.lstrip(os.sep) treepath = treepath.lstrip(os.sep)
node = self.get_node(top, treepath, quiet=True) node = self.get_node(top, treepath, quiet=True)
# node does not exist # node does not exist
@ -116,34 +116,34 @@ class Noder:
if self.hash and node.md5: if self.hash and node.md5:
md5 = self._get_hash(path) md5 = self._get_hash(path)
if md5 != node.md5: if md5 != node.md5:
m = f'\tchange: checksum changed for \"{path}\"' msg = f'\tchange: checksum changed for \"{path}\"'
self._debug(m) self._debug(msg)
return node, True return node, True
self._debug(f'\tchange: no change for \"{path}\"') self._debug(f'\tchange: no change for \"{path}\"')
return node, False return node, False
def _rec_size(self, node, store=True): def _rec_size(self, node, store=True):
''' """
recursively traverse tree and return size recursively traverse tree and return size
@store: store the size in the node @store: store the size in the node
''' """
if node.type == self.TYPE_FILE: if node.type == self.TYPE_FILE:
self._debug(f'getting node size for \"{node.name}\"') self._debug(f'getting node size for \"{node.name}\"')
return node.size return node.size
m = f'getting node size recursively for \"{node.name}\"' msg = f'getting node size recursively for \"{node.name}\"'
self._debug(m) self._debug(msg)
size = 0 size = 0
for i in node.children: for i in node.children:
if node.type == self.TYPE_DIR: if node.type == self.TYPE_DIR:
sz = self._rec_size(i, store=store) size = self._rec_size(i, store=store)
if store: if store:
i.size = sz i.size = size
size += sz size += size
if node.type == self.TYPE_STORAGE: if node.type == self.TYPE_STORAGE:
sz = self._rec_size(i, store=store) size = self._rec_size(i, store=store)
if store: if store:
i.size = sz i.size = size
size += sz size += size
else: else:
continue continue
if store: if store:
@ -151,34 +151,34 @@ class Noder:
return size return size
def rec_size(self, node): def rec_size(self, node):
'''recursively traverse tree and store dir size''' """recursively traverse tree and store dir size"""
return self._rec_size(node, store=True) return self._rec_size(node, store=True)
############################################################### ###############################################################
# public helpers # public helpers
############################################################### ###############################################################
def format_storage_attr(self, attr): def format_storage_attr(self, attr):
'''format the storage attr for saving''' """format the storage attr for saving"""
if not attr: if not attr:
return '' return ''
if type(attr) is list: if isinstance(attr, list):
return ', '.join(attr) return ', '.join(attr)
attr = attr.rstrip() attr = attr.rstrip()
return attr return attr
def set_hashing(self, val): def set_hashing(self, val):
'''hash files when indexing''' """hash files when indexing"""
self.hash = val self.hash = val
############################################################### ###############################################################
# node creationg # node creationg
############################################################### ###############################################################
def new_top_node(self): def new_top_node(self):
'''create a new top node''' """create a new top node"""
return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP) return anytree.AnyNode(name=self.TOPNAME, type=self.TYPE_TOP)
def update_metanode(self, top): def update_metanode(self, top):
'''create or update meta node information''' """create or update meta node information"""
meta = self._get_meta_node(top) meta = self._get_meta_node(top)
epoch = int(time.time()) epoch = int(time.time())
if not meta: if not meta:
@ -192,7 +192,7 @@ class Noder:
return meta return meta
def _get_meta_node(self, top): def _get_meta_node(self, top):
'''return the meta node if any''' """return the meta node if any"""
try: try:
return next(filter(lambda x: x.type == self.TYPE_META, return next(filter(lambda x: x.type == self.TYPE_META,
top.children)) top.children))
@ -200,15 +200,15 @@ class Noder:
return None return None
def file_node(self, name, path, parent, storagepath): def file_node(self, name, path, parent, storagepath):
'''create a new node representing a file''' """create a new node representing a file"""
if not os.path.exists(path): if not os.path.exists(path):
Logger.err(f'File \"{path}\" does not exist') Logger.err(f'File \"{path}\" does not exist')
return None return None
path = os.path.abspath(path) path = os.path.abspath(path)
try: try:
st = os.lstat(path) stat = os.lstat(path)
except OSError as e: except OSError as exc:
Logger.err(f'OSError: {e}') Logger.err(f'OSError: {exc}')
return None return None
md5 = None md5 = None
if self.hash: if self.hash:
@ -216,20 +216,20 @@ class Noder:
relpath = os.sep.join([storagepath, name]) relpath = os.sep.join([storagepath, name])
maccess = os.path.getmtime(path) maccess = os.path.getmtime(path)
n = self._node(name, self.TYPE_FILE, relpath, parent, node = self._node(name, self.TYPE_FILE, relpath, parent,
size=st.st_size, md5=md5, maccess=maccess) size=stat.st_size, md5=md5, maccess=maccess)
if self.arc: if self.arc:
ext = os.path.splitext(path)[1][1:] ext = os.path.splitext(path)[1][1:]
if ext.lower() in self.decomp.get_formats(): if ext.lower() in self.decomp.get_formats():
self._debug(f'{path} is an archive') self._debug(f'{path} is an archive')
names = self.decomp.get_names(path) names = self.decomp.get_names(path)
self.list_to_tree(n, names) self.list_to_tree(node, names)
else: else:
self._debug(f'{path} is NOT an archive') self._debug(f'{path} is NOT an archive')
return n return node
def dir_node(self, name, path, parent, storagepath): def dir_node(self, name, path, parent, storagepath):
'''create a new node representing a directory''' """create a new node representing a directory"""
path = os.path.abspath(path) path = os.path.abspath(path)
relpath = os.sep.join([storagepath, name]) relpath = os.sep.join([storagepath, name])
maccess = os.path.getmtime(path) maccess = os.path.getmtime(path)
@ -237,21 +237,21 @@ class Noder:
parent, maccess=maccess) parent, maccess=maccess)
def clean_not_flagged(self, top): def clean_not_flagged(self, top):
'''remove any node not flagged and clean flags''' """remove any node not flagged and clean flags"""
cnt = 0 cnt = 0
for node in anytree.PreOrderIter(top): for node in anytree.PreOrderIter(top):
if node.type != self.TYPE_FILE and node.type != self.TYPE_DIR: if node.type not in [self.TYPE_FILE, self.TYPE_DIR]:
continue continue
if self._clean(node): if self._clean(node):
cnt += 1 cnt += 1
return cnt return cnt
def flag(self, node): def flag(self, node):
'''flag a node''' """flag a node"""
node.flag = True node.flag = True
def _clean(self, node): def _clean(self, node):
'''remove node if not flagged''' """remove node if not flagged"""
if not self._has_attr(node, 'flag') or \ if not self._has_attr(node, 'flag') or \
not node.flag: not node.flag:
node.parent = None node.parent = None
@ -260,7 +260,7 @@ class Noder:
return False return False
def storage_node(self, name, path, parent, attr=None): def storage_node(self, name, path, parent, attr=None):
'''create a new node representing a storage''' """create a new node representing a storage"""
path = os.path.abspath(path) path = os.path.abspath(path)
free = shutil.disk_usage(path).free free = shutil.disk_usage(path).free
total = shutil.disk_usage(path).total total = shutil.disk_usage(path).total
@ -269,15 +269,15 @@ class Noder:
total=total, parent=parent, attr=attr, ts=epoch) total=total, parent=parent, attr=attr, ts=epoch)
def archive_node(self, name, path, parent, archive): def archive_node(self, name, path, parent, archive):
'''crete a new node for archive data''' """crete a new node for archive data"""
return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path, return anytree.AnyNode(name=name, type=self.TYPE_ARC, relpath=path,
parent=parent, size=0, md5=None, parent=parent, size=0, md5=None,
archive=archive) archive=archive)
def _node(self, name, type, relpath, parent, def _node(self, name, nodetype, relpath, parent,
size=None, md5=None, maccess=None): size=None, md5=None, maccess=None):
'''generic node creation''' """generic node creation"""
return anytree.AnyNode(name=name, type=type, relpath=relpath, return anytree.AnyNode(name=name, type=nodetype, relpath=relpath,
parent=parent, size=size, parent=parent, size=size,
md5=md5, maccess=maccess) md5=md5, maccess=maccess)
@ -285,16 +285,16 @@ class Noder:
# printing # printing
############################################################### ###############################################################
def _node_to_csv(self, node, sep=',', raw=False): def _node_to_csv(self, node, sep=',', raw=False):
''' """
print a node to csv print a node to csv
@node: the node to consider @node: the node to consider
@sep: CSV separator character @sep: CSV separator character
@raw: print raw size rather than human readable @raw: print raw size rather than human readable
''' """
if not node: if not node:
return '' return
if node.type == self.TYPE_TOP: if node.type == self.TYPE_TOP:
return '' return
out = [] out = []
if node.type == self.TYPE_STORAGE: if node.type == self.TYPE_STORAGE:
@ -302,16 +302,16 @@ class Noder:
out.append(node.name) # name out.append(node.name) # name
out.append(node.type) # type out.append(node.type) # type
out.append('') # fake full path out.append('') # fake full path
sz = self._rec_size(node, store=False) size = self._rec_size(node, store=False)
out.append(utils.size_to_str(sz, raw=raw)) # size out.append(size_to_str(size, raw=raw)) # size
out.append(utils.epoch_to_str(node.ts)) # indexed_at out.append(epoch_to_str(node.ts)) # indexed_at
out.append('') # fake maccess out.append('') # fake maccess
out.append('') # fake md5 out.append('') # fake md5
out.append(str(len(node.children))) # nbfiles out.append(str(len(node.children))) # nbfiles
# fake free_space # fake free_space
out.append(utils.size_to_str(node.free, raw=raw)) out.append(size_to_str(node.free, raw=raw))
# fake total_space # fake total_space
out.append(utils.size_to_str(node.total, raw=raw)) out.append(size_to_str(node.total, raw=raw))
out.append(node.attr) # meta out.append(node.attr) # meta
else: else:
# handle other nodes # handle other nodes
@ -322,10 +322,10 @@ class Noder:
fullpath = os.path.join(storage.name, parents) fullpath = os.path.join(storage.name, parents)
out.append(fullpath.replace('"', '""')) # full path out.append(fullpath.replace('"', '""')) # full path
out.append(utils.size_to_str(node.size, raw=raw)) # size out.append(size_to_str(node.size, raw=raw)) # size
out.append(utils.epoch_to_str(storage.ts)) # indexed_at out.append(epoch_to_str(storage.ts)) # indexed_at
if self._has_attr(node, 'maccess'): if self._has_attr(node, 'maccess'):
out.append(utils.epoch_to_str(node.maccess)) # maccess out.append(epoch_to_str(node.maccess)) # maccess
else: else:
out.append('') # fake maccess out.append('') # fake maccess
if node.md5: if node.md5:
@ -347,7 +347,7 @@ class Noder:
def _print_node(self, node, pre='', withpath=False, def _print_node(self, node, pre='', withpath=False,
withdepth=False, withstorage=False, withdepth=False, withstorage=False,
recalcparent=False, raw=False): recalcparent=False, raw=False):
''' """
print a node print a node
@node: the node to print @node: the node to print
@pre: string to print before node @pre: string to print before node
@ -356,7 +356,7 @@ class Noder:
@withstorage: print the node storage it belongs to @withstorage: print the node storage it belongs to
@recalcparent: get relpath from tree instead of relpath field @recalcparent: get relpath from tree instead of relpath field
@raw: print raw size rather than human readable @raw: print raw size rather than human readable
''' """
if node.type == self.TYPE_TOP: if node.type == self.TYPE_TOP:
# top node # top node
Logger.out(f'{pre}{node.name}') Logger.out(f'{pre}{node.name}')
@ -374,8 +374,8 @@ class Noder:
attr = '' attr = ''
if node.md5: if node.md5:
attr = f', md5:{node.md5}' attr = f', md5:{node.md5}'
sz = utils.size_to_str(node.size, raw=raw) size = size_to_str(node.size, raw=raw)
compl = f'size:{sz}{attr}' compl = f'size:{size}{attr}'
if withstorage: if withstorage:
content = Logger.bold(storage.name) content = Logger.bold(storage.name)
compl += f', storage:{content}' compl += f', storage:{content}'
@ -396,35 +396,35 @@ class Noder:
storage = self._get_storage(node) storage = self._get_storage(node)
attr = [] attr = []
if node.size: if node.size:
attr.append(['totsize', utils.size_to_str(node.size, raw=raw)]) attr.append(['totsize', size_to_str(node.size, raw=raw)])
if withstorage: if withstorage:
attr.append(['storage', Logger.bold(storage.name)]) attr.append(['storage', Logger.bold(storage.name)])
Logger.dir(pre, name, depth=depth, attr=attr) Logger.dir(pre, name, depth=depth, attr=attr)
elif node.type == self.TYPE_STORAGE: elif node.type == self.TYPE_STORAGE:
# node of type storage # node of type storage
hf = utils.size_to_str(node.free, raw=raw) szfree = size_to_str(node.free, raw=raw)
ht = utils.size_to_str(node.total, raw=raw) sztotal = size_to_str(node.total, raw=raw)
nbchildren = len(node.children) nbchildren = len(node.children)
pcent = node.free * 100 / node.total pcent = node.free * 100 / node.total
freepercent = f'{pcent:.1f}%' freepercent = f'{pcent:.1f}%'
# get the date # get the date
dt = '' timestamp = ''
if self._has_attr(node, 'ts'): if self._has_attr(node, 'ts'):
dt = 'date:' timestamp = 'date:'
dt += utils.epoch_to_str(node.ts) timestamp += epoch_to_str(node.ts)
ds = '' disksize = ''
# the children size # the children size
sz = self._rec_size(node, store=False) size = self._rec_size(node, store=False)
sz = utils.size_to_str(sz, raw=raw) size = size_to_str(size, raw=raw)
ds = 'totsize:' + f'{sz}' disksize = 'totsize:' + f'{size}'
# format the output # format the output
name = node.name name = node.name
args = [ args = [
'nbfiles:' + f'{nbchildren}', 'nbfiles:' + f'{nbchildren}',
ds, disksize,
f'free:{freepercent}', f'free:{freepercent}',
'du:' + f'{hf}/{ht}', 'du:' + f'{szfree}/{sztotal}',
dt] timestamp]
argsstring = ' | '.join(args) argsstring = ' | '.join(args)
Logger.storage(pre, Logger.storage(pre,
name, name,
@ -437,35 +437,37 @@ class Noder:
else: else:
Logger.err(f'bad node encountered: {node}') Logger.err(f'bad node encountered: {node}')
def print_tree(self, top, node, style=anytree.ContRoundStyle(), def print_tree(self, top, node,
fmt='native', header=False, raw=False): fmt='native',
''' header=False,
raw=False):
"""
print the tree in different format print the tree in different format
@node: start node @node: start node
@style: when fmt=native, defines the tree style @style: when fmt=native, defines the tree style
@fmt: output format @fmt: output format
@header: when fmt=csv, print the header @header: when fmt=csv, print the header
@raw: print the raw size rather than human readable @raw: print the raw size rather than human readable
''' """
if fmt == 'native': if fmt == 'native':
# "tree" style # "tree" style
rend = anytree.RenderTree(node, childiter=self._sort_tree) rend = anytree.RenderTree(node, childiter=self._sort_tree)
for pre, fill, node in rend: for pre, _, thenode in rend:
self._print_node(node, pre=pre, withdepth=True, raw=raw) self._print_node(thenode, pre=pre, withdepth=True, raw=raw)
elif fmt == 'csv': elif fmt == 'csv':
# csv output # csv output
self._to_csv(node, with_header=header, raw=raw) self._to_csv(node, with_header=header, raw=raw)
elif fmt.startswith('fzf'): elif fmt.startswith('fzf'):
# flat # flat
self._to_fzf(top, node) self._to_fzf(top, node, fmt)
def _to_csv(self, node, with_header=False, raw=False): def _to_csv(self, node, with_header=False, raw=False):
'''print the tree to csv''' """print the tree to csv"""
rend = anytree.RenderTree(node, childiter=self._sort_tree) rend = anytree.RenderTree(node, childiter=self._sort_tree)
if with_header: if with_header:
Logger.out(self.CSV_HEADER) Logger.out(self.CSV_HEADER)
for _, _, node in rend: for _, _, item in rend:
self._node_to_csv(node, raw=raw) self._node_to_csv(item, raw=raw)
def _fzf_prompt(self, strings): def _fzf_prompt(self, strings):
# prompt with fzf # prompt with fzf
@ -480,16 +482,16 @@ class Noder:
@node: node to start with @node: node to start with
@fmt: output format for selected nodes @fmt: output format for selected nodes
""" """
rend = anytree.RenderTree(node, childiter=self._sort_tree) rendered = anytree.RenderTree(node, childiter=self._sort_tree)
nodes = dict() nodes = {}
# construct node names list # construct node names list
for _, _, node in rend: for _, _, rend in rendered:
if not node: if not rend:
continue continue
parents = self._get_parents(node) parents = self._get_parents(rend)
storage = self._get_storage(node) storage = self._get_storage(rend)
fullpath = os.path.join(storage.name, parents) fullpath = os.path.join(storage.name, parents)
nodes[fullpath] = node nodes[fullpath] = rend
# prompt with fzf # prompt with fzf
paths = self._fzf_prompt(nodes.keys()) paths = self._fzf_prompt(nodes.keys())
# print the resulting tree # print the resulting tree
@ -499,11 +501,11 @@ class Noder:
continue continue
if path not in nodes: if path not in nodes:
continue continue
node = nodes[path] rend = nodes[path]
self.print_tree(top, node, fmt=subfmt) self.print_tree(top, rend, fmt=subfmt)
def to_dot(self, node, path='tree.dot'): def to_dot(self, node, path='tree.dot'):
'''export to dot for graphing''' """export to dot for graphing"""
anytree.exporter.DotExporter(node).to_dotfile(path) anytree.exporter.DotExporter(node).to_dotfile(path)
Logger.info(f'dot file created under \"{path}\"') Logger.info(f'dot file created under \"{path}\"')
return f'dot {path} -T png -o /tmp/tree.png' return f'dot {path} -T png -o /tmp/tree.png'
@ -515,7 +517,7 @@ class Noder:
script=False, directory=False, script=False, directory=False,
startpath=None, parentfromtree=False, startpath=None, parentfromtree=False,
fmt='native', raw=False): fmt='native', raw=False):
''' """
find files based on their names find files based on their names
@top: top node @top: top node
@key: term to search for @key: term to search for
@ -525,7 +527,7 @@ class Noder:
@parentfromtree: get path from parent instead of stored relpath @parentfromtree: get path from parent instead of stored relpath
@fmt: output format @fmt: output format
@raw: raw size output @raw: raw size output
''' """
self._debug(f'searching for \"{key}\"') self._debug(f'searching for \"{key}\"')
if not key: if not key:
# nothing to search for # nothing to search for
@ -534,49 +536,49 @@ class Noder:
if startpath: if startpath:
start = self.get_node(top, startpath) start = self.get_node(top, startpath)
found = anytree.findall(start, filter_=self._callback_find_name(key)) found = anytree.findall(start, filter_=self._callback_find_name(key))
nb = len(found) nbfound = len(found)
self._debug(f'found {nb} node(s)') self._debug(f'found {nbfound} node(s)')
# compile found nodes # compile found nodes
paths = dict() paths = {}
nodes = [] nodes = []
for f in found: for item in found:
if f.type == self.TYPE_STORAGE: if item.type == self.TYPE_STORAGE:
# ignore storage nodes # ignore storage nodes
continue continue
if directory and f.type != self.TYPE_DIR: if directory and item.type != self.TYPE_DIR:
# ignore non directory # ignore non directory
continue continue
nodes.append(f) nodes.append(item)
if parentfromtree: if parentfromtree:
paths[self._get_parents(f)] = f paths[self._get_parents(item)] = item
else: else:
paths[f.relpath] = f paths[item.relpath] = item
if fmt == 'native': if fmt == 'native':
for n in nodes: for item in nodes:
self._print_node(n, withpath=True, self._print_node(item, withpath=True,
withdepth=True, withdepth=True,
withstorage=True, withstorage=True,
recalcparent=parentfromtree, recalcparent=parentfromtree,
raw=raw) raw=raw)
elif fmt == 'csv': elif fmt == 'csv':
for n in nodes: for item in nodes:
self._node_to_csv(n, raw=raw) self._node_to_csv(item, raw=raw)
elif fmt.startswith('fzf'): elif fmt.startswith('fzf'):
selected = self._fzf_prompt(paths) selected = self._fzf_prompt(paths)
newpaths = dict() newpaths = {}
subfmt = fmt.replace('fzf-', '') subfmt = fmt.replace('fzf-', '')
for s in selected: for item in selected:
if s not in paths: if item not in paths:
continue continue
newpaths[s] = paths[s] newpaths[item] = paths[item]
self.print_tree(top, newpaths[s], fmt=subfmt) self.print_tree(top, newpaths[item], fmt=subfmt)
paths = newpaths paths = newpaths
if script: if script:
tmp = ['${source}/' + x for x in paths.keys()] tmp = ['${source}/' + x for x in paths]
tmpstr = ' '.join(tmp) tmpstr = ' '.join(tmp)
cmd = f'op=file; source=/media/mnt; $op {tmpstr}' cmd = f'op=file; source=/media/mnt; $op {tmpstr}'
Logger.info(cmd) Logger.info(cmd)
@ -584,7 +586,7 @@ class Noder:
return found return found
def _callback_find_name(self, term): def _callback_find_name(self, term):
'''callback for finding files''' """callback for finding files"""
def find_name(node): def find_name(node):
if term.lower() in node.name.lower(): if term.lower() in node.name.lower():
return True return True
@ -596,19 +598,19 @@ class Noder:
############################################################### ###############################################################
def walk(self, top, path, rec=False, fmt='native', def walk(self, top, path, rec=False, fmt='native',
raw=False): raw=False):
''' """
walk the tree for ls based on names walk the tree for ls based on names
@top: start node @top: start node
@rec: recursive walk @rec: recursive walk
@fmt: output format @fmt: output format
@raw: print raw size @raw: print raw size
''' """
self._debug(f'walking path: \"{path}\"') self._debug(f'walking path: \"{path}\"')
r = anytree.resolver.Resolver('name') resolv = anytree.resolver.Resolver('name')
found = [] found = []
try: try:
found = r.glob(top, path) found = resolv.glob(top, path)
if len(found) < 1: if len(found) < 1:
# nothing found # nothing found
return [] return []
@ -627,20 +629,20 @@ class Noder:
withpath=False, withdepth=True, raw=raw) withpath=False, withdepth=True, raw=raw)
elif fmt == 'csv': elif fmt == 'csv':
self._node_to_csv(found[0].parent, raw=raw) self._node_to_csv(found[0].parent, raw=raw)
elif fmt == 'fzf': elif fmt.startswith('fzf'):
pass pass
# print all found nodes # print all found nodes
for f in found: for item in found:
if fmt == 'native': if fmt == 'native':
self._print_node(f, withpath=False, self._print_node(item, withpath=False,
pre='- ', pre='- ',
withdepth=True, withdepth=True,
raw=raw) raw=raw)
elif fmt == 'csv': elif fmt == 'csv':
self._node_to_csv(f, raw=raw) self._node_to_csv(item, raw=raw)
elif fmt == 'fzf': elif fmt.startswith('fzf'):
self._to_fzf(top, f) self._to_fzf(top, item, fmt)
except anytree.resolver.ChildResolverError: except anytree.resolver.ChildResolverError:
pass pass
@ -650,47 +652,47 @@ class Noder:
# tree creation # tree creation
############################################################### ###############################################################
def _add_entry(self, name, top, resolv): def _add_entry(self, name, top, resolv):
'''add an entry to the tree''' """add an entry to the tree"""
entries = name.rstrip(os.sep).split(os.sep) entries = name.rstrip(os.sep).split(os.sep)
if len(entries) == 1: if len(entries) == 1:
self.archive_node(name, name, top, top.name) self.archive_node(name, name, top, top.name)
return return
sub = os.sep.join(entries[:-1]) sub = os.sep.join(entries[:-1])
f = entries[-1] nodename = entries[-1]
try: try:
parent = resolv.get(top, sub) parent = resolv.get(top, sub)
parent = self.archive_node(f, name, parent, top.name) parent = self.archive_node(nodename, name, parent, top.name)
except anytree.resolver.ChildResolverError: except anytree.resolver.ChildResolverError:
self.archive_node(f, name, top, top.name) self.archive_node(nodename, name, top, top.name)
def list_to_tree(self, parent, names): def list_to_tree(self, parent, names):
'''convert list of files to a tree''' """convert list of files to a tree"""
if not names: if not names:
return return
r = anytree.resolver.Resolver('name') resolv = anytree.resolver.Resolver('name')
for name in names: for name in names:
name = name.rstrip(os.sep) name = name.rstrip(os.sep)
self._add_entry(name, parent, r) self._add_entry(name, parent, resolv)
############################################################### ###############################################################
# diverse # diverse
############################################################### ###############################################################
def _sort_tree(self, items): def _sort_tree(self, items):
'''sorting a list of items''' """sorting a list of items"""
return sorted(items, key=self._sort, reverse=self.sortsize) return sorted(items, key=self._sort, reverse=self.sortsize)
def _sort(self, lst): def _sort(self, lst):
'''sort a list''' """sort a list"""
if self.sortsize: if self.sortsize:
return self._sort_size(lst) return self._sort_size(lst)
return self._sort_fs(lst) return self._sort_fs(lst)
def _sort_fs(self, node): def _sort_fs(self, node):
'''sorting nodes dir first and alpha''' """sorting nodes dir first and alpha"""
return (node.type, node.name.lstrip('\.').lower()) return (node.type, node.name.lstrip('.').lower())
def _sort_size(self, node): def _sort_size(self, node):
'''sorting nodes by size''' """sorting nodes by size"""
try: try:
if not node.size: if not node.size:
return 0 return 0
@ -699,7 +701,7 @@ class Noder:
return 0 return 0
def _get_storage(self, node): def _get_storage(self, node):
'''recursively traverse up to find storage''' """recursively traverse up to find storage"""
if node.type == self.TYPE_STORAGE: if node.type == self.TYPE_STORAGE:
return node return node
return node.ancestors[1] return node.ancestors[1]
@ -708,7 +710,7 @@ class Noder:
return attr in node.__dict__.keys() return attr in node.__dict__.keys()
def _get_parents(self, node): def _get_parents(self, node):
'''get all parents recursively''' """get all parents recursively"""
if node.type == self.TYPE_STORAGE: if node.type == self.TYPE_STORAGE:
return '' return ''
if node.type == self.TYPE_TOP: if node.type == self.TYPE_TOP:
@ -720,10 +722,10 @@ class Noder:
def _get_hash(self, path): def _get_hash(self, path):
"""return md5 hash of node""" """return md5 hash of node"""
return utils.md5sum(path) return md5sum(path)
def _debug(self, string): def _debug(self, string):
'''print debug''' """print debug"""
if not self.debug: if not self.debug:
return return
Logger.debug(string) Logger.debug(string)

48
catcli/utils.py
Unescape Escape View File

@ -16,29 +16,29 @@ from catcli.logger import Logger
def md5sum(path): def md5sum(path):
'''calculate md5 sum of a file''' """calculate md5 sum of a file"""
p = os.path.realpath(path) rpath = os.path.realpath(path)
if not os.path.exists(p): if not os.path.exists(rpath):
Logger.err(f'\nmd5sum - file does not exist: {p}') Logger.err(f'\nmd5sum - file does not exist: {rpath}')
return None return None
try: try:
with open(p, mode='rb') as f: with open(rpath, mode='rb') as file:
d = hashlib.md5() hashv = hashlib.md5()
while True: while True:
buf = f.read(4096) buf = file.read(4096)
if not buf: if not buf:
break break
d.update(buf) hashv.update(buf)
return d.hexdigest() return hashv.hexdigest()
except PermissionError: except PermissionError:
pass pass
except OSError as e: except OSError as exc:
Logger.err(f'md5sum error: {e}') Logger.err(f'md5sum error: {exc}')
return None return None
def size_to_str(size, raw=True): def size_to_str(size, raw=True):
'''convert size to string, optionally human readable''' """convert size to string, optionally human readable"""
div = 1024. div = 1024.
suf = ['B', 'K', 'M', 'G', 'T', 'P'] suf = ['B', 'K', 'M', 'G', 'T', 'P']
if raw or size < div: if raw or size < div:
@ -52,28 +52,28 @@ def size_to_str(size, raw=True):
def epoch_to_str(epoch): def epoch_to_str(epoch):
'''convert epoch to string''' """convert epoch to string"""
if not epoch: if not epoch:
return '' return ''
fmt = '%Y-%m-%d %H:%M:%S' fmt = '%Y-%m-%d %H:%M:%S'
t = datetime.datetime.fromtimestamp(float(epoch)) timestamp = datetime.datetime.fromtimestamp(float(epoch))
return t.strftime(fmt) return timestamp.strftime(fmt)
def ask(question): def ask(question):
'''ask the user what to do''' """ask the user what to do"""
resp = input(f'{question} [y|N] ? ') resp = input(f'{question} [y|N] ? ')
return resp.lower() == 'y' return resp.lower() == 'y'
def edit(string): def edit(string):
'''edit the information with the default EDITOR''' """edit the information with the default EDITOR"""
string = string.encode('utf-8') string = string.encode('utf-8')
EDITOR = os.environ.get('EDITOR', 'vim') editor = os.environ.get('EDITOR', 'vim')
with tempfile.NamedTemporaryFile(prefix='catcli', suffix='.tmp') as f: with tempfile.NamedTemporaryFile(prefix='catcli', suffix='.tmp') as file:
f.write(string) file.write(string)
f.flush() file.flush()
subprocess.call([EDITOR, f.name]) subprocess.call([editor, file.name])
f.seek(0) file.seek(0)
new = f.read() new = file.read()
return new.decode('utf-8') return new.decode('utf-8')

6
catcli/version.py
Unescape Escape View File

@ -0,0 +1,6 @@
"""
author: deadc0de6 (https://github.com/deadc0de6)
Copyright (c) 2022, deadc0de6
"""
__version__ = '0.8.7'

85
catcli/walker.py
Unescape Escape View File

@ -12,57 +12,58 @@ from catcli.logger import Logger
class Walker: class Walker:
"""a filesystem walker"""
MAXLINE = 80 - 15 MAXLINE = 80 - 15
def __init__(self, noder, hash=True, debug=False, def __init__(self, noder, usehash=True, debug=False,
logpath=None): logpath=None):
''' """
@noder: the noder to use @noder: the noder to use
@hash: calculate hash of nodes @hash: calculate hash of nodes
@debug: debug mode @debug: debug mode
@logpath: path where to log catalog changes on reindex @logpath: path where to log catalog changes on reindex
''' """
self.noder = noder self.noder = noder
self.hash = hash self.usehash = usehash
self.noder.set_hashing(self.hash) self.noder.set_hashing(self.usehash)
self.debug = debug self.debug = debug
self.lpath = logpath self.lpath = logpath
def index(self, path, parent, name, storagepath=''): def index(self, path, parent, name, storagepath=''):
''' """
index a directory and store in tree index a directory and store in tree
@path: path to index @path: path to index
@parent: parent node @parent: parent node
@name: this stoarge name @name: this stoarge name
''' """
self._debug(f'indexing starting at {path}') self._debug(f'indexing starting at {path}')
if not parent: if not parent:
parent = self.noder.dir_node(name, path, parent) parent = self.noder.dir_node(name, path, parent)
if os.path.islink(path): if os.path.islink(path):
rel = os.readlink(path) rel = os.readlink(path)
ab = os.path.join(path, rel) abspath = os.path.join(path, rel)
if os.path.isdir(ab): if os.path.isdir(abspath):
return parent, 0 return parent, 0
cnt = 0 cnt = 0
for (root, dirs, files) in os.walk(path): for (root, dirs, files) in os.walk(path):
for f in files: for file in files:
self._debug(f'found file {f} under {path}') self._debug(f'found file {file} under {path}')
sub = os.path.join(root, f) sub = os.path.join(root, file)
if not os.path.exists(sub): if not os.path.exists(sub):
continue continue
self._progress(f) self._progress(file)
self._debug(f'index file {sub}') self._debug(f'index file {sub}')
n = self.noder.file_node(os.path.basename(f), sub, node = self.noder.file_node(os.path.basename(file), sub,
parent, storagepath) parent, storagepath)
if n: if node:
cnt += 1 cnt += 1
for d in dirs: for adir in dirs:
self._debug(f'found dir {d} under {path}') self._debug(f'found dir {adir} under {path}')
base = os.path.basename(d) base = os.path.basename(adir)
sub = os.path.join(root, d) sub = os.path.join(root, adir)
self._debug(f'index directory {sub}') self._debug(f'index directory {sub}')
if not os.path.exists(sub): if not os.path.exists(sub):
continue continue
@ -80,40 +81,40 @@ class Walker:
return parent, cnt return parent, cnt
def reindex(self, path, parent, top): def reindex(self, path, parent, top):
'''reindex a directory and store in tree''' """reindex a directory and store in tree"""
cnt = self._reindex(path, parent, top) cnt = self._reindex(path, parent, top)
cnt += self.noder.clean_not_flagged(parent) cnt += self.noder.clean_not_flagged(parent)
return cnt return cnt
def _reindex(self, path, parent, top, storagepath=''): def _reindex(self, path, parent, top, storagepath=''):
''' """
reindex a directory and store in tree reindex a directory and store in tree
@path: directory path to re-index @path: directory path to re-index
@top: top node (storage) @top: top node (storage)
@storagepath: rel path relative to indexed directory @storagepath: rel path relative to indexed directory
''' """
self._debug(f'reindexing starting at {path}') self._debug(f'reindexing starting at {path}')
cnt = 0 cnt = 0
for (root, dirs, files) in os.walk(path): for (root, dirs, files) in os.walk(path):
for f in files: for file in files:
self._debug(f'found file \"{f}\" under {path}') self._debug(f'found file \"{file}\" under {path}')
sub = os.path.join(root, f) sub = os.path.join(root, file)
treepath = os.path.join(storagepath, f) treepath = os.path.join(storagepath, file)
reindex, n = self._need_reindex(parent, sub, treepath) reindex, node = self._need_reindex(parent, sub, treepath)
if not reindex: if not reindex:
self._debug(f'\tskip file {sub}') self._debug(f'\tskip file {sub}')
self.noder.flag(n) self.noder.flag(node)
continue continue
self._log2file(f'update catalog for \"{sub}\"') self._log2file(f'update catalog for \"{sub}\"')
n = self.noder.file_node(os.path.basename(f), sub, node = self.noder.file_node(os.path.basename(file), sub,
parent, storagepath) parent, storagepath)
self.noder.flag(n) self.noder.flag(node)
cnt += 1 cnt += 1
for d in dirs: for adir in dirs:
self._debug(f'found dir \"{d}\" under {path}') self._debug(f'found dir \"{adir}\" under {path}')
base = os.path.basename(d) base = os.path.basename(adir)
sub = os.path.join(root, d) sub = os.path.join(root, adir)
treepath = os.path.join(storagepath, d) treepath = os.path.join(storagepath, adir)
reindex, dummy = self._need_reindex(parent, sub, treepath) reindex, dummy = self._need_reindex(parent, sub, treepath)
if reindex: if reindex:
self._log2file(f'update catalog for \"{sub}\"') self._log2file(f'update catalog for \"{sub}\"')
@ -130,12 +131,12 @@ class Walker:
return cnt return cnt
def _need_reindex(self, top, path, treepath): def _need_reindex(self, top, path, treepath):
''' """
test if node needs re-indexing test if node needs re-indexing
@top: top node (storage) @top: top node (storage)
@path: abs path to file @path: abs path to file
@treepath: rel path from indexed directory @treepath: rel path from indexed directory
''' """
cnode, changed = self.noder.get_node_if_changed(top, path, treepath) cnode, changed = self.noder.get_node_if_changed(top, path, treepath)
if not cnode: if not cnode:
self._debug(f'\t{path} does not exist') self._debug(f'\t{path} does not exist')
@ -152,13 +153,13 @@ class Walker:
return True, cnode return True, cnode
def _debug(self, string): def _debug(self, string):
'''print to debug''' """print to debug"""
if not self.debug: if not self.debug:
return return
Logger.debug(string) Logger.debug(string)
def _progress(self, string): def _progress(self, string):
'''print progress''' """print progress"""
if self.debug: if self.debug:
return return
if not string: if not string:
@ -170,7 +171,7 @@ class Walker:
Logger.progr(f'indexing: {string:80}') Logger.progr(f'indexing: {string:80}')
def _log2file(self, string): def _log2file(self, string):
'''log to file''' """log to file"""
if not self.lpath: if not self.lpath:
return return
line = f'{string}\n' line = f'{string}\n'

18
tests.sh
Unescape Escape View File

@ -13,8 +13,22 @@ pycodestyle tests/
pyflakes catcli/ pyflakes catcli/
pyflakes tests/ pyflakes tests/
pylint catcli/ # R0914: Too many local variables
pylint tests/ # R0913: Too many arguments
# R0912: Too many branches
# R0915: Too many statements
pylint \
--disable=R0914 \
--disable=R0913 \
--disable=R0912 \
--disable=R0915 \
catcli/
pylint \
--disable=W0212 \
--disable=R0914 \
--disable=R0915 \
--disable=R0801 \
tests/
nosebin="nose2" nosebin="nose2"
PYTHONPATH=catcli ${nosebin} --with-coverage --coverage=catcli PYTHONPATH=catcli ${nosebin} --with-coverage --coverage=catcli

56
tests/helpers.py
Unescape Escape View File

@ -21,32 +21,32 @@ TMPSUFFIX = '.catcli'
def get_rnd_string(length): def get_rnd_string(length):
'''Get a random string of specific length ''' """Get a random string of specific length """
alpha = string.ascii_uppercase + string.digits alpha = string.ascii_uppercase + string.digits
return ''.join(random.choice(alpha) for _ in range(length)) return ''.join(random.choice(alpha) for _ in range(length))
def md5sum(path): def md5sum(path):
'''calculate md5 sum of a file''' """calculate md5 sum of a file"""
p = os.path.realpath(path) rpath = os.path.realpath(path)
if not os.path.exists(p): if not os.path.exists(rpath):
return None return None
try: try:
with open(p, mode='rb') as f: with open(rpath, mode='rb') as file:
d = hashlib.md5() val = hashlib.md5()
while True: while True:
buf = f.read(4096) buf = file.read(4096)
if not buf: if not buf:
break break
d.update(buf) val.update(buf)
return d.hexdigest() return val.hexdigest()
except PermissionError: except PermissionError:
pass pass
return None return None
def clean(path): def clean(path):
'''Delete file or folder.''' """Delete file or folder."""
if not os.path.exists(path): if not os.path.exists(path):
return return
if os.path.islink(path): if os.path.islink(path):
@ -58,10 +58,12 @@ def clean(path):
def edit_file(path, newcontent): def edit_file(path, newcontent):
"""edit file content"""
return write_to_file(path, newcontent) return write_to_file(path, newcontent)
def unix_tree(path): def unix_tree(path):
"""print using unix tree tool"""
if not os.path.exists(path): if not os.path.exists(path):
return return
# cmd = ['tree', path] # cmd = ['tree', path]
@ -75,7 +77,7 @@ def unix_tree(path):
def create_tree(): def create_tree():
''' create a random tree of files and directories ''' """ create a random tree of files and directories """
dirpath = get_tempdir() dirpath = get_tempdir()
# create 3 files # create 3 files
create_rnd_file(dirpath, get_rnd_string(5)) create_rnd_file(dirpath, get_rnd_string(5))
@ -83,13 +85,13 @@ def create_tree():
create_rnd_file(dirpath, get_rnd_string(5)) create_rnd_file(dirpath, get_rnd_string(5))
# create 2 directories # create 2 directories
d1 = create_dir(dirpath, get_rnd_string(3)) dir1 = create_dir(dirpath, get_rnd_string(3))
d2 = create_dir(dirpath, get_rnd_string(3)) dir2 = create_dir(dirpath, get_rnd_string(3))
# fill directories # fill directories
create_rnd_file(d1, get_rnd_string(4)) create_rnd_file(dir1, get_rnd_string(4))
create_rnd_file(d1, get_rnd_string(4)) create_rnd_file(dir1, get_rnd_string(4))
create_rnd_file(d2, get_rnd_string(6)) create_rnd_file(dir2, get_rnd_string(6))
return dirpath return dirpath
@ -99,12 +101,12 @@ def create_tree():
def get_tempdir(): def get_tempdir():
'''Get a temporary directory ''' """Get a temporary directory """
return tempfile.mkdtemp(suffix=TMPSUFFIX) return tempfile.mkdtemp(suffix=TMPSUFFIX)
def create_dir(path, dirname): def create_dir(path, dirname):
'''Create a directory ''' """Create a directory """
fpath = os.path.join(path, dirname) fpath = os.path.join(path, dirname)
if not os.path.exists(fpath): if not os.path.exists(fpath):
os.mkdir(fpath) os.mkdir(fpath)
@ -112,7 +114,7 @@ def create_dir(path, dirname):
def create_rnd_file(path, filename, content=None): def create_rnd_file(path, filename, content=None):
'''Create the file filename in path with random content if None ''' """Create the file filename in path with random content if None """
if not content: if not content:
content = get_rnd_string(100) content = get_rnd_string(100)
fpath = os.path.join(path, filename) fpath = os.path.join(path, filename)
@ -120,23 +122,25 @@ def create_rnd_file(path, filename, content=None):
def write_to_file(path, content): def write_to_file(path, content):
with open(path, 'w') as f: """write content to file"""
f.write(content) with open(path, 'w', encoding='UTF-8') as file:
file.write(content)
return path return path
def read_from_file(path): def read_from_file(path):
"""read file content"""
if not os.path.exists(path): if not os.path.exists(path):
return '' return ''
with open(path, 'r') as f: with open(path, 'r', encoding='UTF-8') as file:
content = f.read() content = file.read()
return content return content
############################################################ ############################################################
# fake tree in json # fake tree in json
############################################################ ############################################################
FAKECATALOG = ''' FAKECATALOG = """
{ {
"children": [ "children": [
{ {
@ -214,9 +218,9 @@ FAKECATALOG = '''
"name": "top", "name": "top",
"type": "top" "type": "top"
} }
''' """
def get_fakecatalog(): def get_fakecatalog():
# catalog constructed through test_index """catalog constructed through test_index"""
return FAKECATALOG return FAKECATALOG

3
tests/test_find.py
Unescape Escape View File

@ -14,8 +14,10 @@ from tests.helpers import get_fakecatalog
class TestFind(unittest.TestCase): class TestFind(unittest.TestCase):
"""test find"""
def test_find(self): def test_find(self):
"""test find"""
# init # init
catalog = Catalog('fake', force=True, debug=False) catalog = Catalog('fake', force=True, debug=False)
top = catalog._restore_json(get_fakecatalog()) top = catalog._restore_json(get_fakecatalog())
@ -38,6 +40,7 @@ class TestFind(unittest.TestCase):
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

3
tests/test_graph.py
Unescape Escape View File

@ -16,8 +16,10 @@ from tests.helpers import clean, get_fakecatalog
class TestGraph(unittest.TestCase): class TestGraph(unittest.TestCase):
"""test graph"""
def test_graph(self): def test_graph(self):
"""test graph"""
# init # init
path = 'fake' path = 'fake'
gpath = tempfile.gettempdir() + os.sep + 'graph.dot' gpath = tempfile.gettempdir() + os.sep + 'graph.dot'
@ -38,6 +40,7 @@ class TestGraph(unittest.TestCase):
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

33
tests/test_index.py
Unescape Escape View File

@ -16,8 +16,10 @@ from tests.helpers import get_tempdir, create_rnd_file, clean, \
class TestIndexing(unittest.TestCase): class TestIndexing(unittest.TestCase):
"""test index"""
def test_index(self): def test_index(self):
"""test index"""
# init # init
workingdir = get_tempdir() workingdir = get_tempdir()
catalogpath = create_rnd_file(workingdir, 'catalog.json', content='') catalogpath = create_rnd_file(workingdir, 'catalog.json', content='')
@ -27,18 +29,18 @@ class TestIndexing(unittest.TestCase):
self.addCleanup(clean, dirpath) self.addCleanup(clean, dirpath)
# create 3 files # create 3 files
f1 = create_rnd_file(dirpath, get_rnd_string(5)) file1 = create_rnd_file(dirpath, get_rnd_string(5))
f2 = create_rnd_file(dirpath, get_rnd_string(5)) file2 = create_rnd_file(dirpath, get_rnd_string(5))
f3 = create_rnd_file(dirpath, get_rnd_string(5)) file3 = create_rnd_file(dirpath, get_rnd_string(5))
# create 2 directories # create 2 directories
d1 = create_dir(dirpath, get_rnd_string(3)) dir1 = create_dir(dirpath, get_rnd_string(3))
d2 = create_dir(dirpath, get_rnd_string(3)) dir2 = create_dir(dirpath, get_rnd_string(3))
# fill directories with files # fill directories with files
_ = create_rnd_file(d1, get_rnd_string(4)) _ = create_rnd_file(dir1, get_rnd_string(4))
_ = create_rnd_file(d1, get_rnd_string(4)) _ = create_rnd_file(dir1, get_rnd_string(4))
_ = create_rnd_file(d2, get_rnd_string(6)) _ = create_rnd_file(dir2, get_rnd_string(6))
noder = Noder() noder = Noder()
top = noder.new_top_node() top = noder.new_top_node()
@ -61,20 +63,21 @@ class TestIndexing(unittest.TestCase):
# ensures files and directories are in # ensures files and directories are in
names = [x.name for x in storage.children] names = [x.name for x in storage.children]
self.assertTrue(os.path.basename(f1) in names) self.assertTrue(os.path.basename(file1) in names)
self.assertTrue(os.path.basename(f2) in names) self.assertTrue(os.path.basename(file2) in names)
self.assertTrue(os.path.basename(f3) in names) self.assertTrue(os.path.basename(file3) in names)
self.assertTrue(os.path.basename(d1) in names) self.assertTrue(os.path.basename(dir1) in names)
self.assertTrue(os.path.basename(d2) in names) self.assertTrue(os.path.basename(dir2) in names)
for node in storage.children: for node in storage.children:
if node.name == os.path.basename(d1): if node.name == os.path.basename(dir1):
self.assertTrue(len(node.children) == 2) self.assertTrue(len(node.children) == 2)
elif node.name == os.path.basename(d2): elif node.name == os.path.basename(dir2):
self.assertTrue(len(node.children) == 1) self.assertTrue(len(node.children) == 1)
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

3
tests/test_ls.py
Unescape Escape View File

@ -14,8 +14,10 @@ from tests.helpers import get_fakecatalog, clean
class TestWalking(unittest.TestCase): class TestWalking(unittest.TestCase):
"""test ls"""
def test_ls(self): def test_ls(self):
"""test ls"""
# init # init
path = 'fake' path = 'fake'
self.addCleanup(clean, path) self.addCleanup(clean, path)
@ -56,6 +58,7 @@ class TestWalking(unittest.TestCase):
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

3
tests/test_rm.py
Unescape Escape View File

@ -14,8 +14,10 @@ from tests.helpers import clean, get_fakecatalog
class TestRm(unittest.TestCase): class TestRm(unittest.TestCase):
"""test rm"""
def test_rm(self): def test_rm(self):
"""test rm"""
# init # init
path = 'fake' path = 'fake'
self.addCleanup(clean, path) self.addCleanup(clean, path)
@ -48,6 +50,7 @@ class TestRm(unittest.TestCase):
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

3
tests/test_tree.py
Unescape Escape View File

@ -14,8 +14,10 @@ from tests.helpers import clean, get_fakecatalog
class TestTree(unittest.TestCase): class TestTree(unittest.TestCase):
"""Test the tree"""
def test_tree(self): def test_tree(self):
"""test the tree"""
# init # init
path = 'fake' path = 'fake'
self.addCleanup(clean, path) self.addCleanup(clean, path)
@ -37,6 +39,7 @@ class TestTree(unittest.TestCase):
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

89
tests/test_update.py
Unescape Escape View File

@ -7,18 +7,20 @@ Basic unittest for updating an index
import unittest import unittest
import os import os
import anytree
from catcli.catcli import cmd_index, cmd_update from catcli.catcli import cmd_index, cmd_update
from catcli.noder import Noder from catcli.noder import Noder
from catcli.catalog import Catalog from catcli.catalog import Catalog
from tests.helpers import create_dir, create_rnd_file, get_tempdir, \ from tests.helpers import create_dir, create_rnd_file, get_tempdir, \
clean, unix_tree, edit_file, read_from_file, md5sum clean, unix_tree, edit_file, read_from_file, md5sum
import anytree
class TestIndexing(unittest.TestCase): class TestUpdate(unittest.TestCase):
"""test update"""
def test_index(self): def test_update(self):
"""test update"""
# init # init
workingdir = get_tempdir() workingdir = get_tempdir()
catalogpath = create_rnd_file(workingdir, 'catalog.json', content='') catalogpath = create_rnd_file(workingdir, 'catalog.json', content='')
@ -28,20 +30,20 @@ class TestIndexing(unittest.TestCase):
self.addCleanup(clean, dirpath) self.addCleanup(clean, dirpath)
# create 3 files # create 3 files
f1 = create_rnd_file(dirpath, 'file1') file1 = create_rnd_file(dirpath, 'file1')
f2 = create_rnd_file(dirpath, 'file2') file2 = create_rnd_file(dirpath, 'file2')
f3 = create_rnd_file(dirpath, 'file3') file3 = create_rnd_file(dirpath, 'file3')
f4 = create_rnd_file(dirpath, 'file4') file4 = create_rnd_file(dirpath, 'file4')
# create 2 directories # create 2 directories
d1 = create_dir(dirpath, 'dir1') dir1 = create_dir(dirpath, 'dir1')
d2 = create_dir(dirpath, 'dir2') dir2 = create_dir(dirpath, 'dir2')
# fill directories with files # fill directories with files
d1f1 = create_rnd_file(d1, 'dir1file1') d1f1 = create_rnd_file(dir1, 'dir1file1')
d1f2 = create_rnd_file(d1, 'dir1file2') d1f2 = create_rnd_file(dir1, 'dir1file2')
d2f1 = create_rnd_file(d2, 'dir2file1') d2f1 = create_rnd_file(dir2, 'dir2file1')
d2f2 = create_rnd_file(d2, 'dir2file2') d2f2 = create_rnd_file(dir2, 'dir2file2')
noder = Noder(debug=True) noder = Noder(debug=True)
noder.set_hashing(True) noder.set_hashing(True)
@ -49,7 +51,7 @@ class TestIndexing(unittest.TestCase):
catalog = Catalog(catalogpath, force=True, debug=False) catalog = Catalog(catalogpath, force=True, debug=False)
# get checksums # get checksums
f4_md5 = md5sum(f4) f4_md5 = md5sum(file4)
self.assertTrue(f4_md5) self.assertTrue(f4_md5)
d1f1_md5 = md5sum(d1f1) d1f1_md5 = md5sum(d1f1)
self.assertTrue(d1f1_md5) self.assertTrue(d1f1_md5)
@ -69,7 +71,7 @@ class TestIndexing(unittest.TestCase):
self.assertTrue(os.stat(catalogpath).st_size != 0) self.assertTrue(os.stat(catalogpath).st_size != 0)
# ensure md5 sum are in # ensure md5 sum are in
nods = noder.find_name(top, os.path.basename(f4)) nods = noder.find_name(top, os.path.basename(file4))
self.assertTrue(len(nods) == 1) self.assertTrue(len(nods) == 1)
nod = nods[0] nod = nods[0]
self.assertTrue(nod) self.assertTrue(nod)
@ -79,34 +81,34 @@ class TestIndexing(unittest.TestCase):
noder.print_tree(top, top) noder.print_tree(top, top)
# add some files and directories # add some files and directories
new1 = create_rnd_file(d1, 'newf1') new1 = create_rnd_file(dir1, 'newf1')
new2 = create_rnd_file(dirpath, 'newf2') new2 = create_rnd_file(dirpath, 'newf2')
new3 = create_dir(dirpath, 'newd3') new3 = create_dir(dirpath, 'newd3')
new4 = create_dir(d2, 'newd4') new4 = create_dir(dir2, 'newd4')
new5 = create_rnd_file(new4, 'newf5') new5 = create_rnd_file(new4, 'newf5')
unix_tree(dirpath) unix_tree(dirpath)
# modify files # modify files
EDIT = 'edited' editval = 'edited'
edit_file(d1f1, EDIT) edit_file(d1f1, editval)
d1f1_md5_new = md5sum(d1f1) d1f1_md5_new = md5sum(d1f1)
self.assertTrue(d1f1_md5_new) self.assertTrue(d1f1_md5_new)
self.assertTrue(d1f1_md5_new != d1f1_md5) self.assertTrue(d1f1_md5_new != d1f1_md5)
# change file without mtime # change file without mtime
maccess = os.path.getmtime(f4) maccess = os.path.getmtime(file4)
EDIT = 'edited' editval = 'edited'
edit_file(f4, EDIT) edit_file(file4, editval)
# reset edit time # reset edit time
os.utime(f4, (maccess, maccess)) os.utime(file4, (maccess, maccess))
f4_md5_new = md5sum(d1f1) f4_md5_new = md5sum(d1f1)
self.assertTrue(f4_md5_new) self.assertTrue(f4_md5_new)
self.assertTrue(f4_md5_new != f4_md5) self.assertTrue(f4_md5_new != f4_md5)
# change file without mtime # change file without mtime
maccess = os.path.getmtime(d2f2) maccess = os.path.getmtime(d2f2)
EDIT = 'edited' editval = 'edited'
edit_file(d2f2, EDIT) edit_file(d2f2, editval)
# reset edit time # reset edit time
os.utime(d2f2, (maccess, maccess)) os.utime(d2f2, (maccess, maccess))
d2f2_md5_new = md5sum(d2f2) d2f2_md5_new = md5sum(d2f2)
@ -134,7 +136,7 @@ class TestIndexing(unittest.TestCase):
self.assertTrue(nod.md5 == d1f1_md5_new) self.assertTrue(nod.md5 == d1f1_md5_new)
# ensure f4 md5 sum has changed in catalog # ensure f4 md5 sum has changed in catalog
nods = noder.find_name(top, os.path.basename(f4)) nods = noder.find_name(top, os.path.basename(file4))
self.assertTrue(len(nods) == 1) self.assertTrue(len(nods) == 1)
nod = nods[0] nod = nods[0]
self.assertTrue(nod) self.assertTrue(nod)
@ -152,14 +154,14 @@ class TestIndexing(unittest.TestCase):
# ensures files and directories are in # ensures files and directories are in
names = [node.name for node in anytree.PreOrderIter(storage)] names = [node.name for node in anytree.PreOrderIter(storage)]
print(names) print(names)
self.assertTrue(os.path.basename(f1) in names) self.assertTrue(os.path.basename(file1) in names)
self.assertTrue(os.path.basename(f2) in names) self.assertTrue(os.path.basename(file2) in names)
self.assertTrue(os.path.basename(f3) in names) self.assertTrue(os.path.basename(file3) in names)
self.assertTrue(os.path.basename(f4) in names) self.assertTrue(os.path.basename(file4) in names)
self.assertTrue(os.path.basename(d1) in names) self.assertTrue(os.path.basename(dir1) in names)
self.assertTrue(os.path.basename(d1f1) in names) self.assertTrue(os.path.basename(d1f1) in names)
self.assertTrue(os.path.basename(d1f2) in names) self.assertTrue(os.path.basename(d1f2) in names)
self.assertTrue(os.path.basename(d2) in names) self.assertTrue(os.path.basename(dir2) in names)
self.assertTrue(os.path.basename(d2f1) in names) self.assertTrue(os.path.basename(d2f1) in names)
self.assertTrue(os.path.basename(new1) in names) self.assertTrue(os.path.basename(new1) in names)
self.assertTrue(os.path.basename(new2) in names) self.assertTrue(os.path.basename(new2) in names)
@ -168,19 +170,19 @@ class TestIndexing(unittest.TestCase):
self.assertTrue(os.path.basename(new5) in names) self.assertTrue(os.path.basename(new5) in names)
for node in storage.children: for node in storage.children:
if node.name == os.path.basename(d1): if node.name == os.path.basename(dir1):
self.assertTrue(len(node.children) == 3) self.assertTrue(len(node.children) == 3)
elif node.name == os.path.basename(d2): elif node.name == os.path.basename(dir2):
self.assertTrue(len(node.children) == 3) self.assertTrue(len(node.children) == 3)
elif node.name == os.path.basename(new3): elif node.name == os.path.basename(new3):
self.assertTrue(len(node.children) == 0) self.assertTrue(len(node.children) == 0)
elif node.name == os.path.basename(new4): elif node.name == os.path.basename(new4):
self.assertTrue(len(node.children) == 1) self.assertTrue(len(node.children) == 1)
self.assertTrue(read_from_file(d1f1) == EDIT) self.assertTrue(read_from_file(d1f1) == editval)
# remove some files # remove some files
clean(d1f1) clean(d1f1)
clean(d2) clean(dir2)
clean(new2) clean(new2)
clean(new4) clean(new4)
@ -190,14 +192,14 @@ class TestIndexing(unittest.TestCase):
# ensures files and directories are (not) in # ensures files and directories are (not) in
names = [node.name for node in anytree.PreOrderIter(storage)] names = [node.name for node in anytree.PreOrderIter(storage)]
print(names) print(names)
self.assertTrue(os.path.basename(f1) in names) self.assertTrue(os.path.basename(file1) in names)
self.assertTrue(os.path.basename(f2) in names) self.assertTrue(os.path.basename(file2) in names)
self.assertTrue(os.path.basename(f3) in names) self.assertTrue(os.path.basename(file3) in names)
self.assertTrue(os.path.basename(f4) in names) self.assertTrue(os.path.basename(file4) in names)
self.assertTrue(os.path.basename(d1) in names) self.assertTrue(os.path.basename(dir1) in names)
self.assertTrue(os.path.basename(d1f1) not in names) self.assertTrue(os.path.basename(d1f1) not in names)
self.assertTrue(os.path.basename(d1f2) in names) self.assertTrue(os.path.basename(d1f2) in names)
self.assertTrue(os.path.basename(d2) not in names) self.assertTrue(os.path.basename(dir2) not in names)
self.assertTrue(os.path.basename(d2f1) not in names) self.assertTrue(os.path.basename(d2f1) not in names)
self.assertTrue(os.path.basename(d2f1) not in names) self.assertTrue(os.path.basename(d2f1) not in names)
self.assertTrue(os.path.basename(new1) in names) self.assertTrue(os.path.basename(new1) in names)
@ -206,13 +208,14 @@ class TestIndexing(unittest.TestCase):
self.assertTrue(os.path.basename(new4) not in names) self.assertTrue(os.path.basename(new4) not in names)
self.assertTrue(os.path.basename(new5) not in names) self.assertTrue(os.path.basename(new5) not in names)
for node in storage.children: for node in storage.children:
if node.name == os.path.basename(d1): if node.name == os.path.basename(dir1):
self.assertTrue(len(node.children) == 2) self.assertTrue(len(node.children) == 2)
elif node.name == os.path.basename(new3): elif node.name == os.path.basename(new3):
self.assertTrue(len(node.children) == 0) self.assertTrue(len(node.children) == 0)
def main(): def main():
"""entry point"""
unittest.main() unittest.main()

Write Preview
Loading…
Cancel
Save