Archives
/
catcli
mirror of https://github.com/deadc0de6/catcli
2
1
Fork 0
Code Issues Releases Wiki Activity

wildcard and refactoring

Browse Source
features-42
deadc0de6 4 months ago
parent 0dcbfa94bd
commit e58da8b6bf
15 changed files with 183 additions and 104 deletions
Show Stats Download Patch File Download Diff File

17
catcli/catcli.py
Unescape Escape View File

@ -46,7 +46,8 @@ Usage:
[-aBCbdVs] [--path=<path>] [<term>]
{NAME} index [--catalog=<path>] [--meta=<meta>...]
[-aBCcfnV] <name> <path>
{NAME} update [--catalog=<path>] [-aBCcfnV] [--lpath=<path>] <name> <path>
{NAME} update [--catalog=<path>] [-aBCcfnV]
[--lpath=<path>] <name> <path>
{NAME} mount [--catalog=<path>] [-V] <mountpoint>
{NAME} rm [--catalog=<path>] [-BCfV] <storage>
{NAME} rename [--catalog=<path>] [-BCfV] <storage> <name>
@ -176,8 +177,8 @@ def cmd_ls(args: Dict[str, Any],
raise BadFormatException('fzf is not supported in ls, use find')
found = noder.list(top,
path,
rec=args['--recursive'],
fmt=fmt,
rec=args['--recursive'],
raw=args['--raw-size'])
if not found:
path = args['<path>']
@ -213,12 +214,12 @@ def cmd_find(args: Dict[str, Any],
search_for = args['<term>']
if args['--verbose']:
Logger.debug(f'search for \"{search_for}\" under \"{top.name}\"')
found = noder.find_name(top, search_for,
script=script,
startnode=startpath,
only_dir=directory,
fmt=fmt,
raw=raw)
found = noder.find(top, search_for,
script=script,
startnode=startpath,
only_dir=directory,
fmt=fmt,
raw=raw)
return found

2
catcli/colors.py
Unescape Escape View File

@ -20,7 +20,9 @@ class Colors:
PURPLE = '\033[1;35m'
BLUE = '\033[94m'
GRAY = '\033[0;37m'
CYAN = '\033[36m'
MAGENTA = '\033[95m'
WHITE = '\033[97m'
RESET = '\033[0m'
EMPH = '\033[33m'
BOLD = '\033[1m'

55
catcli/noder.py
Unescape Escape View File

@ -9,6 +9,7 @@ import os
import shutil
import time
from typing import List, Union, Tuple, Any, Optional, Dict, cast
import fnmatch
import anytree # type: ignore
from natsort import os_sort_keygen # type: ignore
@ -17,7 +18,8 @@ from catcli import nodes
from catcli.nodes import NodeAny, NodeStorage, \
NodeTop, NodeFile, NodeArchived, NodeDir, NodeMeta, \
typcast_node
from catcli.utils import md5sum, fix_badchars, has_attr
from catcli.utils import md5sum, fix_badchars, has_attr, \
get_node_fullpath
from catcli.logger import Logger
from catcli.printer_native import NativePrinter
from catcli.printer_csv import CsvPrinter
@ -35,8 +37,7 @@ class Noder:
* "dir" node representing a directory
* "file" node representing a file
"""
PRE = ' '
# pylint: disable=R0904
def __init__(self, debug: bool = False,
sortsize: bool = False,
@ -292,11 +293,11 @@ class Noder:
@sep: CSV separator character
@raw: print raw size rather than human readable
"""
typcast_node(node)
if not node:
return
if node.type == nodes.TYPE_TOP:
return
if node.type == nodes.TYPE_STORAGE:
self.csv_printer.print_storage(node,
sep=sep,
@ -341,20 +342,18 @@ class Noder:
@withstorage: print the node storage it belongs to
@raw: print raw size rather than human readable
"""
typcast_node(node)
if node.type == nodes.TYPE_TOP:
# top node
node.__class__ = NodeTop
self.native_printer.print_top(pre, node.name)
elif node.type == nodes.TYPE_FILE:
# node of type file
node.__class__ = NodeFile
self.native_printer.print_file(pre, node,
withpath=withpath,
withstorage=withstorage,
raw=raw)
elif node.type == nodes.TYPE_DIR:
# node of type directory
node.__class__ = NodeDir
self.native_printer.print_dir(pre,
node,
withpath=withpath,
@ -363,14 +362,11 @@ class Noder:
raw=raw)
elif node.type == nodes.TYPE_STORAGE:
# node of type storage
node.__class__ = NodeStorage
self.native_printer.print_storage(pre,
node,
raw=raw)
elif node.type == nodes.TYPE_ARCHIVED:
# archive node
node.__class__ = NodeArchived
if self.arc:
self.native_printer.print_archive(pre, node.name, node.archive)
else:
@ -458,13 +454,13 @@ class Noder:
###############################################################
# searching
###############################################################
def find_name(self, top: NodeTop,
key: str,
script: bool = False,
only_dir: bool = False,
startnode: Optional[NodeAny] = None,
fmt: str = 'native',
raw: bool = False) -> List[NodeAny]:
def find(self, top: NodeTop,
key: str,
script: bool = False,
only_dir: bool = False,
startnode: Optional[NodeAny] = None,
fmt: str = 'native',
raw: bool = False) -> List[NodeAny]:
"""
find files based on their names
@top: top node
@ -511,7 +507,8 @@ class Noder:
else:
if fmt == 'native':
for _, item in paths.items():
self._print_node_native(item, withpath=True,
self._print_node_native(item,
withpath=True,
withnbchildren=True,
withstorage=True,
raw=raw)
@ -533,6 +530,7 @@ class Noder:
def _callback_find_name(self, term: str, only_dir: bool) -> Any:
"""callback for finding files"""
def find_name(node: NodeAny) -> bool:
path = get_node_fullpath(node)
if node.type == nodes.TYPE_STORAGE:
# ignore storage nodes
return False
@ -549,7 +547,9 @@ class Noder:
# filter
if not term:
return True
if term.lower() in node.name.lower():
if term in path:
return True
if fnmatch.fnmatch(path, term):
return True
# ignore
@ -572,17 +572,18 @@ class Noder:
@fmt: output format
@raw: print raw size
"""
self._debug(f'walking path: \"{path}\" from {top.name}')
self._debug(f'walking path: \"{path}\" from \"{top.name}\"')
resolv = anytree.resolver.Resolver('name')
found = []
try:
if '*' in path or '?' in path:
# we need to handle glob
self._debug(f'glob with top {top.name} and path {path}')
self._debug('glob ls...')
found = resolv.glob(top, path)
else:
# we have a canonical path
self._debug('get ls...')
found = resolv.get(top, path)
if found and self.node_has_subs(found):
# let's find its children as well
@ -604,24 +605,12 @@ class Noder:
# sort found nodes
found = sorted(found, key=os_sort_keygen(self._sort))
# print the parent
if fmt == 'native':
self._print_node_native(found[0].parent,
withpath=False,
withnbchildren=True,
raw=raw)
elif fmt.startswith('csv'):
self._print_node_csv(found[0].parent, raw=raw)
elif fmt.startswith('fzf'):
pass
# print all found nodes
if fmt == 'csv-with-header':
self.csv_printer.print_header()
for item in found:
if fmt == 'native':
self._print_node_native(item, withpath=False,
pre=Noder.PRE,
withnbchildren=True,
raw=raw)
elif fmt.startswith('csv'):

65
catcli/printer_native.py
Unescape Escape View File

@ -6,18 +6,23 @@ Class for printing nodes in native format
"""
import sys
import os
from catcli.nodes import NodeFile, NodeDir, NodeStorage
from catcli.colors import Colors
from catcli.logger import Logger
from catcli.utils import fix_badchars, size_to_str, \
has_attr, epoch_to_ls_str
has_attr, epoch_to_ls_str, get_node_fullpath
TS_LJUST = 13
SIZE_LJUST = 6
NAME_LJUST = 20
COLOR_STORAGE = Colors.YELLOW
COLOR_FILE = Colors.WHITE
COLOR_DIRECTORY = Colors.BLUE
COLOR_ARCHIVE = Colors.PURPLE
COLOR_TS = Colors.CYAN
COLOR_SIZE = Colors.GREEN
FULLPATH_IN_NAME = True
class NativePrinter:
"""a node printer class"""
@ -60,9 +65,9 @@ class NativePrinter:
# print
out = f'{pre}{Colors.UND}{self.STORAGE}{Colors.RESET}: '
out += f'{Colors.PURPLE}{name}{Colors.RESET}'
out += f'{COLOR_STORAGE}{name}{Colors.RESET}'
if attrs:
out += f'\n{" "*len(name)}{Colors.GRAY}{"|".join(attrs)}{Colors.RESET}'
out += f' [{Colors.WHITE}{"|".join(attrs)}{Colors.RESET}]'
sys.stdout.write(f'{out}\n')
def print_file(self, pre: str,
@ -75,32 +80,30 @@ class NativePrinter:
name = node.name
storage = node.get_storage_node()
if withpath:
name = os.sep.join([
storage.name,
node.parent.get_parent_hierarchy(),
name])
name = fix_badchars(name)
name = get_node_fullpath(node)
# construct attributes
attrs = []
if node.md5:
attrs.append(f'md5:{node.md5}')
if withstorage:
content = Logger.get_bold_text(storage.name)
attrs.append(f', storage:{content}')
attrs.append(f'storage:{content}')
# print
out = []
out .append(f'{pre}')
line = name.ljust(NAME_LJUST, ' ')
out.append(f'{line}')
out.append(f'{pre}')
out.append(f'{COLOR_FILE}{name}{Colors.RESET}')
size = 0
if node.nodesize:
size = node.nodesize
line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ')
out.append(f'{Colors.BLUE}{line}{Colors.RESET}')
line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ')
out.append(f'{Colors.PURPLE}{line}{Colors.RESET}')
line = size_to_str(size, raw=raw)
out.append(f'{COLOR_SIZE}{line}{Colors.RESET}')
if has_attr(node, 'maccess'):
line = epoch_to_ls_str(node.maccess)
out.append(f'{COLOR_TS}{line}{Colors.RESET}')
if attrs:
out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}')
out = [x for x in out if x]
sys.stdout.write(f'{" ".join(out)}\n')
def print_dir(self, pre: str,
@ -114,11 +117,7 @@ class NativePrinter:
name = node.name
storage = node.get_storage_node()
if withpath:
name = os.sep.join([
storage.name,
node.parent.get_parent_hierarchy(),
name])
name = fix_badchars(name)
name = get_node_fullpath(node)
# construct attrs
attrs = []
if withnbchildren:
@ -129,23 +128,25 @@ class NativePrinter:
# print
out = []
out.append(f'{pre}')
line = name.ljust(NAME_LJUST, ' ')
out.append(f'{Colors.BLUE}{line}{Colors.RESET}')
out.append(f'{COLOR_DIRECTORY}{name}{Colors.RESET}')
size = 0
if node.nodesize:
size = node.nodesize
line = size_to_str(size, raw=raw).ljust(SIZE_LJUST, ' ')
out.append(f'{Colors.GRAY}{line}{Colors.RESET}')
line = epoch_to_ls_str(node.maccess).ljust(TS_LJUST, ' ')
out.append(f'{Colors.GRAY}{line}{Colors.RESET}')
line = size_to_str(size, raw=raw)
out.append(f'{COLOR_SIZE}{line}{Colors.RESET}')
if has_attr(node, 'maccess'):
line = epoch_to_ls_str(node.maccess)
out.append(f'{COLOR_TS}{line}{Colors.RESET}')
if attrs:
out.append(f'{Colors.GRAY}[{",".join(attrs)}]{Colors.RESET}')
out = [x for x in out if x]
sys.stdout.write(f'{" ".join(out)}\n')
def print_archive(self, pre: str,
name: str, archive: str) -> None:
"""print an archive"""
name = fix_badchars(name)
out = f'{pre}{Colors.YELLOW}{name}{Colors.RESET} '
out = f'{pre}{COLOR_ARCHIVE}{name}{Colors.RESET} '
out += f'{Colors.GRAY}[{self.ARCHIVE}:{archive}]{Colors.RESET}'
sys.stdout.write(f'{out}\n')

14
catcli/utils.py
Unescape Escape View File

@ -136,3 +136,17 @@ def fix_badchars(string: str) -> str:
def has_attr(node: nodes.NodeAny, attr: str) -> bool:
"""return True if node has attr as attribute"""
return attr in node.__dict__.keys()
def get_node_fullpath(node: nodes.NodeAny) -> str:
"""get node full path"""
nodes.typcast_node(node)
path = node.name
parents = node.get_parent_hierarchy()
if parents:
path = os.sep.join([parents, path])
storage = node.get_storage_node()
if storage:
path = os.sep.join([storage.name, path])
path = fix_badchars(path)
return str(path)

3
tests-ng/assets/generate.sh
Unescape Escape View File

@ -8,8 +8,7 @@ rm -f tests-ng/assets/github.catalog.json
python3 -m catcli.catcli index -c github .github --catalog=tests-ng/assets/github.catalog.json
# edit catalog
sed -i 's/"free": .*,/"free": 0,/g' "tests-ng/assets/github.catalog.json"
sed -i 's/"total": .*,/"total": 0,/g' "tests-ng/assets/github.catalog.json"
clean_catalog "tests-ng/assets/github.catalog.json"
# native
python3 -m catcli.catcli ls -r -s -B --catalog=tests-ng/assets/github.catalog.json | \

6
tests-ng/assets/github.catalog.csv.txt
Unescape Escape View File

@ -1,6 +1,6 @@
"github","storage","","1662","","","","3","0","0",""
"workflows","dir","github/workflows","1541","","","","2","","",""
"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","",""
"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","",""
"workflows","dir","github/workflows","0","","","","2","","",""
"pypi-release.yml","file","github/workflows/pypi-release.yml","691","","","57699a7a6a03e20e864f220e19f8e197","","","",""
"testing.yml","file","github/workflows/testing.yml","850","","","691df1a4d2f254b5cd04c152e7c6ccaf","","","",""
"codecov.yml","file","github/codecov.yml","104","","","4203204f75b43cd4bf032402beb3359d","","","",""
"FUNDING.yml","file","github/FUNDING.yml","17","","","0c6407a84d412c514007313fb3bca4de","","","",""

8
tests-ng/assets/github.catalog.json
Unescape Escape View File

@ -36,7 +36,7 @@
],
"maccess": 1704320727.2641916,
"name": "workflows",
"size": 1541,
"size": 0,
"type": "dir"
}
],
@ -44,14 +44,14 @@
"name": "github",
"size": 1662,
"total": 0,
"ts": 1704403613,
"ts": 1704747820,
"type": "storage"
},
{
"attr": {
"access": 1704403613,
"access": 1704747820,
"access_version": "0.9.6",
"created": 1704403613,
"created": 1704747820,
"created_version": "0.9.6"
},
"name": "meta",

13
tests-ng/assets/github.catalog.native.txt
Unescape Escape View File

@ -1,8 +1,7 @@
top
└── storage: github
nbfiles:3 | totsize:1662 | free:0.0% | du:0/0 | date:2023-03-09 16:20:59
├── workflows [nbfiles:2, totsize:1541]
│ ├── pypi-release.yml [size:691, md5:57699a7a6a03e20e864f220e19f8e197]
│ └── testing.yml [size:850, md5:691df1a4d2f254b5cd04c152e7c6ccaf]
├── codecov.yml [size:104, md5:4203204f75b43cd4bf032402beb3359d]
└── FUNDING.yml [size:17, md5:0c6407a84d412c514007313fb3bca4de]
└── storage: github [nbfiles:3|totsize:1662|free:0.0%|du:0/0|date:Jan 08 2024]
├── FUNDING.yml 17 Oct 19 21:00 [md5:0c6407a84d412c514007313fb3bca4de]
├── codecov.yml 104 Jan 03 2024 [md5:4203204f75b43cd4bf032402beb3359d]
└── workflows 0 Jan 03 2024 [nbfiles:2]
├── pypi-release.yml 691 Oct 19 21:00 [md5:57699a7a6a03e20e864f220e19f8e197]
└── testing.yml 850 Jan 04 2024 [md5:691df1a4d2f254b5cd04c152e7c6ccaf]

1
tests-ng/compare.sh
Unescape Escape View File

@ -36,6 +36,7 @@ catalog="${tmpd}/catalog"
# index
${bin} -B index -c --catalog="${catalog}" github .github
clean_catalog "${catalog}"
ls -laR .github
cat "${catalog}"

8
tests-ng/find.sh
Unescape Escape View File

@ -37,14 +37,20 @@ catalog="${tmpd}/catalog"
# index
${bin} -B index -c -f --catalog="${catalog}" github1 .github
${bin} -B index -c -f --catalog="${catalog}" github2 .github
clean_catalog "${catalog}"
#cat "${catalog}"
echo ""
${bin} -B ls -r --catalog="${catalog}"
${bin} -B find --catalog="${catalog}" testing.yml
cnt=$(${bin} -B find --catalog="${catalog}" testing.yml | wc -l)
[ "${cnt}" != "2" ] && echo "should return 2!" && exit 1
[ "${cnt}" != "2" ] && echo "should return 2 (not ${cnt})" && exit 1
${bin} -B find --catalog="${catalog}" '*.yml'
cnt=$(${bin} -B find --catalog="${catalog}" '*.yml' | wc -l)
[ "${cnt}" != "8" ] && echo "should return 8 (not ${cnt})" && exit 1
# the end
echo ""

8
tests-ng/helper
Unescape Escape View File

@ -21,6 +21,14 @@ clear_on_exit()
fi
}
# clear catalog stuff for testing
# $1: catalog path
clean_catalog()
{
sed -i 's/"free": .*,/"free": 0,/g' "${1}"
sed -i 's/"total": .*,/"total": 0,/g' "${1}"
}
# clear files
on_exit()
{

59
tests-ng/ls.sh
Unescape Escape View File

@ -0,0 +1,59 @@
#!/usr/bin/env bash
# author: deadc0de6 (https://github.com/deadc0de6)
# Copyright (c) 2024, deadc0de6
set -e
cur=$(cd "$(dirname "${0}")" && pwd)
prev="${cur}/.."
cd "${prev}"
# coverage
bin="python3 -m catcli.catcli"
if command -v coverage 2>/dev/null; then
mkdir -p coverages/
bin="coverage run -p --data-file coverages/coverage --source=catcli -m catcli.catcli"
fi
echo "current dir: $(pwd)"
echo "pythonpath: ${PYTHONPATH}"
echo "bin: ${bin}"
${bin} --version
# get the helpers
# shellcheck source=tests-ng/helper
source "${cur}"/helper
echo -e "$(tput setaf 6)==> RUNNING $(basename "${BASH_SOURCE[0]}") <==$(tput sgr0)"
##########################################################
# the test
##########################################################
# create temp dirs
tmpd=$(mktemp -d)
clear_on_exit "${tmpd}"
catalog="${tmpd}/catalog"
# index
${bin} -B index -c -f --catalog="${catalog}" github1 .github
${bin} -B index -c -f --catalog="${catalog}" github2 .github
clean_catalog "${catalog}"
#cat "${catalog}"
echo ""
${bin} -B ls -r --catalog="${catalog}"
${bin} -B ls --catalog="${catalog}" 'github1/*.yml'
cnt=$(${bin} -B ls --catalog="${catalog}" 'github1/*.yml' | wc -l)
[ "${cnt}" != "2" ] && echo "should return 2 (not ${cnt})" && exit 1
${bin} -B ls --catalog="${catalog}" 'github*/*.yml'
cnt=$(${bin} -B ls --catalog="${catalog}" 'github*/*.yml' | wc -l)
[ "${cnt}" != "4" ] && echo "should return 4 (not ${cnt})" && exit 1
# the end
echo ""
echo "test \"$(basename "$0")\" success"
cd "${cur}"
exit 0

16
tests-ng/update.sh
Unescape Escape View File

@ -41,12 +41,12 @@ echo "abc" > "${tmpd}/dir/a"
# index
${bin} -B index --catalog="${catalog}" dir "${tmpd}/dir"
${bin} -B ls --catalog="${catalog}" dir
${bin} -B ls --catalog="${catalog}"
# get attributes
freeb=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g')
dub=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g')
dateb=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g')
freeb=$(${bin} -B ls --catalog="${catalog}" | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g')
dub=$(${bin} -B ls --catalog="${catalog}" | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g')
dateb=$(${bin} -B ls --catalog="${catalog}" | grep date: | sed 's/^.*,date: \(.*\)$/\1/g')
echo "before: free:${freeb} | du:${dub} | date:${dateb}"
# change content
@ -61,12 +61,12 @@ sleep 1
# update
${bin} -B update -f --catalog="${catalog}" dir "${tmpu}/dir"
${bin} -B ls --catalog="${catalog}" dir
${bin} -B ls --catalog="${catalog}"
# get new attributes
freea=$(${bin} -B ls --catalog="${catalog}" dir | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g')
dua=$(${bin} -B ls --catalog="${catalog}" dir | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g')
datea=$(${bin} -B ls --catalog="${catalog}" dir | grep date: | sed 's/^.*,date: \(.*\)$/\1/g')
freea=$(${bin} -B ls --catalog="${catalog}" | grep free: | sed 's/^.*,free:\([^ ]*\).*$/\1/g')
dua=$(${bin} -B ls --catalog="${catalog}" | grep du: | sed 's/^.*,du:\([^ ]*\).*$/\1/g')
datea=$(${bin} -B ls --catalog="${catalog}" | grep date: | sed 's/^.*,date: \(.*\)$/\1/g')
echo "after: free:${freea} | du:${dua} | date:${datea}"
# test they are all different

12
tests/test_update.py
Unescape Escape View File

@ -71,11 +71,11 @@ class TestUpdate(unittest.TestCase):
self.assertTrue(os.stat(catalogpath).st_size != 0)
# ensure md5 sum are in
nods = noder.find_name(top, os.path.basename(file4))
self.assertTrue(len(nods) == 1)
nods = noder.find(top, os.path.basename(file4))
self.assertEqual(len(nods), 1)
nod = nods[0]
self.assertTrue(nod)
self.assertTrue(nod.md5 == f4_md5)
self.assertEqual(nod.md5, f4_md5)
# print catalog
noder.print_tree(top)
@ -128,7 +128,7 @@ class TestUpdate(unittest.TestCase):
self.assertEqual(len(storage.children), 8)
# ensure d1f1 md5 sum has changed in catalog
nods = noder.find_name(top, os.path.basename(d1f1))
nods = noder.find(top, os.path.basename(d1f1))
self.assertTrue(len(nods) == 1)
nod = nods[0]
self.assertTrue(nod)
@ -136,7 +136,7 @@ class TestUpdate(unittest.TestCase):
self.assertTrue(nod.md5 == d1f1_md5_new)
# ensure f4 md5 sum has changed in catalog
nods = noder.find_name(top, os.path.basename(file4))
nods = noder.find(top, os.path.basename(file4))
self.assertTrue(len(nods) == 1)
nod = nods[0]
self.assertTrue(nod)
@ -144,7 +144,7 @@ class TestUpdate(unittest.TestCase):
self.assertTrue(nod.md5 == f4_md5_new)
# ensure d2f2 md5 sum has changed in catalog
nods = noder.find_name(top, os.path.basename(d2f2))
nods = noder.find(top, os.path.basename(d2f2))
self.assertTrue(len(nods) == 1)
nod = nods[0]
self.assertTrue(nod)

Write Preview
Loading…
Cancel
Save