diff --git a/catcli/noder.py b/catcli/noder.py index 9bc295a..2a85d69 100644 --- a/catcli/noder.py +++ b/catcli/noder.py @@ -175,6 +175,28 @@ class Noder: return self._node(name, self.TYPE_DIR, relpath, parent, maccess=maccess) + def clean_not_flagged(self, top): + '''remove any node not flagged and clean flags''' + cnt = 0 + for node in anytree.PreOrderIter(top): + if node.type != self.TYPE_FILE and node.type != self.TYPE_DIR: + continue + if self._clean(node): + cnt += 1 + return cnt + + def flag(self, node): + node.flag = True + + def _clean(self, node): + '''remove node if not flagged''' + if not self._has_attr(node, 'flag') or \ + not node.flag: + node.parent = None + return True + del node.flag + return False + def storage_node(self, name, path, parent, attr=None): '''create a new node representing a storage''' path = os.path.abspath(path) @@ -237,7 +259,7 @@ class Noder: hf = utils.human(node.free) ht = utils.human(node.total) dt = '' - if node.ts: + if self._has_attr(node, 'ts'): dt = ', date:' dt += utils.epoch_to_str(node.ts) name = '{} (free:{}, total:{}{})'.format(node.name, hf, ht, dt) @@ -369,3 +391,6 @@ class Noder: def _get_storage(self, node): '''recursively traverse up to find storage''' return node.ancestors[1] + + def _has_attr(self, node, attr): + return attr in node.__dict__.keys() diff --git a/catcli/walker.py b/catcli/walker.py index 018307d..19e84f7 100644 --- a/catcli/walker.py +++ b/catcli/walker.py @@ -52,6 +52,12 @@ class Walker: return parent, cnt def reindex(self, path, parent, top): + '''reindex a directory and store in tree''' + cnt = self._reindex(path, parent, top) + cnt += self.noder.clean_not_flagged(top) + return cnt + + def _reindex(self, path, parent, top): '''reindex a directory and store in tree''' self._debug('reindexing starting at {}'.format(path)) cnt = 0 @@ -60,14 +66,16 @@ class Walker: self._debug('found file {} under {}'.format(f, path)) sub = os.path.join(root, f) maccess = os.path.getmtime(sub) - reindex, _ = self._need_reindex(parent, f, maccess) + reindex, n = self._need_reindex(parent, f, maccess) if not reindex: self._debug('\tignore file {}'.format(sub)) + self.noder.flag(n) continue self._debug('\tre-index file {}'.format(sub)) self._log(f) - self.noder.file_node(os.path.basename(f), sub, - parent, path) + n = self.noder.file_node(os.path.basename(f), sub, + parent, path) + self.noder.flag(n) cnt += 1 for d in dirs: self._debug('found dir {} under {}'.format(d, path)) @@ -78,8 +86,10 @@ class Walker: if reindex: self._debug('\tre-index directory {}'.format(sub)) dummy = self.noder.dir_node(base, sub, parent, path) + cnt += 1 + self.noder.flag(dummy) self._debug('reindexing deeper under {}'.format(sub)) - cnt2 = self.reindex(sub, dummy, top) + cnt2 = self._reindex(sub, dummy, top) cnt += cnt2 break self._log(None) @@ -98,6 +108,7 @@ class Walker: if cnode and newer: # remove this node and re-add self._debug('\tis newer') + self._debug('\tremoving node {}'.format(cnode)) cnode.parent = None self._debug('\tis to be re-indexed') return True, cnode diff --git a/tests/helpers.py b/tests/helpers.py index ae5624b..591a282 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -102,6 +102,18 @@ def create_rnd_file(path, filename, content=None): return write_to_file(fpath, content) +def remove(path): + '''Delete file or directory.''' + if not os.path.exists(path): + return + if os.path.islink(path): + os.remove(path) + elif os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + def write_to_file(path, content): with open(path, 'w') as f: f.write(content) diff --git a/tests/test_update.py b/tests/test_update.py index 3638df8..85e6894 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -12,6 +12,7 @@ from catcli.noder import Noder from catcli.walker import Walker from catcli.catalog import Catalog from tests.helpers import * +import anytree class TestIndexing(unittest.TestCase): @@ -83,14 +84,21 @@ class TestIndexing(unittest.TestCase): self.assertTrue(len(storage.children) == 7) # ensures files and directories are in - names = [x.name for x in storage.children] + names = [node.name for node in anytree.PreOrderIter(storage)] + print(names) self.assertTrue(os.path.basename(f1) in names) self.assertTrue(os.path.basename(f2) in names) self.assertTrue(os.path.basename(f3) in names) self.assertTrue(os.path.basename(d1) in names) + self.assertTrue(os.path.basename(d1f1) in names) + self.assertTrue(os.path.basename(d1f2) in names) self.assertTrue(os.path.basename(d2) in names) - self.assertTrue(os.path.basename(new3) in names) + self.assertTrue(os.path.basename(d2f1) in names) + self.assertTrue(os.path.basename(new1) in names) self.assertTrue(os.path.basename(new2) in names) + self.assertTrue(os.path.basename(new3) in names) + self.assertTrue(os.path.basename(new4) in names) + self.assertTrue(os.path.basename(new5) in names) for node in storage.children: if node.name == os.path.basename(d1): @@ -103,6 +111,37 @@ class TestIndexing(unittest.TestCase): self.assertTrue(len(node.children) == 1) self.assertTrue(read_from_file(d1f1) == EDIT) + # remove some files + clean(d1f1) + clean(d2) + clean(new2) + clean(new4) + + # update storage + cmd_update(args, noder, catalog, top, debug=True) + + # ensures files and directories are (not) in + names = [node.name for node in anytree.PreOrderIter(storage)] + print(names) + self.assertTrue(os.path.basename(f1) in names) + self.assertTrue(os.path.basename(f2) in names) + self.assertTrue(os.path.basename(f3) in names) + self.assertTrue(os.path.basename(d1) in names) + self.assertTrue(os.path.basename(d1f1) not in names) + self.assertTrue(os.path.basename(d1f2) in names) + self.assertTrue(os.path.basename(d2) not in names) + self.assertTrue(os.path.basename(d2f1) not in names) + self.assertTrue(os.path.basename(new1) in names) + self.assertTrue(os.path.basename(new2) not in names) + self.assertTrue(os.path.basename(new3) in names) + self.assertTrue(os.path.basename(new4) not in names) + self.assertTrue(os.path.basename(new5) not in names) + for node in storage.children: + if node.name == os.path.basename(d1): + self.assertTrue(len(node.children) == 2) + elif node.name == os.path.basename(new3): + self.assertTrue(len(node.children) == 0) + def main(): unittest.main()