fix removing files when update #3

catcli/noder.py

@@ -175,6 +175,28 @@ class Noder:
         return self._node(name, self.TYPE_DIR, relpath,
                           parent, maccess=maccess)
 
+    def clean_not_flagged(self, top):
+        '''remove any node not flagged and clean flags'''
+        cnt = 0
+        for node in anytree.PreOrderIter(top):
+            if node.type != self.TYPE_FILE and node.type != self.TYPE_DIR:
+                continue
+            if self._clean(node):
+                cnt += 1
+        return cnt
+
+    def flag(self, node):
+        node.flag = True
+
+    def _clean(self, node):
+        '''remove node if not flagged'''
+        if not self._has_attr(node, 'flag') or \
+                not node.flag:
+            node.parent = None
+            return True
+        del node.flag
+        return False
+
     def storage_node(self, name, path, parent, attr=None):
         '''create a new node representing a storage'''
         path = os.path.abspath(path)
@@ -237,7 +259,7 @@ class Noder:
             hf = utils.human(node.free)
             ht = utils.human(node.total)
             dt = ''
-            if node.ts:
+            if self._has_attr(node, 'ts'):
                 dt = ', date:'
                 dt += utils.epoch_to_str(node.ts)
             name = '{} (free:{}, total:{}{})'.format(node.name, hf, ht, dt)
@@ -369,3 +391,6 @@ class Noder:
     def _get_storage(self, node):
         '''recursively traverse up to find storage'''
         return node.ancestors[1]
+
+    def _has_attr(self, node, attr):
+        return attr in node.__dict__.keys()

catcli/walker.py

@@ -52,6 +52,12 @@ class Walker:
         return parent, cnt
 
     def reindex(self, path, parent, top):
+        '''reindex a directory and store in tree'''
+        cnt = self._reindex(path, parent, top)
+        cnt += self.noder.clean_not_flagged(top)
+        return cnt
+
+    def _reindex(self, path, parent, top):
         '''reindex a directory and store in tree'''
         self._debug('reindexing starting at {}'.format(path))
         cnt = 0
@@ -60,14 +66,16 @@ class Walker:
                 self._debug('found file {} under {}'.format(f, path))
                 sub = os.path.join(root, f)
                 maccess = os.path.getmtime(sub)
-                reindex, _ = self._need_reindex(parent, f, maccess)
+                reindex, n = self._need_reindex(parent, f, maccess)
                 if not reindex:
                     self._debug('\tignore file {}'.format(sub))
+                    self.noder.flag(n)
                     continue
                 self._debug('\tre-index file {}'.format(sub))
                 self._log(f)
-                self.noder.file_node(os.path.basename(f), sub,
+                n = self.noder.file_node(os.path.basename(f), sub,
-                                     parent, path)
+                                         parent, path)
+                self.noder.flag(n)
                 cnt += 1
             for d in dirs:
                 self._debug('found dir {} under {}'.format(d, path))
@@ -78,8 +86,10 @@ class Walker:
                 if reindex:
                     self._debug('\tre-index directory {}'.format(sub))
                     dummy = self.noder.dir_node(base, sub, parent, path)
+                    cnt += 1
+                self.noder.flag(dummy)
                 self._debug('reindexing deeper under {}'.format(sub))
-                cnt2 = self.reindex(sub, dummy, top)
+                cnt2 = self._reindex(sub, dummy, top)
                 cnt += cnt2
             break
         self._log(None)
@@ -98,6 +108,7 @@ class Walker:
         if cnode and newer:
             # remove this node and re-add
             self._debug('\tis newer')
+            self._debug('\tremoving node {}'.format(cnode))
             cnode.parent = None
         self._debug('\tis to be re-indexed')
         return True, cnode

tests/helpers.py

@@ -102,6 +102,18 @@ def create_rnd_file(path, filename, content=None):
     return write_to_file(fpath, content)
 
 
+def remove(path):
+    '''Delete file or directory.'''
+    if not os.path.exists(path):
+        return
+    if os.path.islink(path):
+        os.remove(path)
+    elif os.path.isdir(path):
+        shutil.rmtree(path)
+    else:
+        os.remove(path)
+
+
 def write_to_file(path, content):
     with open(path, 'w') as f:
         f.write(content)

tests/test_update.py

@@ -12,6 +12,7 @@ from catcli.noder import Noder
 from catcli.walker import Walker
 from catcli.catalog import Catalog
 from tests.helpers import *
+import anytree
 
 
 class TestIndexing(unittest.TestCase):
@@ -83,14 +84,21 @@ class TestIndexing(unittest.TestCase):
         self.assertTrue(len(storage.children) == 7)
 
         # ensures files and directories are in
-        names = [x.name for x in storage.children]
+        names = [node.name for node in anytree.PreOrderIter(storage)]
+        print(names)
         self.assertTrue(os.path.basename(f1) in names)
         self.assertTrue(os.path.basename(f2) in names)
         self.assertTrue(os.path.basename(f3) in names)
         self.assertTrue(os.path.basename(d1) in names)
+        self.assertTrue(os.path.basename(d1f1) in names)
+        self.assertTrue(os.path.basename(d1f2) in names)
         self.assertTrue(os.path.basename(d2) in names)
-        self.assertTrue(os.path.basename(new3) in names)
+        self.assertTrue(os.path.basename(d2f1) in names)
+        self.assertTrue(os.path.basename(new1) in names)
         self.assertTrue(os.path.basename(new2) in names)
+        self.assertTrue(os.path.basename(new3) in names)
+        self.assertTrue(os.path.basename(new4) in names)
+        self.assertTrue(os.path.basename(new5) in names)
 
         for node in storage.children:
             if node.name == os.path.basename(d1):
@@ -103,6 +111,37 @@ class TestIndexing(unittest.TestCase):
                 self.assertTrue(len(node.children) == 1)
         self.assertTrue(read_from_file(d1f1) == EDIT)
 
+        # remove some files
+        clean(d1f1)
+        clean(d2)
+        clean(new2)
+        clean(new4)
+
+        # update storage
+        cmd_update(args, noder, catalog, top, debug=True)
+
+        # ensures files and directories are (not) in
+        names = [node.name for node in anytree.PreOrderIter(storage)]
+        print(names)
+        self.assertTrue(os.path.basename(f1) in names)
+        self.assertTrue(os.path.basename(f2) in names)
+        self.assertTrue(os.path.basename(f3) in names)
+        self.assertTrue(os.path.basename(d1) in names)
+        self.assertTrue(os.path.basename(d1f1) not in names)
+        self.assertTrue(os.path.basename(d1f2) in names)
+        self.assertTrue(os.path.basename(d2) not in names)
+        self.assertTrue(os.path.basename(d2f1) not in names)
+        self.assertTrue(os.path.basename(new1) in names)
+        self.assertTrue(os.path.basename(new2) not in names)
+        self.assertTrue(os.path.basename(new3) in names)
+        self.assertTrue(os.path.basename(new4) not in names)
+        self.assertTrue(os.path.basename(new5) not in names)
+        for node in storage.children:
+            if node.name == os.path.basename(d1):
+                self.assertTrue(len(node.children) == 2)
+            elif node.name == os.path.basename(new3):
+                self.assertTrue(len(node.children) == 0)
+
 
 def main():
     unittest.main()