From 3bdd5ce142acda73b64a36aecf25a4af53d0b546 Mon Sep 17 00:00:00 2001
From: nick black <dankamongmen@gmail.com>
Date: Sun, 18 Apr 2021 04:45:34 -0400
Subject: [PATCH] scrub TAM boundaries in sixel+kitty #1553

---
 src/lib/internal.h | 25 +++++++++++++++++++++++++
 src/lib/kitty.c    | 18 +-----------------
 src/lib/sixel.c    | 11 ++++-------
 src/lib/sprite.c   |  4 ++++
 4 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/lib/internal.h b/src/lib/internal.h
index 7d47892e7..8f3f56867 100644
--- a/src/lib/internal.h
+++ b/src/lib/internal.h
@@ -956,6 +956,31 @@ clamp_to_sixelmax(const tinfo* t, int* y, int* x){
   }
 }
 
+// any sprixcell which does not cover the entirety of the underlying cell
+// cannot be SPRIXCELL_OPAQUE. this postprocesses the TAM, flipping any
+// such sprixcells to SPRIXCELL_MIXED.
+static inline void
+scrub_tam_boundaries(sprixcell_e* tam, int leny, int lenx, int cdimy, int cdimx){
+  // any sprixcells which don't cover the full cell underneath them cannot
+  // be SPRIXCELL_OPAQUE
+  const int cols = (lenx + cdimx - 1) / cdimx;
+  if(lenx % cdimx){
+    for(int y = 0 ; y < (leny + cdimy - 1) / cdimy ; ++y){
+      if(tam[y * cols + cols - 1] == SPRIXCELL_OPAQUE){
+        tam[y * cols + cols - 1] = SPRIXCELL_MIXED;
+      }
+    }
+  }
+  if(leny % cdimy){
+    const int y = (leny + cdimy - 1) / cdimy - 1;
+    for(int x = 0 ; x < cols ; ++x){
+      if(tam[y * cols + x] == SPRIXCELL_OPAQUE){
+        tam[y * cols + x] = SPRIXCELL_MIXED;
+      }
+    }
+  }
+}
+
 // get the TAM entry for these (absolute) coordinates
 static inline sprixcell_e
 sprixel_state(sprixel* s, int y, int x){
diff --git a/src/lib/kitty.c b/src/lib/kitty.c
index 31887cf15..93b999eb6 100644
--- a/src/lib/kitty.c
+++ b/src/lib/kitty.c
@@ -307,23 +307,7 @@ write_kitty_data(FILE* fp, int linesize, int leny, int lenx,
   if(fclose(fp) == EOF){
     return -1;
   }
-  // any sprixcells which don't cover the full cell underneath them cannot
-  // be SPRIXCELL_OPAQUE
-  if(lenx % cdimx){
-    for(y = 0 ; y < (leny + cdimy - 1) / cdimy ; ++y){
-      if(tacache[y * cols + cols - 1] == SPRIXCELL_OPAQUE){
-        tacache[y * cols + cols - 1] = SPRIXCELL_MIXED;
-      }
-    }
-  }
-  if(leny % cdimy){
-    y = (leny + cdimy - 1) / cdimy - 1;
-    for(x = 0 ; x < cols ; ++x){
-      if(tacache[y * cols + x] == SPRIXCELL_OPAQUE){
-        tacache[y * cols + x] = SPRIXCELL_MIXED;
-      }
-    }
-  }
+  scrub_tam_boundaries(tacache, leny, lenx, cdimy, cdimx);
   return 0;
 }
 #undef RGBA_MAXLEN
diff --git a/src/lib/sixel.c b/src/lib/sixel.c
index 0379e73a3..24e5dab6a 100644
--- a/src/lib/sixel.c
+++ b/src/lib/sixel.c
@@ -431,6 +431,8 @@ sixel_blit_inner(int leny, int lenx, const sixeltable* stab, int rows, int cols,
     free(buf);
     return -1;
   }
+  scrub_tam_boundaries(tacache, leny, lenx, bargs->u.pixel.celldimy,
+                       bargs->u.pixel.celldimx);
   // take ownership of buf on success
   if(plane_blit_sixel(bargs->u.pixel.spx, buf, size, rows, cols,
                       bargs->placey, bargs->placex,
@@ -514,7 +516,7 @@ int sixel_delete(const notcurses* nc, const ncpile* p, FILE* out, sprixel* s){
   for(int yy = s->movedfromy ; yy < s->movedfromy + s->dimy && yy < p->dimy ; ++yy){
     for(int xx = s->movedfromx ; xx < s->movedfromx + s->dimx && xx < p->dimx ; ++xx){
       struct crender *r = &p->crender[yy * p->dimx + xx];
-      if(!r->sprixel){
+      if(!r->sprixel || sprixel_state(r->sprixel, yy, xx) != SPRIXCELL_OPAQUE){
         r->s.damaged = 1;
       }
     }
@@ -525,12 +527,7 @@ int sixel_delete(const notcurses* nc, const ncpile* p, FILE* out, sprixel* s){
 int sixel_draw(const notcurses* n, const ncpile* p, sprixel* s, FILE* out){
   (void)n;
   if(s->invalidated == SPRIXEL_MOVED){
-    for(int yy = s->movedfromy ; yy < s->movedfromy + s->dimy && yy < p->dimy ; ++yy){
-      for(int xx = s->movedfromx ; xx < s->movedfromx + s->dimx && xx < p->dimx ; ++xx){
-//fprintf(stderr, "DAMAGING DUE TO MOVE: %d/%d (%d)\n", yy, xx, yy * p->dimx + xx);
-        p->crender[yy * p->dimx + xx].s.damaged = 1;
-      }
-    }
+    sixel_delete(n, p, out, s);
     s->invalidated = SPRIXEL_INVALIDATED;
   }else{
     if(fwrite(s->glyph, s->glyphlen, 1, out) != 1){
diff --git a/src/lib/sprite.c b/src/lib/sprite.c
index 896901b2b..0dd946886 100644
--- a/src/lib/sprite.c
+++ b/src/lib/sprite.c
@@ -34,6 +34,10 @@ sprixel* sprixel_recycle(ncplane* n){
 void sprixel_movefrom(sprixel* s, int y, int x){
   if(s->invalidated != SPRIXEL_HIDE){
     if(s->invalidated != SPRIXEL_MOVED){
+    // FIXME if we're Sixel, we need to effect any wipes that were run
+    // (we normally don't because redisplaying sixel doesn't change
+    // what's there--you can't "write transparency"). this is probably
+    // best done by conditionally reblitting the sixel(?).
 //fprintf(stderr, "SETTING TO MOVE: %d/%d was: %d\n", y, x, s->invalidated);
       s->invalidated = SPRIXEL_MOVED;
       s->movedfromy = y;