introduce ANNIHILATED_TRANS, special-case O(1) conversion from TRANSPARENT #1587

src/lib/internal.h

@@ -85,8 +85,9 @@ typedef enum {
 //
 // a glyph above a TRANSPARENT sprixcell requires annihilating the underlying
 // cell, but this is a special annihilation which never requires a wipe nor
-// redisplay, just the state transition. a glyph below a TRANSPARENT sprixcell
-// can be emitted with no change to the sprixcell.
+// redisplay, just the O(1) state transition. a glyph below a TRANSPARENT
+// sprixcell can be emitted with no change to the sprixcell. TRANSPARENT
+// sprixcells move to ANNIHILATED_TRANS upon annihilation.
 //
 // a glyph above an ANNIHILATED sprixcell can be emitted with no change to
 // the sprixcell. it does not make sense to emit a glyph below an ANNIHILATED
@@ -101,13 +102,15 @@ typedef enum {
 //
 // an ANNIHILATED sprixcell with no glyph above it must be restored to its
 // original form (from the most recent RGBA frame). this requires the original
-// pixel data. for Sixel, we must keep the RGB values in an auxiliary vector,
-// hung off the TAM, updated each time we convert an RGBA frame into a
+// pixel data. for Sixel, we must keep the palette indices in an auxiliary
+// vector, hung off the TAM, updated each time we convert an RGBA frame into a
 // partially- or wholly-ANNIHILATED sprixel. for Kitty, we must keep the
-// original alpha values (1/3 the data necessary for Sixel). the new state
-// can be solved from this data. if the new state is either OPAQUE or MIXED,
-// the sprixel must be redisplayed. if the new state is TRANSPARENT, this cell
-// requires no such redisplay.
+// original alpha values. the new state can be solved from this data. if the
+// new state is either OPAQUE or MIXED, the sprixel must be redisplayed. if the
+// new state is TRANSPARENT, this cell requires no such redisplay, and the
+// payload needn't be modified. to special-case this O(1) conversion, we keep a
+// distinct state, ANNIHILATED_TRANS. only a TRANSPARENT sprixcell can enter
+// into this state.
 //
 // when a sprixel is removed from the rendering pile, in Sixel all cells it
 // covered must be marked damaged, so that they are rendered, obliterating
@@ -135,12 +138,13 @@ typedef enum {
 // ANNIHILATED cells which are no longer ANNIHILATED), or at blittime for
 // a new RGBA frame.
 typedef enum {
-  SPRIXCELL_OPAQUE_SIXEL,   // no transparent pixels in this cell
+  SPRIXCELL_OPAQUE_SIXEL,      // no transparent pixels in this cell
   SPRIXCELL_OPAQUE_KITTY,
-  SPRIXCELL_MIXED_SIXEL,    // this cell has both opaque and transparent pixels
+  SPRIXCELL_MIXED_SIXEL,       // this cell has both opaque and transparent pixels
   SPRIXCELL_MIXED_KITTY,
-  SPRIXCELL_TRANSPARENT,    // all pixels are naturally transparent
-  SPRIXCELL_ANNIHILATED,    // this cell has been wiped (all trans)
+  SPRIXCELL_TRANSPARENT,       // all pixels are naturally transparent
+  SPRIXCELL_ANNIHILATED,       // this cell has been wiped (all trans)
+  SPRIXCELL_ANNIHILATED_TRANS, // this transparent cell is covered
 } sprixcell_e;
 
 // a TAM entry is a sprixcell_e state plus a possible auxiliary vector for
@@ -458,7 +462,7 @@ typedef struct tinfo {
   // this means dialing down their alpha to 0 (in equivalent space).
   int (*pixel_cell_wipe)(const struct notcurses* nc, sprixel* s, int y, int x);
   // perform the inverse of pixel_cell_wipe, restoring an annihilated sprixcell.
-  int (*pixel_rebuild)(sprixel* s, int y, int x);
+  int (*pixel_rebuild)(sprixel* s, int y, int x, const uint8_t* auxvec);
   int (*pixel_remove)(int id, FILE* out); // kitty only, issue actual delete command
   int (*pixel_init)(int fd);     // called when support is detected
   int (*pixel_draw)(const struct notcurses* n, const struct ncpile* p, sprixel* s, FILE* out);
@@ -935,8 +939,8 @@ int sixel_wipe(const notcurses* nc, sprixel* s, int ycell, int xcell);
 // throughout to 0. the same trick doesn't work on sixel, but there we
 // can just print directly over the bitmap.
 int kitty_wipe(const notcurses* nc, sprixel* s, int ycell, int xcell);
-int sixel_rebuild(sprixel* s, int ycell, int xcell);
-int kitty_rebuild(sprixel* s, int ycell, int xcell);
+int sixel_rebuild(sprixel* s, int ycell, int xcell, const uint8_t* auxvec);
+int kitty_rebuild(sprixel* s, int ycell, int xcell, const uint8_t* auxvec);
 
 void sprixel_free(sprixel* s);
 void sprixel_hide(sprixel* s);
@@ -986,7 +990,19 @@ sprite_draw(const notcurses* n, const ncpile* p, sprixel* s, FILE* out){
 
 static inline int
 sprite_rebuild(const notcurses* nc, sprixel* s, int ycell, int xcell){
-  return nc->tcache.pixel_rebuild(s, ycell, xcell);
+  int ret = 0;
+  uint8_t* auxvec = s->n->tam[s->dimx * ycell + xcell].auxvector;
+  // special case the transition back to SPRIXCELL_TRANSPARENT; this can be
+  // done in O(1), since the actual glyph needn't change.
+  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED_TRANS){
+    s->n->tam[s->dimx * ycell + xcell].state = SPRIXCELL_TRANSPARENT;
+  }else if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED){
+    // sets the new state itself
+    ret = nc->tcache.pixel_rebuild(s, ycell, xcell, auxvec);
+  }
+  s->n->tam[s->dimx * ycell + xcell].auxvector = NULL;
+  free(auxvec);
+  return ret;
 }
 
 static inline void

src/lib/kitty.c

@@ -261,33 +261,10 @@ kitty_restore(char* triplet, int skip, int max, int pleft,
   return max;
 }
 
-// is the auxvec entirely transparent? if so, we can special case a direct
-// conversion back to SPRIXCELL_TRANSPARENT with no further work.
-static int
-auxvec_all_trans_p(const sprixel* s, const uint8_t* auxvec){
-  for(int i = 0 ; i < s->cellpxx * s->cellpxy ; ++i){
-    if(auxvec[i]){
-      return 0;
-    }
-  }
-  return 1;
-}
-
 #define RGBA_MAXLEN 768 // 768 base64-encoded pixels in 4096 bytes
 // restore an annihilated sprixcell by copying the alpha values from the
 // auxiliary vector back into the actual data. we then free the auxvector.
-int kitty_rebuild(sprixel* s, int ycell, int xcell){
-  if(s->n->tam[s->dimx * ycell + xcell].state != SPRIXCELL_ANNIHILATED){
-//fprintf(stderr, "CACHED WIPE %d %d/%d\n", s->id, ycell, xcell);
-    return 0; // already annihilated, needn't draw glyph in kitty
-  }
-  uint8_t* auxvec = s->n->tam[s->dimx * ycell + xcell].auxvector;
-  if(auxvec_all_trans_p(s, auxvec)){
-    free(auxvec);
-    s->n->tam[s->dimx * ycell + xcell].state = SPRIXCELL_TRANSPARENT;
-    s->n->tam[s->dimx * ycell + xcell].auxvector = NULL;
-    return 0;
-  }
+int kitty_rebuild(sprixel* s, int ycell, int xcell, const uint8_t* auxvec){
   const int totalpixels = s->pixy * s->pixx;
   const int xpixels = s->cellpxx;
   const int ypixels = s->cellpxy;
@@ -336,9 +313,7 @@ int kitty_rebuild(sprixel* s, int ycell, int xcell){
       if(thisrow == 0){
 //fprintf(stderr, "CLEARED ROW, TARGY: %d\n", targy - 1);
         if(--targy == 0){
-          free(auxvec);
           s->n->tam[s->dimx * ycell + xcell].state = state;
-          s->n->tam[s->dimx * ycell + xcell].auxvector = NULL;
           return 0;
         }
         thisrow = targx;
@@ -494,7 +469,7 @@ write_kitty_data(FILE* fp, int linesize, int leny, int lenx,
         int ycell = y / cdimy;
         int tyx = xcell + ycell * cols;
 //fprintf(stderr, "Tyx: %d y: %d (%d) * %d x: %d (%d)\n", tyx, y, y / cdimy, cols, x, x / cdimx);
-        if(tam[tyx].state == SPRIXCELL_ANNIHILATED){
+        if(tam[tyx].state == SPRIXCELL_ANNIHILATED || tam[tyx].state == SPRIXCELL_ANNIHILATED_TRANS){
           wipe[e] = 1;
         }else{
           wipe[e] = 0;

src/lib/render.c

@@ -179,7 +179,8 @@ paint_sprixel(ncplane* p, struct crender* rvec, int starty, int startx,
         if(crender->sprixel == NULL){
           crender->sprixel = s;
         }
-        if(sprixel_state(s, absy, absx) == SPRIXCELL_ANNIHILATED){
+        sprixcell_e state = sprixel_state(s, absy, absx);
+        if(state == SPRIXCELL_ANNIHILATED || state == SPRIXCELL_ANNIHILATED_TRANS){
 //fprintf(stderr, "REBUILDING AT %d/%d\n", y, x);
           sprite_rebuild(nc, s, y, x);
         }

src/lib/sixel.c

@@ -170,7 +170,7 @@ extract_color_table(const uint32_t* data, int linesize, int cols,
       for(int sy = visy ; sy < (begy + leny) && sy < visy + 6 ; ++sy){ // offset within sprixel
         const uint32_t* rgb = (data + (linesize / 4 * sy) + visx);
         int txyidx = (sy / cdimy) * cols + (visx / cdimx);
-        if(tam[txyidx].state == SPRIXCELL_ANNIHILATED){
+        if(tam[txyidx].state == SPRIXCELL_ANNIHILATED || tam[txyidx].state == SPRIXCELL_ANNIHILATED_TRANS){
 //fprintf(stderr, "TRANS SKIP %d %d %d %d (cell: %d %d)\n", visy, visx, sy, txyidx, sy / cdimy, visx / cdimx);
           continue;
         }
@@ -546,7 +546,8 @@ deepclean_output(FILE* fp, const sprixel* s, int y, int *x, int rle,
     unsigned char mask = 0;
     for(int yi = y ; yi < y + 6 ; ++yi){
       const int tidx = (yi / s->cellpxy) * s->dimx + (xi / s->cellpxx);
-      const bool nihil = (s->n->tam[tidx].state == SPRIXCELL_ANNIHILATED);
+      const bool nihil = (s->n->tam[tidx].state == SPRIXCELL_ANNIHILATED) ||
+                         (s->n->tam[tidx].state == SPRIXCELL_ANNIHILATED_TRANS);
       if(!nihil){
         mask |= (1u << (yi - y));
       }
@@ -746,10 +747,11 @@ int sixel_init(int fd){
   return tty_emit("\e[?80;8452h", fd);
 }
 
-int sixel_rebuild(sprixel* s, int ycell, int xcell){
+int sixel_rebuild(sprixel* s, int ycell, int xcell, const uint8_t* auxvec){
   (void)s;
   (void)ycell;
   (void)xcell;
+  (void)auxvec;
   return 0;
 }
 
@@ -758,7 +760,8 @@ int sixel_rebuild(sprixel* s, int ycell, int xcell){
 // redrawn, it's redrawn using P2=1.
 int sixel_wipe(const notcurses* nc, sprixel* s, int ycell, int xcell){
   (void)nc;
-  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED){
+  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED ||
+     s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED_TRANS){
 //fprintf(stderr, "CACHED WIPE %d %d/%d\n", s->id, ycell, xcell);
     return 1; // already annihilated FIXME but 0 breaks things
   }

src/lib/sprite.c

@@ -105,7 +105,8 @@ void sprixel_invalidate(sprixel* s, int y, int x){
     int localx = x - s->n->absx;
 //fprintf(stderr, "INVALIDATING AT %d/%d (%d/%d) TAM: %d\n", y, x, localy, localx, s->n->tam[localy * s->dimx + localx].state);
     if(s->n->tam[localy * s->dimx + localx].state != SPRIXCELL_TRANSPARENT &&
-       s->n->tam[localy * s->dimx + localx].state != SPRIXCELL_ANNIHILATED){
+       s->n->tam[localy * s->dimx + localx].state != SPRIXCELL_ANNIHILATED &&
+       s->n->tam[localy * s->dimx + localx].state != SPRIXCELL_ANNIHILATED_TRANS){
       s->invalidated = SPRIXEL_INVALIDATED;
     }
   }
@@ -174,6 +175,10 @@ int sprite_wipe(const notcurses* nc, sprixel* s, int ycell, int xcell){
   if(s->invalidated == SPRIXEL_HIDE){ // no need to do work if we're killing it
     return 0;
   }
+  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_TRANSPARENT){
+    s->n->tam[s->dimx * ycell + xcell].state = SPRIXCELL_ANNIHILATED_TRANS;
+    return 1;
+  }
 //fprintf(stderr, "ANNIHILATED %p %d\n", s->n->tam, s->dimx * ycell + xcell);
   int r = nc->tcache.pixel_cell_wipe(nc, s, ycell, xcell);
 //fprintf(stderr, "WIPED %d %d/%d ret=%d\n", s->id, ycell, xcell, r);