Fix sixel reload cached wipe display #1557

In Sixel, we must supply a value for P2; essentially, 1
means that there are transparent pixels, while 0 means
there are not, or that we don't care what's already
present (P2=0 is implemented faster by some terminals).
We set P2=1 upon encountering a transparent pixel, or
after wiping a cell, but we weren't doing so when we
reloaded a bitmap and prewiped due to cached ANNIHILATION
values in the TAM. Do so. This fixes up the flickering
we saw in the yield demo, as we no longer printed over
the text. Furthermore, we can now properly return 0 from
sixel_wipe() on a cached wipe, eliminating a great many
invalidations and redraws. Move the ANNIHILATION check
into the shared code of sprite_wipe(), removing it from
sixel_wipe() and kitty_wipe().

Oh, what a happy, happy, happy day!

src/lib/internal.h

@@ -993,7 +993,7 @@ sprite_destroy(const notcurses* nc, const ncpile* p, FILE* out, sprixel* s){
 // precondition: s->invalidated is SPRIXEL_INVALIDATED or SPRIXEL_MOVED.
 static inline int
 sprite_draw(const notcurses* n, const ncpile* p, sprixel* s, FILE* out){
-sprixel_debug(stderr, s);
+//sprixel_debug(stderr, s);
   return n->tcache.pixel_draw(p, s, out);
 }
 

src/lib/kitty.c

@@ -336,13 +336,8 @@ int kitty_rebuild(sprixel* s, int ycell, int xcell, uint8_t* auxvec){
 }
 
 int kitty_wipe(sprixel* s, int ycell, int xcell){
-fprintf(stderr, "WIPING %d/%d\n", ycell, xcell);
-  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED){
-fprintf(stderr, "CACHED WIPE %d %d/%d\n", s->id, ycell, xcell);
-    return 0; // already annihilated, needn't draw glyph in kitty
-  }
+//fprintf(stderr, "NEW WIPE %d %d/%d\n", s->id, ycell, xcell);
   uint8_t* auxvec = sprixel_auxiliary_vector(s);
-fprintf(stderr, "NEW WIPE %d %d/%d\n", s->id, ycell, xcell);
   const int totalpixels = s->pixy * s->pixx;
   const int xpixels = s->cellpxx;
   const int ypixels = s->cellpxy;

src/lib/render.c

@@ -174,7 +174,7 @@ paint_sprixel(ncplane* p, struct crender* rvec, int starty, int startx,
         // if sprite_wipe_cell() fails, we presumably do not have the
         // ability to wipe, and must reprint the character
         if(sprite_wipe(nc, p->sprite, y, x)){
-fprintf(stderr, "damaging due to wipe [%s] %d/%d\n", nccell_extended_gcluster(crender->p, &crender->c), absy, absx);
+//fprintf(stderr, "damaging due to wipe [%s] %d/%d\n", nccell_extended_gcluster(crender->p, &crender->c), absy, absx);
           crender->s.damaged = 1;
         }
         crender->s.p_beats_sprixel = 1;
@@ -441,17 +441,16 @@ postpaint_cell(nccell* lastframe, int dimx, struct crender* crender,
   nccell* targc = &crender->c;
   lock_in_highcontrast(targc, crender);
   nccell* prevcell = &lastframe[fbcellidx(y, dimx, *x)];
-if(y < 5 && *x < 5) fprintf(stderr, "taking a look at %d/%d %08x %08x [%u]\n", y, *x, prevcell->gcluster, crender->c.gcluster, crender->s.damaged);
   if(cellcmp_and_dupfar(pool, prevcell, crender->p, targc) > 0){
 //fprintf(stderr, "damaging due to cmp [%s] %d %d\n", nccell_extended_gcluster(crender->p, &crender->c), y, *x);
     if(crender->sprixel){
       sprixcell_e state = sprixel_state(crender->sprixel, y, *x);
       if(!crender->s.p_beats_sprixel && state != SPRIXCELL_OPAQUE_KITTY && state != SPRIXCELL_OPAQUE_SIXEL){
-fprintf(stderr, "damaged due to opaque\n");
+//fprintf(stderr, "damaged due to opaque\n");
         crender->s.damaged = 1;
       }
     }else{
-fprintf(stderr, "damaged due to opaque else %d %d\n", y, *x);
+//fprintf(stderr, "damaged due to opaque else %d %d\n", y, *x);
       crender->s.damaged = 1;
     }
     assert(!nccell_wide_right_p(targc));
@@ -923,7 +922,7 @@ clean_sprixels(notcurses* nc, ncpile* p, FILE* out){
   int ret = 0;
   while( (s = *parent) ){
     if(s->invalidated == SPRIXEL_HIDE){
-fprintf(stderr, "OUGHT HIDE %d [%dx%d] %p\n", s->id, s->dimy, s->dimx, s);
+//fprintf(stderr, "OUGHT HIDE %d [%dx%d] %p\n", s->id, s->dimy, s->dimx, s);
       if(sprite_destroy(nc, p, out, s) == 0){
         if( (*parent = s->next) ){
           s->next->prev = s->prev;
@@ -1010,7 +1009,7 @@ rasterize_core(notcurses* nc, const ncpile* p, FILE* out, unsigned phase){
           ++x;
         }
       }else if(phase != 0 || !rvec[damageidx].s.p_beats_sprixel){
-fprintf(stderr, "phase %u damaged at %d/%d\n", phase, innery, innerx);
+//fprintf(stderr, "phase %u damaged at %d/%d\n", phase, innery, innerx);
         // in the first text phase, we draw only those glyphs where the glyph
         // was not above a sprixel (and the cell is damaged). in the second
         // phase, we draw everything that remains damaged.
@@ -1085,7 +1084,7 @@ fprintf(stderr, "phase %u damaged at %d/%d\n", phase, innery, innerx);
           nc->rstate.bgdefelidable = false;
           nc->rstate.bgpalelidable = false;
         }
-fprintf(stderr, "RAST %08x [%s] to %d/%d cols: %u %016lx\n", srccell->gcluster, pool_extended_gcluster(&nc->pool, srccell), y, x, srccell->width, srccell->channels);
+//fprintf(stderr, "RAST %08x [%s] to %d/%d cols: %u %016lx\n", srccell->gcluster, pool_extended_gcluster(&nc->pool, srccell), y, x, srccell->width, srccell->channels);
         // this is used to invalidate the sprixel in the first text round,
         // which is only necessary for sixel, not kitty.
         if(rvec[damageidx].sprixel){
@@ -1106,8 +1105,6 @@ fprintf(stderr, "RAST %08x [%s] to %d/%d cols: %u %016lx\n", srccell->gcluster,
           x += srccell->width - 1;
           nc->rstate.x += srccell->width - 1;
         }
-      }else{
-fprintf(stderr, "YEET phase %u damaged at %d/%d\n", phase, innery, innerx);
       }
 //fprintf(stderr, "damageidx: %ld\n", damageidx);
     }

src/lib/sixel.c

@@ -262,7 +262,8 @@ extract_color_table(const uint32_t* data, int linesize, int cols,
         const uint32_t* rgb = (data + (linesize / 4 * sy) + visx);
         int txyidx = (sy / cdimy) * cols + (visx / cdimx);
         if(tam[txyidx].state == SPRIXCELL_ANNIHILATED || tam[txyidx].state == SPRIXCELL_ANNIHILATED_TRANS){
-fprintf(stderr, "TRANS SKIP %d %d %d %d (cell: %d %d)\n", visy, visx, sy, txyidx, sy / cdimy, visx / cdimx);
+//fprintf(stderr, "TRANS SKIP %d %d %d %d (cell: %d %d)\n", visy, visx, sy, txyidx, sy / cdimy, visx / cdimx);
+          stab->p2 = SIXEL_P2_TRANS; // even one forces P2=1
           continue;
         }
         if(rgba_trans_p(*rgb, bargs->transcolor)){
@@ -844,10 +845,6 @@ wipe_color(sixelmap* smap, int color, int sband, int eband,
 // using sixel. we just mark it as partially transparent, so that if it's
 // redrawn, it's redrawn using P2=1.
 int sixel_wipe(sprixel* s, int ycell, int xcell){
-  if(s->n->tam[s->dimx * ycell + xcell].state == SPRIXCELL_ANNIHILATED){
-//fprintf(stderr, "CACHED WIPE %d %d/%d\n", s->id, ycell, xcell);
-    return 1; // already annihilated FIXME but 0 breaks things
-  }
 //fprintf(stderr, "WIPING %d/%d\n", ycell, xcell);
   uint8_t* auxvec = sprixel_auxiliary_vector(s);
   memset(auxvec + s->cellpxx * s->cellpxy, 0xff, s->cellpxx * s->cellpxy);

src/lib/sprite.c

@@ -82,7 +82,6 @@ void sprixel_movefrom(sprixel* s, int y, int x){
 
 void sprixel_hide(sprixel* s){
   if(ncplane_pile(s->n) == NULL){ // ncdirect case; destroy now
-fprintf(stderr, "DESTROY IMMEDIATELY\n");
     sprixel_free(s);
     return;
   }
@@ -187,9 +186,9 @@ int sprite_wipe(const notcurses* nc, sprixel* s, int ycell, int xcell){
     s->n->tam[idx].state = SPRIXCELL_ANNIHILATED_TRANS;
     return 1;
   }
-  if(s->n->tam[idx].state == SPRIXCELL_ANNIHILATED_TRANS){
-    // both handle this correctly; one day, we will also check for ANNIHILATED
-    // here, and return 0 (sixel currently must return 1) FIXME
+  if(s->n->tam[idx].state == SPRIXCELL_ANNIHILATED_TRANS ||
+      s->n->tam[idx].state == SPRIXCELL_ANNIHILATED){
+//fprintf(stderr, "CACHED WIPE %d %d/%d\n", s->id, ycell, xcell);
     return 0;
   }
 //fprintf(stderr, "ANNIHILATING %p %d\n", s->n->tam, idx);