[sixel] streamline extract core

src/lib/sixel.c

@@ -768,6 +768,7 @@ extract_cell_color_table(qstate* qs, long cellid){
   const int lenx = qs->lenx;
   const int cstartx = begx + x * cdimx; // starting pixel col for cell
   const int cstarty = begy + y * cdimy; // starting pixel row for cell
+  typeof(qs->bargs->u.pixel.spx->needs_refresh) rmatrix = qs->bargs->u.pixel.spx->needs_refresh;
   tament* tam = qs->bargs->u.pixel.spx->n->tam;
   int cendy = cstarty + cdimy;    // one past last pixel row for cell
   if(cendy > begy + leny){
@@ -778,62 +779,59 @@ extract_cell_color_table(qstate* qs, long cellid){
     cendx = begx + lenx;
   }
   bool firstpix = true;
+  // we initialize the TAM entry based on the first pixel. if it's transparent,
+  // initialize as transparent, and otherwise as opaque. following that, any
+  // transparent pixel takes opaque to mixed, and any filled pixel takes
+  // transparent to mixed.
+  const uint32_t* rgb = (qs->data + (qs->linesize / 4 * cstarty) + cstartx);
+  if(tam[cellid].state == SPRIXCELL_ANNIHILATED || tam[cellid].state == SPRIXCELL_ANNIHILATED_TRANS){
+    if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
+      update_rmatrix(rmatrix, cellid, tam);
+      tam[cellid].state = SPRIXCELL_ANNIHILATED_TRANS;
+      free(tam[cellid].auxvector);
+      tam[cellid].auxvector = NULL;
+    }else{
+      update_rmatrix(rmatrix, cellid, tam);
+      free(tam[cellid].auxvector);
+      tam[cellid].auxvector = NULL;
+    }
+  }else{
+    if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
+      update_rmatrix(rmatrix, cellid, tam);
+      tam[cellid].state = SPRIXCELL_TRANSPARENT;
+    }else{
+      update_rmatrix(rmatrix, cellid, tam);
+      tam[cellid].state = SPRIXCELL_OPAQUE_SIXEL;
+    }
+  }
   sixeltable* stab = qs->stab;
-  typeof(qs->bargs->u.pixel.spx->needs_refresh) rmatrix = qs->bargs->u.pixel.spx->needs_refresh;
   for(int visy = cstarty ; visy < cendy ; ++visy){   // current abs pixel row
-    const bool lastrow = visy + 1 == cendy;
     for(int visx = cstartx ; visx < cendx ; ++visx){ // current abs pixel col
-      const uint32_t* rgb = (qs->data + (qs->linesize / 4 * visy) + visx);
+      rgb = (qs->data + (qs->linesize / 4 * visy) + visx);
       // we do *not* exempt already-wiped pixels from palette creation. once
       // we're done, we'll call sixel_wipe() on these cells. so they remain
       // one of SPRIXCELL_ANNIHILATED or SPRIXCELL_ANNIHILATED_TRANS.
-      if(tam[cellid].state != SPRIXCELL_ANNIHILATED && tam[cellid].state != SPRIXCELL_ANNIHILATED_TRANS){
-        if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
-          if(firstpix){
-            update_rmatrix(rmatrix, cellid, tam);
-            tam[cellid].state = SPRIXCELL_TRANSPARENT;
-          }else if(tam[cellid].state == SPRIXCELL_OPAQUE_SIXEL){
-            tam[cellid].state = SPRIXCELL_MIXED_SIXEL;
-          }
-          stab->map->p2 = SIXEL_P2_TRANS; // even one forces P2=1
-        }else{
-          if(firstpix){
-            update_rmatrix(rmatrix, cellid, tam);
-            tam[cellid].state = SPRIXCELL_OPAQUE_SIXEL;
-          }else if(tam[cellid].state == SPRIXCELL_TRANSPARENT){
-            tam[cellid].state = SPRIXCELL_MIXED_SIXEL;
-          }
-        }
-      }else{
-//fprintf(stderr, "TRANS SKIP %d %d %d %d (cell: %d %d)\n", visy, visx, sy, cellid, sy / cdimy, visx / cdimx);
-        if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
-          if(firstpix){
-            update_rmatrix(rmatrix, cellid, tam);
-            tam[cellid].state = SPRIXCELL_ANNIHILATED_TRANS;
-            free(tam[cellid].auxvector);
-            tam[cellid].auxvector = NULL;
+      // intentional bitwise or, to avoid dependency
+      if(tam[cellid].state != SPRIXCELL_ANNIHILATED){
+        if(tam[cellid].state == SPRIXCELL_ANNIHILATED_TRANS){
+          if(!rgba_trans_p(*rgb, qs->bargs->transcolor)){
+            tam[cellid].state = SPRIXCELL_ANNIHILATED;
           }
         }else{
-          if(firstpix){
-            update_rmatrix(rmatrix, cellid, tam);
-            free(tam[cellid].auxvector);
-            tam[cellid].auxvector = NULL;
-          }
-          tam[cellid].state = SPRIXCELL_ANNIHILATED;
-        }
-        stab->map->p2 = SIXEL_P2_TRANS; // even one forces P2=1
-      }
-      if(lastrow){
-        bool lastcol = visx + 1 >= begx + lenx;
-        if(lastcol){
-          // if we're opaque, we needn't clear the old cell with a glyph
-          if(tam[cellid].state == SPRIXCELL_OPAQUE_SIXEL){
-            if(rmatrix){
-              rmatrix[cellid] = 0;
+          if(!firstpix){
+            if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
+              if(tam[cellid].state == SPRIXCELL_OPAQUE_SIXEL){
+                tam[cellid].state = SPRIXCELL_MIXED_SIXEL;
+              }
+            }else{
+              if(tam[cellid].state == SPRIXCELL_TRANSPARENT){
+                tam[cellid].state = SPRIXCELL_MIXED_SIXEL;
+              }
             }
           }
         }
       }
+//fprintf(stderr, "vis: %d/%d\n", visy, visx);
       firstpix = false;
       if(rgba_trans_p(*rgb, qs->bargs->transcolor)){
         continue;
@@ -843,6 +841,12 @@ extract_cell_color_table(qstate* qs, long cellid){
       }
     }
   }
+  // if we're opaque, we needn't clear the old cell with a glyph
+  if(tam[cellid].state == SPRIXCELL_OPAQUE_SIXEL){
+    rmatrix[cellid] = 0;
+  }else{
+    stab->map->p2 = SIXEL_P2_TRANS; // even one forces P2=1
+  }
   return 0;
 }