From 49c7b6be3f4738126f33c0c91e4d0c3adfdd0cd7 Mon Sep 17 00:00:00 2001
From: nick black <dankamongmen@gmail.com>
Date: Sun, 28 Feb 2021 11:44:34 -0500
Subject: [PATCH] sixel_blit: dumb quantization algorithm #1378

---
 src/lib/sixel.c | 49 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/lib/sixel.c b/src/lib/sixel.c
index 0aa0a92cd..856e354c4 100644
--- a/src/lib/sixel.c
+++ b/src/lib/sixel.c
@@ -1,10 +1,10 @@
 #include "internal.h"
 
 static inline void
-break_sixel_comps(unsigned char comps[static 3], uint32_t rgba){
-  comps[0] = ncpixel_r(rgba) * 100 / 255;
-  comps[1] = ncpixel_g(rgba) * 100 / 255;
-  comps[2] = ncpixel_b(rgba) * 100 / 255;
+break_sixel_comps(unsigned char comps[static 3], uint32_t rgba, unsigned char mask){
+  comps[0] = (ncpixel_r(rgba) & mask) * 100 / 255;
+  comps[1] = (ncpixel_g(rgba) & mask) * 100 / 255;
+  comps[2] = (ncpixel_b(rgba) & mask) * 100 / 255;
 }
 
 // first pass: extract up to 256 sixelspace colors over arbitrarily many sixels
@@ -56,8 +56,8 @@ find_color(colortable* ctab, unsigned char comps[static 3]){
 // everything in a single pass FIXME.
 // what do we do if every pixel is transparent (0 colors)? FIXME
 static int
-extract_color_table(const uint32_t* data, int linesize, int begy, int begx,
-                    int leny, int lenx, colortable* ctab){
+extract_ctable_inner(const uint32_t* data, int linesize, int begy, int begx,
+                     int leny, int lenx, colortable* ctab, unsigned char mask){
   for(int visy = begy ; visy < (begy + leny) ; visy += 6){
     for(int visx = begx ; visx < (begx + lenx) ; visx += 1){
       for(int sy = visy ; sy < (begy + leny) && sy < visy + 6 ; ++sy){
@@ -66,9 +66,8 @@ extract_color_table(const uint32_t* data, int linesize, int begy, int begx,
           continue;
         }
         unsigned char comps[3];
-        break_sixel_comps(comps, *rgb);
+        break_sixel_comps(comps, *rgb, mask);
         if(find_color(ctab, comps) < 0){
-fprintf(stderr, "FUCK ME; THE COLOR TABLE'S FULL\n");
           return -1;
         }
       }
@@ -78,9 +77,31 @@ fprintf(stderr, "FUCK ME; THE COLOR TABLE'S FULL\n");
   return 0;
 }
 
+static inline void
+initialize_ctable(colortable* ctab){
+  ctab->colors = 0;
+  ctab->sixelcount = 0;
+  memset(ctab->table, 0xff, 3);
+}
+
+static int
+extract_color_table(const uint32_t* data, int linesize, int begy, int begx,
+                    int leny, int lenx, colortable* ctab, unsigned char* mask){
+  *mask = 0xff;
+  while(mask){
+    initialize_ctable(ctab);
+    if(extract_ctable_inner(data, linesize, begy, begx, leny, lenx, ctab, *mask) == 0){
+      return 0;
+    }
+    *mask <<= 1;
+    *mask &= 0xff;
+  }
+  return -1;
+}
+
 static int
 extract_data_table(const uint32_t* data, int linesize, int begy, int begx,
-                   int leny, int lenx, sixeltable* stab){
+                   int leny, int lenx, sixeltable* stab, unsigned char mask){
 //fprintf(stderr, "colors: %d sixelcount: %d\n", stab->ctab->colors, stab->ctab->sixelcount);
   for(int c = 0 ; c < stab->ctab->colors ; ++c){
     int pos = 0;
@@ -96,7 +117,7 @@ extract_data_table(const uint32_t* data, int linesize, int begy, int begx,
             continue;
           }
           unsigned char comps[3];
-          break_sixel_comps(comps, *rgb);
+          break_sixel_comps(comps, *rgb, mask);
 //fprintf(stderr, "%d/%d/%d\n", comps[0], comps[1], comps[2]);
           if(memcmp(comps, stab->ctab->table + c * 3, 3) == 0){
             stab->data[c * stab->ctab->sixelcount + pos] |= (1u << (sy - visy));
@@ -211,10 +232,8 @@ int sixel_blit(ncplane* nc, int placey, int placex, int linesize,
   if(ctab == NULL){
     return -1;
   }
-  ctab->colors = 0;
-  ctab->sixelcount = 0;
-  memset(ctab->table, 0xff, 3);
-  if(extract_color_table(data, linesize, begy, begx, leny, lenx, ctab)){
+  unsigned char mask;
+  if(extract_color_table(data, linesize, begy, begx, leny, lenx, ctab, &mask)){
     free(ctab);
     return -1;
   }
@@ -227,7 +246,7 @@ int sixel_blit(ncplane* nc, int placey, int placex, int linesize,
     return -1;
   }
   memset(stable.data, 0, ctab->colors * ctab->sixelcount);
-  if(extract_data_table(data, linesize, begy, begx, leny, lenx, &stable)){
+  if(extract_data_table(data, linesize, begy, begx, leny, lenx, &stable, mask)){
     free(stable.data);
     free(ctab);
     return -1;