[channels] set_rgb needs clear the palette bit!

USAGE.md

@@ -2984,24 +2984,51 @@ channel_rgb8(uint32_t channel, unsigned* r, unsigned* g, unsigned* b){
 // Set the three 8-bit components of a 32-bit channel, and mark it as not using
 // the default color. Retain the other bits unchanged.
 static inline int
-channel_set_rgb8(unsigned* channel, unsigned r, unsigned g, unsigned b){
+ncchannel_set_rgb8(uint32_t* channel, unsigned r, unsigned g, unsigned b){
   if(r >= 256 || g >= 256 || b >= 256){
     return -1;
   }
-  unsigned c = (r << 16u) | (g << 8u) | b;
-  c |= NC_BGDEFAULT_MASK;
-  const uint64_t mask = NC_BGDEFAULT_MASK | NC_BG_MASK;
-  *channel = (*channel & ~mask) | c;
+  uint32_t c = (r << 16u) | (g << 8u) | b;
+  // clear the existing rgb bits, clear the palette index indicator, set
+  // the not-default bit, and or in the new rgb.
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | c;
   return 0;
 }
 
+// Set the three 8-bit components of a 32-bit channel, and mark it as not using
+// the default color. Retain the other bits unchanged. r, g, and b will be
+// clipped to the range [0..255].
+static inline void
+ncchannel_set_rgb8_clipped(uint32_t* channel, int r, int g, int b){
+  if(r >= 256){
+    r = 255;
+  }
+  if(g >= 256){
+    g = 255;
+  }
+  if(b >= 256){
+    b = 255;
+  }
+  if(r <= -1){
+    r = 0;
+  }
+  if(g <= -1){
+    g = 0;
+  }
+  if(b <= -1){
+    b = 0;
+  }
+  uint32_t c = (r << 16u) | (g << 8u) | b;
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | c;
+}
+
 // Same, but provide an assembled, packed 24 bits of rgb.
 static inline int
-channel_set(unsigned* channel, unsigned rgb){
+ncchannel_set(uint32_t* channel, uint32_t rgb){
   if(rgb > 0xffffffu){
     return -1;
   }
-  *channel = (*channel & ~NC_BG_MASK) | NC_BGDEFAULT_MASK | rgb;
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | rgb;
   return 0;
 }
 

include/notcurses/notcurses.h

@@ -141,22 +141,13 @@ API int notcurses_ucs32_to_utf8(const uint32_t* ucs32, unsigned ucs32count,
 #define NC_NOBACKGROUND_MASK  0x8700000000000000ull
 // if this bit is set, we are *not* using the default background color
 #define NC_BGDEFAULT_MASK     0x0000000040000000ull
-// if this bit is set, we are *not* using the default foreground color
-#define NC_FGDEFAULT_MASK     (NC_BGDEFAULT_MASK << 32u)
 // extract these bits to get the background RGB value
 #define NC_BG_RGB_MASK        0x0000000000ffffffull
-// extract these bits to get the foreground RGB value
-#define NC_FG_RGB_MASK        (NC_BG_RGB_MASK << 32u)
 // if this bit *and* NC_BGDEFAULT_MASK are set, we're using a
 // palette-indexed background color
 #define NC_BG_PALETTE         0x0000000008000000ull
-// if this bit *and* NC_FGDEFAULT_MASK are set, we're using a
-// palette-indexed foreground color
-#define NC_FG_PALETTE         (NC_BG_PALETTE << 32u)
 // extract these bits to get the background alpha mask
 #define NC_BG_ALPHA_MASK      0x30000000ull
-// extract these bits to get the foreground alpha mask
-#define NC_FG_ALPHA_MASK      (NC_BG_ALPHA_MASK << 32u)
 
 // initialize a 32-bit channel pair with specified RGB
 #define NCCHANNEL_INITIALIZER(r, g, b) \
@@ -213,7 +204,9 @@ ncchannel_set_rgb8(uint32_t* channel, unsigned r, unsigned g, unsigned b){
     return -1;
   }
   uint32_t c = (r << 16u) | (g << 8u) | b;
-  *channel = (*channel & ~NC_BG_RGB_MASK) | NC_BGDEFAULT_MASK | c;
+  // clear the existing rgb bits, clear the palette index indicator, set
+  // the not-default bit, and or in the new rgb.
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | c;
   return 0;
 }
 
@@ -241,7 +234,7 @@ ncchannel_set_rgb8_clipped(uint32_t* channel, int r, int g, int b){
     b = 0;
   }
   uint32_t c = (r << 16u) | (g << 8u) | b;
-  *channel = (*channel & ~NC_BG_RGB_MASK) | NC_BGDEFAULT_MASK | c;
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | c;
 }
 
 // Same, but provide an assembled, packed 24 bits of rgb.
@@ -250,7 +243,7 @@ ncchannel_set(uint32_t* channel, uint32_t rgb){
   if(rgb > 0xffffffu){
     return -1;
   }
-  *channel = (*channel & ~NC_BG_RGB_MASK) | NC_BGDEFAULT_MASK | rgb;
+  *channel = (*channel & ~(NC_BG_RGB_MASK | NC_BG_PALETTE)) | NC_BGDEFAULT_MASK | rgb;
   return 0;
 }
 
@@ -321,7 +314,7 @@ static inline uint64_t
 ncchannels_reverse(uint64_t channels){
   const uint64_t raw = ((uint64_t)ncchannels_bchannel(channels) << 32u) +
                        ncchannels_fchannel(channels);
-  const uint64_t statemask = (NC_NOBACKGROUND_MASK | NC_FG_ALPHA_MASK |
+  const uint64_t statemask = (NC_NOBACKGROUND_MASK | (NC_BG_ALPHA_MASK << 32u) |
                               NC_BG_ALPHA_MASK | (NC_NOBACKGROUND_MASK >> 32u));
   uint64_t ret = raw & ~statemask;
   ret |= channels & statemask;