[smoothpixel] check that pixel offsets are within cell geometry bounds #1682

src/lib/direct.c

@@ -659,8 +659,8 @@ ncdirect_render_visual(ncdirect* n, ncvisual* ncv,
   }
   int ymax = vopts->leny / bset->height;
   int xmax = vopts->lenx / bset->width;
-  int dimy = vopts->leny > 0 ? ymax : ncdirect_dim_y(n);
-  int dimx = vopts->lenx > 0 ? xmax : ncdirect_dim_x(n);
+  unsigned dimy = vopts->leny > 0 ? ymax : ncdirect_dim_y(n);
+  unsigned dimx = vopts->lenx > 0 ? xmax : ncdirect_dim_x(n);
   int disprows, dispcols, outy;
   if(vopts->scaling != NCSCALE_NONE && vopts->scaling != NCSCALE_NONE_HIRES){
     if(bset->geom != NCBLIT_PIXEL){

src/lib/internal.h

@@ -416,8 +416,8 @@ typedef int (*ncblitter)(struct ncplane* n, int linesize, const void* data,
 // a system for rendering RGBA pixels as text glyphs or sixel/kitty bitmaps
 struct blitset {
   ncblitter_e geom;
-  int width;        // number of input pixels per output cell, width
-  int height;       // number of input pixels per output cell, height
+  unsigned width;   // number of input pixels per output cell, width
+  unsigned height;  // number of input pixels per output cell, height
   // the EGCs which form the blitter. bits grow left to right, and then top to
   // bottom. the first character is always a space, the last a full block.
   const wchar_t* egcs;

src/lib/notcurses.c

@@ -2900,8 +2900,8 @@ ncplane_as_rgba_internal(const ncplane* nc, ncblitter_e blit,
         ncchannels_bg_rgb8(channels, &br, &bb, &bg);
         ba = ncchannels_bg_alpha(channels);
         // handle each destination pixel from this cell
-        for(int py = 0 ; py < bset->height ; ++py){
-          for(int px = 0 ; px < bset->width ; ++px){
+        for(unsigned py = 0 ; py < bset->height ; ++py){
+          for(unsigned px = 0 ; px < bset->width ; ++px){
             uint32_t* p = &ret[(targy + py) * (lenx * bset->width) + (targx + px)];
             bool background = is_bg_p(idx, py, px, bset->width);
             if(background){

src/lib/plot.c

@@ -458,8 +458,8 @@ create_##T(nc##X##plot* ncpp, ncplane* n, const ncplot_options* opts, const T mi
   ncpp->plot.rangex = opts->rangex; \
   /* if we're sizing the plot based off the plane dimensions, scale it by the \
      plot geometry's width for all calculations */ \
-  const int scaleddim = dimx * (bset->geom == NCBLIT_PIXEL ? ncplane_notcurses(n)->tcache.cellpixx : bset->width); \
-  const int scaledprefixlen = PREFIXCOLUMNS * (bset->geom == NCBLIT_PIXEL ? ncplane_notcurses(n)->tcache.cellpixx : bset->width); \
+  const unsigned scaleddim = dimx * (bset->geom == NCBLIT_PIXEL ? ncplane_notcurses(n)->tcache.cellpixx : bset->width); \
+  const unsigned scaledprefixlen = PREFIXCOLUMNS * (bset->geom == NCBLIT_PIXEL ? ncplane_notcurses(n)->tcache.cellpixx : bset->width); \
   if((ncpp->plot.slotcount = ncpp->plot.rangex) == 0){ \
     ncpp->plot.slotcount = scaleddim; \
   } \

src/lib/termdesc.h

@@ -105,8 +105,8 @@ typedef struct tinfo {
   unsigned pixx;                   // total pixel geometry, width
   // we use the cell's size in pixels for pixel blitting. this information can
   // be acquired on all terminals with pixel support.
-  int cellpixy;                    // cell pixel height, might be 0
-  int cellpixx;                    // cell pixel width, might be 0
+  unsigned cellpixy;               // cell pixel height, might be 0
+  unsigned cellpixx;               // cell pixel width, might be 0
   int dimy, dimx;                  // most recent cell geometry
 
   unsigned supported_styles; // bitmask over NCSTYLE_* driven via sgr/ncv

src/lib/visual.c

@@ -214,6 +214,10 @@ int ncvisual_blitset_geom(const notcurses* nc, const tinfo* tcache,
         }
         // FIXME clamp to sprixel limits
         if(vopts->scaling == NCSCALE_NONE || vopts->scaling == NCSCALE_NONE_HIRES){
+          if(vopts->pxoffy >= nc->tcache.cellpixy){
+            logerror("pixel y-offset %d too tall for cell %d\n", vopts->pxoffy, nc->tcache.cellpixy);
+            return -1;
+          }
           int rows = ((*leny + nc->tcache.cellpixy - 1) / nc->tcache.cellpixy)
                       + !!vopts->pxoffy;
           if(rows > ncplane_dim_y(vopts->n)){
@@ -221,6 +225,10 @@ int ncvisual_blitset_geom(const notcurses* nc, const tinfo* tcache,
                      *leny + vopts->pxoffy, ncplane_dim_y(vopts->n) * nc->tcache.cellpixy);
             return -1;
           }
+          if(vopts->pxoffx >= nc->tcache.cellpixx){
+            logerror("pixel x-offset %d too wide for cell %d\n", vopts->pxoffx, nc->tcache.cellpixx);
+            return -1;
+          }
           int cols = ((*lenx + nc->tcache.cellpixx - 1) / nc->tcache.cellpixx)
                       + !!vopts->pxoffx;
           if(cols > ncplane_dim_x(vopts->n)){
@@ -1004,7 +1012,7 @@ ncplane* ncvisual_render_pixels(notcurses* nc, ncvisual* ncv, const struct blits
     }
     clamp_to_sixelmax(&nc->tcache, &disppixy, &disppixx, &outy, scaling);
     // FIXME use a closed form
-    while((outy + nc->tcache.cellpixy - 1) / nc->tcache.cellpixy > ncplane_dim_y(n)){
+    while((outy + nc->tcache.cellpixy - 1) / nc->tcache.cellpixy > (unsigned)ncplane_dim_y(n)){
       outy -= nc->tcache.sprixel_scale_height;
       disppixy = outy;
     }