# HG changeset patch
# User Markus Mützel <markus.muetzel@gmx.de>
# Date 1635678803 -3600
# Node ID f43902a87bf1fe2c6850ff54bc069a3bcbfeb7bf
# Parent  a49c635b179df113a5bcc0d51562c2601e182ae1
gl2ps-print.cc: Avoid integer overflow in multiplication.

* libinterp/corefcn/gl2ps-print.cc (gl2ps_renderer::draw_image,
  gl2ps_renderer::draw_pixels): Cast integer size argument of
  OCTAVE_LOCAL_BUFFER to size_t before multiplication to avoid integer overflow.
  The overflow is very unlikely to happen. But it's an easy fix that doesn't
  clutter the code too much.

diff -r a49c635b179d -r f43902a87bf1 libinterp/corefcn/gl2ps-print.cc
--- a/libinterp/corefcn/gl2ps-print.cc	Fri Oct 29 20:48:47 2021 -0400
+++ b/libinterp/corefcn/gl2ps-print.cc	Sun Oct 31 12:13:23 2021 +0100
@@ -1366,7 +1366,8 @@
           {
             const NDArray xcdata = cdata.array_value ();
 
-            OCTAVE_LOCAL_BUFFER (GLfloat, a, 3*(j1-j0)*(i1-i0));
+            OCTAVE_LOCAL_BUFFER (GLfloat, a,
+                                 static_cast<size_t> (3)*(j1-j0)*(i1-i0));
 
             for (int i = i0; i < i1; i++)
               {
@@ -1395,7 +1396,8 @@
           {
             const FloatNDArray xcdata = cdata.float_array_value ();
 
-            OCTAVE_LOCAL_BUFFER (GLfloat, a, 3*(j1-j0)*(i1-i0));
+            OCTAVE_LOCAL_BUFFER (GLfloat, a,
+                                 static_cast<size_t> (3)*(j1-j0)*(i1-i0));
 
             for (int i = i0; i < i1; i++)
               {
@@ -1424,7 +1426,8 @@
           {
             const uint8NDArray xcdata = cdata.uint8_array_value ();
 
-            OCTAVE_LOCAL_BUFFER (GLubyte, a, 3*(j1-j0)*(i1-i0));
+            OCTAVE_LOCAL_BUFFER (GLubyte, a,
+                                 static_cast<size_t> (3)*(j1-j0)*(i1-i0));
 
             for (int i = i0; i < i1; i++)
               {
@@ -1453,7 +1456,8 @@
           {
             const uint16NDArray xcdata = cdata.uint16_array_value ();
 
-            OCTAVE_LOCAL_BUFFER (GLushort, a, 3*(j1-j0)*(i1-i0));
+            OCTAVE_LOCAL_BUFFER (GLushort, a,
+                                 static_cast<size_t> (3)*(j1-j0)*(i1-i0));
 
             for (int i = i0; i < i1; i++)
               {
@@ -1490,7 +1494,7 @@
   gl2ps_renderer::draw_pixels (int w, int h, const float *data)
   {
     // Clip data between 0 and 1 for float values
-    OCTAVE_LOCAL_BUFFER (float, tmp_data, 3*w*h);
+    OCTAVE_LOCAL_BUFFER (float, tmp_data, static_cast<size_t> (3)*w*h);
 
     for (int i = 0; i < 3*h*w; i++)
       tmp_data[i] = (data[i] < 0.0f ? 0.0f : (data[i] > 1.0f ? 1.0f : data[i]));
@@ -1503,7 +1507,7 @@
   {
     // gl2psDrawPixels only supports the GL_FLOAT type.
 
-    OCTAVE_LOCAL_BUFFER (float, tmp_data, 3*w*h);
+    OCTAVE_LOCAL_BUFFER (float, tmp_data, static_cast<size_t> (3)*w*h);
 
     static const float maxval = std::numeric_limits<uint8_t>::max ();
 
@@ -1518,7 +1522,7 @@
   {
     // gl2psDrawPixels only supports the GL_FLOAT type.
 
-    OCTAVE_LOCAL_BUFFER (float, tmp_data, 3*w*h);
+    OCTAVE_LOCAL_BUFFER (float, tmp_data, static_cast<size_t> (3)*w*h);
 
     static const float maxval = std::numeric_limits<uint16_t>::max ();