Description: fix denial of service via zero scaling factor
Origin: backport, https://github.com/newsoft/libvncserver/commit/05a9bd41a8ec0a9d580a8f420f41718bdd235446
Origin: backport, https://github.com/newsoft/libvncserver/commit/f18f24ce65f5cac22ddcf3ed51417e477f9bad09
Origin: backport, https://github.com/newsoft/libvncserver/commit/5dee1cbcd83920370a487c4fd2718aa4d3eba548
Origin: backport, https://github.com/newsoft/libvncserver/commit/819481c5e2003cd36d002336c248de8c75de362e
Origin: backport, https://github.com/newsoft/libvncserver/commit/e5d9b6a07257c12bf3b6242ddea79ea1c95353a8

Signed-off-by: Gustavo Zacarias <gustavo@zacarias.com.ar>

Index: libvncserver-0.9.9+dfsg/libvncserver/rfbserver.c
===================================================================
--- libvncserver-0.9.9+dfsg.orig/libvncserver/rfbserver.c	2014-09-25 11:19:54.464070151 -0400
+++ libvncserver-0.9.9+dfsg/libvncserver/rfbserver.c	2014-09-25 11:20:04.344070416 -0400
@@ -2487,6 +2487,13 @@
           rfbCloseClient(cl);
           return;
       }
+
+      if (msg.ssc.scale == 0) {
+          rfbLogPerror("rfbProcessClientNormalMessage: will not accept a scale factor of zero");
+          rfbCloseClient(cl);
+          return;
+      }
+
       rfbStatRecordMessageRcvd(cl, msg.type, sz_rfbSetScaleMsg, sz_rfbSetScaleMsg);
       rfbLog("rfbSetScale(%d)\n", msg.ssc.scale);
       rfbScalingSetup(cl,cl->screen->width/msg.ssc.scale, cl->screen->height/msg.ssc.scale);
@@ -2503,6 +2510,13 @@
           rfbCloseClient(cl);
           return;
       }
+
+      if (msg.ssc.scale == 0) {
+          rfbLogPerror("rfbProcessClientNormalMessage: will not accept a scale factor of zero");
+          rfbCloseClient(cl);
+          return;
+      }
+
       rfbStatRecordMessageRcvd(cl, msg.type, sz_rfbSetScaleMsg, sz_rfbSetScaleMsg);
       rfbLog("rfbSetScale(%d)\n", msg.ssc.scale);
       rfbScalingSetup(cl,cl->screen->width/msg.ssc.scale, cl->screen->height/msg.ssc.scale);
Index: libvncserver-0.9.9+dfsg/libvncserver/scale.c
===================================================================
--- libvncserver-0.9.9+dfsg.orig/libvncserver/scale.c	2012-05-04 10:19:00.000000000 -0400
+++ libvncserver-0.9.9+dfsg/libvncserver/scale.c	2014-09-25 11:20:13.580070663 -0400
@@ -66,6 +66,10 @@
         (double) ((int) (x)) : (double) ((int) (x) + 1) )
 #define FLOOR(x) ( (double) ((int) (x)) )
 
+static inline int pad4(int value) {
+    int remainder = value & 3;
+    return value + (remainder == 0 ? 0 : 4 - remainder);
+}
 
 int ScaleX(rfbScreenInfoPtr from, rfbScreenInfoPtr to, int x)
 {
@@ -281,14 +285,29 @@
     ptr = malloc(sizeof(rfbScreenInfo));
     if (ptr!=NULL)
     {
+        int allocSize;
+
         /* copy *everything* (we don't use most of it, but just in case) */
         memcpy(ptr, cl->screen, sizeof(rfbScreenInfo));
+
+        /* SECURITY: make sure that no integer overflow will occur afterwards.
+         * Note: this is defensive coding, as the check should have already been
+         * performed during initial, non-scaled screen setup.
+         */
+        allocSize = pad4(width * (ptr->bitsPerPixel/8)); /* per protocol, width<2**16 and bpp<256 */
+        if ((height == 0) || (allocSize >= (SIZE_MAX / height)))
+        {
+          free(ptr);
+          return NULL; /* malloc() will allocate an incorrect buffer size - early abort */
+        }
+
+        /* Resume copy everything */
         ptr->width = width;
         ptr->height = height;
         ptr->paddedWidthInBytes = (ptr->bitsPerPixel/8)*ptr->width;
 
         /* Need to by multiples of 4 for Sparc systems */
-        ptr->paddedWidthInBytes += (ptr->paddedWidthInBytes % 4);
+        ptr->paddedWidthInBytes = pad4(ptr->paddedWidthInBytes);
 
         /* Reset the reference count to 0! */
         ptr->scaledScreenRefCount = 0;