Backport of:

From ec66c7c584f0b41c98e93758d9b53bd6dd582df2 Mon Sep 17 00:00:00 2001
From: Matthew Waters <matthew@centricular.com>
Date: Tue, 22 Nov 2016 19:05:00 +1100
Subject: flxdec: add some write bounds checking

Without checking the bounds of the frame we are writing into, we can
write off the end of the destination buffer.

https://scarybeastsecurity.blogspot.dk/2016/11/0day-exploit-advancing-exploitation.html

https://bugzilla.gnome.org/show_bug.cgi?id=774834

Index: gst-plugins-good0.10-0.10.31/gst/flx/gstflxdec.c
===================================================================
--- gst-plugins-good0.10-0.10.31.orig/gst/flx/gstflxdec.c	2016-11-22 08:49:30.987370013 -0500
+++ gst-plugins-good0.10-0.10.31/gst/flx/gstflxdec.c	2016-11-22 08:52:35.889496961 -0500
@@ -70,9 +70,9 @@
 static gboolean gst_flxdec_sink_event_handler (GstPad * pad, GstEvent * event);
 
 static void flx_decode_color (GstFlxDec *, guchar *, guchar *, gint);
-static void flx_decode_brun (GstFlxDec *, guchar *, guchar *);
-static void flx_decode_delta_fli (GstFlxDec *, guchar *, guchar *);
-static void flx_decode_delta_flc (GstFlxDec *, guchar *, guchar *);
+static gboolean flx_decode_brun (GstFlxDec *, guchar *, guchar *);
+static gboolean flx_decode_delta_fli (GstFlxDec *, guchar *, guchar *);
+static gboolean flx_decode_delta_flc (GstFlxDec *, guchar *, guchar *);
 
 #define rndalign(off) ((off) + ((off) & 1))
 
@@ -225,13 +225,14 @@
   return ret;
 }
 
-static void
+static gboolean
 flx_decode_chunks (GstFlxDec * flxdec, gulong count, guchar * data,
     guchar * dest)
 {
   FlxFrameChunk *hdr;
+  gboolean ret = TRUE;
 
-  g_return_if_fail (data != NULL);
+  g_return_val_if_fail (data != NULL, FALSE);
 
   while (count--) {
     hdr = (FlxFrameChunk *) data;
@@ -250,17 +251,17 @@
         break;
 
       case FLX_BRUN:
-        flx_decode_brun (flxdec, data, dest);
+        ret = flx_decode_brun (flxdec, data, dest);
         data += rndalign (hdr->size) - FlxFrameChunkSize;
         break;
 
       case FLX_LC:
-        flx_decode_delta_fli (flxdec, data, dest);
+        ret = flx_decode_delta_fli (flxdec, data, dest);
         data += rndalign (hdr->size) - FlxFrameChunkSize;
         break;
 
       case FLX_SS2:
-        flx_decode_delta_flc (flxdec, data, dest);
+        ret = flx_decode_delta_flc (flxdec, data, dest);
         data += rndalign (hdr->size) - FlxFrameChunkSize;
         break;
 
@@ -278,7 +279,12 @@
         data += rndalign (hdr->size) - FlxFrameChunkSize;
         break;
     }
+
+    if (!ret)
+      break;
   }
+
+  return ret;
 }
 
 
@@ -311,13 +317,13 @@
   }
 }
 
-static void
+static gboolean
 flx_decode_brun (GstFlxDec * flxdec, guchar * data, guchar * dest)
 {
   gulong count, lines, row;
   guchar x;
 
-  g_return_if_fail (flxdec != NULL);
+  g_return_val_if_fail (flxdec != NULL, FALSE);
 
   lines = flxdec->hdr.height;
   while (lines--) {
@@ -335,12 +341,21 @@
       if (count > 0x7f) {
         /* literal run */
         count = 0x100 - count;
+        if ((glong) row - count < 0) {
+          GST_ERROR_OBJECT (flxdec, "Invalid BRUN packet detected.");
+          return FALSE;
+        }
         row -= count;
 
         while (count--)
           *dest++ = *data++;
 
       } else {
+        if ((glong) row - count < 0) {
+          GST_ERROR_OBJECT (flxdec, "Invalid BRUN packet detected.");
+          return FALSE;
+        }
+
         /* replicate run */
         row -= count;
         x = *data++;
@@ -350,16 +365,18 @@
       }
     }
   }
+
+  return TRUE;
 }
 
-static void
+static gboolean
 flx_decode_delta_fli (GstFlxDec * flxdec, guchar * data, guchar * dest)
 {
   gulong count, packets, lines, start_line;
   guchar *start_p, x;
 
-  g_return_if_fail (flxdec != NULL);
-  g_return_if_fail (flxdec->delta != NULL);
+  g_return_val_if_fail (flxdec != NULL, FALSE);
+  g_return_val_if_fail (flxdec->delta != NULL, FALSE);
 
   /* use last frame for delta */
   memcpy (dest, GST_BUFFER_DATA (flxdec->delta),
@@ -367,6 +382,10 @@
 
   start_line = (data[0] + (data[1] << 8));
   lines = (data[2] + (data[3] << 8));
+  if (start_line + lines > flxdec->hdr.height) {
+    GST_ERROR_OBJECT (flxdec, "Invalid FLI packet detected. too many lines.");
+    return FALSE;
+  }
   data += 4;
 
   /* start position of delta */
@@ -379,7 +398,8 @@
 
     while (packets--) {
       /* skip count */
-      dest += *data++;
+      guchar skip = *data++;
+      dest += skip;
 
       /* RLE count */
       count = *data++;
@@ -387,12 +407,24 @@
       if (count > 0x7f) {
         /* literal run */
         count = 0x100 - count;
-        x = *data++;
 
+        if (skip + count > flxdec->hdr.width) {
+          GST_ERROR_OBJECT (flxdec, "Invalid FLI packet detected. "
+              "line too long.");
+          return FALSE;
+        }
+
+        x = *data++;
         while (count--)
           *dest++ = x;
 
       } else {
+        if (skip + count > flxdec->hdr.width) {
+          GST_ERROR_OBJECT (flxdec, "Invalid FLI packet detected. "
+              "line too long.");
+          return FALSE;
+        }
+
         /* replicate run */
         while (count--)
           *dest++ = *data++;
@@ -401,22 +433,28 @@
     start_p += flxdec->hdr.width;
     dest = start_p;
   }
+
+  return TRUE;
 }
 
-static void
+static gboolean
 flx_decode_delta_flc (GstFlxDec * flxdec, guchar * data, guchar * dest)
 {
   gulong count, lines, start_l, opcode;
   guchar *start_p;
 
-  g_return_if_fail (flxdec != NULL);
-  g_return_if_fail (flxdec->delta != NULL);
+  g_return_val_if_fail (flxdec != NULL, FALSE);
+  g_return_val_if_fail (flxdec->delta != NULL, FALSE);
 
   /* use last frame for delta */
   memcpy (dest, GST_BUFFER_DATA (flxdec->delta),
       GST_BUFFER_SIZE (flxdec->delta));
 
   lines = (data[0] + (data[1] << 8));
+  if (lines > flxdec->hdr.height) {
+    GST_ERROR_OBJECT (flxdec, "Invalid FLC packet detected. too many lines.");
+    return FALSE;
+  }
   data += 2;
 
   start_p = dest;
@@ -429,9 +467,15 @@
     while ((opcode = (data[0] + (data[1] << 8))) & 0xc000) {
       data += 2;
       if ((opcode & 0xc000) == 0xc000) {
-        /* skip count */
-        start_l += (0x10000 - opcode);
-        dest += flxdec->hdr.width * (0x10000 - opcode);
+        /* line skip count */
+        gulong skip = (0x10000 - opcode);
+        if (skip > flxdec->hdr.height) {
+          GST_ERROR_OBJECT (flxdec, "Invalid FLC packet detected. "
+              "skip line count too big.");
+          return FALSE;
+        }
+        start_l += skip;
+        dest += flxdec->hdr.width * skip;
       } else {
         /* last pixel */
         dest += flxdec->hdr.width;
@@ -443,7 +487,8 @@
     /* last opcode is the packet count */
     while (opcode--) {
       /* skip count */
-      dest += *data++;
+      guchar skip = *data++;
+      dest += skip;
 
       /* RLE count */
       count = *data++;
@@ -451,12 +496,25 @@
       if (count > 0x7f) {
         /* replicate word run */
         count = 0x100 - count;
+
+        if (skip + count > flxdec->hdr.width) {
+          GST_ERROR_OBJECT (flxdec, "Invalid FLC packet detected. "
+              "line too long.");
+          return FALSE;
+        }
+
         while (count--) {
           *dest++ = data[0];
           *dest++ = data[1];
         }
         data += 2;
       } else {
+        if (skip + count > flxdec->hdr.width) {
+          GST_ERROR_OBJECT (flxdec, "Invalid FLC packet detected. "
+              "line too long.");
+          return FALSE;
+        }
+
         /* literal word run */
         while (count--) {
           *dest++ = *data++;
@@ -466,6 +524,8 @@
     }
     lines--;
   }
+
+  return TRUE;
 }
 
 static GstFlowReturn
@@ -593,9 +653,13 @@
             break;
 
           /* decode chunks */
-          flx_decode_chunks (flxdec,
-              ((FlxFrameType *) chunk)->chunks,
-              chunk + FlxFrameTypeSize, GST_BUFFER_DATA (flxdec->frame));
+          if (!flx_decode_chunks (flxdec,
+                  ((FlxFrameType *) chunk)->chunks,
+                  chunk + FlxFrameTypeSize, GST_BUFFER_DATA (flxdec->frame))) {
+            GST_ELEMENT_ERROR (flxdec, STREAM, DECODE,
+                ("%s", "Could not decode chunk"), NULL);
+            return GST_FLOW_ERROR;
+          }
 
           /* save copy of the current frame for possible delta. */
           memcpy (GST_BUFFER_DATA (flxdec->delta),
