shithub: libvpx

Info  •  Files  •  Log  •  Branches

fail early on oversize frames

Even though frame_size is calculated in uint64_t, it winds up in an int
size value.

This was exposed with the msan test because the memset is called with
(int)frame_size, leading to a segfault.

Change-Id: I7fd930360dca274adb8f3e43e5e6785204808861

--- a/vpx_scale/generic/yv12config.c

+++ b/vpx_scale/generic/yv12config.c

@@ -9,6 +9,7 @@

  */

 #include <assert.h>

+#include <limits.h>

 #include "vpx_scale/yv12config.h"

 #include "vpx_mem/vpx_mem.h"

@@ -165,6 +166,12 @@

     uint8_t *buf = NULL;

+    // frame_size is stored in buffer_alloc_sz, which is an int. If it won't

+    // fit, fail early.

+    if (frame_size > INT_MAX) {

+      return -1;

+    }

+

     if (cb != NULL) {

       const int align_addr_extra_size = 31;

       const uint64_t external_frame_size = frame_size + align_addr_extra_size;

@@ -192,8 +199,6 @@

       // Allocation to hold larger frame, or first allocation.

       vpx_free(ybf->buffer_alloc);

       ybf->buffer_alloc = NULL;

-

-      if (frame_size != (size_t)frame_size) return -1;

       ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);

       if (!ybf->buffer_alloc) return -1;