shithub: libvpx

Download patch

ref: 2361c05e8935a74edf698ec651071d58e3f77c21
parent: 1e58bdb41d886be6246f85913b2cd5ec6aae57bd
author: Wan-Teh Chang <wtc@google.com>
date: Tue Nov 20 04:30:54 EST 2018

Validate the |border| parameter earlier.

vpx_realloc_frame_buffer() should validate the |border| parameter
earlier, before it allocates the buffer and preferrably before it uses
|border|.

This backports libaom commit 2860b3ae8b764bdfa2b8c7a06df2673e907b993f:
https://aomedia-review.googlesource.com/c/aom/+/74324

Change-Id: Ib9d59d74e27430ccb1e83c6ad5424aff9672c989

--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -154,6 +154,13 @@
   if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;
 #endif
 
+  /* Only support allocating buffers that have a border that's a multiple
+   * of 32. The border restriction is required to get 16-byte alignment of
+   * the start of the chroma rows without introducing an arbitrary gap
+   * between planes, which would break the semantics of things like
+   * vpx_img_set_rect(). */
+  if (border & 0x1f) return -3;
+
   if (ybf) {
     const int vp9_byte_align = (byte_alignment == 0) ? 1 : byte_alignment;
     const int aligned_width = (width + 7) & ~7;
@@ -222,13 +229,6 @@
       // removed if border is totally removed.
       memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz);
     }
-
-    /* Only support allocating buffers that have a border that's a multiple
-     * of 32. The border restriction is required to get 16-byte alignment of
-     * the start of the chroma rows without introducing an arbitrary gap
-     * between planes, which would break the semantics of things like
-     * vpx_img_set_rect(). */
-    if (border & 0x1f) return -3;
 
     ybf->y_crop_width = width;
     ybf->y_crop_height = height;