shithub: libvpx

Info  •  Files  •  Log  •  Branches

Validate the |border| parameter earlier.

vpx_realloc_frame_buffer() should validate the |border| parameter
earlier, before it allocates the buffer and preferrably before it uses
|border|.

This backports libaom commit 2860b3ae8b764bdfa2b8c7a06df2673e907b993f:
https://aomedia-review.googlesource.com/c/aom/+/74324

Change-Id: Ib9d59d74e27430ccb1e83c6ad5424aff9672c989

--- a/vpx_scale/generic/yv12config.c

+++ b/vpx_scale/generic/yv12config.c

@@ -154,6 +154,13 @@

   if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;

 #endif

+  /* Only support allocating buffers that have a border that's a multiple

+   * of 32. The border restriction is required to get 16-byte alignment of

+   * the start of the chroma rows without introducing an arbitrary gap

+   * between planes, which would break the semantics of things like

+   * vpx_img_set_rect(). */

+  if (border & 0x1f) return -3;

+

   if (ybf) {

     const int vp9_byte_align = (byte_alignment == 0) ? 1 : byte_alignment;

     const int aligned_width = (width + 7) & ~7;

@@ -222,13 +229,6 @@

       // removed if border is totally removed.

       memset(ybf->buffer_alloc, 0, ybf->buffer_alloc_sz);

     }

-

-    /* Only support allocating buffers that have a border that's a multiple

-     * of 32. The border restriction is required to get 16-byte alignment of

-     * the start of the chroma rows without introducing an arbitrary gap

-     * between planes, which would break the semantics of things like

-     * vpx_img_set_rect(). */

-    if (border & 0x1f) return -3;

     ybf->y_crop_width = width;

     ybf->y_crop_height = height;