shithub: libvpx

Info  •  Files  •  Log  •  Branches

Align internal mfqe framebuffer dimensions

MFQE postproc crashed with stream dimensions not a multiple of 16.
The buffer was memset unconditionally, so if the buffer allocation
fails we end up trying to write to NULL.

This patch traps an allocation failure with vpx_internal_error(),
and aligns the buffer dimensions to what vp8_yv12_alloc_frame_buffer()
expects.

Change-Id: I3915d597cd66886a24f4ef39752751ebe6425066

--- a/vp8/common/postproc.c

+++ b/vp8/common/postproc.c

@@ -930,10 +930,16 @@

     {

         if ((flags & VP8D_DEBLOCK) || (flags & VP8D_DEMACROBLOCK))

         {

-            if (vp8_yv12_alloc_frame_buffer(&oci->post_proc_buffer_int, oci->Width, oci->Height, VP8BORDERINPIXELS) >= 0)

-            {

-                oci->post_proc_buffer_int_used = 1;

-            }

+            int width = (oci->Width + 15) & ~15;

+            int height = (oci->Height + 15) & ~15;

+

+            if (vp8_yv12_alloc_frame_buffer(&oci->post_proc_buffer_int,

+                                            width, height, VP8BORDERINPIXELS))

+                vpx_internal_error(&oci->error, VPX_CODEC_MEM_ERROR,

+                                   "Failed to allocate MFQE framebuffer");

+

+            oci->post_proc_buffer_int_used = 1;

+

             // insure that postproc is set to all 0's so that post proc

             // doesn't pull random data in from edge

             vpx_memset((&oci->post_proc_buffer_int)->buffer_alloc,126,(&oci->post_proc_buffer)->frame_size);