shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix buffer overrun of postproc_state.limits

Always allocate cpi->common.postproc_state.limits using unscaled width.

With ./configure --enable-pic --enable-decode-perf-tests
--enable-encode-perf-tests --enable-encode-perf-tests
--enable-vp9-highbitdepth --enable-better-hw-compatibility
--enable-internal-stats --enable-postproc --enable-vp9-postproc
--enable-error-concealment --enable-coefficient-range-checking
--enable-postproc-visualizer --enable-multi-res-encodin
--enable-vp9-temporal-denoising --enable-webm-io --enable-libyuv
segfaults tend to occur in VP9/DatarateOnePassCbrSvcSingleBR.* tests.

This is an analogue to issue
https://bugs.chromium.org/p/webm/issues/detail?id=1374
where a buffer allocated using a scaled width is reused after scaling
back to the original size. Unfortunately, in this case the unscaled
width doesn't appear to be known in the immediated context of the
allocation, so the the signature of vp9_post_proc_frame needs to be
changed to provide that information in order to provide a similar fix
as in #1374.

Signed-off-by: Matthias Räncker <theonetruecamper@gmx.de>
Change-Id: I6f943aafbb3484ee94c5b38d7fcdd9d53fce3e5f

--- a/vp9/common/vp9_postproc.c

+++ b/vp9/common/vp9_postproc.c

@@ -293,7 +293,7 @@

 }

 int vp9_post_proc_frame(struct VP9Common *cm, YV12_BUFFER_CONFIG *dest,

-                        vp9_ppflags_t *ppflags) {

+                        vp9_ppflags_t *ppflags, int unscaled_width) {

   const int q = VPXMIN(105, cm->lf.filter_level * 2);

   const int flags = ppflags->post_proc_flag;

   YV12_BUFFER_CONFIG *const ppbuf = &cm->post_proc_buffer;

@@ -359,7 +359,7 @@

   if (flags & (VP9D_DEMACROBLOCK | VP9D_DEBLOCK)) {

     if (!cm->postproc_state.limits) {

       cm->postproc_state.limits =

-          vpx_calloc(cm->width, sizeof(*cm->postproc_state.limits));

+          vpx_calloc(unscaled_width, sizeof(*cm->postproc_state.limits));

     }

   }

--- a/vp9/common/vp9_postproc.h

+++ b/vp9/common/vp9_postproc.h

@@ -38,7 +38,7 @@

 #define MFQE_PRECISION 4

 int vp9_post_proc_frame(struct VP9Common *cm, YV12_BUFFER_CONFIG *dest,

-                        vp9_ppflags_t *flags);

+                        vp9_ppflags_t *flags, int unscaled_width);

 void vp9_denoise(const YV12_BUFFER_CONFIG *src, YV12_BUFFER_CONFIG *dst, int q,

                  uint8_t *limits);

--- a/vp9/decoder/vp9_decoder.c

+++ b/vp9/decoder/vp9_decoder.c

@@ -397,7 +397,7 @@

 #if CONFIG_VP9_POSTPROC

   if (!cm->show_existing_frame) {

-    ret = vp9_post_proc_frame(cm, sd, flags);

+    ret = vp9_post_proc_frame(cm, sd, flags, cm->width);

   } else {

     *sd = *cm->frame_to_show;

     ret = 0;

--- a/vp9/encoder/vp9_encoder.c

+++ b/vp9/encoder/vp9_encoder.c

@@ -6308,7 +6308,8 @@

             ppflags.post_proc_flag = VP9D_DEBLOCK;

             ppflags.deblocking_level = 0;  // not used in vp9_post_proc_frame()

             ppflags.noise_level = 0;       // not used in vp9_post_proc_frame()

-            vp9_post_proc_frame(cm, pp, &ppflags);

+            vp9_post_proc_frame(cm, pp, &ppflags,

+                                cpi->un_scaled_source->y_width);

           }

 #endif

           vpx_clear_system_state();

@@ -6441,7 +6442,7 @@

   } else {

     int ret;

 #if CONFIG_VP9_POSTPROC

-    ret = vp9_post_proc_frame(cm, dest, flags);

+    ret = vp9_post_proc_frame(cm, dest, flags, cpi->un_scaled_source->y_width);

 #else

     if (cm->frame_to_show) {

       *dest = *cm->frame_to_show;