shithub: libvpx

Info  •  Files  •  Log  •  Branches

variance aq: Fix a variance calculation overflow bug.

This is an actual overflow where the result of the calculation is
materially changed, not just a negative value that is stored in an
unsigned.

Caught with fsanitize=integer on the VP9/AqSegmentTest.TestNoMisMatchAQ2/1 test.

Change-Id: I514b0ef4ae7ad50e3e08c0079aa204d59fa679aa

--- a/vp9/encoder/vp9_aq_variance.c

+++ b/vp9/encoder/vp9_aq_variance.c

@@ -167,7 +167,7 @@

                 vp9_64_zeros, 0, bw, bh, &sse, &avg);

 #endif  // CONFIG_VP9_HIGHBITDEPTH

     var = sse - (((int64_t)avg * avg) / (bw * bh));

-    return (256 * var) / (bw * bh);

+    return (unsigned int)(((uint64_t)256 * var) / (bw * bh));

   } else {

 #if CONFIG_VP9_HIGHBITDEPTH

     if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {

@@ -185,7 +185,7 @@

                              x->plane[0].src.stride,

                              vp9_64_zeros, 0, &sse);

 #endif  // CONFIG_VP9_HIGHBITDEPTH

-    return (256 * var) >> num_pels_log2_lookup[bs];

+    return (unsigned int)(((uint64_t)256 * var) >> num_pels_log2_lookup[bs]);

   }

 }