shithub: libvpx

Info  •  Files  •  Log  •  Branches

Prevent overflow in variance32x32.

Change-Id: I478878c78ef8a770186622d987d318176827ef5f

--- a/vp9/encoder/variance_c.c

+++ b/vp9/encoder/variance_c.c

@@ -12,8 +12,8 @@

 #include "variance.h"

 #include "vp9/common/filter.h"

 #include "vp9/common/subpelvar.h"

+#include "vpx/vpx_integer.h"

-

 unsigned int vp9_get_mb_ss_c(const short *src_ptr) {

   unsigned int i, sum = 0;

@@ -37,8 +37,7 @@

   variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);

   *sse = var;

-  // TODO(rbultje): in extreme cases these products will rollover.

-  return (var - (((unsigned int)avg * avg) >> 10));

+  return (var - (((int64_t)avg * avg) >> 10));

 }

 #endif