shithub: libvpx

Info  •  Files  •  Log  •  Branches

variance_avx2.c: UBSAN/IOC fix

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1222

Change-Id: Ifb3bedf9b4e1b007b21aebaa4beb9ba50424efef

--- a/vpx_dsp/x86/variance_avx2.c

+++ b/vpx_dsp/x86/variance_avx2.c

@@ -45,7 +45,7 @@

   int sum;

   variance_avx2(src, src_stride, ref, ref_stride, 16, 16,

                 sse, &sum, vpx_get16x16var_avx2, 16);

-  return *sse - (((unsigned int)sum * sum) >> 8);

+  return *sse - (((uint32_t)((int64_t)sum * sum)) >> 8);

 }

 unsigned int vpx_mse16x16_avx2(const uint8_t *src, int src_stride,