shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge "VP9 HBD: Fix integer overflow problem in variance calc."

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -378,7 +378,8 @@

 static void get_variance(var *v) {

   v->variance =

       (int)(256 * (v->sum_square_error -

-                   ((v->sum_error * v->sum_error) >> v->log2_count)) >>

+                   (uint32_t)(((int64_t)v->sum_error * v->sum_error) >>

+                              v->log2_count)) >>

             v->log2_count);

 }