shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix variance (signed integer) overflow

based on change made in experimental:
  9847344 Fix variance (signed integer) overflow

Change-Id: I36f4ba5700f6f4615057daf7e70868f68a86669f

--- a/vp8/common/variance_c.c

+++ b/vp8/common/variance_c.c

@@ -75,7 +75,7 @@

     variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 8));

+    return (var - (((unsigned int)avg * avg) >> 8));

 }

 unsigned int vp8_variance8x16_c(

@@ -91,7 +91,7 @@

     variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

 unsigned int vp8_variance16x8_c(

@@ -107,7 +107,7 @@

     variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

@@ -124,7 +124,7 @@

     variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 6));

+    return (var - (((unsigned int)avg * avg) >> 6));

 }

 unsigned int vp8_variance4x4_c(

@@ -140,7 +140,7 @@

     variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg);

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 4));

+    return (var - (((unsigned int)avg * avg) >> 4));

 }

--- a/vp8/common/x86/variance_mmx.c

+++ b/vp8/common/x86/variance_mmx.c

@@ -91,7 +91,7 @@

     vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 4));

+    return (var - (((unsigned int)avg * avg) >> 4));

 }

@@ -108,7 +108,7 @@

     vp8_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 6));

+    return (var - (((unsigned int)avg * avg) >> 6));

 }

@@ -153,7 +153,7 @@

     var = sse0 + sse1 + sse2 + sse3;

     avg = sum0 + sum1 + sum2 + sum3;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 8));

+    return (var - (((unsigned int)avg * avg) >> 8));

 }

 unsigned int vp8_variance16x8_mmx(

@@ -172,7 +172,7 @@

     var = sse0 + sse1;

     avg = sum0 + sum1;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

@@ -194,7 +194,7 @@

     avg = sum0 + sum1;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

@@ -219,7 +219,7 @@

         &xsum, &xxsum

     );

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 4));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 4));

 }

@@ -244,7 +244,7 @@

         &xsum, &xxsum

     );

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 6));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 6));

 }

 unsigned int vp8_sub_pixel_variance16x16_mmx

@@ -282,7 +282,7 @@

     xxsum0 += xxsum1;

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

@@ -335,7 +335,7 @@

     xxsum0 += xxsum1;

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));

 }

 unsigned int vp8_sub_pixel_variance8x16_mmx

@@ -358,7 +358,7 @@

         &xsum, &xxsum

     );

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 7));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 7));

 }

--- a/vp8/common/x86/variance_sse2.c

+++ b/vp8/common/x86/variance_sse2.c

@@ -148,7 +148,7 @@

     vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 4));

+    return (var - (((unsigned int)avg * avg) >> 4));

 }

@@ -165,7 +165,7 @@

     vp8_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 6));

+    return (var - (((unsigned int)avg * avg) >> 6));

 }

@@ -184,7 +184,7 @@

     vp8_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0) ;

     *sse = sse0;

-    return (sse0 - ((unsigned int)(sum0 * sum0) >> 8));

+    return (sse0 - (((unsigned int)sum0 * sum0) >> 8));

 }

 unsigned int vp8_mse16x16_wmt(

     const unsigned char *src_ptr,

@@ -220,7 +220,7 @@

     var = sse0 + sse1;

     avg = sum0 + sum1;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

@@ -241,7 +241,7 @@

     var = sse0 + sse1;

     avg = sum0 + sum1;

     *sse = var;

-    return (var - ((unsigned int)(avg * avg) >> 7));

+    return (var - (((unsigned int)avg * avg) >> 7));

 }

@@ -265,7 +265,7 @@

         &xsum, &xxsum

     );

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 4));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 4));

 }

@@ -314,7 +314,7 @@

     }

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 6));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 6));

 }

 unsigned int vp8_sub_pixel_variance16x16_wmt

@@ -376,7 +376,7 @@

     }

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

 unsigned int vp8_sub_pixel_mse16x16_wmt(

@@ -447,7 +447,7 @@

     }

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));

 }

 unsigned int vp8_sub_pixel_variance8x16_wmt

@@ -495,7 +495,7 @@

     }

     *sse = xxsum;

-    return (xxsum - ((unsigned int)(xsum * xsum) >> 7));

+    return (xxsum - (((unsigned int)xsum * xsum) >> 7));

 }

@@ -515,7 +515,7 @@

         &xsum0, &xxsum0);

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

@@ -534,7 +534,7 @@

         &xsum0, &xxsum0);

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

@@ -554,5 +554,5 @@

         &xsum0, &xxsum0);

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

--- a/vp8/common/x86/variance_ssse3.c

+++ b/vp8/common/x86/variance_ssse3.c

@@ -113,7 +113,7 @@

     }

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));

 }

 unsigned int vp8_sub_pixel_variance16x8_ssse3

@@ -162,5 +162,5 @@

     }

     *sse = xxsum0;

-    return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7));

+    return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));

 }