shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix rounding in ARNR calculation

The rounding of the ARNR filter output prior to
normalization by the filter strength was incorrect
when strength = 0.

In this case 1 << (strength - 1) would not create the
required rounding of 0, rather it would outrange. This
patch fixes this issue.

Change-Id: I771809ba34d6052b17d34c870ea11ff67b418dab

--- a/vp8/encoder/temporal_filter.c

+++ b/vp8/encoder/temporal_filter.c

@@ -98,6 +98,7 @@

     unsigned int i, j, k;

     int modifier;

     int byte = 0;

+    const int rounding = strength > 0 ? 1 << (strength - 1) : 0;

     for (i = 0,k = 0; i < block_size; i++)

     {

@@ -114,7 +115,7 @@

              */

             modifier  *= modifier;

             modifier  *= 3;

-            modifier  += 1 << (strength - 1);

+            modifier  += rounding;

             modifier >>= strength;

             if (modifier > 16)

--- a/vp9/encoder/vp9_temporal_filter.c

+++ b/vp9/encoder/vp9_temporal_filter.c

@@ -99,6 +99,7 @@

   unsigned int i, j, k;

   int modifier;

   int byte = 0;

+  const int rounding = strength > 0 ? 1 << (strength - 1) : 0;

   for (i = 0, k = 0; i < block_size; i++) {

     for (j = 0; j < block_size; j++, k++) {

@@ -111,7 +112,7 @@

       // modifier =  (int)roundf(coeff > 16 ? 0 : 16-coeff);

       modifier  *= modifier;

       modifier  *= 3;

-      modifier  += 1 << (strength - 1);

+      modifier  += rounding;

       modifier >>= strength;

       if (modifier > 16)

-- 

⑨