shithub: libvpx

Info  •  Files  •  Log  •  Branches

Attempt to fix an issue related to 8x8 transform and segfeature

logically this commit should NOT change anything, but seems to help
revert the 3DB loss on bowing in the following commit:

https://on2-git.corp.google.com/g/#change,6193

This is still debugging in progress. Need further investigation to
understand the root cause of the issue.

Change-Id: I0b49d1ef3a311dfff58c6acd3eaebdb3bda6257c

--- a/vp8/common/idctllm.c

+++ b/vp8/common/idctllm.c

@@ -399,14 +399,14 @@

   x4 = (181 * (x4 - x5) + 128) >> 8;

   /* fourth stage */

-  blk[8 * 0] = (x7 + x1 ) >> 15;

-  blk[8 * 1] = (x3 + x2 ) >> 15;

-  blk[8 * 2] = (x0 + x4 ) >> 15;

-  blk[8 * 3] = (x8 + x6 ) >> 15;

-  blk[8 * 4] = (x8 - x6 ) >> 15;

-  blk[8 * 5] = (x0 - x4 ) >> 15;

-  blk[8 * 6] = (x3 - x2 ) >> 15;

-  blk[8 * 7] = (x7 - x1 ) >> 15;

+  blk[8 * 0] = (x7 + x1 ) >> 14;

+  blk[8 * 1] = (x3 + x2 ) >> 14;

+  blk[8 * 2] = (x0 + x4 ) >> 14;

+  blk[8 * 3] = (x8 + x6 ) >> 14;

+  blk[8 * 4] = (x8 - x6 ) >> 14;

+  blk[8 * 5] = (x0 - x4 ) >> 14;

+  blk[8 * 6] = (x3 - x2 ) >> 14;

+  blk[8 * 7] = (x7 - x1 ) >> 14;

 }

 #define TX_DIM 8

@@ -434,7 +434,7 @@

     {

         for (j = 0; j < TX_DIM; j++)

         {

-             block[i*shortpitch+j]  = X[i * TX_DIM + j];

+             block[i*shortpitch+j]  = X[i * TX_DIM + j]>>1;

         }

     }

 }