shithub: libvpx

Info  •  Files  •  Log  •  Branches

moved scaling from dequantization to inverse transform for T8x8

Previously, the scaling related to extended quantize range happens in
dequantization stage, which implies the coefficients form forward
transform are in different scale(4x) from dequantization coefficients
This worked fine when there was not distortion computation done based
on 8x8 transform, but it completely wracked the distortion estimation
based on transform coefficients and dequantized transform coefficients
introduced in commit f64725a00 for macroblocks using 8x8 transform.
This commit fixed the issue by moving the scaling into the stage of
inverse 8x8 transform.

TODO: Test&Verify the transform/quantization pipeline accuracy.

Change-Id: Iff77b36a965c2a6b247e59b9c59df93eba5d60e2

--- a/vp8/common/idctllm.c

+++ b/vp8/common/idctllm.c

@@ -211,7 +211,8 @@

     short *op = output;

     short *orig_op = output;

     int shortpitch = pitch >> 1;

-    a1 = ((input[0] + 4) >> 3);

+    //a1 = ((input[0] + 4) >> 3);

+    a1 = ((input[0] + 16) >> 5);

     for (b = 0; b < 4; b++)

     {

         for (i = 0; i < 4; i++)

@@ -228,7 +229,8 @@

 void vp8_dc_only_idct_add_8x8_c(short input_dc, unsigned char *pred_ptr, unsigned char *dst_ptr, int pitch, int stride)

 {

-    int a1 = ((input_dc + 4) >> 3);

+    //int a1 = ((input_dc + 4) >> 3);

+    int a1 = ((input_dc + 16) >> 5);

     int r, c, b;

     unsigned char *orig_pred = pred_ptr;

     unsigned char *orig_dst = dst_ptr;

@@ -394,7 +396,7 @@

     {

         for (j = 0; j < TX_DIM; j++)

         {

-             X[i * TX_DIM + j] = (int)coefs[i * TX_DIM + j];

+             X[i * TX_DIM + j] = (int)(coefs[i * TX_DIM + j]+2)>>2;

         }

     }

   for (i = 0; i < 8; i++)

--- a/vp8/decoder/dequantize.c

+++ b/vp8/decoder/dequantize.c

@@ -131,7 +131,7 @@

     for (i = 0; i < 16; i++)

     {

-        DQ[i] = (short)((Q[i] * DQC[i]+2)>>2);

+        DQ[i] = (short)((Q[i] * DQC[i]));

     }

 #ifdef DEC_DEBUG

     if (dec_debug) {

@@ -164,12 +164,12 @@

     }

 #endif

-    input[0]= (input[0] * dq[0]+2)>>2;

+    input[0]= input[0] * dq[0];

     // recover quantizer for 4 4x4 blocks

     for (i = 1; i < 64; i++)

     {

-      input[i]=(input[i] * dq[1]+2)>>2;

+      input[i]=input[i] * dq[1];

     }

 #ifdef DEC_DEBUG

     if (dec_debug) {

@@ -262,7 +262,7 @@

 #endif

     for (i = 1; i < 64; i++)

     {

-        input[i]=(input[i] * dq[1]+2)>>2;

+        input[i]=input[i] * dq[1];

     }

 #ifdef DEC_DEBUG

--- a/vp8/encoder/encodemb.c

+++ b/vp8/encoder/encodemb.c

@@ -1035,7 +1035,7 @@

             final_eob = i;

         rc = vp8_default_zig_zag1d_8x8[i];

         qcoeff_ptr[rc] = x;

-        dqcoeff_ptr[rc] = (x * dequant_ptr[rc!=0]+2)>>2;

+        dqcoeff_ptr[rc] = (x * dequant_ptr[rc!=0]);

         next = tokens[i][best].next;

         best = (best_mask[best] >> i) & 1;

--- a/vp8/encoder/onyx_if.c

+++ b/vp8/encoder/onyx_if.c

@@ -3411,7 +3411,6 @@

     scale_and_extend_source(cpi->un_scaled_source, cpi);

     vp8_first_pass(cpi);

 }

-

 //#define WRITE_RECON_BUFFER 1

 #if WRITE_RECON_BUFFER

 void write_cx_frame_to_file(YV12_BUFFER_CONFIG *frame, int this_frame)

--- a/vp8/encoder/quantize.c

+++ b/vp8/encoder/quantize.c

@@ -566,7 +566,6 @@

       qcoeff_ptr[rc]  = x;                         // write to destination

       //dqcoeff_ptr[rc] = x * dequant_ptr[rc]/q2nd;        // dequantized value

       dqcoeff_ptr[rc] = x * dequant_ptr[rc];        // dequantized value

-      dqcoeff_ptr[rc] = (dqcoeff_ptr[rc]+2)>>2;

       if (y)

@@ -627,7 +626,6 @@

       qcoeff_ptr[rc]  = x;                         // write to destination

       //dqcoeff_ptr[rc] = x * dequant_ptr[rc!=0] / q1st;        // dequantized value

       dqcoeff_ptr[rc] = x * dequant_ptr[rc!=0];        // dequantized value

-      dqcoeff_ptr[rc] = (dqcoeff_ptr[rc]+2)>>2;

       if (y)

       {

--- a/vp8/encoder/rdopt.c

+++ b/vp8/encoder/rdopt.c

@@ -777,11 +777,7 @@

     mb->e_mbd.dqcoeff[192] = 0;

     d = ENCODEMB_INVOKE(&rtcd->encodemb, mberr)(mb, 0) << 2;

-#if CONFIG_EXTEND_QRANGE

-    d += ENCODEMB_INVOKE(rtcd, berr)(mb_y2->coeff, x_y2->dqcoeff)<<2;

-#else

-    d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff);

-#endif

+    d += ENCODEMB_INVOKE(&rtcd->encodemb, berr)(mb_y2->coeff, x_y2->dqcoeff)<<2;

     *Distortion = (d >> 4);

     // rate