shithub: libvpx

Info  •  Files  •  Log  •  Branches

quantize: normalize intermediate types

Despite abs_coeff being a positive value, all the other implementations
treat it as signed which simplifies restoring the sign.

HBD builds cast qcoeff to avoid a visual studio warning. Match
vp9_quantize.c style of casting the entire expression.

Change-Id: I62b539b8df05364df3d7644311e325288da7c5b5

--- a/vp9/encoder/vp9_quantize.c

+++ b/vp9/encoder/vp9_quantize.c

@@ -78,7 +78,7 @@

       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;

       const int64_t tmp = abs_coeff + round_ptr[rc != 0];

       const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 16);

-      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);

+      qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;

       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];

       if (abs_qcoeff) eob = i;

     }

--- a/vpx_dsp/quantize.c

+++ b/vpx_dsp/quantize.c

@@ -201,11 +201,8 @@

       if (abs_coeff >= zbins[rc != 0]) {

         const int64_t tmp1 = abs_coeff + round_ptr[rc != 0];

         const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;

-        const uint32_t abs_qcoeff =

-            (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);

-        // Restoring the sign triggers unsigned overflow warnings with negative

-        // values because the result of the xor operation is unsigned.

-        qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;

+        const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);

+        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);

         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];

         if (abs_qcoeff) eob = i;

       }

@@ -310,9 +307,8 @@

       const int64_t tmp1 =

           abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);

       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;

-      const uint32_t abs_qcoeff =

-          (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);

-      qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;

+      const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);

+      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);

       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;

       if (abs_qcoeff) eob = idx_arr[i];

     }