shithub: libvpx

Info  •  Files  •  Log  •  Branches

Use 64-bit integer to store distortion in optimize_b

This fixes the overflow issue.

Bug=webm:1241

Change-Id: Ia168b7fae1ad214a6837aaa785a08bf8506987dd

--- a/vp9/encoder/vp9_encodemb.c

+++ b/vp9/encoder/vp9_encodemb.c

@@ -54,9 +54,10 @@

   (((1 << (VP9_PROB_COST_SHIFT - 1)) + (R) * (RM)) & \

    ((1 << VP9_PROB_COST_SHIFT) - 1))

+// TODO(aconverse): Re-pack this structure.

 typedef struct vp9_token_state {

   int           rate;

-  int           error;

+  int64_t       error;

   int           next;

   int16_t       token;

   tran_low_t    qc;

@@ -112,7 +113,8 @@

   const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][type]) >> 1;

   const int64_t rddiv = mb->rddiv;

   int64_t rd_cost0, rd_cost1;

-  int rate0, rate1, error0, error1;

+  int rate0, rate1;

+  int64_t error0, error1;

   int16_t t0, t1;

   EXTRABIT e0;

   int best, band, pt, i, final_eob;