shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge "Fix integer overflow issue in rtc coding flow intra mode search"

--- a/vp9/encoder/vp9_pickmode.c

+++ b/vp9/encoder/vp9_pickmode.c

@@ -658,7 +658,8 @@

   block = 0;

   *rate = 0;

   *dist = 0;

-  *sse = (*sse << 6) >> shift;

+  if (*sse < INT64_MAX)

+    *sse = (*sse << 6) >> shift;

   for (r = 0; r < max_blocks_high; r += block_step) {

     for (c = 0; c < num_4x4_w; c += block_step) {

       if (c < max_blocks_wide) {

@@ -912,6 +913,8 @@

   // TODO(jingning): This needs further refactoring.

   block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0,

             bsize_tx, MIN(tx_size, TX_16X16));

+  // this_sse is a dummy variable here. Its value should remain INT64_MAX.

+  assert(this_sse == INT64_MAX);

   x->skip_txfm[0] = is_skippable;

   rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable);