shithub: libvpx

Download patch

ref: ff1ae7f713dc079de7e124a023ee3dc6aa9b616d
parent: fe533c97413313a4e4c75ce1a1a6ec7451b5acc3
author: Yaowu Xu <yaowu@google.com>
date: Tue Sep 24 05:37:55 EDT 2013 

Prevent using uninitialized value in RD decision

INT64_MAX may be assigned as RDCOST when RDCSOST computation is skipped
for speed, this commit to prevent INT64_MAX from being used as real
RDCOST in transform size decision.

Change-Id: I89a945134191bbdea1f1431ade70424ac079eaac


--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -3093,8 +3093,12 @@
         vp9_cost_bit(vp9_get_pred_prob_mbskip(cm, xd), 0);
     *returndist = dist_y + dist_uv;
     if (cpi->sf.tx_size_search_method == USE_FULL_RD)
-      for (i = 0; i < TX_MODES; i++)
-        ctx->tx_rd_diff[i] = tx_cache[i] - tx_cache[cm->tx_mode];
+      for (i = 0; i < TX_MODES; i++) {
+        if (tx_cache[i] < INT64_MAX && tx_cache[cm->tx_mode] < INT64_MAX)
+          ctx->tx_rd_diff[i] = tx_cache[i] - tx_cache[cm->tx_mode];
+        else
+          ctx->tx_rd_diff[i] = 0;
+      }
   }
 
   ctx->mic = *xd->this_mi;
-- 
⑨